1a88b5ba8SSam Ravnborg /* ldc.c: Logical Domain Channel link-layer protocol driver. 2a88b5ba8SSam Ravnborg * 3a88b5ba8SSam Ravnborg * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> 4a88b5ba8SSam Ravnborg */ 5a88b5ba8SSam Ravnborg 6a88b5ba8SSam Ravnborg #include <linux/kernel.h> 7066bcacaSPaul Gortmaker #include <linux/export.h> 8a88b5ba8SSam Ravnborg #include <linux/slab.h> 9a88b5ba8SSam Ravnborg #include <linux/spinlock.h> 10a88b5ba8SSam Ravnborg #include <linux/delay.h> 11a88b5ba8SSam Ravnborg #include <linux/errno.h> 12a88b5ba8SSam Ravnborg #include <linux/string.h> 13a88b5ba8SSam Ravnborg #include <linux/scatterlist.h> 14a88b5ba8SSam Ravnborg #include <linux/interrupt.h> 15a88b5ba8SSam Ravnborg #include <linux/list.h> 16a88b5ba8SSam Ravnborg #include <linux/init.h> 17e756fd80SAkinobu Mita #include <linux/bitmap.h> 180ae53ed1SSowmini Varadhan #include <linux/iommu-common.h> 19a88b5ba8SSam Ravnborg 20a88b5ba8SSam Ravnborg #include <asm/hypervisor.h> 21a88b5ba8SSam Ravnborg #include <asm/iommu.h> 22a88b5ba8SSam Ravnborg #include <asm/page.h> 23a88b5ba8SSam Ravnborg #include <asm/ldc.h> 24a88b5ba8SSam Ravnborg #include <asm/mdesc.h> 25a88b5ba8SSam Ravnborg 26a88b5ba8SSam Ravnborg #define DRV_MODULE_NAME "ldc" 27a88b5ba8SSam Ravnborg #define PFX DRV_MODULE_NAME ": " 28a88b5ba8SSam Ravnborg #define DRV_MODULE_VERSION "1.1" 29a88b5ba8SSam Ravnborg #define DRV_MODULE_RELDATE "July 22, 2008" 30a88b5ba8SSam Ravnborg 310ae53ed1SSowmini Varadhan #define COOKIE_PGSZ_CODE 0xf000000000000000ULL 320ae53ed1SSowmini Varadhan #define COOKIE_PGSZ_CODE_SHIFT 60ULL 330ae53ed1SSowmini Varadhan 340ae53ed1SSowmini Varadhan 357c9503b8SGreg Kroah-Hartman static char version[] = 36a88b5ba8SSam Ravnborg DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 37a88b5ba8SSam Ravnborg #define LDC_PACKET_SIZE 64 38a88b5ba8SSam Ravnborg 39a88b5ba8SSam Ravnborg /* Packet header layout for unreliable and reliable mode frames. 40a88b5ba8SSam Ravnborg * When in RAW mode, packets are simply straight 64-byte payloads 41a88b5ba8SSam Ravnborg * with no headers. 42a88b5ba8SSam Ravnborg */ 43a88b5ba8SSam Ravnborg struct ldc_packet { 44a88b5ba8SSam Ravnborg u8 type; 45a88b5ba8SSam Ravnborg #define LDC_CTRL 0x01 46a88b5ba8SSam Ravnborg #define LDC_DATA 0x02 47a88b5ba8SSam Ravnborg #define LDC_ERR 0x10 48a88b5ba8SSam Ravnborg 49a88b5ba8SSam Ravnborg u8 stype; 50a88b5ba8SSam Ravnborg #define LDC_INFO 0x01 51a88b5ba8SSam Ravnborg #define LDC_ACK 0x02 52a88b5ba8SSam Ravnborg #define LDC_NACK 0x04 53a88b5ba8SSam Ravnborg 54a88b5ba8SSam Ravnborg u8 ctrl; 55a88b5ba8SSam Ravnborg #define LDC_VERS 0x01 /* Link Version */ 56a88b5ba8SSam Ravnborg #define LDC_RTS 0x02 /* Request To Send */ 57a88b5ba8SSam Ravnborg #define LDC_RTR 0x03 /* Ready To Receive */ 58a88b5ba8SSam Ravnborg #define LDC_RDX 0x04 /* Ready for Data eXchange */ 59a88b5ba8SSam Ravnborg #define LDC_CTRL_MSK 0x0f 60a88b5ba8SSam Ravnborg 61a88b5ba8SSam Ravnborg u8 env; 62a88b5ba8SSam Ravnborg #define LDC_LEN 0x3f 63a88b5ba8SSam Ravnborg #define LDC_FRAG_MASK 0xc0 64a88b5ba8SSam Ravnborg #define LDC_START 0x40 65a88b5ba8SSam Ravnborg #define LDC_STOP 0x80 66a88b5ba8SSam Ravnborg 67a88b5ba8SSam Ravnborg u32 seqid; 68a88b5ba8SSam Ravnborg 69a88b5ba8SSam Ravnborg union { 70a88b5ba8SSam Ravnborg u8 u_data[LDC_PACKET_SIZE - 8]; 71a88b5ba8SSam Ravnborg struct { 72a88b5ba8SSam Ravnborg u32 pad; 73a88b5ba8SSam Ravnborg u32 ackid; 74a88b5ba8SSam Ravnborg u8 r_data[LDC_PACKET_SIZE - 8 - 8]; 75a88b5ba8SSam Ravnborg } r; 76a88b5ba8SSam Ravnborg } u; 77a88b5ba8SSam Ravnborg }; 78a88b5ba8SSam Ravnborg 79a88b5ba8SSam Ravnborg struct ldc_version { 80a88b5ba8SSam Ravnborg u16 major; 81a88b5ba8SSam Ravnborg u16 minor; 82a88b5ba8SSam Ravnborg }; 83a88b5ba8SSam Ravnborg 84a88b5ba8SSam Ravnborg /* Ordered from largest major to lowest. */ 85a88b5ba8SSam Ravnborg static struct ldc_version ver_arr[] = { 86a88b5ba8SSam Ravnborg { .major = 1, .minor = 0 }, 87a88b5ba8SSam Ravnborg }; 88a88b5ba8SSam Ravnborg 89a88b5ba8SSam Ravnborg #define LDC_DEFAULT_MTU (4 * LDC_PACKET_SIZE) 90a88b5ba8SSam Ravnborg #define LDC_DEFAULT_NUM_ENTRIES (PAGE_SIZE / LDC_PACKET_SIZE) 91a88b5ba8SSam Ravnborg 92a88b5ba8SSam Ravnborg struct ldc_channel; 93a88b5ba8SSam Ravnborg 94a88b5ba8SSam Ravnborg struct ldc_mode_ops { 95a88b5ba8SSam Ravnborg int (*write)(struct ldc_channel *, const void *, unsigned int); 96a88b5ba8SSam Ravnborg int (*read)(struct ldc_channel *, void *, unsigned int); 97a88b5ba8SSam Ravnborg }; 98a88b5ba8SSam Ravnborg 99a88b5ba8SSam Ravnborg static const struct ldc_mode_ops raw_ops; 100a88b5ba8SSam Ravnborg static const struct ldc_mode_ops nonraw_ops; 101a88b5ba8SSam Ravnborg static const struct ldc_mode_ops stream_ops; 102a88b5ba8SSam Ravnborg 103a88b5ba8SSam Ravnborg int ldom_domaining_enabled; 104a88b5ba8SSam Ravnborg 105a88b5ba8SSam Ravnborg struct ldc_iommu { 1060ae53ed1SSowmini Varadhan /* Protects ldc_unmap. */ 107a88b5ba8SSam Ravnborg spinlock_t lock; 108a88b5ba8SSam Ravnborg struct ldc_mtable_entry *page_table; 1090ae53ed1SSowmini Varadhan struct iommu_map_table iommu_map_table; 110a88b5ba8SSam Ravnborg }; 111a88b5ba8SSam Ravnborg 112a88b5ba8SSam Ravnborg struct ldc_channel { 113a88b5ba8SSam Ravnborg /* Protects all operations that depend upon channel state. */ 114a88b5ba8SSam Ravnborg spinlock_t lock; 115a88b5ba8SSam Ravnborg 116a88b5ba8SSam Ravnborg unsigned long id; 117a88b5ba8SSam Ravnborg 118a88b5ba8SSam Ravnborg u8 *mssbuf; 119a88b5ba8SSam Ravnborg u32 mssbuf_len; 120a88b5ba8SSam Ravnborg u32 mssbuf_off; 121a88b5ba8SSam Ravnborg 122a88b5ba8SSam Ravnborg struct ldc_packet *tx_base; 123a88b5ba8SSam Ravnborg unsigned long tx_head; 124a88b5ba8SSam Ravnborg unsigned long tx_tail; 125a88b5ba8SSam Ravnborg unsigned long tx_num_entries; 126a88b5ba8SSam Ravnborg unsigned long tx_ra; 127a88b5ba8SSam Ravnborg 128a88b5ba8SSam Ravnborg unsigned long tx_acked; 129a88b5ba8SSam Ravnborg 130a88b5ba8SSam Ravnborg struct ldc_packet *rx_base; 131a88b5ba8SSam Ravnborg unsigned long rx_head; 132a88b5ba8SSam Ravnborg unsigned long rx_tail; 133a88b5ba8SSam Ravnborg unsigned long rx_num_entries; 134a88b5ba8SSam Ravnborg unsigned long rx_ra; 135a88b5ba8SSam Ravnborg 136a88b5ba8SSam Ravnborg u32 rcv_nxt; 137a88b5ba8SSam Ravnborg u32 snd_nxt; 138a88b5ba8SSam Ravnborg 139a88b5ba8SSam Ravnborg unsigned long chan_state; 140a88b5ba8SSam Ravnborg 141a88b5ba8SSam Ravnborg struct ldc_channel_config cfg; 142a88b5ba8SSam Ravnborg void *event_arg; 143a88b5ba8SSam Ravnborg 144a88b5ba8SSam Ravnborg const struct ldc_mode_ops *mops; 145a88b5ba8SSam Ravnborg 146a88b5ba8SSam Ravnborg struct ldc_iommu iommu; 147a88b5ba8SSam Ravnborg 148a88b5ba8SSam Ravnborg struct ldc_version ver; 149a88b5ba8SSam Ravnborg 150a88b5ba8SSam Ravnborg u8 hs_state; 151a88b5ba8SSam Ravnborg #define LDC_HS_CLOSED 0x00 152a88b5ba8SSam Ravnborg #define LDC_HS_OPEN 0x01 153a88b5ba8SSam Ravnborg #define LDC_HS_GOTVERS 0x02 154a88b5ba8SSam Ravnborg #define LDC_HS_SENTRTR 0x03 155a88b5ba8SSam Ravnborg #define LDC_HS_GOTRTR 0x04 156a88b5ba8SSam Ravnborg #define LDC_HS_COMPLETE 0x10 157a88b5ba8SSam Ravnborg 158a88b5ba8SSam Ravnborg u8 flags; 159a88b5ba8SSam Ravnborg #define LDC_FLAG_ALLOCED_QUEUES 0x01 160a88b5ba8SSam Ravnborg #define LDC_FLAG_REGISTERED_QUEUES 0x02 161a88b5ba8SSam Ravnborg #define LDC_FLAG_REGISTERED_IRQS 0x04 162a88b5ba8SSam Ravnborg #define LDC_FLAG_RESET 0x10 163a88b5ba8SSam Ravnborg 164a88b5ba8SSam Ravnborg u8 mss; 165a88b5ba8SSam Ravnborg u8 state; 166a88b5ba8SSam Ravnborg 167a88b5ba8SSam Ravnborg #define LDC_IRQ_NAME_MAX 32 168a88b5ba8SSam Ravnborg char rx_irq_name[LDC_IRQ_NAME_MAX]; 169a88b5ba8SSam Ravnborg char tx_irq_name[LDC_IRQ_NAME_MAX]; 170a88b5ba8SSam Ravnborg 171a88b5ba8SSam Ravnborg struct hlist_head mh_list; 172a88b5ba8SSam Ravnborg 173a88b5ba8SSam Ravnborg struct hlist_node list; 174a88b5ba8SSam Ravnborg }; 175a88b5ba8SSam Ravnborg 176a88b5ba8SSam Ravnborg #define ldcdbg(TYPE, f, a...) \ 177a88b5ba8SSam Ravnborg do { if (lp->cfg.debug & LDC_DEBUG_##TYPE) \ 178a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \ 179a88b5ba8SSam Ravnborg } while (0) 180a88b5ba8SSam Ravnborg 181a88b5ba8SSam Ravnborg static const char *state_to_str(u8 state) 182a88b5ba8SSam Ravnborg { 183a88b5ba8SSam Ravnborg switch (state) { 184a88b5ba8SSam Ravnborg case LDC_STATE_INVALID: 185a88b5ba8SSam Ravnborg return "INVALID"; 186a88b5ba8SSam Ravnborg case LDC_STATE_INIT: 187a88b5ba8SSam Ravnborg return "INIT"; 188a88b5ba8SSam Ravnborg case LDC_STATE_BOUND: 189a88b5ba8SSam Ravnborg return "BOUND"; 190a88b5ba8SSam Ravnborg case LDC_STATE_READY: 191a88b5ba8SSam Ravnborg return "READY"; 192a88b5ba8SSam Ravnborg case LDC_STATE_CONNECTED: 193a88b5ba8SSam Ravnborg return "CONNECTED"; 194a88b5ba8SSam Ravnborg default: 195a88b5ba8SSam Ravnborg return "<UNKNOWN>"; 196a88b5ba8SSam Ravnborg } 197a88b5ba8SSam Ravnborg } 198a88b5ba8SSam Ravnborg 199a88b5ba8SSam Ravnborg static void ldc_set_state(struct ldc_channel *lp, u8 state) 200a88b5ba8SSam Ravnborg { 201a88b5ba8SSam Ravnborg ldcdbg(STATE, "STATE (%s) --> (%s)\n", 202a88b5ba8SSam Ravnborg state_to_str(lp->state), 203a88b5ba8SSam Ravnborg state_to_str(state)); 204a88b5ba8SSam Ravnborg 205a88b5ba8SSam Ravnborg lp->state = state; 206a88b5ba8SSam Ravnborg } 207a88b5ba8SSam Ravnborg 208a88b5ba8SSam Ravnborg static unsigned long __advance(unsigned long off, unsigned long num_entries) 209a88b5ba8SSam Ravnborg { 210a88b5ba8SSam Ravnborg off += LDC_PACKET_SIZE; 211a88b5ba8SSam Ravnborg if (off == (num_entries * LDC_PACKET_SIZE)) 212a88b5ba8SSam Ravnborg off = 0; 213a88b5ba8SSam Ravnborg 214a88b5ba8SSam Ravnborg return off; 215a88b5ba8SSam Ravnborg } 216a88b5ba8SSam Ravnborg 217a88b5ba8SSam Ravnborg static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off) 218a88b5ba8SSam Ravnborg { 219a88b5ba8SSam Ravnborg return __advance(off, lp->rx_num_entries); 220a88b5ba8SSam Ravnborg } 221a88b5ba8SSam Ravnborg 222a88b5ba8SSam Ravnborg static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off) 223a88b5ba8SSam Ravnborg { 224a88b5ba8SSam Ravnborg return __advance(off, lp->tx_num_entries); 225a88b5ba8SSam Ravnborg } 226a88b5ba8SSam Ravnborg 227a88b5ba8SSam Ravnborg static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp, 228a88b5ba8SSam Ravnborg unsigned long *new_tail) 229a88b5ba8SSam Ravnborg { 230a88b5ba8SSam Ravnborg struct ldc_packet *p; 231a88b5ba8SSam Ravnborg unsigned long t; 232a88b5ba8SSam Ravnborg 233a88b5ba8SSam Ravnborg t = tx_advance(lp, lp->tx_tail); 234a88b5ba8SSam Ravnborg if (t == lp->tx_head) 235a88b5ba8SSam Ravnborg return NULL; 236a88b5ba8SSam Ravnborg 237a88b5ba8SSam Ravnborg *new_tail = t; 238a88b5ba8SSam Ravnborg 239a88b5ba8SSam Ravnborg p = lp->tx_base; 240a88b5ba8SSam Ravnborg return p + (lp->tx_tail / LDC_PACKET_SIZE); 241a88b5ba8SSam Ravnborg } 242a88b5ba8SSam Ravnborg 243a88b5ba8SSam Ravnborg /* When we are in reliable or stream mode, have to track the next packet 244a88b5ba8SSam Ravnborg * we haven't gotten an ACK for in the TX queue using tx_acked. We have 245a88b5ba8SSam Ravnborg * to be careful not to stomp over the queue past that point. During 246a88b5ba8SSam Ravnborg * the handshake, we don't have TX data packets pending in the queue 247a88b5ba8SSam Ravnborg * and that's why handshake_get_tx_packet() need not be mindful of 248a88b5ba8SSam Ravnborg * lp->tx_acked. 249a88b5ba8SSam Ravnborg */ 250a88b5ba8SSam Ravnborg static unsigned long head_for_data(struct ldc_channel *lp) 251a88b5ba8SSam Ravnborg { 252a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_STREAM) 253a88b5ba8SSam Ravnborg return lp->tx_acked; 254a88b5ba8SSam Ravnborg return lp->tx_head; 255a88b5ba8SSam Ravnborg } 256a88b5ba8SSam Ravnborg 257a88b5ba8SSam Ravnborg static int tx_has_space_for(struct ldc_channel *lp, unsigned int size) 258a88b5ba8SSam Ravnborg { 259a88b5ba8SSam Ravnborg unsigned long limit, tail, new_tail, diff; 260a88b5ba8SSam Ravnborg unsigned int mss; 261a88b5ba8SSam Ravnborg 262a88b5ba8SSam Ravnborg limit = head_for_data(lp); 263a88b5ba8SSam Ravnborg tail = lp->tx_tail; 264a88b5ba8SSam Ravnborg new_tail = tx_advance(lp, tail); 265a88b5ba8SSam Ravnborg if (new_tail == limit) 266a88b5ba8SSam Ravnborg return 0; 267a88b5ba8SSam Ravnborg 268a88b5ba8SSam Ravnborg if (limit > new_tail) 269a88b5ba8SSam Ravnborg diff = limit - new_tail; 270a88b5ba8SSam Ravnborg else 271a88b5ba8SSam Ravnborg diff = (limit + 272a88b5ba8SSam Ravnborg ((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail)); 273a88b5ba8SSam Ravnborg diff /= LDC_PACKET_SIZE; 274a88b5ba8SSam Ravnborg mss = lp->mss; 275a88b5ba8SSam Ravnborg 276a88b5ba8SSam Ravnborg if (diff * mss < size) 277a88b5ba8SSam Ravnborg return 0; 278a88b5ba8SSam Ravnborg 279a88b5ba8SSam Ravnborg return 1; 280a88b5ba8SSam Ravnborg } 281a88b5ba8SSam Ravnborg 282a88b5ba8SSam Ravnborg static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp, 283a88b5ba8SSam Ravnborg unsigned long *new_tail) 284a88b5ba8SSam Ravnborg { 285a88b5ba8SSam Ravnborg struct ldc_packet *p; 286a88b5ba8SSam Ravnborg unsigned long h, t; 287a88b5ba8SSam Ravnborg 288a88b5ba8SSam Ravnborg h = head_for_data(lp); 289a88b5ba8SSam Ravnborg t = tx_advance(lp, lp->tx_tail); 290a88b5ba8SSam Ravnborg if (t == h) 291a88b5ba8SSam Ravnborg return NULL; 292a88b5ba8SSam Ravnborg 293a88b5ba8SSam Ravnborg *new_tail = t; 294a88b5ba8SSam Ravnborg 295a88b5ba8SSam Ravnborg p = lp->tx_base; 296a88b5ba8SSam Ravnborg return p + (lp->tx_tail / LDC_PACKET_SIZE); 297a88b5ba8SSam Ravnborg } 298a88b5ba8SSam Ravnborg 299a88b5ba8SSam Ravnborg static int set_tx_tail(struct ldc_channel *lp, unsigned long tail) 300a88b5ba8SSam Ravnborg { 301a88b5ba8SSam Ravnborg unsigned long orig_tail = lp->tx_tail; 302a88b5ba8SSam Ravnborg int limit = 1000; 303a88b5ba8SSam Ravnborg 304a88b5ba8SSam Ravnborg lp->tx_tail = tail; 305a88b5ba8SSam Ravnborg while (limit-- > 0) { 306a88b5ba8SSam Ravnborg unsigned long err; 307a88b5ba8SSam Ravnborg 308a88b5ba8SSam Ravnborg err = sun4v_ldc_tx_set_qtail(lp->id, tail); 309a88b5ba8SSam Ravnborg if (!err) 310a88b5ba8SSam Ravnborg return 0; 311a88b5ba8SSam Ravnborg 312a88b5ba8SSam Ravnborg if (err != HV_EWOULDBLOCK) { 313a88b5ba8SSam Ravnborg lp->tx_tail = orig_tail; 314a88b5ba8SSam Ravnborg return -EINVAL; 315a88b5ba8SSam Ravnborg } 316a88b5ba8SSam Ravnborg udelay(1); 317a88b5ba8SSam Ravnborg } 318a88b5ba8SSam Ravnborg 319a88b5ba8SSam Ravnborg lp->tx_tail = orig_tail; 320a88b5ba8SSam Ravnborg return -EBUSY; 321a88b5ba8SSam Ravnborg } 322a88b5ba8SSam Ravnborg 323a88b5ba8SSam Ravnborg /* This just updates the head value in the hypervisor using 324a88b5ba8SSam Ravnborg * a polling loop with a timeout. The caller takes care of 325a88b5ba8SSam Ravnborg * upating software state representing the head change, if any. 326a88b5ba8SSam Ravnborg */ 327a88b5ba8SSam Ravnborg static int __set_rx_head(struct ldc_channel *lp, unsigned long head) 328a88b5ba8SSam Ravnborg { 329a88b5ba8SSam Ravnborg int limit = 1000; 330a88b5ba8SSam Ravnborg 331a88b5ba8SSam Ravnborg while (limit-- > 0) { 332a88b5ba8SSam Ravnborg unsigned long err; 333a88b5ba8SSam Ravnborg 334a88b5ba8SSam Ravnborg err = sun4v_ldc_rx_set_qhead(lp->id, head); 335a88b5ba8SSam Ravnborg if (!err) 336a88b5ba8SSam Ravnborg return 0; 337a88b5ba8SSam Ravnborg 338a88b5ba8SSam Ravnborg if (err != HV_EWOULDBLOCK) 339a88b5ba8SSam Ravnborg return -EINVAL; 340a88b5ba8SSam Ravnborg 341a88b5ba8SSam Ravnborg udelay(1); 342a88b5ba8SSam Ravnborg } 343a88b5ba8SSam Ravnborg 344a88b5ba8SSam Ravnborg return -EBUSY; 345a88b5ba8SSam Ravnborg } 346a88b5ba8SSam Ravnborg 347a88b5ba8SSam Ravnborg static int send_tx_packet(struct ldc_channel *lp, 348a88b5ba8SSam Ravnborg struct ldc_packet *p, 349a88b5ba8SSam Ravnborg unsigned long new_tail) 350a88b5ba8SSam Ravnborg { 351a88b5ba8SSam Ravnborg BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE))); 352a88b5ba8SSam Ravnborg 353a88b5ba8SSam Ravnborg return set_tx_tail(lp, new_tail); 354a88b5ba8SSam Ravnborg } 355a88b5ba8SSam Ravnborg 356a88b5ba8SSam Ravnborg static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp, 357a88b5ba8SSam Ravnborg u8 stype, u8 ctrl, 358a88b5ba8SSam Ravnborg void *data, int dlen, 359a88b5ba8SSam Ravnborg unsigned long *new_tail) 360a88b5ba8SSam Ravnborg { 361a88b5ba8SSam Ravnborg struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail); 362a88b5ba8SSam Ravnborg 363a88b5ba8SSam Ravnborg if (p) { 364a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p)); 365a88b5ba8SSam Ravnborg p->type = LDC_CTRL; 366a88b5ba8SSam Ravnborg p->stype = stype; 367a88b5ba8SSam Ravnborg p->ctrl = ctrl; 368a88b5ba8SSam Ravnborg if (data) 369a88b5ba8SSam Ravnborg memcpy(p->u.u_data, data, dlen); 370a88b5ba8SSam Ravnborg } 371a88b5ba8SSam Ravnborg return p; 372a88b5ba8SSam Ravnborg } 373a88b5ba8SSam Ravnborg 374a88b5ba8SSam Ravnborg static int start_handshake(struct ldc_channel *lp) 375a88b5ba8SSam Ravnborg { 376a88b5ba8SSam Ravnborg struct ldc_packet *p; 377a88b5ba8SSam Ravnborg struct ldc_version *ver; 378a88b5ba8SSam Ravnborg unsigned long new_tail; 379a88b5ba8SSam Ravnborg 380a88b5ba8SSam Ravnborg ver = &ver_arr[0]; 381a88b5ba8SSam Ravnborg 382a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n", 383a88b5ba8SSam Ravnborg ver->major, ver->minor); 384a88b5ba8SSam Ravnborg 385a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS, 386a88b5ba8SSam Ravnborg ver, sizeof(*ver), &new_tail); 387a88b5ba8SSam Ravnborg if (p) { 388a88b5ba8SSam Ravnborg int err = send_tx_packet(lp, p, new_tail); 389a88b5ba8SSam Ravnborg if (!err) 390a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_RESET; 391a88b5ba8SSam Ravnborg return err; 392a88b5ba8SSam Ravnborg } 393a88b5ba8SSam Ravnborg return -EBUSY; 394a88b5ba8SSam Ravnborg } 395a88b5ba8SSam Ravnborg 396a88b5ba8SSam Ravnborg static int send_version_nack(struct ldc_channel *lp, 397a88b5ba8SSam Ravnborg u16 major, u16 minor) 398a88b5ba8SSam Ravnborg { 399a88b5ba8SSam Ravnborg struct ldc_packet *p; 400a88b5ba8SSam Ravnborg struct ldc_version ver; 401a88b5ba8SSam Ravnborg unsigned long new_tail; 402a88b5ba8SSam Ravnborg 403a88b5ba8SSam Ravnborg ver.major = major; 404a88b5ba8SSam Ravnborg ver.minor = minor; 405a88b5ba8SSam Ravnborg 406a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS, 407a88b5ba8SSam Ravnborg &ver, sizeof(ver), &new_tail); 408a88b5ba8SSam Ravnborg if (p) { 409a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n", 410a88b5ba8SSam Ravnborg ver.major, ver.minor); 411a88b5ba8SSam Ravnborg 412a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 413a88b5ba8SSam Ravnborg } 414a88b5ba8SSam Ravnborg return -EBUSY; 415a88b5ba8SSam Ravnborg } 416a88b5ba8SSam Ravnborg 417a88b5ba8SSam Ravnborg static int send_version_ack(struct ldc_channel *lp, 418a88b5ba8SSam Ravnborg struct ldc_version *vp) 419a88b5ba8SSam Ravnborg { 420a88b5ba8SSam Ravnborg struct ldc_packet *p; 421a88b5ba8SSam Ravnborg unsigned long new_tail; 422a88b5ba8SSam Ravnborg 423a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS, 424a88b5ba8SSam Ravnborg vp, sizeof(*vp), &new_tail); 425a88b5ba8SSam Ravnborg if (p) { 426a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n", 427a88b5ba8SSam Ravnborg vp->major, vp->minor); 428a88b5ba8SSam Ravnborg 429a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 430a88b5ba8SSam Ravnborg } 431a88b5ba8SSam Ravnborg return -EBUSY; 432a88b5ba8SSam Ravnborg } 433a88b5ba8SSam Ravnborg 434a88b5ba8SSam Ravnborg static int send_rts(struct ldc_channel *lp) 435a88b5ba8SSam Ravnborg { 436a88b5ba8SSam Ravnborg struct ldc_packet *p; 437a88b5ba8SSam Ravnborg unsigned long new_tail; 438a88b5ba8SSam Ravnborg 439a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0, 440a88b5ba8SSam Ravnborg &new_tail); 441a88b5ba8SSam Ravnborg if (p) { 442a88b5ba8SSam Ravnborg p->env = lp->cfg.mode; 443a88b5ba8SSam Ravnborg p->seqid = 0; 444a88b5ba8SSam Ravnborg lp->rcv_nxt = 0; 445a88b5ba8SSam Ravnborg 446a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n", 447a88b5ba8SSam Ravnborg p->env, p->seqid); 448a88b5ba8SSam Ravnborg 449a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 450a88b5ba8SSam Ravnborg } 451a88b5ba8SSam Ravnborg return -EBUSY; 452a88b5ba8SSam Ravnborg } 453a88b5ba8SSam Ravnborg 454a88b5ba8SSam Ravnborg static int send_rtr(struct ldc_channel *lp) 455a88b5ba8SSam Ravnborg { 456a88b5ba8SSam Ravnborg struct ldc_packet *p; 457a88b5ba8SSam Ravnborg unsigned long new_tail; 458a88b5ba8SSam Ravnborg 459a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0, 460a88b5ba8SSam Ravnborg &new_tail); 461a88b5ba8SSam Ravnborg if (p) { 462a88b5ba8SSam Ravnborg p->env = lp->cfg.mode; 463a88b5ba8SSam Ravnborg p->seqid = 0; 464a88b5ba8SSam Ravnborg 465a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n", 466a88b5ba8SSam Ravnborg p->env, p->seqid); 467a88b5ba8SSam Ravnborg 468a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 469a88b5ba8SSam Ravnborg } 470a88b5ba8SSam Ravnborg return -EBUSY; 471a88b5ba8SSam Ravnborg } 472a88b5ba8SSam Ravnborg 473a88b5ba8SSam Ravnborg static int send_rdx(struct ldc_channel *lp) 474a88b5ba8SSam Ravnborg { 475a88b5ba8SSam Ravnborg struct ldc_packet *p; 476a88b5ba8SSam Ravnborg unsigned long new_tail; 477a88b5ba8SSam Ravnborg 478a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0, 479a88b5ba8SSam Ravnborg &new_tail); 480a88b5ba8SSam Ravnborg if (p) { 481a88b5ba8SSam Ravnborg p->env = 0; 482a88b5ba8SSam Ravnborg p->seqid = ++lp->snd_nxt; 483a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt; 484a88b5ba8SSam Ravnborg 485a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n", 486a88b5ba8SSam Ravnborg p->env, p->seqid, p->u.r.ackid); 487a88b5ba8SSam Ravnborg 488a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 489a88b5ba8SSam Ravnborg } 490a88b5ba8SSam Ravnborg return -EBUSY; 491a88b5ba8SSam Ravnborg } 492a88b5ba8SSam Ravnborg 493a88b5ba8SSam Ravnborg static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt) 494a88b5ba8SSam Ravnborg { 495a88b5ba8SSam Ravnborg struct ldc_packet *p; 496a88b5ba8SSam Ravnborg unsigned long new_tail; 497a88b5ba8SSam Ravnborg int err; 498a88b5ba8SSam Ravnborg 499a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail); 500a88b5ba8SSam Ravnborg if (!p) 501a88b5ba8SSam Ravnborg return -EBUSY; 502a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p)); 503a88b5ba8SSam Ravnborg p->type = data_pkt->type; 504a88b5ba8SSam Ravnborg p->stype = LDC_NACK; 505a88b5ba8SSam Ravnborg p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK; 506a88b5ba8SSam Ravnborg p->seqid = lp->snd_nxt + 1; 507a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt; 508a88b5ba8SSam Ravnborg 509a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n", 510a88b5ba8SSam Ravnborg p->type, p->ctrl, p->seqid, p->u.r.ackid); 511a88b5ba8SSam Ravnborg 512a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail); 513a88b5ba8SSam Ravnborg if (!err) 514a88b5ba8SSam Ravnborg lp->snd_nxt++; 515a88b5ba8SSam Ravnborg 516a88b5ba8SSam Ravnborg return err; 517a88b5ba8SSam Ravnborg } 518a88b5ba8SSam Ravnborg 519a88b5ba8SSam Ravnborg static int ldc_abort(struct ldc_channel *lp) 520a88b5ba8SSam Ravnborg { 521a88b5ba8SSam Ravnborg unsigned long hv_err; 522a88b5ba8SSam Ravnborg 523a88b5ba8SSam Ravnborg ldcdbg(STATE, "ABORT\n"); 524a88b5ba8SSam Ravnborg 525a88b5ba8SSam Ravnborg /* We report but do not act upon the hypervisor errors because 526a88b5ba8SSam Ravnborg * there really isn't much we can do if they fail at this point. 527a88b5ba8SSam Ravnborg */ 528a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); 529a88b5ba8SSam Ravnborg if (hv_err) 530a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 531a88b5ba8SSam Ravnborg "sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n", 532a88b5ba8SSam Ravnborg lp->id, lp->tx_ra, lp->tx_num_entries, hv_err); 533a88b5ba8SSam Ravnborg 534a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, 535a88b5ba8SSam Ravnborg &lp->tx_head, 536a88b5ba8SSam Ravnborg &lp->tx_tail, 537a88b5ba8SSam Ravnborg &lp->chan_state); 538a88b5ba8SSam Ravnborg if (hv_err) 539a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 540a88b5ba8SSam Ravnborg "sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n", 541a88b5ba8SSam Ravnborg lp->id, hv_err); 542a88b5ba8SSam Ravnborg 543a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); 544a88b5ba8SSam Ravnborg if (hv_err) 545a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 546a88b5ba8SSam Ravnborg "sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n", 547a88b5ba8SSam Ravnborg lp->id, lp->rx_ra, lp->rx_num_entries, hv_err); 548a88b5ba8SSam Ravnborg 549a88b5ba8SSam Ravnborg /* Refetch the RX queue state as well, because we could be invoked 550a88b5ba8SSam Ravnborg * here in the queue processing context. 551a88b5ba8SSam Ravnborg */ 552a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 553a88b5ba8SSam Ravnborg &lp->rx_head, 554a88b5ba8SSam Ravnborg &lp->rx_tail, 555a88b5ba8SSam Ravnborg &lp->chan_state); 556a88b5ba8SSam Ravnborg if (hv_err) 557a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 558a88b5ba8SSam Ravnborg "sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n", 559a88b5ba8SSam Ravnborg lp->id, hv_err); 560a88b5ba8SSam Ravnborg 561a88b5ba8SSam Ravnborg return -ECONNRESET; 562a88b5ba8SSam Ravnborg } 563a88b5ba8SSam Ravnborg 564a88b5ba8SSam Ravnborg static struct ldc_version *find_by_major(u16 major) 565a88b5ba8SSam Ravnborg { 566a88b5ba8SSam Ravnborg struct ldc_version *ret = NULL; 567a88b5ba8SSam Ravnborg int i; 568a88b5ba8SSam Ravnborg 569a88b5ba8SSam Ravnborg for (i = 0; i < ARRAY_SIZE(ver_arr); i++) { 570a88b5ba8SSam Ravnborg struct ldc_version *v = &ver_arr[i]; 571a88b5ba8SSam Ravnborg if (v->major <= major) { 572a88b5ba8SSam Ravnborg ret = v; 573a88b5ba8SSam Ravnborg break; 574a88b5ba8SSam Ravnborg } 575a88b5ba8SSam Ravnborg } 576a88b5ba8SSam Ravnborg return ret; 577a88b5ba8SSam Ravnborg } 578a88b5ba8SSam Ravnborg 579a88b5ba8SSam Ravnborg static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp) 580a88b5ba8SSam Ravnborg { 581a88b5ba8SSam Ravnborg struct ldc_version *vap; 582a88b5ba8SSam Ravnborg int err; 583a88b5ba8SSam Ravnborg 584a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n", 585a88b5ba8SSam Ravnborg vp->major, vp->minor); 586a88b5ba8SSam Ravnborg 587a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_GOTVERS) { 588a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN; 589a88b5ba8SSam Ravnborg memset(&lp->ver, 0, sizeof(lp->ver)); 590a88b5ba8SSam Ravnborg } 591a88b5ba8SSam Ravnborg 592a88b5ba8SSam Ravnborg vap = find_by_major(vp->major); 593a88b5ba8SSam Ravnborg if (!vap) { 594a88b5ba8SSam Ravnborg err = send_version_nack(lp, 0, 0); 595a88b5ba8SSam Ravnborg } else if (vap->major != vp->major) { 596a88b5ba8SSam Ravnborg err = send_version_nack(lp, vap->major, vap->minor); 597a88b5ba8SSam Ravnborg } else { 598a88b5ba8SSam Ravnborg struct ldc_version ver = *vp; 599a88b5ba8SSam Ravnborg if (ver.minor > vap->minor) 600a88b5ba8SSam Ravnborg ver.minor = vap->minor; 601a88b5ba8SSam Ravnborg err = send_version_ack(lp, &ver); 602a88b5ba8SSam Ravnborg if (!err) { 603a88b5ba8SSam Ravnborg lp->ver = ver; 604a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_GOTVERS; 605a88b5ba8SSam Ravnborg } 606a88b5ba8SSam Ravnborg } 607a88b5ba8SSam Ravnborg if (err) 608a88b5ba8SSam Ravnborg return ldc_abort(lp); 609a88b5ba8SSam Ravnborg 610a88b5ba8SSam Ravnborg return 0; 611a88b5ba8SSam Ravnborg } 612a88b5ba8SSam Ravnborg 613a88b5ba8SSam Ravnborg static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp) 614a88b5ba8SSam Ravnborg { 615a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n", 616a88b5ba8SSam Ravnborg vp->major, vp->minor); 617a88b5ba8SSam Ravnborg 618a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_GOTVERS) { 619a88b5ba8SSam Ravnborg if (lp->ver.major != vp->major || 620a88b5ba8SSam Ravnborg lp->ver.minor != vp->minor) 621a88b5ba8SSam Ravnborg return ldc_abort(lp); 622a88b5ba8SSam Ravnborg } else { 623a88b5ba8SSam Ravnborg lp->ver = *vp; 624a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_GOTVERS; 625a88b5ba8SSam Ravnborg } 626a88b5ba8SSam Ravnborg if (send_rts(lp)) 627a88b5ba8SSam Ravnborg return ldc_abort(lp); 628a88b5ba8SSam Ravnborg return 0; 629a88b5ba8SSam Ravnborg } 630a88b5ba8SSam Ravnborg 631a88b5ba8SSam Ravnborg static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp) 632a88b5ba8SSam Ravnborg { 633a88b5ba8SSam Ravnborg struct ldc_version *vap; 634a88b5ba8SSam Ravnborg struct ldc_packet *p; 635a88b5ba8SSam Ravnborg unsigned long new_tail; 636a88b5ba8SSam Ravnborg 637b2c0805fSSteven Rostedt if (vp->major == 0 && vp->minor == 0) 638b2c0805fSSteven Rostedt return ldc_abort(lp); 639b2c0805fSSteven Rostedt 640b2c0805fSSteven Rostedt vap = find_by_major(vp->major); 641b2c0805fSSteven Rostedt if (!vap) 642b2c0805fSSteven Rostedt return ldc_abort(lp); 643b2c0805fSSteven Rostedt 644a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS, 645a88b5ba8SSam Ravnborg vap, sizeof(*vap), 646a88b5ba8SSam Ravnborg &new_tail); 647b2c0805fSSteven Rostedt if (!p) 648a88b5ba8SSam Ravnborg return ldc_abort(lp); 649b2c0805fSSteven Rostedt 650b2c0805fSSteven Rostedt return send_tx_packet(lp, p, new_tail); 651a88b5ba8SSam Ravnborg } 652a88b5ba8SSam Ravnborg 653a88b5ba8SSam Ravnborg static int process_version(struct ldc_channel *lp, 654a88b5ba8SSam Ravnborg struct ldc_packet *p) 655a88b5ba8SSam Ravnborg { 656a88b5ba8SSam Ravnborg struct ldc_version *vp; 657a88b5ba8SSam Ravnborg 658a88b5ba8SSam Ravnborg vp = (struct ldc_version *) p->u.u_data; 659a88b5ba8SSam Ravnborg 660a88b5ba8SSam Ravnborg switch (p->stype) { 661a88b5ba8SSam Ravnborg case LDC_INFO: 662a88b5ba8SSam Ravnborg return process_ver_info(lp, vp); 663a88b5ba8SSam Ravnborg 664a88b5ba8SSam Ravnborg case LDC_ACK: 665a88b5ba8SSam Ravnborg return process_ver_ack(lp, vp); 666a88b5ba8SSam Ravnborg 667a88b5ba8SSam Ravnborg case LDC_NACK: 668a88b5ba8SSam Ravnborg return process_ver_nack(lp, vp); 669a88b5ba8SSam Ravnborg 670a88b5ba8SSam Ravnborg default: 671a88b5ba8SSam Ravnborg return ldc_abort(lp); 672a88b5ba8SSam Ravnborg } 673a88b5ba8SSam Ravnborg } 674a88b5ba8SSam Ravnborg 675a88b5ba8SSam Ravnborg static int process_rts(struct ldc_channel *lp, 676a88b5ba8SSam Ravnborg struct ldc_packet *p) 677a88b5ba8SSam Ravnborg { 678a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n", 679a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env); 680a88b5ba8SSam Ravnborg 681a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO || 682a88b5ba8SSam Ravnborg lp->hs_state != LDC_HS_GOTVERS || 683a88b5ba8SSam Ravnborg p->env != lp->cfg.mode) 684a88b5ba8SSam Ravnborg return ldc_abort(lp); 685a88b5ba8SSam Ravnborg 686a88b5ba8SSam Ravnborg lp->snd_nxt = p->seqid; 687a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid; 688a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_SENTRTR; 689a88b5ba8SSam Ravnborg if (send_rtr(lp)) 690a88b5ba8SSam Ravnborg return ldc_abort(lp); 691a88b5ba8SSam Ravnborg 692a88b5ba8SSam Ravnborg return 0; 693a88b5ba8SSam Ravnborg } 694a88b5ba8SSam Ravnborg 695a88b5ba8SSam Ravnborg static int process_rtr(struct ldc_channel *lp, 696a88b5ba8SSam Ravnborg struct ldc_packet *p) 697a88b5ba8SSam Ravnborg { 698a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n", 699a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env); 700a88b5ba8SSam Ravnborg 701a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO || 702a88b5ba8SSam Ravnborg p->env != lp->cfg.mode) 703a88b5ba8SSam Ravnborg return ldc_abort(lp); 704a88b5ba8SSam Ravnborg 705a88b5ba8SSam Ravnborg lp->snd_nxt = p->seqid; 706a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 707a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 708a88b5ba8SSam Ravnborg send_rdx(lp); 709a88b5ba8SSam Ravnborg 710a88b5ba8SSam Ravnborg return LDC_EVENT_UP; 711a88b5ba8SSam Ravnborg } 712a88b5ba8SSam Ravnborg 713a88b5ba8SSam Ravnborg static int rx_seq_ok(struct ldc_channel *lp, u32 seqid) 714a88b5ba8SSam Ravnborg { 715a88b5ba8SSam Ravnborg return lp->rcv_nxt + 1 == seqid; 716a88b5ba8SSam Ravnborg } 717a88b5ba8SSam Ravnborg 718a88b5ba8SSam Ravnborg static int process_rdx(struct ldc_channel *lp, 719a88b5ba8SSam Ravnborg struct ldc_packet *p) 720a88b5ba8SSam Ravnborg { 721a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n", 722a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env, p->u.r.ackid); 723a88b5ba8SSam Ravnborg 724a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO || 725a88b5ba8SSam Ravnborg !(rx_seq_ok(lp, p->seqid))) 726a88b5ba8SSam Ravnborg return ldc_abort(lp); 727a88b5ba8SSam Ravnborg 728a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid; 729a88b5ba8SSam Ravnborg 730a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 731a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 732a88b5ba8SSam Ravnborg 733a88b5ba8SSam Ravnborg return LDC_EVENT_UP; 734a88b5ba8SSam Ravnborg } 735a88b5ba8SSam Ravnborg 736a88b5ba8SSam Ravnborg static int process_control_frame(struct ldc_channel *lp, 737a88b5ba8SSam Ravnborg struct ldc_packet *p) 738a88b5ba8SSam Ravnborg { 739a88b5ba8SSam Ravnborg switch (p->ctrl) { 740a88b5ba8SSam Ravnborg case LDC_VERS: 741a88b5ba8SSam Ravnborg return process_version(lp, p); 742a88b5ba8SSam Ravnborg 743a88b5ba8SSam Ravnborg case LDC_RTS: 744a88b5ba8SSam Ravnborg return process_rts(lp, p); 745a88b5ba8SSam Ravnborg 746a88b5ba8SSam Ravnborg case LDC_RTR: 747a88b5ba8SSam Ravnborg return process_rtr(lp, p); 748a88b5ba8SSam Ravnborg 749a88b5ba8SSam Ravnborg case LDC_RDX: 750a88b5ba8SSam Ravnborg return process_rdx(lp, p); 751a88b5ba8SSam Ravnborg 752a88b5ba8SSam Ravnborg default: 753a88b5ba8SSam Ravnborg return ldc_abort(lp); 754a88b5ba8SSam Ravnborg } 755a88b5ba8SSam Ravnborg } 756a88b5ba8SSam Ravnborg 757a88b5ba8SSam Ravnborg static int process_error_frame(struct ldc_channel *lp, 758a88b5ba8SSam Ravnborg struct ldc_packet *p) 759a88b5ba8SSam Ravnborg { 760a88b5ba8SSam Ravnborg return ldc_abort(lp); 761a88b5ba8SSam Ravnborg } 762a88b5ba8SSam Ravnborg 763a88b5ba8SSam Ravnborg static int process_data_ack(struct ldc_channel *lp, 764a88b5ba8SSam Ravnborg struct ldc_packet *ack) 765a88b5ba8SSam Ravnborg { 766a88b5ba8SSam Ravnborg unsigned long head = lp->tx_acked; 767a88b5ba8SSam Ravnborg u32 ackid = ack->u.r.ackid; 768a88b5ba8SSam Ravnborg 769a88b5ba8SSam Ravnborg while (1) { 770a88b5ba8SSam Ravnborg struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE); 771a88b5ba8SSam Ravnborg 772a88b5ba8SSam Ravnborg head = tx_advance(lp, head); 773a88b5ba8SSam Ravnborg 774a88b5ba8SSam Ravnborg if (p->seqid == ackid) { 775a88b5ba8SSam Ravnborg lp->tx_acked = head; 776a88b5ba8SSam Ravnborg return 0; 777a88b5ba8SSam Ravnborg } 778a88b5ba8SSam Ravnborg if (head == lp->tx_tail) 779a88b5ba8SSam Ravnborg return ldc_abort(lp); 780a88b5ba8SSam Ravnborg } 781a88b5ba8SSam Ravnborg 782a88b5ba8SSam Ravnborg return 0; 783a88b5ba8SSam Ravnborg } 784a88b5ba8SSam Ravnborg 785a88b5ba8SSam Ravnborg static void send_events(struct ldc_channel *lp, unsigned int event_mask) 786a88b5ba8SSam Ravnborg { 787a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_RESET) 788a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_RESET); 789a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_UP) 790a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_UP); 791a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_DATA_READY) 792a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY); 793a88b5ba8SSam Ravnborg } 794a88b5ba8SSam Ravnborg 795a88b5ba8SSam Ravnborg static irqreturn_t ldc_rx(int irq, void *dev_id) 796a88b5ba8SSam Ravnborg { 797a88b5ba8SSam Ravnborg struct ldc_channel *lp = dev_id; 798c6fee081SDavid S. Miller unsigned long orig_state, flags; 799a88b5ba8SSam Ravnborg unsigned int event_mask; 800a88b5ba8SSam Ravnborg 801a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 802a88b5ba8SSam Ravnborg 803a88b5ba8SSam Ravnborg orig_state = lp->chan_state; 804c6fee081SDavid S. Miller 805c6fee081SDavid S. Miller /* We should probably check for hypervisor errors here and 806c6fee081SDavid S. Miller * reset the LDC channel if we get one. 807c6fee081SDavid S. Miller */ 808c6fee081SDavid S. Miller sun4v_ldc_rx_get_state(lp->id, 809a88b5ba8SSam Ravnborg &lp->rx_head, 810a88b5ba8SSam Ravnborg &lp->rx_tail, 811a88b5ba8SSam Ravnborg &lp->chan_state); 812a88b5ba8SSam Ravnborg 813a88b5ba8SSam Ravnborg ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n", 814a88b5ba8SSam Ravnborg orig_state, lp->chan_state, lp->rx_head, lp->rx_tail); 815a88b5ba8SSam Ravnborg 816a88b5ba8SSam Ravnborg event_mask = 0; 817a88b5ba8SSam Ravnborg 818a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW && 819a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_UP) { 820a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 821a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 822a88b5ba8SSam Ravnborg 823a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_UP; 824a88b5ba8SSam Ravnborg 825a88b5ba8SSam Ravnborg orig_state = lp->chan_state; 826a88b5ba8SSam Ravnborg } 827a88b5ba8SSam Ravnborg 828a88b5ba8SSam Ravnborg /* If we are in reset state, flush the RX queue and ignore 829a88b5ba8SSam Ravnborg * everything. 830a88b5ba8SSam Ravnborg */ 831a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_RESET) { 832a88b5ba8SSam Ravnborg (void) __set_rx_head(lp, lp->rx_tail); 833a88b5ba8SSam Ravnborg goto out; 834a88b5ba8SSam Ravnborg } 835a88b5ba8SSam Ravnborg 836a88b5ba8SSam Ravnborg /* Once we finish the handshake, we let the ldc_read() 837a88b5ba8SSam Ravnborg * paths do all of the control frame and state management. 838a88b5ba8SSam Ravnborg * Just trigger the callback. 839a88b5ba8SSam Ravnborg */ 840a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_COMPLETE) { 841a88b5ba8SSam Ravnborg handshake_complete: 842a88b5ba8SSam Ravnborg if (lp->chan_state != orig_state) { 843a88b5ba8SSam Ravnborg unsigned int event = LDC_EVENT_RESET; 844a88b5ba8SSam Ravnborg 845a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_UP) 846a88b5ba8SSam Ravnborg event = LDC_EVENT_UP; 847a88b5ba8SSam Ravnborg 848a88b5ba8SSam Ravnborg event_mask |= event; 849a88b5ba8SSam Ravnborg } 850a88b5ba8SSam Ravnborg if (lp->rx_head != lp->rx_tail) 851a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_DATA_READY; 852a88b5ba8SSam Ravnborg 853a88b5ba8SSam Ravnborg goto out; 854a88b5ba8SSam Ravnborg } 855a88b5ba8SSam Ravnborg 856a88b5ba8SSam Ravnborg if (lp->chan_state != orig_state) 857a88b5ba8SSam Ravnborg goto out; 858a88b5ba8SSam Ravnborg 859a88b5ba8SSam Ravnborg while (lp->rx_head != lp->rx_tail) { 860a88b5ba8SSam Ravnborg struct ldc_packet *p; 861a88b5ba8SSam Ravnborg unsigned long new; 862a88b5ba8SSam Ravnborg int err; 863a88b5ba8SSam Ravnborg 864a88b5ba8SSam Ravnborg p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE); 865a88b5ba8SSam Ravnborg 866a88b5ba8SSam Ravnborg switch (p->type) { 867a88b5ba8SSam Ravnborg case LDC_CTRL: 868a88b5ba8SSam Ravnborg err = process_control_frame(lp, p); 869a88b5ba8SSam Ravnborg if (err > 0) 870a88b5ba8SSam Ravnborg event_mask |= err; 871a88b5ba8SSam Ravnborg break; 872a88b5ba8SSam Ravnborg 873a88b5ba8SSam Ravnborg case LDC_DATA: 874a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_DATA_READY; 875a88b5ba8SSam Ravnborg err = 0; 876a88b5ba8SSam Ravnborg break; 877a88b5ba8SSam Ravnborg 878a88b5ba8SSam Ravnborg case LDC_ERR: 879a88b5ba8SSam Ravnborg err = process_error_frame(lp, p); 880a88b5ba8SSam Ravnborg break; 881a88b5ba8SSam Ravnborg 882a88b5ba8SSam Ravnborg default: 883a88b5ba8SSam Ravnborg err = ldc_abort(lp); 884a88b5ba8SSam Ravnborg break; 885a88b5ba8SSam Ravnborg } 886a88b5ba8SSam Ravnborg 887a88b5ba8SSam Ravnborg if (err < 0) 888a88b5ba8SSam Ravnborg break; 889a88b5ba8SSam Ravnborg 890a88b5ba8SSam Ravnborg new = lp->rx_head; 891a88b5ba8SSam Ravnborg new += LDC_PACKET_SIZE; 892a88b5ba8SSam Ravnborg if (new == (lp->rx_num_entries * LDC_PACKET_SIZE)) 893a88b5ba8SSam Ravnborg new = 0; 894a88b5ba8SSam Ravnborg lp->rx_head = new; 895a88b5ba8SSam Ravnborg 896a88b5ba8SSam Ravnborg err = __set_rx_head(lp, new); 897a88b5ba8SSam Ravnborg if (err < 0) { 898a88b5ba8SSam Ravnborg (void) ldc_abort(lp); 899a88b5ba8SSam Ravnborg break; 900a88b5ba8SSam Ravnborg } 901a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_COMPLETE) 902a88b5ba8SSam Ravnborg goto handshake_complete; 903a88b5ba8SSam Ravnborg } 904a88b5ba8SSam Ravnborg 905a88b5ba8SSam Ravnborg out: 906a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 907a88b5ba8SSam Ravnborg 908a88b5ba8SSam Ravnborg send_events(lp, event_mask); 909a88b5ba8SSam Ravnborg 910a88b5ba8SSam Ravnborg return IRQ_HANDLED; 911a88b5ba8SSam Ravnborg } 912a88b5ba8SSam Ravnborg 913a88b5ba8SSam Ravnborg static irqreturn_t ldc_tx(int irq, void *dev_id) 914a88b5ba8SSam Ravnborg { 915a88b5ba8SSam Ravnborg struct ldc_channel *lp = dev_id; 916c6fee081SDavid S. Miller unsigned long flags, orig_state; 917a88b5ba8SSam Ravnborg unsigned int event_mask = 0; 918a88b5ba8SSam Ravnborg 919a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 920a88b5ba8SSam Ravnborg 921a88b5ba8SSam Ravnborg orig_state = lp->chan_state; 922c6fee081SDavid S. Miller 923c6fee081SDavid S. Miller /* We should probably check for hypervisor errors here and 924c6fee081SDavid S. Miller * reset the LDC channel if we get one. 925c6fee081SDavid S. Miller */ 926c6fee081SDavid S. Miller sun4v_ldc_tx_get_state(lp->id, 927a88b5ba8SSam Ravnborg &lp->tx_head, 928a88b5ba8SSam Ravnborg &lp->tx_tail, 929a88b5ba8SSam Ravnborg &lp->chan_state); 930a88b5ba8SSam Ravnborg 931a88b5ba8SSam Ravnborg ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n", 932a88b5ba8SSam Ravnborg orig_state, lp->chan_state, lp->tx_head, lp->tx_tail); 933a88b5ba8SSam Ravnborg 934a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW && 935a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_UP) { 936a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 937a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 938a88b5ba8SSam Ravnborg 939a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_UP; 940a88b5ba8SSam Ravnborg } 941a88b5ba8SSam Ravnborg 942a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 943a88b5ba8SSam Ravnborg 944a88b5ba8SSam Ravnborg send_events(lp, event_mask); 945a88b5ba8SSam Ravnborg 946a88b5ba8SSam Ravnborg return IRQ_HANDLED; 947a88b5ba8SSam Ravnborg } 948a88b5ba8SSam Ravnborg 949a88b5ba8SSam Ravnborg /* XXX ldc_alloc() and ldc_free() needs to run under a mutex so 950a88b5ba8SSam Ravnborg * XXX that addition and removal from the ldc_channel_list has 951a88b5ba8SSam Ravnborg * XXX atomicity, otherwise the __ldc_channel_exists() check is 952a88b5ba8SSam Ravnborg * XXX totally pointless as another thread can slip into ldc_alloc() 953a88b5ba8SSam Ravnborg * XXX and add a channel with the same ID. There also needs to be 954a88b5ba8SSam Ravnborg * XXX a spinlock for ldc_channel_list. 955a88b5ba8SSam Ravnborg */ 956a88b5ba8SSam Ravnborg static HLIST_HEAD(ldc_channel_list); 957a88b5ba8SSam Ravnborg 958a88b5ba8SSam Ravnborg static int __ldc_channel_exists(unsigned long id) 959a88b5ba8SSam Ravnborg { 960a88b5ba8SSam Ravnborg struct ldc_channel *lp; 961a88b5ba8SSam Ravnborg 962b67bfe0dSSasha Levin hlist_for_each_entry(lp, &ldc_channel_list, list) { 963a88b5ba8SSam Ravnborg if (lp->id == id) 964a88b5ba8SSam Ravnborg return 1; 965a88b5ba8SSam Ravnborg } 966a88b5ba8SSam Ravnborg return 0; 967a88b5ba8SSam Ravnborg } 968a88b5ba8SSam Ravnborg 969a88b5ba8SSam Ravnborg static int alloc_queue(const char *name, unsigned long num_entries, 970a88b5ba8SSam Ravnborg struct ldc_packet **base, unsigned long *ra) 971a88b5ba8SSam Ravnborg { 972a88b5ba8SSam Ravnborg unsigned long size, order; 973a88b5ba8SSam Ravnborg void *q; 974a88b5ba8SSam Ravnborg 975a88b5ba8SSam Ravnborg size = num_entries * LDC_PACKET_SIZE; 976a88b5ba8SSam Ravnborg order = get_order(size); 977a88b5ba8SSam Ravnborg 978a88b5ba8SSam Ravnborg q = (void *) __get_free_pages(GFP_KERNEL, order); 979a88b5ba8SSam Ravnborg if (!q) { 980a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of %s queue failed with " 981a88b5ba8SSam Ravnborg "size=%lu order=%lu\n", name, size, order); 982a88b5ba8SSam Ravnborg return -ENOMEM; 983a88b5ba8SSam Ravnborg } 984a88b5ba8SSam Ravnborg 985a88b5ba8SSam Ravnborg memset(q, 0, PAGE_SIZE << order); 986a88b5ba8SSam Ravnborg 987a88b5ba8SSam Ravnborg *base = q; 988a88b5ba8SSam Ravnborg *ra = __pa(q); 989a88b5ba8SSam Ravnborg 990a88b5ba8SSam Ravnborg return 0; 991a88b5ba8SSam Ravnborg } 992a88b5ba8SSam Ravnborg 993a88b5ba8SSam Ravnborg static void free_queue(unsigned long num_entries, struct ldc_packet *q) 994a88b5ba8SSam Ravnborg { 995a88b5ba8SSam Ravnborg unsigned long size, order; 996a88b5ba8SSam Ravnborg 997a88b5ba8SSam Ravnborg if (!q) 998a88b5ba8SSam Ravnborg return; 999a88b5ba8SSam Ravnborg 1000a88b5ba8SSam Ravnborg size = num_entries * LDC_PACKET_SIZE; 1001a88b5ba8SSam Ravnborg order = get_order(size); 1002a88b5ba8SSam Ravnborg 1003a88b5ba8SSam Ravnborg free_pages((unsigned long)q, order); 1004a88b5ba8SSam Ravnborg } 1005a88b5ba8SSam Ravnborg 10060ae53ed1SSowmini Varadhan static unsigned long ldc_cookie_to_index(u64 cookie, void *arg) 10070ae53ed1SSowmini Varadhan { 10080ae53ed1SSowmini Varadhan u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT; 10090ae53ed1SSowmini Varadhan /* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */ 10100ae53ed1SSowmini Varadhan 10110ae53ed1SSowmini Varadhan cookie &= ~COOKIE_PGSZ_CODE; 10120ae53ed1SSowmini Varadhan 10130ae53ed1SSowmini Varadhan return (cookie >> (13ULL + (szcode * 3ULL))); 10140ae53ed1SSowmini Varadhan } 10150ae53ed1SSowmini Varadhan 10160ae53ed1SSowmini Varadhan static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie, 10170ae53ed1SSowmini Varadhan unsigned long entry, unsigned long npages) 10180ae53ed1SSowmini Varadhan { 10190ae53ed1SSowmini Varadhan struct ldc_mtable_entry *base; 10200ae53ed1SSowmini Varadhan unsigned long i, shift; 10210ae53ed1SSowmini Varadhan 10220ae53ed1SSowmini Varadhan shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3; 10230ae53ed1SSowmini Varadhan base = iommu->page_table + entry; 10240ae53ed1SSowmini Varadhan for (i = 0; i < npages; i++) { 10250ae53ed1SSowmini Varadhan if (base->cookie) 10260ae53ed1SSowmini Varadhan sun4v_ldc_revoke(id, cookie + (i << shift), 10270ae53ed1SSowmini Varadhan base->cookie); 10280ae53ed1SSowmini Varadhan base->mte = 0; 10290ae53ed1SSowmini Varadhan } 10300ae53ed1SSowmini Varadhan } 10310ae53ed1SSowmini Varadhan 1032a88b5ba8SSam Ravnborg /* XXX Make this configurable... XXX */ 1033a88b5ba8SSam Ravnborg #define LDC_IOTABLE_SIZE (8 * 1024) 1034a88b5ba8SSam Ravnborg 10350ae53ed1SSowmini Varadhan static int ldc_iommu_init(const char *name, struct ldc_channel *lp) 1036a88b5ba8SSam Ravnborg { 1037a88b5ba8SSam Ravnborg unsigned long sz, num_tsb_entries, tsbsize, order; 10380ae53ed1SSowmini Varadhan struct ldc_iommu *ldc_iommu = &lp->iommu; 10390ae53ed1SSowmini Varadhan struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table; 1040a88b5ba8SSam Ravnborg struct ldc_mtable_entry *table; 1041a88b5ba8SSam Ravnborg unsigned long hv_err; 1042a88b5ba8SSam Ravnborg int err; 1043a88b5ba8SSam Ravnborg 1044a88b5ba8SSam Ravnborg num_tsb_entries = LDC_IOTABLE_SIZE; 1045a88b5ba8SSam Ravnborg tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); 10460ae53ed1SSowmini Varadhan spin_lock_init(&ldc_iommu->lock); 1047a88b5ba8SSam Ravnborg 1048a88b5ba8SSam Ravnborg sz = num_tsb_entries / 8; 1049a88b5ba8SSam Ravnborg sz = (sz + 7UL) & ~7UL; 10500ae53ed1SSowmini Varadhan iommu->map = kzalloc(sz, GFP_KERNEL); 10510ae53ed1SSowmini Varadhan if (!iommu->map) { 1052a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz); 1053a88b5ba8SSam Ravnborg return -ENOMEM; 1054a88b5ba8SSam Ravnborg } 10550ae53ed1SSowmini Varadhan iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT, 10560ae53ed1SSowmini Varadhan NULL, false /* no large pool */, 10570ae53ed1SSowmini Varadhan 1 /* npools */, 10580ae53ed1SSowmini Varadhan true /* skip span boundary check */); 1059a88b5ba8SSam Ravnborg 1060a88b5ba8SSam Ravnborg order = get_order(tsbsize); 1061a88b5ba8SSam Ravnborg 1062a88b5ba8SSam Ravnborg table = (struct ldc_mtable_entry *) 1063a88b5ba8SSam Ravnborg __get_free_pages(GFP_KERNEL, order); 1064a88b5ba8SSam Ravnborg err = -ENOMEM; 1065a88b5ba8SSam Ravnborg if (!table) { 1066a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of MTE table failed, " 1067a88b5ba8SSam Ravnborg "size=%lu order=%lu\n", tsbsize, order); 1068a88b5ba8SSam Ravnborg goto out_free_map; 1069a88b5ba8SSam Ravnborg } 1070a88b5ba8SSam Ravnborg 1071a88b5ba8SSam Ravnborg memset(table, 0, PAGE_SIZE << order); 1072a88b5ba8SSam Ravnborg 10730ae53ed1SSowmini Varadhan ldc_iommu->page_table = table; 1074a88b5ba8SSam Ravnborg 1075a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table), 1076a88b5ba8SSam Ravnborg num_tsb_entries); 1077a88b5ba8SSam Ravnborg err = -EINVAL; 1078a88b5ba8SSam Ravnborg if (hv_err) 1079a88b5ba8SSam Ravnborg goto out_free_table; 1080a88b5ba8SSam Ravnborg 1081a88b5ba8SSam Ravnborg return 0; 1082a88b5ba8SSam Ravnborg 1083a88b5ba8SSam Ravnborg out_free_table: 1084a88b5ba8SSam Ravnborg free_pages((unsigned long) table, order); 10850ae53ed1SSowmini Varadhan ldc_iommu->page_table = NULL; 1086a88b5ba8SSam Ravnborg 1087a88b5ba8SSam Ravnborg out_free_map: 10880ae53ed1SSowmini Varadhan kfree(iommu->map); 10890ae53ed1SSowmini Varadhan iommu->map = NULL; 1090a88b5ba8SSam Ravnborg 1091a88b5ba8SSam Ravnborg return err; 1092a88b5ba8SSam Ravnborg } 1093a88b5ba8SSam Ravnborg 1094a88b5ba8SSam Ravnborg static void ldc_iommu_release(struct ldc_channel *lp) 1095a88b5ba8SSam Ravnborg { 10960ae53ed1SSowmini Varadhan struct ldc_iommu *ldc_iommu = &lp->iommu; 10970ae53ed1SSowmini Varadhan struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table; 1098a88b5ba8SSam Ravnborg unsigned long num_tsb_entries, tsbsize, order; 1099a88b5ba8SSam Ravnborg 1100a88b5ba8SSam Ravnborg (void) sun4v_ldc_set_map_table(lp->id, 0, 0); 1101a88b5ba8SSam Ravnborg 11020ae53ed1SSowmini Varadhan num_tsb_entries = iommu->poolsize * iommu->nr_pools; 1103a88b5ba8SSam Ravnborg tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); 1104a88b5ba8SSam Ravnborg order = get_order(tsbsize); 1105a88b5ba8SSam Ravnborg 11060ae53ed1SSowmini Varadhan free_pages((unsigned long) ldc_iommu->page_table, order); 11070ae53ed1SSowmini Varadhan ldc_iommu->page_table = NULL; 1108a88b5ba8SSam Ravnborg 11090ae53ed1SSowmini Varadhan kfree(iommu->map); 11100ae53ed1SSowmini Varadhan iommu->map = NULL; 1111a88b5ba8SSam Ravnborg } 1112a88b5ba8SSam Ravnborg 1113a88b5ba8SSam Ravnborg struct ldc_channel *ldc_alloc(unsigned long id, 1114a88b5ba8SSam Ravnborg const struct ldc_channel_config *cfgp, 1115c21c4ab0SSowmini Varadhan void *event_arg, 1116c21c4ab0SSowmini Varadhan const char *name) 1117a88b5ba8SSam Ravnborg { 1118a88b5ba8SSam Ravnborg struct ldc_channel *lp; 1119a88b5ba8SSam Ravnborg const struct ldc_mode_ops *mops; 1120a88b5ba8SSam Ravnborg unsigned long dummy1, dummy2, hv_err; 1121a88b5ba8SSam Ravnborg u8 mss, *mssbuf; 1122a88b5ba8SSam Ravnborg int err; 1123a88b5ba8SSam Ravnborg 1124a88b5ba8SSam Ravnborg err = -ENODEV; 1125a88b5ba8SSam Ravnborg if (!ldom_domaining_enabled) 1126a88b5ba8SSam Ravnborg goto out_err; 1127a88b5ba8SSam Ravnborg 1128a88b5ba8SSam Ravnborg err = -EINVAL; 1129a88b5ba8SSam Ravnborg if (!cfgp) 1130a88b5ba8SSam Ravnborg goto out_err; 1131c21c4ab0SSowmini Varadhan if (!name) 1132c21c4ab0SSowmini Varadhan goto out_err; 1133a88b5ba8SSam Ravnborg 1134a88b5ba8SSam Ravnborg switch (cfgp->mode) { 1135a88b5ba8SSam Ravnborg case LDC_MODE_RAW: 1136a88b5ba8SSam Ravnborg mops = &raw_ops; 1137a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE; 1138a88b5ba8SSam Ravnborg break; 1139a88b5ba8SSam Ravnborg 1140a88b5ba8SSam Ravnborg case LDC_MODE_UNRELIABLE: 1141a88b5ba8SSam Ravnborg mops = &nonraw_ops; 1142a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE - 8; 1143a88b5ba8SSam Ravnborg break; 1144a88b5ba8SSam Ravnborg 1145a88b5ba8SSam Ravnborg case LDC_MODE_STREAM: 1146a88b5ba8SSam Ravnborg mops = &stream_ops; 1147a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE - 8 - 8; 1148a88b5ba8SSam Ravnborg break; 1149a88b5ba8SSam Ravnborg 1150a88b5ba8SSam Ravnborg default: 1151a88b5ba8SSam Ravnborg goto out_err; 1152a88b5ba8SSam Ravnborg } 1153a88b5ba8SSam Ravnborg 1154a88b5ba8SSam Ravnborg if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq) 1155a88b5ba8SSam Ravnborg goto out_err; 1156a88b5ba8SSam Ravnborg 1157a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2); 1158a88b5ba8SSam Ravnborg err = -ENODEV; 1159a88b5ba8SSam Ravnborg if (hv_err == HV_ECHANNEL) 1160a88b5ba8SSam Ravnborg goto out_err; 1161a88b5ba8SSam Ravnborg 1162a88b5ba8SSam Ravnborg err = -EEXIST; 1163a88b5ba8SSam Ravnborg if (__ldc_channel_exists(id)) 1164a88b5ba8SSam Ravnborg goto out_err; 1165a88b5ba8SSam Ravnborg 1166a88b5ba8SSam Ravnborg mssbuf = NULL; 1167a88b5ba8SSam Ravnborg 1168a88b5ba8SSam Ravnborg lp = kzalloc(sizeof(*lp), GFP_KERNEL); 1169a88b5ba8SSam Ravnborg err = -ENOMEM; 1170a88b5ba8SSam Ravnborg if (!lp) 1171a88b5ba8SSam Ravnborg goto out_err; 1172a88b5ba8SSam Ravnborg 1173a88b5ba8SSam Ravnborg spin_lock_init(&lp->lock); 1174a88b5ba8SSam Ravnborg 1175a88b5ba8SSam Ravnborg lp->id = id; 1176a88b5ba8SSam Ravnborg 11770ae53ed1SSowmini Varadhan err = ldc_iommu_init(name, lp); 1178a88b5ba8SSam Ravnborg if (err) 1179a88b5ba8SSam Ravnborg goto out_free_ldc; 1180a88b5ba8SSam Ravnborg 1181a88b5ba8SSam Ravnborg lp->mops = mops; 1182a88b5ba8SSam Ravnborg lp->mss = mss; 1183a88b5ba8SSam Ravnborg 1184a88b5ba8SSam Ravnborg lp->cfg = *cfgp; 1185a88b5ba8SSam Ravnborg if (!lp->cfg.mtu) 1186a88b5ba8SSam Ravnborg lp->cfg.mtu = LDC_DEFAULT_MTU; 1187a88b5ba8SSam Ravnborg 1188a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_STREAM) { 1189a88b5ba8SSam Ravnborg mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL); 1190a88b5ba8SSam Ravnborg if (!mssbuf) { 1191a88b5ba8SSam Ravnborg err = -ENOMEM; 1192a88b5ba8SSam Ravnborg goto out_free_iommu; 1193a88b5ba8SSam Ravnborg } 1194a88b5ba8SSam Ravnborg lp->mssbuf = mssbuf; 1195a88b5ba8SSam Ravnborg } 1196a88b5ba8SSam Ravnborg 1197a88b5ba8SSam Ravnborg lp->event_arg = event_arg; 1198a88b5ba8SSam Ravnborg 1199a88b5ba8SSam Ravnborg /* XXX allow setting via ldc_channel_config to override defaults 1200a88b5ba8SSam Ravnborg * XXX or use some formula based upon mtu 1201a88b5ba8SSam Ravnborg */ 1202a88b5ba8SSam Ravnborg lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES; 1203a88b5ba8SSam Ravnborg lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES; 1204a88b5ba8SSam Ravnborg 1205a88b5ba8SSam Ravnborg err = alloc_queue("TX", lp->tx_num_entries, 1206a88b5ba8SSam Ravnborg &lp->tx_base, &lp->tx_ra); 1207a88b5ba8SSam Ravnborg if (err) 1208a88b5ba8SSam Ravnborg goto out_free_mssbuf; 1209a88b5ba8SSam Ravnborg 1210a88b5ba8SSam Ravnborg err = alloc_queue("RX", lp->rx_num_entries, 1211a88b5ba8SSam Ravnborg &lp->rx_base, &lp->rx_ra); 1212a88b5ba8SSam Ravnborg if (err) 1213a88b5ba8SSam Ravnborg goto out_free_txq; 1214a88b5ba8SSam Ravnborg 1215a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_ALLOCED_QUEUES; 1216a88b5ba8SSam Ravnborg 1217a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_CLOSED; 1218a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_INIT); 1219a88b5ba8SSam Ravnborg 1220a88b5ba8SSam Ravnborg INIT_HLIST_NODE(&lp->list); 1221a88b5ba8SSam Ravnborg hlist_add_head(&lp->list, &ldc_channel_list); 1222a88b5ba8SSam Ravnborg 1223a88b5ba8SSam Ravnborg INIT_HLIST_HEAD(&lp->mh_list); 1224a88b5ba8SSam Ravnborg 1225c21c4ab0SSowmini Varadhan snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name); 1226c21c4ab0SSowmini Varadhan snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name); 1227c21c4ab0SSowmini Varadhan 1228c21c4ab0SSowmini Varadhan err = request_irq(lp->cfg.rx_irq, ldc_rx, 0, 1229c21c4ab0SSowmini Varadhan lp->rx_irq_name, lp); 1230c21c4ab0SSowmini Varadhan if (err) 1231c21c4ab0SSowmini Varadhan goto out_free_txq; 1232c21c4ab0SSowmini Varadhan 1233c21c4ab0SSowmini Varadhan err = request_irq(lp->cfg.tx_irq, ldc_tx, 0, 1234c21c4ab0SSowmini Varadhan lp->tx_irq_name, lp); 1235c21c4ab0SSowmini Varadhan if (err) { 1236c21c4ab0SSowmini Varadhan free_irq(lp->cfg.rx_irq, lp); 1237c21c4ab0SSowmini Varadhan goto out_free_txq; 1238c21c4ab0SSowmini Varadhan } 1239c21c4ab0SSowmini Varadhan 1240a88b5ba8SSam Ravnborg return lp; 1241a88b5ba8SSam Ravnborg 1242a88b5ba8SSam Ravnborg out_free_txq: 1243a88b5ba8SSam Ravnborg free_queue(lp->tx_num_entries, lp->tx_base); 1244a88b5ba8SSam Ravnborg 1245a88b5ba8SSam Ravnborg out_free_mssbuf: 1246a88b5ba8SSam Ravnborg kfree(mssbuf); 1247a88b5ba8SSam Ravnborg 1248a88b5ba8SSam Ravnborg out_free_iommu: 1249a88b5ba8SSam Ravnborg ldc_iommu_release(lp); 1250a88b5ba8SSam Ravnborg 1251a88b5ba8SSam Ravnborg out_free_ldc: 1252a88b5ba8SSam Ravnborg kfree(lp); 1253a88b5ba8SSam Ravnborg 1254a88b5ba8SSam Ravnborg out_err: 1255a88b5ba8SSam Ravnborg return ERR_PTR(err); 1256a88b5ba8SSam Ravnborg } 1257a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_alloc); 1258a88b5ba8SSam Ravnborg 12591678c2bdSDwight Engen void ldc_unbind(struct ldc_channel *lp) 1260a88b5ba8SSam Ravnborg { 1261a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_REGISTERED_IRQS) { 1262a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1263a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp); 12641678c2bdSDwight Engen lp->flags &= ~LDC_FLAG_REGISTERED_IRQS; 1265a88b5ba8SSam Ravnborg } 1266a88b5ba8SSam Ravnborg 1267a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) { 1268a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0); 1269a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0); 1270a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES; 1271a88b5ba8SSam Ravnborg } 1272a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) { 1273a88b5ba8SSam Ravnborg free_queue(lp->tx_num_entries, lp->tx_base); 1274a88b5ba8SSam Ravnborg free_queue(lp->rx_num_entries, lp->rx_base); 1275a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES; 1276a88b5ba8SSam Ravnborg } 1277a88b5ba8SSam Ravnborg 12781678c2bdSDwight Engen ldc_set_state(lp, LDC_STATE_INIT); 12791678c2bdSDwight Engen } 12801678c2bdSDwight Engen EXPORT_SYMBOL(ldc_unbind); 12811678c2bdSDwight Engen 12821678c2bdSDwight Engen void ldc_free(struct ldc_channel *lp) 12831678c2bdSDwight Engen { 12841678c2bdSDwight Engen ldc_unbind(lp); 1285a88b5ba8SSam Ravnborg hlist_del(&lp->list); 1286a88b5ba8SSam Ravnborg kfree(lp->mssbuf); 1287a88b5ba8SSam Ravnborg ldc_iommu_release(lp); 1288a88b5ba8SSam Ravnborg 1289a88b5ba8SSam Ravnborg kfree(lp); 1290a88b5ba8SSam Ravnborg } 1291a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_free); 1292a88b5ba8SSam Ravnborg 1293a88b5ba8SSam Ravnborg /* Bind the channel. This registers the LDC queues with 1294a88b5ba8SSam Ravnborg * the hypervisor and puts the channel into a pseudo-listening 1295a88b5ba8SSam Ravnborg * state. This does not initiate a handshake, ldc_connect() does 1296a88b5ba8SSam Ravnborg * that. 1297a88b5ba8SSam Ravnborg */ 1298c21c4ab0SSowmini Varadhan int ldc_bind(struct ldc_channel *lp) 1299a88b5ba8SSam Ravnborg { 1300a88b5ba8SSam Ravnborg unsigned long hv_err, flags; 1301a88b5ba8SSam Ravnborg int err = -EINVAL; 1302a88b5ba8SSam Ravnborg 1303c21c4ab0SSowmini Varadhan if (lp->state != LDC_STATE_INIT) 1304a88b5ba8SSam Ravnborg return -EINVAL; 1305a88b5ba8SSam Ravnborg 1306a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1307a88b5ba8SSam Ravnborg 1308a88b5ba8SSam Ravnborg enable_irq(lp->cfg.rx_irq); 1309a88b5ba8SSam Ravnborg enable_irq(lp->cfg.tx_irq); 1310a88b5ba8SSam Ravnborg 1311a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_REGISTERED_IRQS; 1312a88b5ba8SSam Ravnborg 1313a88b5ba8SSam Ravnborg err = -ENODEV; 1314a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0); 1315a88b5ba8SSam Ravnborg if (hv_err) 1316a88b5ba8SSam Ravnborg goto out_free_irqs; 1317a88b5ba8SSam Ravnborg 1318a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); 1319a88b5ba8SSam Ravnborg if (hv_err) 1320a88b5ba8SSam Ravnborg goto out_free_irqs; 1321a88b5ba8SSam Ravnborg 1322a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0); 1323a88b5ba8SSam Ravnborg if (hv_err) 1324a88b5ba8SSam Ravnborg goto out_unmap_tx; 1325a88b5ba8SSam Ravnborg 1326a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); 1327a88b5ba8SSam Ravnborg if (hv_err) 1328a88b5ba8SSam Ravnborg goto out_unmap_tx; 1329a88b5ba8SSam Ravnborg 1330a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_REGISTERED_QUEUES; 1331a88b5ba8SSam Ravnborg 1332a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, 1333a88b5ba8SSam Ravnborg &lp->tx_head, 1334a88b5ba8SSam Ravnborg &lp->tx_tail, 1335a88b5ba8SSam Ravnborg &lp->chan_state); 1336a88b5ba8SSam Ravnborg err = -EBUSY; 1337a88b5ba8SSam Ravnborg if (hv_err) 1338a88b5ba8SSam Ravnborg goto out_unmap_rx; 1339a88b5ba8SSam Ravnborg 1340a88b5ba8SSam Ravnborg lp->tx_acked = lp->tx_head; 1341a88b5ba8SSam Ravnborg 1342a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN; 1343a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_BOUND); 1344a88b5ba8SSam Ravnborg 1345a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1346a88b5ba8SSam Ravnborg 1347a88b5ba8SSam Ravnborg return 0; 1348a88b5ba8SSam Ravnborg 1349a88b5ba8SSam Ravnborg out_unmap_rx: 1350a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES; 1351a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0); 1352a88b5ba8SSam Ravnborg 1353a88b5ba8SSam Ravnborg out_unmap_tx: 1354a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0); 1355a88b5ba8SSam Ravnborg 1356a88b5ba8SSam Ravnborg out_free_irqs: 1357a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_IRQS; 1358a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp); 1359a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1360a88b5ba8SSam Ravnborg 1361a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1362a88b5ba8SSam Ravnborg 1363a88b5ba8SSam Ravnborg return err; 1364a88b5ba8SSam Ravnborg } 1365a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_bind); 1366a88b5ba8SSam Ravnborg 1367a88b5ba8SSam Ravnborg int ldc_connect(struct ldc_channel *lp) 1368a88b5ba8SSam Ravnborg { 1369a88b5ba8SSam Ravnborg unsigned long flags; 1370a88b5ba8SSam Ravnborg int err; 1371a88b5ba8SSam Ravnborg 1372a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW) 1373a88b5ba8SSam Ravnborg return -EINVAL; 1374a88b5ba8SSam Ravnborg 1375a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1376a88b5ba8SSam Ravnborg 1377a88b5ba8SSam Ravnborg if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || 1378a88b5ba8SSam Ravnborg !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) || 1379a88b5ba8SSam Ravnborg lp->hs_state != LDC_HS_OPEN) 13804ec1b010SSowmini Varadhan err = ((lp->hs_state > LDC_HS_OPEN) ? 0 : -EINVAL); 1381a88b5ba8SSam Ravnborg else 1382a88b5ba8SSam Ravnborg err = start_handshake(lp); 1383a88b5ba8SSam Ravnborg 1384a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1385a88b5ba8SSam Ravnborg 1386a88b5ba8SSam Ravnborg return err; 1387a88b5ba8SSam Ravnborg } 1388a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_connect); 1389a88b5ba8SSam Ravnborg 1390a88b5ba8SSam Ravnborg int ldc_disconnect(struct ldc_channel *lp) 1391a88b5ba8SSam Ravnborg { 1392a88b5ba8SSam Ravnborg unsigned long hv_err, flags; 1393a88b5ba8SSam Ravnborg int err; 1394a88b5ba8SSam Ravnborg 1395a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW) 1396a88b5ba8SSam Ravnborg return -EINVAL; 1397a88b5ba8SSam Ravnborg 1398a88b5ba8SSam Ravnborg if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || 1399a88b5ba8SSam Ravnborg !(lp->flags & LDC_FLAG_REGISTERED_QUEUES)) 1400a88b5ba8SSam Ravnborg return -EINVAL; 1401a88b5ba8SSam Ravnborg 1402a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1403a88b5ba8SSam Ravnborg 1404a88b5ba8SSam Ravnborg err = -ENODEV; 1405a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0); 1406a88b5ba8SSam Ravnborg if (hv_err) 1407a88b5ba8SSam Ravnborg goto out_err; 1408a88b5ba8SSam Ravnborg 1409a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); 1410a88b5ba8SSam Ravnborg if (hv_err) 1411a88b5ba8SSam Ravnborg goto out_err; 1412a88b5ba8SSam Ravnborg 1413a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0); 1414a88b5ba8SSam Ravnborg if (hv_err) 1415a88b5ba8SSam Ravnborg goto out_err; 1416a88b5ba8SSam Ravnborg 1417a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); 1418a88b5ba8SSam Ravnborg if (hv_err) 1419a88b5ba8SSam Ravnborg goto out_err; 1420a88b5ba8SSam Ravnborg 1421a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_BOUND); 1422a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN; 1423a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_RESET; 1424a88b5ba8SSam Ravnborg 1425a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1426a88b5ba8SSam Ravnborg 1427a88b5ba8SSam Ravnborg return 0; 1428a88b5ba8SSam Ravnborg 1429a88b5ba8SSam Ravnborg out_err: 1430a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0); 1431a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0); 1432a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp); 1433a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1434a88b5ba8SSam Ravnborg lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS | 1435a88b5ba8SSam Ravnborg LDC_FLAG_REGISTERED_QUEUES); 1436a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_INIT); 1437a88b5ba8SSam Ravnborg 1438a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1439a88b5ba8SSam Ravnborg 1440a88b5ba8SSam Ravnborg return err; 1441a88b5ba8SSam Ravnborg } 1442a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_disconnect); 1443a88b5ba8SSam Ravnborg 1444a88b5ba8SSam Ravnborg int ldc_state(struct ldc_channel *lp) 1445a88b5ba8SSam Ravnborg { 1446a88b5ba8SSam Ravnborg return lp->state; 1447a88b5ba8SSam Ravnborg } 1448a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_state); 1449a88b5ba8SSam Ravnborg 1450a88b5ba8SSam Ravnborg static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size) 1451a88b5ba8SSam Ravnborg { 1452a88b5ba8SSam Ravnborg struct ldc_packet *p; 1453a88b5ba8SSam Ravnborg unsigned long new_tail; 1454a88b5ba8SSam Ravnborg int err; 1455a88b5ba8SSam Ravnborg 1456a88b5ba8SSam Ravnborg if (size > LDC_PACKET_SIZE) 1457a88b5ba8SSam Ravnborg return -EMSGSIZE; 1458a88b5ba8SSam Ravnborg 1459a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail); 1460a88b5ba8SSam Ravnborg if (!p) 1461a88b5ba8SSam Ravnborg return -EAGAIN; 1462a88b5ba8SSam Ravnborg 1463a88b5ba8SSam Ravnborg memcpy(p, buf, size); 1464a88b5ba8SSam Ravnborg 1465a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail); 1466a88b5ba8SSam Ravnborg if (!err) 1467a88b5ba8SSam Ravnborg err = size; 1468a88b5ba8SSam Ravnborg 1469a88b5ba8SSam Ravnborg return err; 1470a88b5ba8SSam Ravnborg } 1471a88b5ba8SSam Ravnborg 1472a88b5ba8SSam Ravnborg static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size) 1473a88b5ba8SSam Ravnborg { 1474a88b5ba8SSam Ravnborg struct ldc_packet *p; 1475a88b5ba8SSam Ravnborg unsigned long hv_err, new; 1476a88b5ba8SSam Ravnborg int err; 1477a88b5ba8SSam Ravnborg 1478a88b5ba8SSam Ravnborg if (size < LDC_PACKET_SIZE) 1479a88b5ba8SSam Ravnborg return -EINVAL; 1480a88b5ba8SSam Ravnborg 1481a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 1482a88b5ba8SSam Ravnborg &lp->rx_head, 1483a88b5ba8SSam Ravnborg &lp->rx_tail, 1484a88b5ba8SSam Ravnborg &lp->chan_state); 1485a88b5ba8SSam Ravnborg if (hv_err) 1486a88b5ba8SSam Ravnborg return ldc_abort(lp); 1487a88b5ba8SSam Ravnborg 1488a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN || 1489a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING) 1490a88b5ba8SSam Ravnborg return -ECONNRESET; 1491a88b5ba8SSam Ravnborg 1492a88b5ba8SSam Ravnborg if (lp->rx_head == lp->rx_tail) 1493a88b5ba8SSam Ravnborg return 0; 1494a88b5ba8SSam Ravnborg 1495a88b5ba8SSam Ravnborg p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE); 1496a88b5ba8SSam Ravnborg memcpy(buf, p, LDC_PACKET_SIZE); 1497a88b5ba8SSam Ravnborg 1498a88b5ba8SSam Ravnborg new = rx_advance(lp, lp->rx_head); 1499a88b5ba8SSam Ravnborg lp->rx_head = new; 1500a88b5ba8SSam Ravnborg 1501a88b5ba8SSam Ravnborg err = __set_rx_head(lp, new); 1502a88b5ba8SSam Ravnborg if (err < 0) 1503a88b5ba8SSam Ravnborg err = -ECONNRESET; 1504a88b5ba8SSam Ravnborg else 1505a88b5ba8SSam Ravnborg err = LDC_PACKET_SIZE; 1506a88b5ba8SSam Ravnborg 1507a88b5ba8SSam Ravnborg return err; 1508a88b5ba8SSam Ravnborg } 1509a88b5ba8SSam Ravnborg 1510a88b5ba8SSam Ravnborg static const struct ldc_mode_ops raw_ops = { 1511a88b5ba8SSam Ravnborg .write = write_raw, 1512a88b5ba8SSam Ravnborg .read = read_raw, 1513a88b5ba8SSam Ravnborg }; 1514a88b5ba8SSam Ravnborg 1515a88b5ba8SSam Ravnborg static int write_nonraw(struct ldc_channel *lp, const void *buf, 1516a88b5ba8SSam Ravnborg unsigned int size) 1517a88b5ba8SSam Ravnborg { 1518a88b5ba8SSam Ravnborg unsigned long hv_err, tail; 1519a88b5ba8SSam Ravnborg unsigned int copied; 1520a88b5ba8SSam Ravnborg u32 seq; 1521a88b5ba8SSam Ravnborg int err; 1522a88b5ba8SSam Ravnborg 1523a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail, 1524a88b5ba8SSam Ravnborg &lp->chan_state); 1525a88b5ba8SSam Ravnborg if (unlikely(hv_err)) 1526a88b5ba8SSam Ravnborg return -EBUSY; 1527a88b5ba8SSam Ravnborg 1528a88b5ba8SSam Ravnborg if (unlikely(lp->chan_state != LDC_CHANNEL_UP)) 1529a88b5ba8SSam Ravnborg return ldc_abort(lp); 1530a88b5ba8SSam Ravnborg 1531a88b5ba8SSam Ravnborg if (!tx_has_space_for(lp, size)) 1532a88b5ba8SSam Ravnborg return -EAGAIN; 1533a88b5ba8SSam Ravnborg 1534a88b5ba8SSam Ravnborg seq = lp->snd_nxt; 1535a88b5ba8SSam Ravnborg copied = 0; 1536a88b5ba8SSam Ravnborg tail = lp->tx_tail; 1537a88b5ba8SSam Ravnborg while (copied < size) { 1538a88b5ba8SSam Ravnborg struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE); 1539a88b5ba8SSam Ravnborg u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ? 1540a88b5ba8SSam Ravnborg p->u.u_data : 1541a88b5ba8SSam Ravnborg p->u.r.r_data); 1542a88b5ba8SSam Ravnborg int data_len; 1543a88b5ba8SSam Ravnborg 1544a88b5ba8SSam Ravnborg p->type = LDC_DATA; 1545a88b5ba8SSam Ravnborg p->stype = LDC_INFO; 1546a88b5ba8SSam Ravnborg p->ctrl = 0; 1547a88b5ba8SSam Ravnborg 1548a88b5ba8SSam Ravnborg data_len = size - copied; 1549a88b5ba8SSam Ravnborg if (data_len > lp->mss) 1550a88b5ba8SSam Ravnborg data_len = lp->mss; 1551a88b5ba8SSam Ravnborg 1552a88b5ba8SSam Ravnborg BUG_ON(data_len > LDC_LEN); 1553a88b5ba8SSam Ravnborg 1554a88b5ba8SSam Ravnborg p->env = (data_len | 1555a88b5ba8SSam Ravnborg (copied == 0 ? LDC_START : 0) | 1556a88b5ba8SSam Ravnborg (data_len == size - copied ? LDC_STOP : 0)); 1557a88b5ba8SSam Ravnborg 1558a88b5ba8SSam Ravnborg p->seqid = ++seq; 1559a88b5ba8SSam Ravnborg 1560a88b5ba8SSam Ravnborg ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n", 1561a88b5ba8SSam Ravnborg p->type, 1562a88b5ba8SSam Ravnborg p->stype, 1563a88b5ba8SSam Ravnborg p->ctrl, 1564a88b5ba8SSam Ravnborg p->env, 1565a88b5ba8SSam Ravnborg p->seqid); 1566a88b5ba8SSam Ravnborg 1567a88b5ba8SSam Ravnborg memcpy(data, buf, data_len); 1568a88b5ba8SSam Ravnborg buf += data_len; 1569a88b5ba8SSam Ravnborg copied += data_len; 1570a88b5ba8SSam Ravnborg 1571a88b5ba8SSam Ravnborg tail = tx_advance(lp, tail); 1572a88b5ba8SSam Ravnborg } 1573a88b5ba8SSam Ravnborg 1574a88b5ba8SSam Ravnborg err = set_tx_tail(lp, tail); 1575a88b5ba8SSam Ravnborg if (!err) { 1576a88b5ba8SSam Ravnborg lp->snd_nxt = seq; 1577a88b5ba8SSam Ravnborg err = size; 1578a88b5ba8SSam Ravnborg } 1579a88b5ba8SSam Ravnborg 1580a88b5ba8SSam Ravnborg return err; 1581a88b5ba8SSam Ravnborg } 1582a88b5ba8SSam Ravnborg 1583a88b5ba8SSam Ravnborg static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p, 1584a88b5ba8SSam Ravnborg struct ldc_packet *first_frag) 1585a88b5ba8SSam Ravnborg { 1586a88b5ba8SSam Ravnborg int err; 1587a88b5ba8SSam Ravnborg 1588a88b5ba8SSam Ravnborg if (first_frag) 1589a88b5ba8SSam Ravnborg lp->rcv_nxt = first_frag->seqid - 1; 1590a88b5ba8SSam Ravnborg 1591a88b5ba8SSam Ravnborg err = send_data_nack(lp, p); 1592a88b5ba8SSam Ravnborg if (err) 1593a88b5ba8SSam Ravnborg return err; 1594a88b5ba8SSam Ravnborg 1595a88b5ba8SSam Ravnborg err = __set_rx_head(lp, lp->rx_tail); 1596a88b5ba8SSam Ravnborg if (err < 0) 1597a88b5ba8SSam Ravnborg return ldc_abort(lp); 1598a88b5ba8SSam Ravnborg 1599a88b5ba8SSam Ravnborg return 0; 1600a88b5ba8SSam Ravnborg } 1601a88b5ba8SSam Ravnborg 1602a88b5ba8SSam Ravnborg static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p) 1603a88b5ba8SSam Ravnborg { 1604a88b5ba8SSam Ravnborg if (p->stype & LDC_ACK) { 1605a88b5ba8SSam Ravnborg int err = process_data_ack(lp, p); 1606a88b5ba8SSam Ravnborg if (err) 1607a88b5ba8SSam Ravnborg return err; 1608a88b5ba8SSam Ravnborg } 1609a88b5ba8SSam Ravnborg if (p->stype & LDC_NACK) 1610a88b5ba8SSam Ravnborg return ldc_abort(lp); 1611a88b5ba8SSam Ravnborg 1612a88b5ba8SSam Ravnborg return 0; 1613a88b5ba8SSam Ravnborg } 1614a88b5ba8SSam Ravnborg 1615a88b5ba8SSam Ravnborg static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head) 1616a88b5ba8SSam Ravnborg { 1617a88b5ba8SSam Ravnborg unsigned long dummy; 1618a88b5ba8SSam Ravnborg int limit = 1000; 1619a88b5ba8SSam Ravnborg 1620a88b5ba8SSam Ravnborg ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n", 1621a88b5ba8SSam Ravnborg cur_head, lp->rx_head, lp->rx_tail); 1622a88b5ba8SSam Ravnborg while (limit-- > 0) { 1623a88b5ba8SSam Ravnborg unsigned long hv_err; 1624a88b5ba8SSam Ravnborg 1625a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 1626a88b5ba8SSam Ravnborg &dummy, 1627a88b5ba8SSam Ravnborg &lp->rx_tail, 1628a88b5ba8SSam Ravnborg &lp->chan_state); 1629a88b5ba8SSam Ravnborg if (hv_err) 1630a88b5ba8SSam Ravnborg return ldc_abort(lp); 1631a88b5ba8SSam Ravnborg 1632a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN || 1633a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING) 1634a88b5ba8SSam Ravnborg return -ECONNRESET; 1635a88b5ba8SSam Ravnborg 1636a88b5ba8SSam Ravnborg if (cur_head != lp->rx_tail) { 1637a88b5ba8SSam Ravnborg ldcdbg(DATA, "DATA WAIT DONE " 1638a88b5ba8SSam Ravnborg "head[%lx] tail[%lx] chan_state[%lx]\n", 1639a88b5ba8SSam Ravnborg dummy, lp->rx_tail, lp->chan_state); 1640a88b5ba8SSam Ravnborg return 0; 1641a88b5ba8SSam Ravnborg } 1642a88b5ba8SSam Ravnborg 1643a88b5ba8SSam Ravnborg udelay(1); 1644a88b5ba8SSam Ravnborg } 1645a88b5ba8SSam Ravnborg return -EAGAIN; 1646a88b5ba8SSam Ravnborg } 1647a88b5ba8SSam Ravnborg 1648a88b5ba8SSam Ravnborg static int rx_set_head(struct ldc_channel *lp, unsigned long head) 1649a88b5ba8SSam Ravnborg { 1650a88b5ba8SSam Ravnborg int err = __set_rx_head(lp, head); 1651a88b5ba8SSam Ravnborg 1652a88b5ba8SSam Ravnborg if (err < 0) 1653a88b5ba8SSam Ravnborg return ldc_abort(lp); 1654a88b5ba8SSam Ravnborg 1655a88b5ba8SSam Ravnborg lp->rx_head = head; 1656a88b5ba8SSam Ravnborg return 0; 1657a88b5ba8SSam Ravnborg } 1658a88b5ba8SSam Ravnborg 1659a88b5ba8SSam Ravnborg static void send_data_ack(struct ldc_channel *lp) 1660a88b5ba8SSam Ravnborg { 1661a88b5ba8SSam Ravnborg unsigned long new_tail; 1662a88b5ba8SSam Ravnborg struct ldc_packet *p; 1663a88b5ba8SSam Ravnborg 1664a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail); 1665a88b5ba8SSam Ravnborg if (likely(p)) { 1666a88b5ba8SSam Ravnborg int err; 1667a88b5ba8SSam Ravnborg 1668a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p)); 1669a88b5ba8SSam Ravnborg p->type = LDC_DATA; 1670a88b5ba8SSam Ravnborg p->stype = LDC_ACK; 1671a88b5ba8SSam Ravnborg p->ctrl = 0; 1672a88b5ba8SSam Ravnborg p->seqid = lp->snd_nxt + 1; 1673a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt; 1674a88b5ba8SSam Ravnborg 1675a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail); 1676a88b5ba8SSam Ravnborg if (!err) 1677a88b5ba8SSam Ravnborg lp->snd_nxt++; 1678a88b5ba8SSam Ravnborg } 1679a88b5ba8SSam Ravnborg } 1680a88b5ba8SSam Ravnborg 1681a88b5ba8SSam Ravnborg static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size) 1682a88b5ba8SSam Ravnborg { 1683a88b5ba8SSam Ravnborg struct ldc_packet *first_frag; 1684a88b5ba8SSam Ravnborg unsigned long hv_err, new; 1685a88b5ba8SSam Ravnborg int err, copied; 1686a88b5ba8SSam Ravnborg 1687a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 1688a88b5ba8SSam Ravnborg &lp->rx_head, 1689a88b5ba8SSam Ravnborg &lp->rx_tail, 1690a88b5ba8SSam Ravnborg &lp->chan_state); 1691a88b5ba8SSam Ravnborg if (hv_err) 1692a88b5ba8SSam Ravnborg return ldc_abort(lp); 1693a88b5ba8SSam Ravnborg 1694a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN || 1695a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING) 1696a88b5ba8SSam Ravnborg return -ECONNRESET; 1697a88b5ba8SSam Ravnborg 1698a88b5ba8SSam Ravnborg if (lp->rx_head == lp->rx_tail) 1699a88b5ba8SSam Ravnborg return 0; 1700a88b5ba8SSam Ravnborg 1701a88b5ba8SSam Ravnborg first_frag = NULL; 1702a88b5ba8SSam Ravnborg copied = err = 0; 1703a88b5ba8SSam Ravnborg new = lp->rx_head; 1704a88b5ba8SSam Ravnborg while (1) { 1705a88b5ba8SSam Ravnborg struct ldc_packet *p; 1706a88b5ba8SSam Ravnborg int pkt_len; 1707a88b5ba8SSam Ravnborg 1708a88b5ba8SSam Ravnborg BUG_ON(new == lp->rx_tail); 1709a88b5ba8SSam Ravnborg p = lp->rx_base + (new / LDC_PACKET_SIZE); 1710a88b5ba8SSam Ravnborg 1711a88b5ba8SSam Ravnborg ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] " 1712a88b5ba8SSam Ravnborg "rcv_nxt[%08x]\n", 1713a88b5ba8SSam Ravnborg p->type, 1714a88b5ba8SSam Ravnborg p->stype, 1715a88b5ba8SSam Ravnborg p->ctrl, 1716a88b5ba8SSam Ravnborg p->env, 1717a88b5ba8SSam Ravnborg p->seqid, 1718a88b5ba8SSam Ravnborg p->u.r.ackid, 1719a88b5ba8SSam Ravnborg lp->rcv_nxt); 1720a88b5ba8SSam Ravnborg 1721a88b5ba8SSam Ravnborg if (unlikely(!rx_seq_ok(lp, p->seqid))) { 1722a88b5ba8SSam Ravnborg err = rx_bad_seq(lp, p, first_frag); 1723a88b5ba8SSam Ravnborg copied = 0; 1724a88b5ba8SSam Ravnborg break; 1725a88b5ba8SSam Ravnborg } 1726a88b5ba8SSam Ravnborg 1727a88b5ba8SSam Ravnborg if (p->type & LDC_CTRL) { 1728a88b5ba8SSam Ravnborg err = process_control_frame(lp, p); 1729a88b5ba8SSam Ravnborg if (err < 0) 1730a88b5ba8SSam Ravnborg break; 1731a88b5ba8SSam Ravnborg err = 0; 1732a88b5ba8SSam Ravnborg } 1733a88b5ba8SSam Ravnborg 1734a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid; 1735a88b5ba8SSam Ravnborg 1736a88b5ba8SSam Ravnborg if (!(p->type & LDC_DATA)) { 1737a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1738a88b5ba8SSam Ravnborg goto no_data; 1739a88b5ba8SSam Ravnborg } 1740a88b5ba8SSam Ravnborg if (p->stype & (LDC_ACK | LDC_NACK)) { 1741a88b5ba8SSam Ravnborg err = data_ack_nack(lp, p); 1742a88b5ba8SSam Ravnborg if (err) 1743a88b5ba8SSam Ravnborg break; 1744a88b5ba8SSam Ravnborg } 1745a88b5ba8SSam Ravnborg if (!(p->stype & LDC_INFO)) { 1746a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1747a88b5ba8SSam Ravnborg err = rx_set_head(lp, new); 1748a88b5ba8SSam Ravnborg if (err) 1749a88b5ba8SSam Ravnborg break; 1750a88b5ba8SSam Ravnborg goto no_data; 1751a88b5ba8SSam Ravnborg } 1752a88b5ba8SSam Ravnborg 1753a88b5ba8SSam Ravnborg pkt_len = p->env & LDC_LEN; 1754a88b5ba8SSam Ravnborg 1755a88b5ba8SSam Ravnborg /* Every initial packet starts with the START bit set. 1756a88b5ba8SSam Ravnborg * 1757a88b5ba8SSam Ravnborg * Singleton packets will have both START+STOP set. 1758a88b5ba8SSam Ravnborg * 1759a88b5ba8SSam Ravnborg * Fragments will have START set in the first frame, STOP 1760a88b5ba8SSam Ravnborg * set in the last frame, and neither bit set in middle 1761a88b5ba8SSam Ravnborg * frames of the packet. 1762a88b5ba8SSam Ravnborg * 1763a88b5ba8SSam Ravnborg * Therefore if we are at the beginning of a packet and 1764a88b5ba8SSam Ravnborg * we don't see START, or we are in the middle of a fragmented 1765a88b5ba8SSam Ravnborg * packet and do see START, we are unsynchronized and should 1766a88b5ba8SSam Ravnborg * flush the RX queue. 1767a88b5ba8SSam Ravnborg */ 1768a88b5ba8SSam Ravnborg if ((first_frag == NULL && !(p->env & LDC_START)) || 1769a88b5ba8SSam Ravnborg (first_frag != NULL && (p->env & LDC_START))) { 1770a88b5ba8SSam Ravnborg if (!first_frag) 1771a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1772a88b5ba8SSam Ravnborg 1773a88b5ba8SSam Ravnborg err = rx_set_head(lp, new); 1774a88b5ba8SSam Ravnborg if (err) 1775a88b5ba8SSam Ravnborg break; 1776a88b5ba8SSam Ravnborg 1777a88b5ba8SSam Ravnborg if (!first_frag) 1778a88b5ba8SSam Ravnborg goto no_data; 1779a88b5ba8SSam Ravnborg } 1780a88b5ba8SSam Ravnborg if (!first_frag) 1781a88b5ba8SSam Ravnborg first_frag = p; 1782a88b5ba8SSam Ravnborg 1783a88b5ba8SSam Ravnborg if (pkt_len > size - copied) { 1784a88b5ba8SSam Ravnborg /* User didn't give us a big enough buffer, 1785a88b5ba8SSam Ravnborg * what to do? This is a pretty serious error. 1786a88b5ba8SSam Ravnborg * 1787a88b5ba8SSam Ravnborg * Since we haven't updated the RX ring head to 1788a88b5ba8SSam Ravnborg * consume any of the packets, signal the error 1789a88b5ba8SSam Ravnborg * to the user and just leave the RX ring alone. 1790a88b5ba8SSam Ravnborg * 1791a88b5ba8SSam Ravnborg * This seems the best behavior because this allows 1792a88b5ba8SSam Ravnborg * a user of the LDC layer to start with a small 1793a88b5ba8SSam Ravnborg * RX buffer for ldc_read() calls and use -EMSGSIZE 1794a88b5ba8SSam Ravnborg * as a cue to enlarge it's read buffer. 1795a88b5ba8SSam Ravnborg */ 1796a88b5ba8SSam Ravnborg err = -EMSGSIZE; 1797a88b5ba8SSam Ravnborg break; 1798a88b5ba8SSam Ravnborg } 1799a88b5ba8SSam Ravnborg 1800a88b5ba8SSam Ravnborg /* Ok, we are gonna eat this one. */ 1801a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1802a88b5ba8SSam Ravnborg 1803a88b5ba8SSam Ravnborg memcpy(buf, 1804a88b5ba8SSam Ravnborg (lp->cfg.mode == LDC_MODE_UNRELIABLE ? 1805a88b5ba8SSam Ravnborg p->u.u_data : p->u.r.r_data), pkt_len); 1806a88b5ba8SSam Ravnborg buf += pkt_len; 1807a88b5ba8SSam Ravnborg copied += pkt_len; 1808a88b5ba8SSam Ravnborg 1809a88b5ba8SSam Ravnborg if (p->env & LDC_STOP) 1810a88b5ba8SSam Ravnborg break; 1811a88b5ba8SSam Ravnborg 1812a88b5ba8SSam Ravnborg no_data: 1813a88b5ba8SSam Ravnborg if (new == lp->rx_tail) { 1814a88b5ba8SSam Ravnborg err = rx_data_wait(lp, new); 1815a88b5ba8SSam Ravnborg if (err) 1816a88b5ba8SSam Ravnborg break; 1817a88b5ba8SSam Ravnborg } 1818a88b5ba8SSam Ravnborg } 1819a88b5ba8SSam Ravnborg 1820a88b5ba8SSam Ravnborg if (!err) 1821a88b5ba8SSam Ravnborg err = rx_set_head(lp, new); 1822a88b5ba8SSam Ravnborg 1823a88b5ba8SSam Ravnborg if (err && first_frag) 1824a88b5ba8SSam Ravnborg lp->rcv_nxt = first_frag->seqid - 1; 1825a88b5ba8SSam Ravnborg 1826a88b5ba8SSam Ravnborg if (!err) { 1827a88b5ba8SSam Ravnborg err = copied; 1828a88b5ba8SSam Ravnborg if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE) 1829a88b5ba8SSam Ravnborg send_data_ack(lp); 1830a88b5ba8SSam Ravnborg } 1831a88b5ba8SSam Ravnborg 1832a88b5ba8SSam Ravnborg return err; 1833a88b5ba8SSam Ravnborg } 1834a88b5ba8SSam Ravnborg 1835a88b5ba8SSam Ravnborg static const struct ldc_mode_ops nonraw_ops = { 1836a88b5ba8SSam Ravnborg .write = write_nonraw, 1837a88b5ba8SSam Ravnborg .read = read_nonraw, 1838a88b5ba8SSam Ravnborg }; 1839a88b5ba8SSam Ravnborg 1840a88b5ba8SSam Ravnborg static int write_stream(struct ldc_channel *lp, const void *buf, 1841a88b5ba8SSam Ravnborg unsigned int size) 1842a88b5ba8SSam Ravnborg { 1843a88b5ba8SSam Ravnborg if (size > lp->cfg.mtu) 1844a88b5ba8SSam Ravnborg size = lp->cfg.mtu; 1845a88b5ba8SSam Ravnborg return write_nonraw(lp, buf, size); 1846a88b5ba8SSam Ravnborg } 1847a88b5ba8SSam Ravnborg 1848a88b5ba8SSam Ravnborg static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size) 1849a88b5ba8SSam Ravnborg { 1850a88b5ba8SSam Ravnborg if (!lp->mssbuf_len) { 1851a88b5ba8SSam Ravnborg int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu); 1852a88b5ba8SSam Ravnborg if (err < 0) 1853a88b5ba8SSam Ravnborg return err; 1854a88b5ba8SSam Ravnborg 1855a88b5ba8SSam Ravnborg lp->mssbuf_len = err; 1856a88b5ba8SSam Ravnborg lp->mssbuf_off = 0; 1857a88b5ba8SSam Ravnborg } 1858a88b5ba8SSam Ravnborg 1859a88b5ba8SSam Ravnborg if (size > lp->mssbuf_len) 1860a88b5ba8SSam Ravnborg size = lp->mssbuf_len; 1861a88b5ba8SSam Ravnborg memcpy(buf, lp->mssbuf + lp->mssbuf_off, size); 1862a88b5ba8SSam Ravnborg 1863a88b5ba8SSam Ravnborg lp->mssbuf_off += size; 1864a88b5ba8SSam Ravnborg lp->mssbuf_len -= size; 1865a88b5ba8SSam Ravnborg 1866a88b5ba8SSam Ravnborg return size; 1867a88b5ba8SSam Ravnborg } 1868a88b5ba8SSam Ravnborg 1869a88b5ba8SSam Ravnborg static const struct ldc_mode_ops stream_ops = { 1870a88b5ba8SSam Ravnborg .write = write_stream, 1871a88b5ba8SSam Ravnborg .read = read_stream, 1872a88b5ba8SSam Ravnborg }; 1873a88b5ba8SSam Ravnborg 1874a88b5ba8SSam Ravnborg int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size) 1875a88b5ba8SSam Ravnborg { 1876a88b5ba8SSam Ravnborg unsigned long flags; 1877a88b5ba8SSam Ravnborg int err; 1878a88b5ba8SSam Ravnborg 1879a88b5ba8SSam Ravnborg if (!buf) 1880a88b5ba8SSam Ravnborg return -EINVAL; 1881a88b5ba8SSam Ravnborg 1882a88b5ba8SSam Ravnborg if (!size) 1883a88b5ba8SSam Ravnborg return 0; 1884a88b5ba8SSam Ravnborg 1885a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1886a88b5ba8SSam Ravnborg 1887a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE) 1888a88b5ba8SSam Ravnborg err = -ENOTCONN; 1889a88b5ba8SSam Ravnborg else 1890a88b5ba8SSam Ravnborg err = lp->mops->write(lp, buf, size); 1891a88b5ba8SSam Ravnborg 1892a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1893a88b5ba8SSam Ravnborg 1894a88b5ba8SSam Ravnborg return err; 1895a88b5ba8SSam Ravnborg } 1896a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_write); 1897a88b5ba8SSam Ravnborg 1898a88b5ba8SSam Ravnborg int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size) 1899a88b5ba8SSam Ravnborg { 1900a88b5ba8SSam Ravnborg unsigned long flags; 1901a88b5ba8SSam Ravnborg int err; 1902a88b5ba8SSam Ravnborg 1903a88b5ba8SSam Ravnborg if (!buf) 1904a88b5ba8SSam Ravnborg return -EINVAL; 1905a88b5ba8SSam Ravnborg 1906a88b5ba8SSam Ravnborg if (!size) 1907a88b5ba8SSam Ravnborg return 0; 1908a88b5ba8SSam Ravnborg 1909a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1910a88b5ba8SSam Ravnborg 1911a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE) 1912a88b5ba8SSam Ravnborg err = -ENOTCONN; 1913a88b5ba8SSam Ravnborg else 1914a88b5ba8SSam Ravnborg err = lp->mops->read(lp, buf, size); 1915a88b5ba8SSam Ravnborg 1916a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1917a88b5ba8SSam Ravnborg 1918a88b5ba8SSam Ravnborg return err; 1919a88b5ba8SSam Ravnborg } 1920a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_read); 1921a88b5ba8SSam Ravnborg 1922a88b5ba8SSam Ravnborg static u64 pagesize_code(void) 1923a88b5ba8SSam Ravnborg { 1924a88b5ba8SSam Ravnborg switch (PAGE_SIZE) { 1925a88b5ba8SSam Ravnborg default: 1926a88b5ba8SSam Ravnborg case (8ULL * 1024ULL): 1927a88b5ba8SSam Ravnborg return 0; 1928a88b5ba8SSam Ravnborg case (64ULL * 1024ULL): 1929a88b5ba8SSam Ravnborg return 1; 1930a88b5ba8SSam Ravnborg case (512ULL * 1024ULL): 1931a88b5ba8SSam Ravnborg return 2; 1932a88b5ba8SSam Ravnborg case (4ULL * 1024ULL * 1024ULL): 1933a88b5ba8SSam Ravnborg return 3; 1934a88b5ba8SSam Ravnborg case (32ULL * 1024ULL * 1024ULL): 1935a88b5ba8SSam Ravnborg return 4; 1936a88b5ba8SSam Ravnborg case (256ULL * 1024ULL * 1024ULL): 1937a88b5ba8SSam Ravnborg return 5; 1938a88b5ba8SSam Ravnborg } 1939a88b5ba8SSam Ravnborg } 1940a88b5ba8SSam Ravnborg 1941a88b5ba8SSam Ravnborg static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset) 1942a88b5ba8SSam Ravnborg { 1943a88b5ba8SSam Ravnborg return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) | 1944a88b5ba8SSam Ravnborg (index << PAGE_SHIFT) | 1945a88b5ba8SSam Ravnborg page_offset); 1946a88b5ba8SSam Ravnborg } 1947a88b5ba8SSam Ravnborg 1948a88b5ba8SSam Ravnborg 1949a88b5ba8SSam Ravnborg static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu, 1950a88b5ba8SSam Ravnborg unsigned long npages) 1951a88b5ba8SSam Ravnborg { 1952a88b5ba8SSam Ravnborg long entry; 1953a88b5ba8SSam Ravnborg 19540ae53ed1SSowmini Varadhan entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table, 19550ae53ed1SSowmini Varadhan npages, NULL, (unsigned long)-1, 0); 1956a88b5ba8SSam Ravnborg if (unlikely(entry < 0)) 1957a88b5ba8SSam Ravnborg return NULL; 1958a88b5ba8SSam Ravnborg 1959a88b5ba8SSam Ravnborg return iommu->page_table + entry; 1960a88b5ba8SSam Ravnborg } 1961a88b5ba8SSam Ravnborg 1962a88b5ba8SSam Ravnborg static u64 perm_to_mte(unsigned int map_perm) 1963a88b5ba8SSam Ravnborg { 1964a88b5ba8SSam Ravnborg u64 mte_base; 1965a88b5ba8SSam Ravnborg 1966a88b5ba8SSam Ravnborg mte_base = pagesize_code(); 1967a88b5ba8SSam Ravnborg 1968a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_SHADOW) { 1969a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R) 1970a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_COPY_R; 1971a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W) 1972a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_COPY_W; 1973a88b5ba8SSam Ravnborg } 1974a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_DIRECT) { 1975a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R) 1976a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_READ; 1977a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W) 1978a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_WRITE; 1979a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_X) 1980a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_EXEC; 1981a88b5ba8SSam Ravnborg } 1982a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_IO) { 1983a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R) 1984a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_IOMMU_R; 1985a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W) 1986a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_IOMMU_W; 1987a88b5ba8SSam Ravnborg } 1988a88b5ba8SSam Ravnborg 1989a88b5ba8SSam Ravnborg return mte_base; 1990a88b5ba8SSam Ravnborg } 1991a88b5ba8SSam Ravnborg 1992a88b5ba8SSam Ravnborg static int pages_in_region(unsigned long base, long len) 1993a88b5ba8SSam Ravnborg { 1994a88b5ba8SSam Ravnborg int count = 0; 1995a88b5ba8SSam Ravnborg 1996a88b5ba8SSam Ravnborg do { 1997a88b5ba8SSam Ravnborg unsigned long new = (base + PAGE_SIZE) & PAGE_MASK; 1998a88b5ba8SSam Ravnborg 1999a88b5ba8SSam Ravnborg len -= (new - base); 2000a88b5ba8SSam Ravnborg base = new; 2001a88b5ba8SSam Ravnborg count++; 2002a88b5ba8SSam Ravnborg } while (len > 0); 2003a88b5ba8SSam Ravnborg 2004a88b5ba8SSam Ravnborg return count; 2005a88b5ba8SSam Ravnborg } 2006a88b5ba8SSam Ravnborg 2007a88b5ba8SSam Ravnborg struct cookie_state { 2008a88b5ba8SSam Ravnborg struct ldc_mtable_entry *page_table; 2009a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies; 2010a88b5ba8SSam Ravnborg u64 mte_base; 2011a88b5ba8SSam Ravnborg u64 prev_cookie; 2012a88b5ba8SSam Ravnborg u32 pte_idx; 2013a88b5ba8SSam Ravnborg u32 nc; 2014a88b5ba8SSam Ravnborg }; 2015a88b5ba8SSam Ravnborg 2016a88b5ba8SSam Ravnborg static void fill_cookies(struct cookie_state *sp, unsigned long pa, 2017a88b5ba8SSam Ravnborg unsigned long off, unsigned long len) 2018a88b5ba8SSam Ravnborg { 2019a88b5ba8SSam Ravnborg do { 2020a88b5ba8SSam Ravnborg unsigned long tlen, new = pa + PAGE_SIZE; 2021a88b5ba8SSam Ravnborg u64 this_cookie; 2022a88b5ba8SSam Ravnborg 2023a88b5ba8SSam Ravnborg sp->page_table[sp->pte_idx].mte = sp->mte_base | pa; 2024a88b5ba8SSam Ravnborg 2025a88b5ba8SSam Ravnborg tlen = PAGE_SIZE; 2026a88b5ba8SSam Ravnborg if (off) 2027a88b5ba8SSam Ravnborg tlen = PAGE_SIZE - off; 2028a88b5ba8SSam Ravnborg if (tlen > len) 2029a88b5ba8SSam Ravnborg tlen = len; 2030a88b5ba8SSam Ravnborg 2031a88b5ba8SSam Ravnborg this_cookie = make_cookie(sp->pte_idx, 2032a88b5ba8SSam Ravnborg pagesize_code(), off); 2033a88b5ba8SSam Ravnborg 2034a88b5ba8SSam Ravnborg off = 0; 2035a88b5ba8SSam Ravnborg 2036a88b5ba8SSam Ravnborg if (this_cookie == sp->prev_cookie) { 2037a88b5ba8SSam Ravnborg sp->cookies[sp->nc - 1].cookie_size += tlen; 2038a88b5ba8SSam Ravnborg } else { 2039a88b5ba8SSam Ravnborg sp->cookies[sp->nc].cookie_addr = this_cookie; 2040a88b5ba8SSam Ravnborg sp->cookies[sp->nc].cookie_size = tlen; 2041a88b5ba8SSam Ravnborg sp->nc++; 2042a88b5ba8SSam Ravnborg } 2043a88b5ba8SSam Ravnborg sp->prev_cookie = this_cookie + tlen; 2044a88b5ba8SSam Ravnborg 2045a88b5ba8SSam Ravnborg sp->pte_idx++; 2046a88b5ba8SSam Ravnborg 2047a88b5ba8SSam Ravnborg len -= tlen; 2048a88b5ba8SSam Ravnborg pa = new; 2049a88b5ba8SSam Ravnborg } while (len > 0); 2050a88b5ba8SSam Ravnborg } 2051a88b5ba8SSam Ravnborg 2052a88b5ba8SSam Ravnborg static int sg_count_one(struct scatterlist *sg) 2053a88b5ba8SSam Ravnborg { 2054a88b5ba8SSam Ravnborg unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT; 2055a88b5ba8SSam Ravnborg long len = sg->length; 2056a88b5ba8SSam Ravnborg 2057a88b5ba8SSam Ravnborg if ((sg->offset | len) & (8UL - 1)) 2058a88b5ba8SSam Ravnborg return -EFAULT; 2059a88b5ba8SSam Ravnborg 2060a88b5ba8SSam Ravnborg return pages_in_region(base + sg->offset, len); 2061a88b5ba8SSam Ravnborg } 2062a88b5ba8SSam Ravnborg 2063a88b5ba8SSam Ravnborg static int sg_count_pages(struct scatterlist *sg, int num_sg) 2064a88b5ba8SSam Ravnborg { 2065a88b5ba8SSam Ravnborg int count; 2066a88b5ba8SSam Ravnborg int i; 2067a88b5ba8SSam Ravnborg 2068a88b5ba8SSam Ravnborg count = 0; 2069a88b5ba8SSam Ravnborg for (i = 0; i < num_sg; i++) { 2070a88b5ba8SSam Ravnborg int err = sg_count_one(sg + i); 2071a88b5ba8SSam Ravnborg if (err < 0) 2072a88b5ba8SSam Ravnborg return err; 2073a88b5ba8SSam Ravnborg count += err; 2074a88b5ba8SSam Ravnborg } 2075a88b5ba8SSam Ravnborg 2076a88b5ba8SSam Ravnborg return count; 2077a88b5ba8SSam Ravnborg } 2078a88b5ba8SSam Ravnborg 2079a88b5ba8SSam Ravnborg int ldc_map_sg(struct ldc_channel *lp, 2080a88b5ba8SSam Ravnborg struct scatterlist *sg, int num_sg, 2081a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies, 2082a88b5ba8SSam Ravnborg unsigned int map_perm) 2083a88b5ba8SSam Ravnborg { 20840ae53ed1SSowmini Varadhan unsigned long i, npages; 2085a88b5ba8SSam Ravnborg struct ldc_mtable_entry *base; 2086a88b5ba8SSam Ravnborg struct cookie_state state; 2087a88b5ba8SSam Ravnborg struct ldc_iommu *iommu; 2088a88b5ba8SSam Ravnborg int err; 20898c07a308SAkinobu Mita struct scatterlist *s; 2090a88b5ba8SSam Ravnborg 2091a88b5ba8SSam Ravnborg if (map_perm & ~LDC_MAP_ALL) 2092a88b5ba8SSam Ravnborg return -EINVAL; 2093a88b5ba8SSam Ravnborg 2094a88b5ba8SSam Ravnborg err = sg_count_pages(sg, num_sg); 2095a88b5ba8SSam Ravnborg if (err < 0) 2096a88b5ba8SSam Ravnborg return err; 2097a88b5ba8SSam Ravnborg 2098a88b5ba8SSam Ravnborg npages = err; 2099a88b5ba8SSam Ravnborg if (err > ncookies) 2100a88b5ba8SSam Ravnborg return -EMSGSIZE; 2101a88b5ba8SSam Ravnborg 2102a88b5ba8SSam Ravnborg iommu = &lp->iommu; 2103a88b5ba8SSam Ravnborg 2104a88b5ba8SSam Ravnborg base = alloc_npages(iommu, npages); 2105a88b5ba8SSam Ravnborg 2106a88b5ba8SSam Ravnborg if (!base) 2107a88b5ba8SSam Ravnborg return -ENOMEM; 2108a88b5ba8SSam Ravnborg 2109a88b5ba8SSam Ravnborg state.page_table = iommu->page_table; 2110a88b5ba8SSam Ravnborg state.cookies = cookies; 2111a88b5ba8SSam Ravnborg state.mte_base = perm_to_mte(map_perm); 2112a88b5ba8SSam Ravnborg state.prev_cookie = ~(u64)0; 2113a88b5ba8SSam Ravnborg state.pte_idx = (base - iommu->page_table); 2114a88b5ba8SSam Ravnborg state.nc = 0; 2115a88b5ba8SSam Ravnborg 21168c07a308SAkinobu Mita for_each_sg(sg, s, num_sg, i) { 21178c07a308SAkinobu Mita fill_cookies(&state, page_to_pfn(sg_page(s)) << PAGE_SHIFT, 21188c07a308SAkinobu Mita s->offset, s->length); 21198c07a308SAkinobu Mita } 2120a88b5ba8SSam Ravnborg 2121a88b5ba8SSam Ravnborg return state.nc; 2122a88b5ba8SSam Ravnborg } 2123a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_map_sg); 2124a88b5ba8SSam Ravnborg 2125a88b5ba8SSam Ravnborg int ldc_map_single(struct ldc_channel *lp, 2126a88b5ba8SSam Ravnborg void *buf, unsigned int len, 2127a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies, 2128a88b5ba8SSam Ravnborg unsigned int map_perm) 2129a88b5ba8SSam Ravnborg { 21300ae53ed1SSowmini Varadhan unsigned long npages, pa; 2131a88b5ba8SSam Ravnborg struct ldc_mtable_entry *base; 2132a88b5ba8SSam Ravnborg struct cookie_state state; 2133a88b5ba8SSam Ravnborg struct ldc_iommu *iommu; 2134a88b5ba8SSam Ravnborg 2135a88b5ba8SSam Ravnborg if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1)) 2136a88b5ba8SSam Ravnborg return -EINVAL; 2137a88b5ba8SSam Ravnborg 2138a88b5ba8SSam Ravnborg pa = __pa(buf); 2139a88b5ba8SSam Ravnborg if ((pa | len) & (8UL - 1)) 2140a88b5ba8SSam Ravnborg return -EFAULT; 2141a88b5ba8SSam Ravnborg 2142a88b5ba8SSam Ravnborg npages = pages_in_region(pa, len); 2143a88b5ba8SSam Ravnborg 2144a88b5ba8SSam Ravnborg iommu = &lp->iommu; 2145a88b5ba8SSam Ravnborg 2146a88b5ba8SSam Ravnborg base = alloc_npages(iommu, npages); 2147a88b5ba8SSam Ravnborg 2148a88b5ba8SSam Ravnborg if (!base) 2149a88b5ba8SSam Ravnborg return -ENOMEM; 2150a88b5ba8SSam Ravnborg 2151a88b5ba8SSam Ravnborg state.page_table = iommu->page_table; 2152a88b5ba8SSam Ravnborg state.cookies = cookies; 2153a88b5ba8SSam Ravnborg state.mte_base = perm_to_mte(map_perm); 2154a88b5ba8SSam Ravnborg state.prev_cookie = ~(u64)0; 2155a88b5ba8SSam Ravnborg state.pte_idx = (base - iommu->page_table); 2156a88b5ba8SSam Ravnborg state.nc = 0; 2157a88b5ba8SSam Ravnborg fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len); 215842db672dSDavid L Stevens BUG_ON(state.nc > ncookies); 2159a88b5ba8SSam Ravnborg 2160a88b5ba8SSam Ravnborg return state.nc; 2161a88b5ba8SSam Ravnborg } 2162a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_map_single); 2163a88b5ba8SSam Ravnborg 21640ae53ed1SSowmini Varadhan 2165a88b5ba8SSam Ravnborg static void free_npages(unsigned long id, struct ldc_iommu *iommu, 2166a88b5ba8SSam Ravnborg u64 cookie, u64 size) 2167a88b5ba8SSam Ravnborg { 21680ae53ed1SSowmini Varadhan unsigned long npages, entry; 2169a88b5ba8SSam Ravnborg 2170a88b5ba8SSam Ravnborg npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT; 2171a88b5ba8SSam Ravnborg 21720ae53ed1SSowmini Varadhan entry = ldc_cookie_to_index(cookie, iommu); 21730ae53ed1SSowmini Varadhan ldc_demap(iommu, id, cookie, entry, npages); 21740ae53ed1SSowmini Varadhan iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry); 2175a88b5ba8SSam Ravnborg } 2176a88b5ba8SSam Ravnborg 2177a88b5ba8SSam Ravnborg void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, 2178a88b5ba8SSam Ravnborg int ncookies) 2179a88b5ba8SSam Ravnborg { 2180a88b5ba8SSam Ravnborg struct ldc_iommu *iommu = &lp->iommu; 2181c12f048fSDavid S. Miller int i; 21820ae53ed1SSowmini Varadhan unsigned long flags; 2183a88b5ba8SSam Ravnborg 2184a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags); 2185a88b5ba8SSam Ravnborg for (i = 0; i < ncookies; i++) { 2186a88b5ba8SSam Ravnborg u64 addr = cookies[i].cookie_addr; 2187a88b5ba8SSam Ravnborg u64 size = cookies[i].cookie_size; 2188a88b5ba8SSam Ravnborg 2189a88b5ba8SSam Ravnborg free_npages(lp->id, iommu, addr, size); 2190a88b5ba8SSam Ravnborg } 2191a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags); 2192a88b5ba8SSam Ravnborg } 2193a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_unmap); 2194a88b5ba8SSam Ravnborg 2195a88b5ba8SSam Ravnborg int ldc_copy(struct ldc_channel *lp, int copy_dir, 2196a88b5ba8SSam Ravnborg void *buf, unsigned int len, unsigned long offset, 2197a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies) 2198a88b5ba8SSam Ravnborg { 2199a88b5ba8SSam Ravnborg unsigned int orig_len; 2200a88b5ba8SSam Ravnborg unsigned long ra; 2201a88b5ba8SSam Ravnborg int i; 2202a88b5ba8SSam Ravnborg 2203a88b5ba8SSam Ravnborg if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) { 2204a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n", 2205a88b5ba8SSam Ravnborg lp->id, copy_dir); 2206a88b5ba8SSam Ravnborg return -EINVAL; 2207a88b5ba8SSam Ravnborg } 2208a88b5ba8SSam Ravnborg 2209a88b5ba8SSam Ravnborg ra = __pa(buf); 2210a88b5ba8SSam Ravnborg if ((ra | len | offset) & (8UL - 1)) { 2211a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer " 2212a88b5ba8SSam Ravnborg "ra[%lx] len[%x] offset[%lx]\n", 2213a88b5ba8SSam Ravnborg lp->id, ra, len, offset); 2214a88b5ba8SSam Ravnborg return -EFAULT; 2215a88b5ba8SSam Ravnborg } 2216a88b5ba8SSam Ravnborg 2217a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE || 2218a88b5ba8SSam Ravnborg (lp->flags & LDC_FLAG_RESET)) { 2219a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] " 2220a88b5ba8SSam Ravnborg "flags[%x]\n", lp->id, lp->hs_state, lp->flags); 2221a88b5ba8SSam Ravnborg return -ECONNRESET; 2222a88b5ba8SSam Ravnborg } 2223a88b5ba8SSam Ravnborg 2224a88b5ba8SSam Ravnborg orig_len = len; 2225a88b5ba8SSam Ravnborg for (i = 0; i < ncookies; i++) { 2226a88b5ba8SSam Ravnborg unsigned long cookie_raddr = cookies[i].cookie_addr; 2227a88b5ba8SSam Ravnborg unsigned long this_len = cookies[i].cookie_size; 2228a88b5ba8SSam Ravnborg unsigned long actual_len; 2229a88b5ba8SSam Ravnborg 2230a88b5ba8SSam Ravnborg if (unlikely(offset)) { 2231a88b5ba8SSam Ravnborg unsigned long this_off = offset; 2232a88b5ba8SSam Ravnborg 2233a88b5ba8SSam Ravnborg if (this_off > this_len) 2234a88b5ba8SSam Ravnborg this_off = this_len; 2235a88b5ba8SSam Ravnborg 2236a88b5ba8SSam Ravnborg offset -= this_off; 2237a88b5ba8SSam Ravnborg this_len -= this_off; 2238a88b5ba8SSam Ravnborg if (!this_len) 2239a88b5ba8SSam Ravnborg continue; 2240a88b5ba8SSam Ravnborg cookie_raddr += this_off; 2241a88b5ba8SSam Ravnborg } 2242a88b5ba8SSam Ravnborg 2243a88b5ba8SSam Ravnborg if (this_len > len) 2244a88b5ba8SSam Ravnborg this_len = len; 2245a88b5ba8SSam Ravnborg 2246a88b5ba8SSam Ravnborg while (1) { 2247a88b5ba8SSam Ravnborg unsigned long hv_err; 2248a88b5ba8SSam Ravnborg 2249a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_copy(lp->id, copy_dir, 2250a88b5ba8SSam Ravnborg cookie_raddr, ra, 2251a88b5ba8SSam Ravnborg this_len, &actual_len); 2252a88b5ba8SSam Ravnborg if (unlikely(hv_err)) { 2253a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] " 2254a88b5ba8SSam Ravnborg "HV error %lu\n", 2255a88b5ba8SSam Ravnborg lp->id, hv_err); 2256a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE || 2257a88b5ba8SSam Ravnborg (lp->flags & LDC_FLAG_RESET)) 2258a88b5ba8SSam Ravnborg return -ECONNRESET; 2259a88b5ba8SSam Ravnborg else 2260a88b5ba8SSam Ravnborg return -EFAULT; 2261a88b5ba8SSam Ravnborg } 2262a88b5ba8SSam Ravnborg 2263a88b5ba8SSam Ravnborg cookie_raddr += actual_len; 2264a88b5ba8SSam Ravnborg ra += actual_len; 2265a88b5ba8SSam Ravnborg len -= actual_len; 2266a88b5ba8SSam Ravnborg if (actual_len == this_len) 2267a88b5ba8SSam Ravnborg break; 2268a88b5ba8SSam Ravnborg 2269a88b5ba8SSam Ravnborg this_len -= actual_len; 2270a88b5ba8SSam Ravnborg } 2271a88b5ba8SSam Ravnborg 2272a88b5ba8SSam Ravnborg if (!len) 2273a88b5ba8SSam Ravnborg break; 2274a88b5ba8SSam Ravnborg } 2275a88b5ba8SSam Ravnborg 2276a88b5ba8SSam Ravnborg /* It is caller policy what to do about short copies. 2277a88b5ba8SSam Ravnborg * For example, a networking driver can declare the 2278a88b5ba8SSam Ravnborg * packet a runt and drop it. 2279a88b5ba8SSam Ravnborg */ 2280a88b5ba8SSam Ravnborg 2281a88b5ba8SSam Ravnborg return orig_len - len; 2282a88b5ba8SSam Ravnborg } 2283a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_copy); 2284a88b5ba8SSam Ravnborg 2285a88b5ba8SSam Ravnborg void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len, 2286a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int *ncookies, 2287a88b5ba8SSam Ravnborg unsigned int map_perm) 2288a88b5ba8SSam Ravnborg { 2289a88b5ba8SSam Ravnborg void *buf; 2290a88b5ba8SSam Ravnborg int err; 2291a88b5ba8SSam Ravnborg 2292a88b5ba8SSam Ravnborg if (len & (8UL - 1)) 2293a88b5ba8SSam Ravnborg return ERR_PTR(-EINVAL); 2294a88b5ba8SSam Ravnborg 22950edfad59SSowmini Varadhan buf = kzalloc(len, GFP_ATOMIC); 2296a88b5ba8SSam Ravnborg if (!buf) 2297a88b5ba8SSam Ravnborg return ERR_PTR(-ENOMEM); 2298a88b5ba8SSam Ravnborg 2299a88b5ba8SSam Ravnborg err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm); 2300a88b5ba8SSam Ravnborg if (err < 0) { 2301a88b5ba8SSam Ravnborg kfree(buf); 2302a88b5ba8SSam Ravnborg return ERR_PTR(err); 2303a88b5ba8SSam Ravnborg } 2304a88b5ba8SSam Ravnborg *ncookies = err; 2305a88b5ba8SSam Ravnborg 2306a88b5ba8SSam Ravnborg return buf; 2307a88b5ba8SSam Ravnborg } 2308a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_alloc_exp_dring); 2309a88b5ba8SSam Ravnborg 2310a88b5ba8SSam Ravnborg void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len, 2311a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies) 2312a88b5ba8SSam Ravnborg { 2313a88b5ba8SSam Ravnborg ldc_unmap(lp, cookies, ncookies); 2314a88b5ba8SSam Ravnborg kfree(buf); 2315a88b5ba8SSam Ravnborg } 2316a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_free_exp_dring); 2317a88b5ba8SSam Ravnborg 2318a88b5ba8SSam Ravnborg static int __init ldc_init(void) 2319a88b5ba8SSam Ravnborg { 2320a88b5ba8SSam Ravnborg unsigned long major, minor; 2321a88b5ba8SSam Ravnborg struct mdesc_handle *hp; 2322a88b5ba8SSam Ravnborg const u64 *v; 2323a88b5ba8SSam Ravnborg int err; 2324a88b5ba8SSam Ravnborg u64 mp; 2325a88b5ba8SSam Ravnborg 2326a88b5ba8SSam Ravnborg hp = mdesc_grab(); 2327a88b5ba8SSam Ravnborg if (!hp) 2328a88b5ba8SSam Ravnborg return -ENODEV; 2329a88b5ba8SSam Ravnborg 2330a88b5ba8SSam Ravnborg mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform"); 2331a88b5ba8SSam Ravnborg err = -ENODEV; 2332a88b5ba8SSam Ravnborg if (mp == MDESC_NODE_NULL) 2333a88b5ba8SSam Ravnborg goto out; 2334a88b5ba8SSam Ravnborg 2335a88b5ba8SSam Ravnborg v = mdesc_get_property(hp, mp, "domaining-enabled", NULL); 2336a88b5ba8SSam Ravnborg if (!v) 2337a88b5ba8SSam Ravnborg goto out; 2338a88b5ba8SSam Ravnborg 2339a88b5ba8SSam Ravnborg major = 1; 2340a88b5ba8SSam Ravnborg minor = 0; 2341a88b5ba8SSam Ravnborg if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) { 2342a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "Could not register LDOM hvapi.\n"); 2343a88b5ba8SSam Ravnborg goto out; 2344a88b5ba8SSam Ravnborg } 2345a88b5ba8SSam Ravnborg 2346a88b5ba8SSam Ravnborg printk(KERN_INFO "%s", version); 2347a88b5ba8SSam Ravnborg 2348a88b5ba8SSam Ravnborg if (!*v) { 2349a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "Domaining disabled.\n"); 2350a88b5ba8SSam Ravnborg goto out; 2351a88b5ba8SSam Ravnborg } 2352a88b5ba8SSam Ravnborg ldom_domaining_enabled = 1; 2353a88b5ba8SSam Ravnborg err = 0; 2354a88b5ba8SSam Ravnborg 2355a88b5ba8SSam Ravnborg out: 2356a88b5ba8SSam Ravnborg mdesc_release(hp); 2357a88b5ba8SSam Ravnborg return err; 2358a88b5ba8SSam Ravnborg } 2359a88b5ba8SSam Ravnborg 2360a88b5ba8SSam Ravnborg core_initcall(ldc_init); 2361