1a88b5ba8SSam Ravnborg /* ldc.c: Logical Domain Channel link-layer protocol driver. 2a88b5ba8SSam Ravnborg * 3a88b5ba8SSam Ravnborg * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> 4a88b5ba8SSam Ravnborg */ 5a88b5ba8SSam Ravnborg 6a88b5ba8SSam Ravnborg #include <linux/kernel.h> 7a88b5ba8SSam Ravnborg #include <linux/module.h> 8a88b5ba8SSam Ravnborg #include <linux/slab.h> 9a88b5ba8SSam Ravnborg #include <linux/spinlock.h> 10a88b5ba8SSam Ravnborg #include <linux/delay.h> 11a88b5ba8SSam Ravnborg #include <linux/errno.h> 12a88b5ba8SSam Ravnborg #include <linux/string.h> 13a88b5ba8SSam Ravnborg #include <linux/scatterlist.h> 14a88b5ba8SSam Ravnborg #include <linux/interrupt.h> 15a88b5ba8SSam Ravnborg #include <linux/list.h> 16a88b5ba8SSam Ravnborg #include <linux/init.h> 17a88b5ba8SSam Ravnborg 18a88b5ba8SSam Ravnborg #include <asm/hypervisor.h> 19a88b5ba8SSam Ravnborg #include <asm/iommu.h> 20a88b5ba8SSam Ravnborg #include <asm/page.h> 21a88b5ba8SSam Ravnborg #include <asm/ldc.h> 22a88b5ba8SSam Ravnborg #include <asm/mdesc.h> 23a88b5ba8SSam Ravnborg 24a88b5ba8SSam Ravnborg #define DRV_MODULE_NAME "ldc" 25a88b5ba8SSam Ravnborg #define PFX DRV_MODULE_NAME ": " 26a88b5ba8SSam Ravnborg #define DRV_MODULE_VERSION "1.1" 27a88b5ba8SSam Ravnborg #define DRV_MODULE_RELDATE "July 22, 2008" 28a88b5ba8SSam Ravnborg 29a88b5ba8SSam Ravnborg static char version[] __devinitdata = 30a88b5ba8SSam Ravnborg DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 31a88b5ba8SSam Ravnborg #define LDC_PACKET_SIZE 64 32a88b5ba8SSam Ravnborg 33a88b5ba8SSam Ravnborg /* Packet header layout for unreliable and reliable mode frames. 34a88b5ba8SSam Ravnborg * When in RAW mode, packets are simply straight 64-byte payloads 35a88b5ba8SSam Ravnborg * with no headers. 36a88b5ba8SSam Ravnborg */ 37a88b5ba8SSam Ravnborg struct ldc_packet { 38a88b5ba8SSam Ravnborg u8 type; 39a88b5ba8SSam Ravnborg #define LDC_CTRL 0x01 40a88b5ba8SSam Ravnborg #define LDC_DATA 0x02 41a88b5ba8SSam Ravnborg #define LDC_ERR 0x10 42a88b5ba8SSam Ravnborg 43a88b5ba8SSam Ravnborg u8 stype; 44a88b5ba8SSam Ravnborg #define LDC_INFO 0x01 45a88b5ba8SSam Ravnborg #define LDC_ACK 0x02 46a88b5ba8SSam Ravnborg #define LDC_NACK 0x04 47a88b5ba8SSam Ravnborg 48a88b5ba8SSam Ravnborg u8 ctrl; 49a88b5ba8SSam Ravnborg #define LDC_VERS 0x01 /* Link Version */ 50a88b5ba8SSam Ravnborg #define LDC_RTS 0x02 /* Request To Send */ 51a88b5ba8SSam Ravnborg #define LDC_RTR 0x03 /* Ready To Receive */ 52a88b5ba8SSam Ravnborg #define LDC_RDX 0x04 /* Ready for Data eXchange */ 53a88b5ba8SSam Ravnborg #define LDC_CTRL_MSK 0x0f 54a88b5ba8SSam Ravnborg 55a88b5ba8SSam Ravnborg u8 env; 56a88b5ba8SSam Ravnborg #define LDC_LEN 0x3f 57a88b5ba8SSam Ravnborg #define LDC_FRAG_MASK 0xc0 58a88b5ba8SSam Ravnborg #define LDC_START 0x40 59a88b5ba8SSam Ravnborg #define LDC_STOP 0x80 60a88b5ba8SSam Ravnborg 61a88b5ba8SSam Ravnborg u32 seqid; 62a88b5ba8SSam Ravnborg 63a88b5ba8SSam Ravnborg union { 64a88b5ba8SSam Ravnborg u8 u_data[LDC_PACKET_SIZE - 8]; 65a88b5ba8SSam Ravnborg struct { 66a88b5ba8SSam Ravnborg u32 pad; 67a88b5ba8SSam Ravnborg u32 ackid; 68a88b5ba8SSam Ravnborg u8 r_data[LDC_PACKET_SIZE - 8 - 8]; 69a88b5ba8SSam Ravnborg } r; 70a88b5ba8SSam Ravnborg } u; 71a88b5ba8SSam Ravnborg }; 72a88b5ba8SSam Ravnborg 73a88b5ba8SSam Ravnborg struct ldc_version { 74a88b5ba8SSam Ravnborg u16 major; 75a88b5ba8SSam Ravnborg u16 minor; 76a88b5ba8SSam Ravnborg }; 77a88b5ba8SSam Ravnborg 78a88b5ba8SSam Ravnborg /* Ordered from largest major to lowest. */ 79a88b5ba8SSam Ravnborg static struct ldc_version ver_arr[] = { 80a88b5ba8SSam Ravnborg { .major = 1, .minor = 0 }, 81a88b5ba8SSam Ravnborg }; 82a88b5ba8SSam Ravnborg 83a88b5ba8SSam Ravnborg #define LDC_DEFAULT_MTU (4 * LDC_PACKET_SIZE) 84a88b5ba8SSam Ravnborg #define LDC_DEFAULT_NUM_ENTRIES (PAGE_SIZE / LDC_PACKET_SIZE) 85a88b5ba8SSam Ravnborg 86a88b5ba8SSam Ravnborg struct ldc_channel; 87a88b5ba8SSam Ravnborg 88a88b5ba8SSam Ravnborg struct ldc_mode_ops { 89a88b5ba8SSam Ravnborg int (*write)(struct ldc_channel *, const void *, unsigned int); 90a88b5ba8SSam Ravnborg int (*read)(struct ldc_channel *, void *, unsigned int); 91a88b5ba8SSam Ravnborg }; 92a88b5ba8SSam Ravnborg 93a88b5ba8SSam Ravnborg static const struct ldc_mode_ops raw_ops; 94a88b5ba8SSam Ravnborg static const struct ldc_mode_ops nonraw_ops; 95a88b5ba8SSam Ravnborg static const struct ldc_mode_ops stream_ops; 96a88b5ba8SSam Ravnborg 97a88b5ba8SSam Ravnborg int ldom_domaining_enabled; 98a88b5ba8SSam Ravnborg 99a88b5ba8SSam Ravnborg struct ldc_iommu { 100a88b5ba8SSam Ravnborg /* Protects arena alloc/free. */ 101a88b5ba8SSam Ravnborg spinlock_t lock; 102a88b5ba8SSam Ravnborg struct iommu_arena arena; 103a88b5ba8SSam Ravnborg struct ldc_mtable_entry *page_table; 104a88b5ba8SSam Ravnborg }; 105a88b5ba8SSam Ravnborg 106a88b5ba8SSam Ravnborg struct ldc_channel { 107a88b5ba8SSam Ravnborg /* Protects all operations that depend upon channel state. */ 108a88b5ba8SSam Ravnborg spinlock_t lock; 109a88b5ba8SSam Ravnborg 110a88b5ba8SSam Ravnborg unsigned long id; 111a88b5ba8SSam Ravnborg 112a88b5ba8SSam Ravnborg u8 *mssbuf; 113a88b5ba8SSam Ravnborg u32 mssbuf_len; 114a88b5ba8SSam Ravnborg u32 mssbuf_off; 115a88b5ba8SSam Ravnborg 116a88b5ba8SSam Ravnborg struct ldc_packet *tx_base; 117a88b5ba8SSam Ravnborg unsigned long tx_head; 118a88b5ba8SSam Ravnborg unsigned long tx_tail; 119a88b5ba8SSam Ravnborg unsigned long tx_num_entries; 120a88b5ba8SSam Ravnborg unsigned long tx_ra; 121a88b5ba8SSam Ravnborg 122a88b5ba8SSam Ravnborg unsigned long tx_acked; 123a88b5ba8SSam Ravnborg 124a88b5ba8SSam Ravnborg struct ldc_packet *rx_base; 125a88b5ba8SSam Ravnborg unsigned long rx_head; 126a88b5ba8SSam Ravnborg unsigned long rx_tail; 127a88b5ba8SSam Ravnborg unsigned long rx_num_entries; 128a88b5ba8SSam Ravnborg unsigned long rx_ra; 129a88b5ba8SSam Ravnborg 130a88b5ba8SSam Ravnborg u32 rcv_nxt; 131a88b5ba8SSam Ravnborg u32 snd_nxt; 132a88b5ba8SSam Ravnborg 133a88b5ba8SSam Ravnborg unsigned long chan_state; 134a88b5ba8SSam Ravnborg 135a88b5ba8SSam Ravnborg struct ldc_channel_config cfg; 136a88b5ba8SSam Ravnborg void *event_arg; 137a88b5ba8SSam Ravnborg 138a88b5ba8SSam Ravnborg const struct ldc_mode_ops *mops; 139a88b5ba8SSam Ravnborg 140a88b5ba8SSam Ravnborg struct ldc_iommu iommu; 141a88b5ba8SSam Ravnborg 142a88b5ba8SSam Ravnborg struct ldc_version ver; 143a88b5ba8SSam Ravnborg 144a88b5ba8SSam Ravnborg u8 hs_state; 145a88b5ba8SSam Ravnborg #define LDC_HS_CLOSED 0x00 146a88b5ba8SSam Ravnborg #define LDC_HS_OPEN 0x01 147a88b5ba8SSam Ravnborg #define LDC_HS_GOTVERS 0x02 148a88b5ba8SSam Ravnborg #define LDC_HS_SENTRTR 0x03 149a88b5ba8SSam Ravnborg #define LDC_HS_GOTRTR 0x04 150a88b5ba8SSam Ravnborg #define LDC_HS_COMPLETE 0x10 151a88b5ba8SSam Ravnborg 152a88b5ba8SSam Ravnborg u8 flags; 153a88b5ba8SSam Ravnborg #define LDC_FLAG_ALLOCED_QUEUES 0x01 154a88b5ba8SSam Ravnborg #define LDC_FLAG_REGISTERED_QUEUES 0x02 155a88b5ba8SSam Ravnborg #define LDC_FLAG_REGISTERED_IRQS 0x04 156a88b5ba8SSam Ravnborg #define LDC_FLAG_RESET 0x10 157a88b5ba8SSam Ravnborg 158a88b5ba8SSam Ravnborg u8 mss; 159a88b5ba8SSam Ravnborg u8 state; 160a88b5ba8SSam Ravnborg 161a88b5ba8SSam Ravnborg #define LDC_IRQ_NAME_MAX 32 162a88b5ba8SSam Ravnborg char rx_irq_name[LDC_IRQ_NAME_MAX]; 163a88b5ba8SSam Ravnborg char tx_irq_name[LDC_IRQ_NAME_MAX]; 164a88b5ba8SSam Ravnborg 165a88b5ba8SSam Ravnborg struct hlist_head mh_list; 166a88b5ba8SSam Ravnborg 167a88b5ba8SSam Ravnborg struct hlist_node list; 168a88b5ba8SSam Ravnborg }; 169a88b5ba8SSam Ravnborg 170a88b5ba8SSam Ravnborg #define ldcdbg(TYPE, f, a...) \ 171a88b5ba8SSam Ravnborg do { if (lp->cfg.debug & LDC_DEBUG_##TYPE) \ 172a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \ 173a88b5ba8SSam Ravnborg } while (0) 174a88b5ba8SSam Ravnborg 175a88b5ba8SSam Ravnborg static const char *state_to_str(u8 state) 176a88b5ba8SSam Ravnborg { 177a88b5ba8SSam Ravnborg switch (state) { 178a88b5ba8SSam Ravnborg case LDC_STATE_INVALID: 179a88b5ba8SSam Ravnborg return "INVALID"; 180a88b5ba8SSam Ravnborg case LDC_STATE_INIT: 181a88b5ba8SSam Ravnborg return "INIT"; 182a88b5ba8SSam Ravnborg case LDC_STATE_BOUND: 183a88b5ba8SSam Ravnborg return "BOUND"; 184a88b5ba8SSam Ravnborg case LDC_STATE_READY: 185a88b5ba8SSam Ravnborg return "READY"; 186a88b5ba8SSam Ravnborg case LDC_STATE_CONNECTED: 187a88b5ba8SSam Ravnborg return "CONNECTED"; 188a88b5ba8SSam Ravnborg default: 189a88b5ba8SSam Ravnborg return "<UNKNOWN>"; 190a88b5ba8SSam Ravnborg } 191a88b5ba8SSam Ravnborg } 192a88b5ba8SSam Ravnborg 193a88b5ba8SSam Ravnborg static void ldc_set_state(struct ldc_channel *lp, u8 state) 194a88b5ba8SSam Ravnborg { 195a88b5ba8SSam Ravnborg ldcdbg(STATE, "STATE (%s) --> (%s)\n", 196a88b5ba8SSam Ravnborg state_to_str(lp->state), 197a88b5ba8SSam Ravnborg state_to_str(state)); 198a88b5ba8SSam Ravnborg 199a88b5ba8SSam Ravnborg lp->state = state; 200a88b5ba8SSam Ravnborg } 201a88b5ba8SSam Ravnborg 202a88b5ba8SSam Ravnborg static unsigned long __advance(unsigned long off, unsigned long num_entries) 203a88b5ba8SSam Ravnborg { 204a88b5ba8SSam Ravnborg off += LDC_PACKET_SIZE; 205a88b5ba8SSam Ravnborg if (off == (num_entries * LDC_PACKET_SIZE)) 206a88b5ba8SSam Ravnborg off = 0; 207a88b5ba8SSam Ravnborg 208a88b5ba8SSam Ravnborg return off; 209a88b5ba8SSam Ravnborg } 210a88b5ba8SSam Ravnborg 211a88b5ba8SSam Ravnborg static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off) 212a88b5ba8SSam Ravnborg { 213a88b5ba8SSam Ravnborg return __advance(off, lp->rx_num_entries); 214a88b5ba8SSam Ravnborg } 215a88b5ba8SSam Ravnborg 216a88b5ba8SSam Ravnborg static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off) 217a88b5ba8SSam Ravnborg { 218a88b5ba8SSam Ravnborg return __advance(off, lp->tx_num_entries); 219a88b5ba8SSam Ravnborg } 220a88b5ba8SSam Ravnborg 221a88b5ba8SSam Ravnborg static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp, 222a88b5ba8SSam Ravnborg unsigned long *new_tail) 223a88b5ba8SSam Ravnborg { 224a88b5ba8SSam Ravnborg struct ldc_packet *p; 225a88b5ba8SSam Ravnborg unsigned long t; 226a88b5ba8SSam Ravnborg 227a88b5ba8SSam Ravnborg t = tx_advance(lp, lp->tx_tail); 228a88b5ba8SSam Ravnborg if (t == lp->tx_head) 229a88b5ba8SSam Ravnborg return NULL; 230a88b5ba8SSam Ravnborg 231a88b5ba8SSam Ravnborg *new_tail = t; 232a88b5ba8SSam Ravnborg 233a88b5ba8SSam Ravnborg p = lp->tx_base; 234a88b5ba8SSam Ravnborg return p + (lp->tx_tail / LDC_PACKET_SIZE); 235a88b5ba8SSam Ravnborg } 236a88b5ba8SSam Ravnborg 237a88b5ba8SSam Ravnborg /* When we are in reliable or stream mode, have to track the next packet 238a88b5ba8SSam Ravnborg * we haven't gotten an ACK for in the TX queue using tx_acked. We have 239a88b5ba8SSam Ravnborg * to be careful not to stomp over the queue past that point. During 240a88b5ba8SSam Ravnborg * the handshake, we don't have TX data packets pending in the queue 241a88b5ba8SSam Ravnborg * and that's why handshake_get_tx_packet() need not be mindful of 242a88b5ba8SSam Ravnborg * lp->tx_acked. 243a88b5ba8SSam Ravnborg */ 244a88b5ba8SSam Ravnborg static unsigned long head_for_data(struct ldc_channel *lp) 245a88b5ba8SSam Ravnborg { 246a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_STREAM) 247a88b5ba8SSam Ravnborg return lp->tx_acked; 248a88b5ba8SSam Ravnborg return lp->tx_head; 249a88b5ba8SSam Ravnborg } 250a88b5ba8SSam Ravnborg 251a88b5ba8SSam Ravnborg static int tx_has_space_for(struct ldc_channel *lp, unsigned int size) 252a88b5ba8SSam Ravnborg { 253a88b5ba8SSam Ravnborg unsigned long limit, tail, new_tail, diff; 254a88b5ba8SSam Ravnborg unsigned int mss; 255a88b5ba8SSam Ravnborg 256a88b5ba8SSam Ravnborg limit = head_for_data(lp); 257a88b5ba8SSam Ravnborg tail = lp->tx_tail; 258a88b5ba8SSam Ravnborg new_tail = tx_advance(lp, tail); 259a88b5ba8SSam Ravnborg if (new_tail == limit) 260a88b5ba8SSam Ravnborg return 0; 261a88b5ba8SSam Ravnborg 262a88b5ba8SSam Ravnborg if (limit > new_tail) 263a88b5ba8SSam Ravnborg diff = limit - new_tail; 264a88b5ba8SSam Ravnborg else 265a88b5ba8SSam Ravnborg diff = (limit + 266a88b5ba8SSam Ravnborg ((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail)); 267a88b5ba8SSam Ravnborg diff /= LDC_PACKET_SIZE; 268a88b5ba8SSam Ravnborg mss = lp->mss; 269a88b5ba8SSam Ravnborg 270a88b5ba8SSam Ravnborg if (diff * mss < size) 271a88b5ba8SSam Ravnborg return 0; 272a88b5ba8SSam Ravnborg 273a88b5ba8SSam Ravnborg return 1; 274a88b5ba8SSam Ravnborg } 275a88b5ba8SSam Ravnborg 276a88b5ba8SSam Ravnborg static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp, 277a88b5ba8SSam Ravnborg unsigned long *new_tail) 278a88b5ba8SSam Ravnborg { 279a88b5ba8SSam Ravnborg struct ldc_packet *p; 280a88b5ba8SSam Ravnborg unsigned long h, t; 281a88b5ba8SSam Ravnborg 282a88b5ba8SSam Ravnborg h = head_for_data(lp); 283a88b5ba8SSam Ravnborg t = tx_advance(lp, lp->tx_tail); 284a88b5ba8SSam Ravnborg if (t == h) 285a88b5ba8SSam Ravnborg return NULL; 286a88b5ba8SSam Ravnborg 287a88b5ba8SSam Ravnborg *new_tail = t; 288a88b5ba8SSam Ravnborg 289a88b5ba8SSam Ravnborg p = lp->tx_base; 290a88b5ba8SSam Ravnborg return p + (lp->tx_tail / LDC_PACKET_SIZE); 291a88b5ba8SSam Ravnborg } 292a88b5ba8SSam Ravnborg 293a88b5ba8SSam Ravnborg static int set_tx_tail(struct ldc_channel *lp, unsigned long tail) 294a88b5ba8SSam Ravnborg { 295a88b5ba8SSam Ravnborg unsigned long orig_tail = lp->tx_tail; 296a88b5ba8SSam Ravnborg int limit = 1000; 297a88b5ba8SSam Ravnborg 298a88b5ba8SSam Ravnborg lp->tx_tail = tail; 299a88b5ba8SSam Ravnborg while (limit-- > 0) { 300a88b5ba8SSam Ravnborg unsigned long err; 301a88b5ba8SSam Ravnborg 302a88b5ba8SSam Ravnborg err = sun4v_ldc_tx_set_qtail(lp->id, tail); 303a88b5ba8SSam Ravnborg if (!err) 304a88b5ba8SSam Ravnborg return 0; 305a88b5ba8SSam Ravnborg 306a88b5ba8SSam Ravnborg if (err != HV_EWOULDBLOCK) { 307a88b5ba8SSam Ravnborg lp->tx_tail = orig_tail; 308a88b5ba8SSam Ravnborg return -EINVAL; 309a88b5ba8SSam Ravnborg } 310a88b5ba8SSam Ravnborg udelay(1); 311a88b5ba8SSam Ravnborg } 312a88b5ba8SSam Ravnborg 313a88b5ba8SSam Ravnborg lp->tx_tail = orig_tail; 314a88b5ba8SSam Ravnborg return -EBUSY; 315a88b5ba8SSam Ravnborg } 316a88b5ba8SSam Ravnborg 317a88b5ba8SSam Ravnborg /* This just updates the head value in the hypervisor using 318a88b5ba8SSam Ravnborg * a polling loop with a timeout. The caller takes care of 319a88b5ba8SSam Ravnborg * upating software state representing the head change, if any. 320a88b5ba8SSam Ravnborg */ 321a88b5ba8SSam Ravnborg static int __set_rx_head(struct ldc_channel *lp, unsigned long head) 322a88b5ba8SSam Ravnborg { 323a88b5ba8SSam Ravnborg int limit = 1000; 324a88b5ba8SSam Ravnborg 325a88b5ba8SSam Ravnborg while (limit-- > 0) { 326a88b5ba8SSam Ravnborg unsigned long err; 327a88b5ba8SSam Ravnborg 328a88b5ba8SSam Ravnborg err = sun4v_ldc_rx_set_qhead(lp->id, head); 329a88b5ba8SSam Ravnborg if (!err) 330a88b5ba8SSam Ravnborg return 0; 331a88b5ba8SSam Ravnborg 332a88b5ba8SSam Ravnborg if (err != HV_EWOULDBLOCK) 333a88b5ba8SSam Ravnborg return -EINVAL; 334a88b5ba8SSam Ravnborg 335a88b5ba8SSam Ravnborg udelay(1); 336a88b5ba8SSam Ravnborg } 337a88b5ba8SSam Ravnborg 338a88b5ba8SSam Ravnborg return -EBUSY; 339a88b5ba8SSam Ravnborg } 340a88b5ba8SSam Ravnborg 341a88b5ba8SSam Ravnborg static int send_tx_packet(struct ldc_channel *lp, 342a88b5ba8SSam Ravnborg struct ldc_packet *p, 343a88b5ba8SSam Ravnborg unsigned long new_tail) 344a88b5ba8SSam Ravnborg { 345a88b5ba8SSam Ravnborg BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE))); 346a88b5ba8SSam Ravnborg 347a88b5ba8SSam Ravnborg return set_tx_tail(lp, new_tail); 348a88b5ba8SSam Ravnborg } 349a88b5ba8SSam Ravnborg 350a88b5ba8SSam Ravnborg static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp, 351a88b5ba8SSam Ravnborg u8 stype, u8 ctrl, 352a88b5ba8SSam Ravnborg void *data, int dlen, 353a88b5ba8SSam Ravnborg unsigned long *new_tail) 354a88b5ba8SSam Ravnborg { 355a88b5ba8SSam Ravnborg struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail); 356a88b5ba8SSam Ravnborg 357a88b5ba8SSam Ravnborg if (p) { 358a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p)); 359a88b5ba8SSam Ravnborg p->type = LDC_CTRL; 360a88b5ba8SSam Ravnborg p->stype = stype; 361a88b5ba8SSam Ravnborg p->ctrl = ctrl; 362a88b5ba8SSam Ravnborg if (data) 363a88b5ba8SSam Ravnborg memcpy(p->u.u_data, data, dlen); 364a88b5ba8SSam Ravnborg } 365a88b5ba8SSam Ravnborg return p; 366a88b5ba8SSam Ravnborg } 367a88b5ba8SSam Ravnborg 368a88b5ba8SSam Ravnborg static int start_handshake(struct ldc_channel *lp) 369a88b5ba8SSam Ravnborg { 370a88b5ba8SSam Ravnborg struct ldc_packet *p; 371a88b5ba8SSam Ravnborg struct ldc_version *ver; 372a88b5ba8SSam Ravnborg unsigned long new_tail; 373a88b5ba8SSam Ravnborg 374a88b5ba8SSam Ravnborg ver = &ver_arr[0]; 375a88b5ba8SSam Ravnborg 376a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n", 377a88b5ba8SSam Ravnborg ver->major, ver->minor); 378a88b5ba8SSam Ravnborg 379a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS, 380a88b5ba8SSam Ravnborg ver, sizeof(*ver), &new_tail); 381a88b5ba8SSam Ravnborg if (p) { 382a88b5ba8SSam Ravnborg int err = send_tx_packet(lp, p, new_tail); 383a88b5ba8SSam Ravnborg if (!err) 384a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_RESET; 385a88b5ba8SSam Ravnborg return err; 386a88b5ba8SSam Ravnborg } 387a88b5ba8SSam Ravnborg return -EBUSY; 388a88b5ba8SSam Ravnborg } 389a88b5ba8SSam Ravnborg 390a88b5ba8SSam Ravnborg static int send_version_nack(struct ldc_channel *lp, 391a88b5ba8SSam Ravnborg u16 major, u16 minor) 392a88b5ba8SSam Ravnborg { 393a88b5ba8SSam Ravnborg struct ldc_packet *p; 394a88b5ba8SSam Ravnborg struct ldc_version ver; 395a88b5ba8SSam Ravnborg unsigned long new_tail; 396a88b5ba8SSam Ravnborg 397a88b5ba8SSam Ravnborg ver.major = major; 398a88b5ba8SSam Ravnborg ver.minor = minor; 399a88b5ba8SSam Ravnborg 400a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS, 401a88b5ba8SSam Ravnborg &ver, sizeof(ver), &new_tail); 402a88b5ba8SSam Ravnborg if (p) { 403a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n", 404a88b5ba8SSam Ravnborg ver.major, ver.minor); 405a88b5ba8SSam Ravnborg 406a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 407a88b5ba8SSam Ravnborg } 408a88b5ba8SSam Ravnborg return -EBUSY; 409a88b5ba8SSam Ravnborg } 410a88b5ba8SSam Ravnborg 411a88b5ba8SSam Ravnborg static int send_version_ack(struct ldc_channel *lp, 412a88b5ba8SSam Ravnborg struct ldc_version *vp) 413a88b5ba8SSam Ravnborg { 414a88b5ba8SSam Ravnborg struct ldc_packet *p; 415a88b5ba8SSam Ravnborg unsigned long new_tail; 416a88b5ba8SSam Ravnborg 417a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS, 418a88b5ba8SSam Ravnborg vp, sizeof(*vp), &new_tail); 419a88b5ba8SSam Ravnborg if (p) { 420a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n", 421a88b5ba8SSam Ravnborg vp->major, vp->minor); 422a88b5ba8SSam Ravnborg 423a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 424a88b5ba8SSam Ravnborg } 425a88b5ba8SSam Ravnborg return -EBUSY; 426a88b5ba8SSam Ravnborg } 427a88b5ba8SSam Ravnborg 428a88b5ba8SSam Ravnborg static int send_rts(struct ldc_channel *lp) 429a88b5ba8SSam Ravnborg { 430a88b5ba8SSam Ravnborg struct ldc_packet *p; 431a88b5ba8SSam Ravnborg unsigned long new_tail; 432a88b5ba8SSam Ravnborg 433a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0, 434a88b5ba8SSam Ravnborg &new_tail); 435a88b5ba8SSam Ravnborg if (p) { 436a88b5ba8SSam Ravnborg p->env = lp->cfg.mode; 437a88b5ba8SSam Ravnborg p->seqid = 0; 438a88b5ba8SSam Ravnborg lp->rcv_nxt = 0; 439a88b5ba8SSam Ravnborg 440a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n", 441a88b5ba8SSam Ravnborg p->env, p->seqid); 442a88b5ba8SSam Ravnborg 443a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 444a88b5ba8SSam Ravnborg } 445a88b5ba8SSam Ravnborg return -EBUSY; 446a88b5ba8SSam Ravnborg } 447a88b5ba8SSam Ravnborg 448a88b5ba8SSam Ravnborg static int send_rtr(struct ldc_channel *lp) 449a88b5ba8SSam Ravnborg { 450a88b5ba8SSam Ravnborg struct ldc_packet *p; 451a88b5ba8SSam Ravnborg unsigned long new_tail; 452a88b5ba8SSam Ravnborg 453a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0, 454a88b5ba8SSam Ravnborg &new_tail); 455a88b5ba8SSam Ravnborg if (p) { 456a88b5ba8SSam Ravnborg p->env = lp->cfg.mode; 457a88b5ba8SSam Ravnborg p->seqid = 0; 458a88b5ba8SSam Ravnborg 459a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n", 460a88b5ba8SSam Ravnborg p->env, p->seqid); 461a88b5ba8SSam Ravnborg 462a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 463a88b5ba8SSam Ravnborg } 464a88b5ba8SSam Ravnborg return -EBUSY; 465a88b5ba8SSam Ravnborg } 466a88b5ba8SSam Ravnborg 467a88b5ba8SSam Ravnborg static int send_rdx(struct ldc_channel *lp) 468a88b5ba8SSam Ravnborg { 469a88b5ba8SSam Ravnborg struct ldc_packet *p; 470a88b5ba8SSam Ravnborg unsigned long new_tail; 471a88b5ba8SSam Ravnborg 472a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0, 473a88b5ba8SSam Ravnborg &new_tail); 474a88b5ba8SSam Ravnborg if (p) { 475a88b5ba8SSam Ravnborg p->env = 0; 476a88b5ba8SSam Ravnborg p->seqid = ++lp->snd_nxt; 477a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt; 478a88b5ba8SSam Ravnborg 479a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n", 480a88b5ba8SSam Ravnborg p->env, p->seqid, p->u.r.ackid); 481a88b5ba8SSam Ravnborg 482a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 483a88b5ba8SSam Ravnborg } 484a88b5ba8SSam Ravnborg return -EBUSY; 485a88b5ba8SSam Ravnborg } 486a88b5ba8SSam Ravnborg 487a88b5ba8SSam Ravnborg static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt) 488a88b5ba8SSam Ravnborg { 489a88b5ba8SSam Ravnborg struct ldc_packet *p; 490a88b5ba8SSam Ravnborg unsigned long new_tail; 491a88b5ba8SSam Ravnborg int err; 492a88b5ba8SSam Ravnborg 493a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail); 494a88b5ba8SSam Ravnborg if (!p) 495a88b5ba8SSam Ravnborg return -EBUSY; 496a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p)); 497a88b5ba8SSam Ravnborg p->type = data_pkt->type; 498a88b5ba8SSam Ravnborg p->stype = LDC_NACK; 499a88b5ba8SSam Ravnborg p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK; 500a88b5ba8SSam Ravnborg p->seqid = lp->snd_nxt + 1; 501a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt; 502a88b5ba8SSam Ravnborg 503a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n", 504a88b5ba8SSam Ravnborg p->type, p->ctrl, p->seqid, p->u.r.ackid); 505a88b5ba8SSam Ravnborg 506a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail); 507a88b5ba8SSam Ravnborg if (!err) 508a88b5ba8SSam Ravnborg lp->snd_nxt++; 509a88b5ba8SSam Ravnborg 510a88b5ba8SSam Ravnborg return err; 511a88b5ba8SSam Ravnborg } 512a88b5ba8SSam Ravnborg 513a88b5ba8SSam Ravnborg static int ldc_abort(struct ldc_channel *lp) 514a88b5ba8SSam Ravnborg { 515a88b5ba8SSam Ravnborg unsigned long hv_err; 516a88b5ba8SSam Ravnborg 517a88b5ba8SSam Ravnborg ldcdbg(STATE, "ABORT\n"); 518a88b5ba8SSam Ravnborg 519a88b5ba8SSam Ravnborg /* We report but do not act upon the hypervisor errors because 520a88b5ba8SSam Ravnborg * there really isn't much we can do if they fail at this point. 521a88b5ba8SSam Ravnborg */ 522a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); 523a88b5ba8SSam Ravnborg if (hv_err) 524a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 525a88b5ba8SSam Ravnborg "sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n", 526a88b5ba8SSam Ravnborg lp->id, lp->tx_ra, lp->tx_num_entries, hv_err); 527a88b5ba8SSam Ravnborg 528a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, 529a88b5ba8SSam Ravnborg &lp->tx_head, 530a88b5ba8SSam Ravnborg &lp->tx_tail, 531a88b5ba8SSam Ravnborg &lp->chan_state); 532a88b5ba8SSam Ravnborg if (hv_err) 533a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 534a88b5ba8SSam Ravnborg "sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n", 535a88b5ba8SSam Ravnborg lp->id, hv_err); 536a88b5ba8SSam Ravnborg 537a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); 538a88b5ba8SSam Ravnborg if (hv_err) 539a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 540a88b5ba8SSam Ravnborg "sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n", 541a88b5ba8SSam Ravnborg lp->id, lp->rx_ra, lp->rx_num_entries, hv_err); 542a88b5ba8SSam Ravnborg 543a88b5ba8SSam Ravnborg /* Refetch the RX queue state as well, because we could be invoked 544a88b5ba8SSam Ravnborg * here in the queue processing context. 545a88b5ba8SSam Ravnborg */ 546a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 547a88b5ba8SSam Ravnborg &lp->rx_head, 548a88b5ba8SSam Ravnborg &lp->rx_tail, 549a88b5ba8SSam Ravnborg &lp->chan_state); 550a88b5ba8SSam Ravnborg if (hv_err) 551a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 552a88b5ba8SSam Ravnborg "sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n", 553a88b5ba8SSam Ravnborg lp->id, hv_err); 554a88b5ba8SSam Ravnborg 555a88b5ba8SSam Ravnborg return -ECONNRESET; 556a88b5ba8SSam Ravnborg } 557a88b5ba8SSam Ravnborg 558a88b5ba8SSam Ravnborg static struct ldc_version *find_by_major(u16 major) 559a88b5ba8SSam Ravnborg { 560a88b5ba8SSam Ravnborg struct ldc_version *ret = NULL; 561a88b5ba8SSam Ravnborg int i; 562a88b5ba8SSam Ravnborg 563a88b5ba8SSam Ravnborg for (i = 0; i < ARRAY_SIZE(ver_arr); i++) { 564a88b5ba8SSam Ravnborg struct ldc_version *v = &ver_arr[i]; 565a88b5ba8SSam Ravnborg if (v->major <= major) { 566a88b5ba8SSam Ravnborg ret = v; 567a88b5ba8SSam Ravnborg break; 568a88b5ba8SSam Ravnborg } 569a88b5ba8SSam Ravnborg } 570a88b5ba8SSam Ravnborg return ret; 571a88b5ba8SSam Ravnborg } 572a88b5ba8SSam Ravnborg 573a88b5ba8SSam Ravnborg static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp) 574a88b5ba8SSam Ravnborg { 575a88b5ba8SSam Ravnborg struct ldc_version *vap; 576a88b5ba8SSam Ravnborg int err; 577a88b5ba8SSam Ravnborg 578a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n", 579a88b5ba8SSam Ravnborg vp->major, vp->minor); 580a88b5ba8SSam Ravnborg 581a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_GOTVERS) { 582a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN; 583a88b5ba8SSam Ravnborg memset(&lp->ver, 0, sizeof(lp->ver)); 584a88b5ba8SSam Ravnborg } 585a88b5ba8SSam Ravnborg 586a88b5ba8SSam Ravnborg vap = find_by_major(vp->major); 587a88b5ba8SSam Ravnborg if (!vap) { 588a88b5ba8SSam Ravnborg err = send_version_nack(lp, 0, 0); 589a88b5ba8SSam Ravnborg } else if (vap->major != vp->major) { 590a88b5ba8SSam Ravnborg err = send_version_nack(lp, vap->major, vap->minor); 591a88b5ba8SSam Ravnborg } else { 592a88b5ba8SSam Ravnborg struct ldc_version ver = *vp; 593a88b5ba8SSam Ravnborg if (ver.minor > vap->minor) 594a88b5ba8SSam Ravnborg ver.minor = vap->minor; 595a88b5ba8SSam Ravnborg err = send_version_ack(lp, &ver); 596a88b5ba8SSam Ravnborg if (!err) { 597a88b5ba8SSam Ravnborg lp->ver = ver; 598a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_GOTVERS; 599a88b5ba8SSam Ravnborg } 600a88b5ba8SSam Ravnborg } 601a88b5ba8SSam Ravnborg if (err) 602a88b5ba8SSam Ravnborg return ldc_abort(lp); 603a88b5ba8SSam Ravnborg 604a88b5ba8SSam Ravnborg return 0; 605a88b5ba8SSam Ravnborg } 606a88b5ba8SSam Ravnborg 607a88b5ba8SSam Ravnborg static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp) 608a88b5ba8SSam Ravnborg { 609a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n", 610a88b5ba8SSam Ravnborg vp->major, vp->minor); 611a88b5ba8SSam Ravnborg 612a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_GOTVERS) { 613a88b5ba8SSam Ravnborg if (lp->ver.major != vp->major || 614a88b5ba8SSam Ravnborg lp->ver.minor != vp->minor) 615a88b5ba8SSam Ravnborg return ldc_abort(lp); 616a88b5ba8SSam Ravnborg } else { 617a88b5ba8SSam Ravnborg lp->ver = *vp; 618a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_GOTVERS; 619a88b5ba8SSam Ravnborg } 620a88b5ba8SSam Ravnborg if (send_rts(lp)) 621a88b5ba8SSam Ravnborg return ldc_abort(lp); 622a88b5ba8SSam Ravnborg return 0; 623a88b5ba8SSam Ravnborg } 624a88b5ba8SSam Ravnborg 625a88b5ba8SSam Ravnborg static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp) 626a88b5ba8SSam Ravnborg { 627a88b5ba8SSam Ravnborg struct ldc_version *vap; 628a88b5ba8SSam Ravnborg 629a88b5ba8SSam Ravnborg if ((vp->major == 0 && vp->minor == 0) || 630a88b5ba8SSam Ravnborg !(vap = find_by_major(vp->major))) { 631a88b5ba8SSam Ravnborg return ldc_abort(lp); 632a88b5ba8SSam Ravnborg } else { 633a88b5ba8SSam Ravnborg struct ldc_packet *p; 634a88b5ba8SSam Ravnborg unsigned long new_tail; 635a88b5ba8SSam Ravnborg 636a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS, 637a88b5ba8SSam Ravnborg vap, sizeof(*vap), 638a88b5ba8SSam Ravnborg &new_tail); 639a88b5ba8SSam Ravnborg if (p) 640a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 641a88b5ba8SSam Ravnborg else 642a88b5ba8SSam Ravnborg return ldc_abort(lp); 643a88b5ba8SSam Ravnborg } 644a88b5ba8SSam Ravnborg } 645a88b5ba8SSam Ravnborg 646a88b5ba8SSam Ravnborg static int process_version(struct ldc_channel *lp, 647a88b5ba8SSam Ravnborg struct ldc_packet *p) 648a88b5ba8SSam Ravnborg { 649a88b5ba8SSam Ravnborg struct ldc_version *vp; 650a88b5ba8SSam Ravnborg 651a88b5ba8SSam Ravnborg vp = (struct ldc_version *) p->u.u_data; 652a88b5ba8SSam Ravnborg 653a88b5ba8SSam Ravnborg switch (p->stype) { 654a88b5ba8SSam Ravnborg case LDC_INFO: 655a88b5ba8SSam Ravnborg return process_ver_info(lp, vp); 656a88b5ba8SSam Ravnborg 657a88b5ba8SSam Ravnborg case LDC_ACK: 658a88b5ba8SSam Ravnborg return process_ver_ack(lp, vp); 659a88b5ba8SSam Ravnborg 660a88b5ba8SSam Ravnborg case LDC_NACK: 661a88b5ba8SSam Ravnborg return process_ver_nack(lp, vp); 662a88b5ba8SSam Ravnborg 663a88b5ba8SSam Ravnborg default: 664a88b5ba8SSam Ravnborg return ldc_abort(lp); 665a88b5ba8SSam Ravnborg } 666a88b5ba8SSam Ravnborg } 667a88b5ba8SSam Ravnborg 668a88b5ba8SSam Ravnborg static int process_rts(struct ldc_channel *lp, 669a88b5ba8SSam Ravnborg struct ldc_packet *p) 670a88b5ba8SSam Ravnborg { 671a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n", 672a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env); 673a88b5ba8SSam Ravnborg 674a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO || 675a88b5ba8SSam Ravnborg lp->hs_state != LDC_HS_GOTVERS || 676a88b5ba8SSam Ravnborg p->env != lp->cfg.mode) 677a88b5ba8SSam Ravnborg return ldc_abort(lp); 678a88b5ba8SSam Ravnborg 679a88b5ba8SSam Ravnborg lp->snd_nxt = p->seqid; 680a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid; 681a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_SENTRTR; 682a88b5ba8SSam Ravnborg if (send_rtr(lp)) 683a88b5ba8SSam Ravnborg return ldc_abort(lp); 684a88b5ba8SSam Ravnborg 685a88b5ba8SSam Ravnborg return 0; 686a88b5ba8SSam Ravnborg } 687a88b5ba8SSam Ravnborg 688a88b5ba8SSam Ravnborg static int process_rtr(struct ldc_channel *lp, 689a88b5ba8SSam Ravnborg struct ldc_packet *p) 690a88b5ba8SSam Ravnborg { 691a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n", 692a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env); 693a88b5ba8SSam Ravnborg 694a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO || 695a88b5ba8SSam Ravnborg p->env != lp->cfg.mode) 696a88b5ba8SSam Ravnborg return ldc_abort(lp); 697a88b5ba8SSam Ravnborg 698a88b5ba8SSam Ravnborg lp->snd_nxt = p->seqid; 699a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 700a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 701a88b5ba8SSam Ravnborg send_rdx(lp); 702a88b5ba8SSam Ravnborg 703a88b5ba8SSam Ravnborg return LDC_EVENT_UP; 704a88b5ba8SSam Ravnborg } 705a88b5ba8SSam Ravnborg 706a88b5ba8SSam Ravnborg static int rx_seq_ok(struct ldc_channel *lp, u32 seqid) 707a88b5ba8SSam Ravnborg { 708a88b5ba8SSam Ravnborg return lp->rcv_nxt + 1 == seqid; 709a88b5ba8SSam Ravnborg } 710a88b5ba8SSam Ravnborg 711a88b5ba8SSam Ravnborg static int process_rdx(struct ldc_channel *lp, 712a88b5ba8SSam Ravnborg struct ldc_packet *p) 713a88b5ba8SSam Ravnborg { 714a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n", 715a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env, p->u.r.ackid); 716a88b5ba8SSam Ravnborg 717a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO || 718a88b5ba8SSam Ravnborg !(rx_seq_ok(lp, p->seqid))) 719a88b5ba8SSam Ravnborg return ldc_abort(lp); 720a88b5ba8SSam Ravnborg 721a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid; 722a88b5ba8SSam Ravnborg 723a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 724a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 725a88b5ba8SSam Ravnborg 726a88b5ba8SSam Ravnborg return LDC_EVENT_UP; 727a88b5ba8SSam Ravnborg } 728a88b5ba8SSam Ravnborg 729a88b5ba8SSam Ravnborg static int process_control_frame(struct ldc_channel *lp, 730a88b5ba8SSam Ravnborg struct ldc_packet *p) 731a88b5ba8SSam Ravnborg { 732a88b5ba8SSam Ravnborg switch (p->ctrl) { 733a88b5ba8SSam Ravnborg case LDC_VERS: 734a88b5ba8SSam Ravnborg return process_version(lp, p); 735a88b5ba8SSam Ravnborg 736a88b5ba8SSam Ravnborg case LDC_RTS: 737a88b5ba8SSam Ravnborg return process_rts(lp, p); 738a88b5ba8SSam Ravnborg 739a88b5ba8SSam Ravnborg case LDC_RTR: 740a88b5ba8SSam Ravnborg return process_rtr(lp, p); 741a88b5ba8SSam Ravnborg 742a88b5ba8SSam Ravnborg case LDC_RDX: 743a88b5ba8SSam Ravnborg return process_rdx(lp, p); 744a88b5ba8SSam Ravnborg 745a88b5ba8SSam Ravnborg default: 746a88b5ba8SSam Ravnborg return ldc_abort(lp); 747a88b5ba8SSam Ravnborg } 748a88b5ba8SSam Ravnborg } 749a88b5ba8SSam Ravnborg 750a88b5ba8SSam Ravnborg static int process_error_frame(struct ldc_channel *lp, 751a88b5ba8SSam Ravnborg struct ldc_packet *p) 752a88b5ba8SSam Ravnborg { 753a88b5ba8SSam Ravnborg return ldc_abort(lp); 754a88b5ba8SSam Ravnborg } 755a88b5ba8SSam Ravnborg 756a88b5ba8SSam Ravnborg static int process_data_ack(struct ldc_channel *lp, 757a88b5ba8SSam Ravnborg struct ldc_packet *ack) 758a88b5ba8SSam Ravnborg { 759a88b5ba8SSam Ravnborg unsigned long head = lp->tx_acked; 760a88b5ba8SSam Ravnborg u32 ackid = ack->u.r.ackid; 761a88b5ba8SSam Ravnborg 762a88b5ba8SSam Ravnborg while (1) { 763a88b5ba8SSam Ravnborg struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE); 764a88b5ba8SSam Ravnborg 765a88b5ba8SSam Ravnborg head = tx_advance(lp, head); 766a88b5ba8SSam Ravnborg 767a88b5ba8SSam Ravnborg if (p->seqid == ackid) { 768a88b5ba8SSam Ravnborg lp->tx_acked = head; 769a88b5ba8SSam Ravnborg return 0; 770a88b5ba8SSam Ravnborg } 771a88b5ba8SSam Ravnborg if (head == lp->tx_tail) 772a88b5ba8SSam Ravnborg return ldc_abort(lp); 773a88b5ba8SSam Ravnborg } 774a88b5ba8SSam Ravnborg 775a88b5ba8SSam Ravnborg return 0; 776a88b5ba8SSam Ravnborg } 777a88b5ba8SSam Ravnborg 778a88b5ba8SSam Ravnborg static void send_events(struct ldc_channel *lp, unsigned int event_mask) 779a88b5ba8SSam Ravnborg { 780a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_RESET) 781a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_RESET); 782a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_UP) 783a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_UP); 784a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_DATA_READY) 785a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY); 786a88b5ba8SSam Ravnborg } 787a88b5ba8SSam Ravnborg 788a88b5ba8SSam Ravnborg static irqreturn_t ldc_rx(int irq, void *dev_id) 789a88b5ba8SSam Ravnborg { 790a88b5ba8SSam Ravnborg struct ldc_channel *lp = dev_id; 791a88b5ba8SSam Ravnborg unsigned long orig_state, hv_err, flags; 792a88b5ba8SSam Ravnborg unsigned int event_mask; 793a88b5ba8SSam Ravnborg 794a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 795a88b5ba8SSam Ravnborg 796a88b5ba8SSam Ravnborg orig_state = lp->chan_state; 797a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 798a88b5ba8SSam Ravnborg &lp->rx_head, 799a88b5ba8SSam Ravnborg &lp->rx_tail, 800a88b5ba8SSam Ravnborg &lp->chan_state); 801a88b5ba8SSam Ravnborg 802a88b5ba8SSam Ravnborg ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n", 803a88b5ba8SSam Ravnborg orig_state, lp->chan_state, lp->rx_head, lp->rx_tail); 804a88b5ba8SSam Ravnborg 805a88b5ba8SSam Ravnborg event_mask = 0; 806a88b5ba8SSam Ravnborg 807a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW && 808a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_UP) { 809a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 810a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 811a88b5ba8SSam Ravnborg 812a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_UP; 813a88b5ba8SSam Ravnborg 814a88b5ba8SSam Ravnborg orig_state = lp->chan_state; 815a88b5ba8SSam Ravnborg } 816a88b5ba8SSam Ravnborg 817a88b5ba8SSam Ravnborg /* If we are in reset state, flush the RX queue and ignore 818a88b5ba8SSam Ravnborg * everything. 819a88b5ba8SSam Ravnborg */ 820a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_RESET) { 821a88b5ba8SSam Ravnborg (void) __set_rx_head(lp, lp->rx_tail); 822a88b5ba8SSam Ravnborg goto out; 823a88b5ba8SSam Ravnborg } 824a88b5ba8SSam Ravnborg 825a88b5ba8SSam Ravnborg /* Once we finish the handshake, we let the ldc_read() 826a88b5ba8SSam Ravnborg * paths do all of the control frame and state management. 827a88b5ba8SSam Ravnborg * Just trigger the callback. 828a88b5ba8SSam Ravnborg */ 829a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_COMPLETE) { 830a88b5ba8SSam Ravnborg handshake_complete: 831a88b5ba8SSam Ravnborg if (lp->chan_state != orig_state) { 832a88b5ba8SSam Ravnborg unsigned int event = LDC_EVENT_RESET; 833a88b5ba8SSam Ravnborg 834a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_UP) 835a88b5ba8SSam Ravnborg event = LDC_EVENT_UP; 836a88b5ba8SSam Ravnborg 837a88b5ba8SSam Ravnborg event_mask |= event; 838a88b5ba8SSam Ravnborg } 839a88b5ba8SSam Ravnborg if (lp->rx_head != lp->rx_tail) 840a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_DATA_READY; 841a88b5ba8SSam Ravnborg 842a88b5ba8SSam Ravnborg goto out; 843a88b5ba8SSam Ravnborg } 844a88b5ba8SSam Ravnborg 845a88b5ba8SSam Ravnborg if (lp->chan_state != orig_state) 846a88b5ba8SSam Ravnborg goto out; 847a88b5ba8SSam Ravnborg 848a88b5ba8SSam Ravnborg while (lp->rx_head != lp->rx_tail) { 849a88b5ba8SSam Ravnborg struct ldc_packet *p; 850a88b5ba8SSam Ravnborg unsigned long new; 851a88b5ba8SSam Ravnborg int err; 852a88b5ba8SSam Ravnborg 853a88b5ba8SSam Ravnborg p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE); 854a88b5ba8SSam Ravnborg 855a88b5ba8SSam Ravnborg switch (p->type) { 856a88b5ba8SSam Ravnborg case LDC_CTRL: 857a88b5ba8SSam Ravnborg err = process_control_frame(lp, p); 858a88b5ba8SSam Ravnborg if (err > 0) 859a88b5ba8SSam Ravnborg event_mask |= err; 860a88b5ba8SSam Ravnborg break; 861a88b5ba8SSam Ravnborg 862a88b5ba8SSam Ravnborg case LDC_DATA: 863a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_DATA_READY; 864a88b5ba8SSam Ravnborg err = 0; 865a88b5ba8SSam Ravnborg break; 866a88b5ba8SSam Ravnborg 867a88b5ba8SSam Ravnborg case LDC_ERR: 868a88b5ba8SSam Ravnborg err = process_error_frame(lp, p); 869a88b5ba8SSam Ravnborg break; 870a88b5ba8SSam Ravnborg 871a88b5ba8SSam Ravnborg default: 872a88b5ba8SSam Ravnborg err = ldc_abort(lp); 873a88b5ba8SSam Ravnborg break; 874a88b5ba8SSam Ravnborg } 875a88b5ba8SSam Ravnborg 876a88b5ba8SSam Ravnborg if (err < 0) 877a88b5ba8SSam Ravnborg break; 878a88b5ba8SSam Ravnborg 879a88b5ba8SSam Ravnborg new = lp->rx_head; 880a88b5ba8SSam Ravnborg new += LDC_PACKET_SIZE; 881a88b5ba8SSam Ravnborg if (new == (lp->rx_num_entries * LDC_PACKET_SIZE)) 882a88b5ba8SSam Ravnborg new = 0; 883a88b5ba8SSam Ravnborg lp->rx_head = new; 884a88b5ba8SSam Ravnborg 885a88b5ba8SSam Ravnborg err = __set_rx_head(lp, new); 886a88b5ba8SSam Ravnborg if (err < 0) { 887a88b5ba8SSam Ravnborg (void) ldc_abort(lp); 888a88b5ba8SSam Ravnborg break; 889a88b5ba8SSam Ravnborg } 890a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_COMPLETE) 891a88b5ba8SSam Ravnborg goto handshake_complete; 892a88b5ba8SSam Ravnborg } 893a88b5ba8SSam Ravnborg 894a88b5ba8SSam Ravnborg out: 895a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 896a88b5ba8SSam Ravnborg 897a88b5ba8SSam Ravnborg send_events(lp, event_mask); 898a88b5ba8SSam Ravnborg 899a88b5ba8SSam Ravnborg return IRQ_HANDLED; 900a88b5ba8SSam Ravnborg } 901a88b5ba8SSam Ravnborg 902a88b5ba8SSam Ravnborg static irqreturn_t ldc_tx(int irq, void *dev_id) 903a88b5ba8SSam Ravnborg { 904a88b5ba8SSam Ravnborg struct ldc_channel *lp = dev_id; 905a88b5ba8SSam Ravnborg unsigned long flags, hv_err, orig_state; 906a88b5ba8SSam Ravnborg unsigned int event_mask = 0; 907a88b5ba8SSam Ravnborg 908a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 909a88b5ba8SSam Ravnborg 910a88b5ba8SSam Ravnborg orig_state = lp->chan_state; 911a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, 912a88b5ba8SSam Ravnborg &lp->tx_head, 913a88b5ba8SSam Ravnborg &lp->tx_tail, 914a88b5ba8SSam Ravnborg &lp->chan_state); 915a88b5ba8SSam Ravnborg 916a88b5ba8SSam Ravnborg ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n", 917a88b5ba8SSam Ravnborg orig_state, lp->chan_state, lp->tx_head, lp->tx_tail); 918a88b5ba8SSam Ravnborg 919a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW && 920a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_UP) { 921a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 922a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 923a88b5ba8SSam Ravnborg 924a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_UP; 925a88b5ba8SSam Ravnborg } 926a88b5ba8SSam Ravnborg 927a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 928a88b5ba8SSam Ravnborg 929a88b5ba8SSam Ravnborg send_events(lp, event_mask); 930a88b5ba8SSam Ravnborg 931a88b5ba8SSam Ravnborg return IRQ_HANDLED; 932a88b5ba8SSam Ravnborg } 933a88b5ba8SSam Ravnborg 934a88b5ba8SSam Ravnborg /* XXX ldc_alloc() and ldc_free() needs to run under a mutex so 935a88b5ba8SSam Ravnborg * XXX that addition and removal from the ldc_channel_list has 936a88b5ba8SSam Ravnborg * XXX atomicity, otherwise the __ldc_channel_exists() check is 937a88b5ba8SSam Ravnborg * XXX totally pointless as another thread can slip into ldc_alloc() 938a88b5ba8SSam Ravnborg * XXX and add a channel with the same ID. There also needs to be 939a88b5ba8SSam Ravnborg * XXX a spinlock for ldc_channel_list. 940a88b5ba8SSam Ravnborg */ 941a88b5ba8SSam Ravnborg static HLIST_HEAD(ldc_channel_list); 942a88b5ba8SSam Ravnborg 943a88b5ba8SSam Ravnborg static int __ldc_channel_exists(unsigned long id) 944a88b5ba8SSam Ravnborg { 945a88b5ba8SSam Ravnborg struct ldc_channel *lp; 946a88b5ba8SSam Ravnborg struct hlist_node *n; 947a88b5ba8SSam Ravnborg 948a88b5ba8SSam Ravnborg hlist_for_each_entry(lp, n, &ldc_channel_list, list) { 949a88b5ba8SSam Ravnborg if (lp->id == id) 950a88b5ba8SSam Ravnborg return 1; 951a88b5ba8SSam Ravnborg } 952a88b5ba8SSam Ravnborg return 0; 953a88b5ba8SSam Ravnborg } 954a88b5ba8SSam Ravnborg 955a88b5ba8SSam Ravnborg static int alloc_queue(const char *name, unsigned long num_entries, 956a88b5ba8SSam Ravnborg struct ldc_packet **base, unsigned long *ra) 957a88b5ba8SSam Ravnborg { 958a88b5ba8SSam Ravnborg unsigned long size, order; 959a88b5ba8SSam Ravnborg void *q; 960a88b5ba8SSam Ravnborg 961a88b5ba8SSam Ravnborg size = num_entries * LDC_PACKET_SIZE; 962a88b5ba8SSam Ravnborg order = get_order(size); 963a88b5ba8SSam Ravnborg 964a88b5ba8SSam Ravnborg q = (void *) __get_free_pages(GFP_KERNEL, order); 965a88b5ba8SSam Ravnborg if (!q) { 966a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of %s queue failed with " 967a88b5ba8SSam Ravnborg "size=%lu order=%lu\n", name, size, order); 968a88b5ba8SSam Ravnborg return -ENOMEM; 969a88b5ba8SSam Ravnborg } 970a88b5ba8SSam Ravnborg 971a88b5ba8SSam Ravnborg memset(q, 0, PAGE_SIZE << order); 972a88b5ba8SSam Ravnborg 973a88b5ba8SSam Ravnborg *base = q; 974a88b5ba8SSam Ravnborg *ra = __pa(q); 975a88b5ba8SSam Ravnborg 976a88b5ba8SSam Ravnborg return 0; 977a88b5ba8SSam Ravnborg } 978a88b5ba8SSam Ravnborg 979a88b5ba8SSam Ravnborg static void free_queue(unsigned long num_entries, struct ldc_packet *q) 980a88b5ba8SSam Ravnborg { 981a88b5ba8SSam Ravnborg unsigned long size, order; 982a88b5ba8SSam Ravnborg 983a88b5ba8SSam Ravnborg if (!q) 984a88b5ba8SSam Ravnborg return; 985a88b5ba8SSam Ravnborg 986a88b5ba8SSam Ravnborg size = num_entries * LDC_PACKET_SIZE; 987a88b5ba8SSam Ravnborg order = get_order(size); 988a88b5ba8SSam Ravnborg 989a88b5ba8SSam Ravnborg free_pages((unsigned long)q, order); 990a88b5ba8SSam Ravnborg } 991a88b5ba8SSam Ravnborg 992a88b5ba8SSam Ravnborg /* XXX Make this configurable... XXX */ 993a88b5ba8SSam Ravnborg #define LDC_IOTABLE_SIZE (8 * 1024) 994a88b5ba8SSam Ravnborg 995a88b5ba8SSam Ravnborg static int ldc_iommu_init(struct ldc_channel *lp) 996a88b5ba8SSam Ravnborg { 997a88b5ba8SSam Ravnborg unsigned long sz, num_tsb_entries, tsbsize, order; 998a88b5ba8SSam Ravnborg struct ldc_iommu *iommu = &lp->iommu; 999a88b5ba8SSam Ravnborg struct ldc_mtable_entry *table; 1000a88b5ba8SSam Ravnborg unsigned long hv_err; 1001a88b5ba8SSam Ravnborg int err; 1002a88b5ba8SSam Ravnborg 1003a88b5ba8SSam Ravnborg num_tsb_entries = LDC_IOTABLE_SIZE; 1004a88b5ba8SSam Ravnborg tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); 1005a88b5ba8SSam Ravnborg 1006a88b5ba8SSam Ravnborg spin_lock_init(&iommu->lock); 1007a88b5ba8SSam Ravnborg 1008a88b5ba8SSam Ravnborg sz = num_tsb_entries / 8; 1009a88b5ba8SSam Ravnborg sz = (sz + 7UL) & ~7UL; 1010a88b5ba8SSam Ravnborg iommu->arena.map = kzalloc(sz, GFP_KERNEL); 1011a88b5ba8SSam Ravnborg if (!iommu->arena.map) { 1012a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz); 1013a88b5ba8SSam Ravnborg return -ENOMEM; 1014a88b5ba8SSam Ravnborg } 1015a88b5ba8SSam Ravnborg 1016a88b5ba8SSam Ravnborg iommu->arena.limit = num_tsb_entries; 1017a88b5ba8SSam Ravnborg 1018a88b5ba8SSam Ravnborg order = get_order(tsbsize); 1019a88b5ba8SSam Ravnborg 1020a88b5ba8SSam Ravnborg table = (struct ldc_mtable_entry *) 1021a88b5ba8SSam Ravnborg __get_free_pages(GFP_KERNEL, order); 1022a88b5ba8SSam Ravnborg err = -ENOMEM; 1023a88b5ba8SSam Ravnborg if (!table) { 1024a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of MTE table failed, " 1025a88b5ba8SSam Ravnborg "size=%lu order=%lu\n", tsbsize, order); 1026a88b5ba8SSam Ravnborg goto out_free_map; 1027a88b5ba8SSam Ravnborg } 1028a88b5ba8SSam Ravnborg 1029a88b5ba8SSam Ravnborg memset(table, 0, PAGE_SIZE << order); 1030a88b5ba8SSam Ravnborg 1031a88b5ba8SSam Ravnborg iommu->page_table = table; 1032a88b5ba8SSam Ravnborg 1033a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table), 1034a88b5ba8SSam Ravnborg num_tsb_entries); 1035a88b5ba8SSam Ravnborg err = -EINVAL; 1036a88b5ba8SSam Ravnborg if (hv_err) 1037a88b5ba8SSam Ravnborg goto out_free_table; 1038a88b5ba8SSam Ravnborg 1039a88b5ba8SSam Ravnborg return 0; 1040a88b5ba8SSam Ravnborg 1041a88b5ba8SSam Ravnborg out_free_table: 1042a88b5ba8SSam Ravnborg free_pages((unsigned long) table, order); 1043a88b5ba8SSam Ravnborg iommu->page_table = NULL; 1044a88b5ba8SSam Ravnborg 1045a88b5ba8SSam Ravnborg out_free_map: 1046a88b5ba8SSam Ravnborg kfree(iommu->arena.map); 1047a88b5ba8SSam Ravnborg iommu->arena.map = NULL; 1048a88b5ba8SSam Ravnborg 1049a88b5ba8SSam Ravnborg return err; 1050a88b5ba8SSam Ravnborg } 1051a88b5ba8SSam Ravnborg 1052a88b5ba8SSam Ravnborg static void ldc_iommu_release(struct ldc_channel *lp) 1053a88b5ba8SSam Ravnborg { 1054a88b5ba8SSam Ravnborg struct ldc_iommu *iommu = &lp->iommu; 1055a88b5ba8SSam Ravnborg unsigned long num_tsb_entries, tsbsize, order; 1056a88b5ba8SSam Ravnborg 1057a88b5ba8SSam Ravnborg (void) sun4v_ldc_set_map_table(lp->id, 0, 0); 1058a88b5ba8SSam Ravnborg 1059a88b5ba8SSam Ravnborg num_tsb_entries = iommu->arena.limit; 1060a88b5ba8SSam Ravnborg tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); 1061a88b5ba8SSam Ravnborg order = get_order(tsbsize); 1062a88b5ba8SSam Ravnborg 1063a88b5ba8SSam Ravnborg free_pages((unsigned long) iommu->page_table, order); 1064a88b5ba8SSam Ravnborg iommu->page_table = NULL; 1065a88b5ba8SSam Ravnborg 1066a88b5ba8SSam Ravnborg kfree(iommu->arena.map); 1067a88b5ba8SSam Ravnborg iommu->arena.map = NULL; 1068a88b5ba8SSam Ravnborg } 1069a88b5ba8SSam Ravnborg 1070a88b5ba8SSam Ravnborg struct ldc_channel *ldc_alloc(unsigned long id, 1071a88b5ba8SSam Ravnborg const struct ldc_channel_config *cfgp, 1072a88b5ba8SSam Ravnborg void *event_arg) 1073a88b5ba8SSam Ravnborg { 1074a88b5ba8SSam Ravnborg struct ldc_channel *lp; 1075a88b5ba8SSam Ravnborg const struct ldc_mode_ops *mops; 1076a88b5ba8SSam Ravnborg unsigned long dummy1, dummy2, hv_err; 1077a88b5ba8SSam Ravnborg u8 mss, *mssbuf; 1078a88b5ba8SSam Ravnborg int err; 1079a88b5ba8SSam Ravnborg 1080a88b5ba8SSam Ravnborg err = -ENODEV; 1081a88b5ba8SSam Ravnborg if (!ldom_domaining_enabled) 1082a88b5ba8SSam Ravnborg goto out_err; 1083a88b5ba8SSam Ravnborg 1084a88b5ba8SSam Ravnborg err = -EINVAL; 1085a88b5ba8SSam Ravnborg if (!cfgp) 1086a88b5ba8SSam Ravnborg goto out_err; 1087a88b5ba8SSam Ravnborg 1088a88b5ba8SSam Ravnborg switch (cfgp->mode) { 1089a88b5ba8SSam Ravnborg case LDC_MODE_RAW: 1090a88b5ba8SSam Ravnborg mops = &raw_ops; 1091a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE; 1092a88b5ba8SSam Ravnborg break; 1093a88b5ba8SSam Ravnborg 1094a88b5ba8SSam Ravnborg case LDC_MODE_UNRELIABLE: 1095a88b5ba8SSam Ravnborg mops = &nonraw_ops; 1096a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE - 8; 1097a88b5ba8SSam Ravnborg break; 1098a88b5ba8SSam Ravnborg 1099a88b5ba8SSam Ravnborg case LDC_MODE_STREAM: 1100a88b5ba8SSam Ravnborg mops = &stream_ops; 1101a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE - 8 - 8; 1102a88b5ba8SSam Ravnborg break; 1103a88b5ba8SSam Ravnborg 1104a88b5ba8SSam Ravnborg default: 1105a88b5ba8SSam Ravnborg goto out_err; 1106a88b5ba8SSam Ravnborg } 1107a88b5ba8SSam Ravnborg 1108a88b5ba8SSam Ravnborg if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq) 1109a88b5ba8SSam Ravnborg goto out_err; 1110a88b5ba8SSam Ravnborg 1111a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2); 1112a88b5ba8SSam Ravnborg err = -ENODEV; 1113a88b5ba8SSam Ravnborg if (hv_err == HV_ECHANNEL) 1114a88b5ba8SSam Ravnborg goto out_err; 1115a88b5ba8SSam Ravnborg 1116a88b5ba8SSam Ravnborg err = -EEXIST; 1117a88b5ba8SSam Ravnborg if (__ldc_channel_exists(id)) 1118a88b5ba8SSam Ravnborg goto out_err; 1119a88b5ba8SSam Ravnborg 1120a88b5ba8SSam Ravnborg mssbuf = NULL; 1121a88b5ba8SSam Ravnborg 1122a88b5ba8SSam Ravnborg lp = kzalloc(sizeof(*lp), GFP_KERNEL); 1123a88b5ba8SSam Ravnborg err = -ENOMEM; 1124a88b5ba8SSam Ravnborg if (!lp) 1125a88b5ba8SSam Ravnborg goto out_err; 1126a88b5ba8SSam Ravnborg 1127a88b5ba8SSam Ravnborg spin_lock_init(&lp->lock); 1128a88b5ba8SSam Ravnborg 1129a88b5ba8SSam Ravnborg lp->id = id; 1130a88b5ba8SSam Ravnborg 1131a88b5ba8SSam Ravnborg err = ldc_iommu_init(lp); 1132a88b5ba8SSam Ravnborg if (err) 1133a88b5ba8SSam Ravnborg goto out_free_ldc; 1134a88b5ba8SSam Ravnborg 1135a88b5ba8SSam Ravnborg lp->mops = mops; 1136a88b5ba8SSam Ravnborg lp->mss = mss; 1137a88b5ba8SSam Ravnborg 1138a88b5ba8SSam Ravnborg lp->cfg = *cfgp; 1139a88b5ba8SSam Ravnborg if (!lp->cfg.mtu) 1140a88b5ba8SSam Ravnborg lp->cfg.mtu = LDC_DEFAULT_MTU; 1141a88b5ba8SSam Ravnborg 1142a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_STREAM) { 1143a88b5ba8SSam Ravnborg mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL); 1144a88b5ba8SSam Ravnborg if (!mssbuf) { 1145a88b5ba8SSam Ravnborg err = -ENOMEM; 1146a88b5ba8SSam Ravnborg goto out_free_iommu; 1147a88b5ba8SSam Ravnborg } 1148a88b5ba8SSam Ravnborg lp->mssbuf = mssbuf; 1149a88b5ba8SSam Ravnborg } 1150a88b5ba8SSam Ravnborg 1151a88b5ba8SSam Ravnborg lp->event_arg = event_arg; 1152a88b5ba8SSam Ravnborg 1153a88b5ba8SSam Ravnborg /* XXX allow setting via ldc_channel_config to override defaults 1154a88b5ba8SSam Ravnborg * XXX or use some formula based upon mtu 1155a88b5ba8SSam Ravnborg */ 1156a88b5ba8SSam Ravnborg lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES; 1157a88b5ba8SSam Ravnborg lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES; 1158a88b5ba8SSam Ravnborg 1159a88b5ba8SSam Ravnborg err = alloc_queue("TX", lp->tx_num_entries, 1160a88b5ba8SSam Ravnborg &lp->tx_base, &lp->tx_ra); 1161a88b5ba8SSam Ravnborg if (err) 1162a88b5ba8SSam Ravnborg goto out_free_mssbuf; 1163a88b5ba8SSam Ravnborg 1164a88b5ba8SSam Ravnborg err = alloc_queue("RX", lp->rx_num_entries, 1165a88b5ba8SSam Ravnborg &lp->rx_base, &lp->rx_ra); 1166a88b5ba8SSam Ravnborg if (err) 1167a88b5ba8SSam Ravnborg goto out_free_txq; 1168a88b5ba8SSam Ravnborg 1169a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_ALLOCED_QUEUES; 1170a88b5ba8SSam Ravnborg 1171a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_CLOSED; 1172a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_INIT); 1173a88b5ba8SSam Ravnborg 1174a88b5ba8SSam Ravnborg INIT_HLIST_NODE(&lp->list); 1175a88b5ba8SSam Ravnborg hlist_add_head(&lp->list, &ldc_channel_list); 1176a88b5ba8SSam Ravnborg 1177a88b5ba8SSam Ravnborg INIT_HLIST_HEAD(&lp->mh_list); 1178a88b5ba8SSam Ravnborg 1179a88b5ba8SSam Ravnborg return lp; 1180a88b5ba8SSam Ravnborg 1181a88b5ba8SSam Ravnborg out_free_txq: 1182a88b5ba8SSam Ravnborg free_queue(lp->tx_num_entries, lp->tx_base); 1183a88b5ba8SSam Ravnborg 1184a88b5ba8SSam Ravnborg out_free_mssbuf: 1185a88b5ba8SSam Ravnborg if (mssbuf) 1186a88b5ba8SSam Ravnborg kfree(mssbuf); 1187a88b5ba8SSam Ravnborg 1188a88b5ba8SSam Ravnborg out_free_iommu: 1189a88b5ba8SSam Ravnborg ldc_iommu_release(lp); 1190a88b5ba8SSam Ravnborg 1191a88b5ba8SSam Ravnborg out_free_ldc: 1192a88b5ba8SSam Ravnborg kfree(lp); 1193a88b5ba8SSam Ravnborg 1194a88b5ba8SSam Ravnborg out_err: 1195a88b5ba8SSam Ravnborg return ERR_PTR(err); 1196a88b5ba8SSam Ravnborg } 1197a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_alloc); 1198a88b5ba8SSam Ravnborg 1199a88b5ba8SSam Ravnborg void ldc_free(struct ldc_channel *lp) 1200a88b5ba8SSam Ravnborg { 1201a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_REGISTERED_IRQS) { 1202a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1203a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp); 1204a88b5ba8SSam Ravnborg } 1205a88b5ba8SSam Ravnborg 1206a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) { 1207a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0); 1208a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0); 1209a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES; 1210a88b5ba8SSam Ravnborg } 1211a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) { 1212a88b5ba8SSam Ravnborg free_queue(lp->tx_num_entries, lp->tx_base); 1213a88b5ba8SSam Ravnborg free_queue(lp->rx_num_entries, lp->rx_base); 1214a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES; 1215a88b5ba8SSam Ravnborg } 1216a88b5ba8SSam Ravnborg 1217a88b5ba8SSam Ravnborg hlist_del(&lp->list); 1218a88b5ba8SSam Ravnborg 1219a88b5ba8SSam Ravnborg if (lp->mssbuf) 1220a88b5ba8SSam Ravnborg kfree(lp->mssbuf); 1221a88b5ba8SSam Ravnborg 1222a88b5ba8SSam Ravnborg ldc_iommu_release(lp); 1223a88b5ba8SSam Ravnborg 1224a88b5ba8SSam Ravnborg kfree(lp); 1225a88b5ba8SSam Ravnborg } 1226a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_free); 1227a88b5ba8SSam Ravnborg 1228a88b5ba8SSam Ravnborg /* Bind the channel. This registers the LDC queues with 1229a88b5ba8SSam Ravnborg * the hypervisor and puts the channel into a pseudo-listening 1230a88b5ba8SSam Ravnborg * state. This does not initiate a handshake, ldc_connect() does 1231a88b5ba8SSam Ravnborg * that. 1232a88b5ba8SSam Ravnborg */ 1233a88b5ba8SSam Ravnborg int ldc_bind(struct ldc_channel *lp, const char *name) 1234a88b5ba8SSam Ravnborg { 1235a88b5ba8SSam Ravnborg unsigned long hv_err, flags; 1236a88b5ba8SSam Ravnborg int err = -EINVAL; 1237a88b5ba8SSam Ravnborg 1238a88b5ba8SSam Ravnborg if (!name || 1239a88b5ba8SSam Ravnborg (lp->state != LDC_STATE_INIT)) 1240a88b5ba8SSam Ravnborg return -EINVAL; 1241a88b5ba8SSam Ravnborg 1242a88b5ba8SSam Ravnborg snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name); 1243a88b5ba8SSam Ravnborg snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name); 1244a88b5ba8SSam Ravnborg 1245a88b5ba8SSam Ravnborg err = request_irq(lp->cfg.rx_irq, ldc_rx, 1246a88b5ba8SSam Ravnborg IRQF_SAMPLE_RANDOM | IRQF_SHARED, 1247a88b5ba8SSam Ravnborg lp->rx_irq_name, lp); 1248a88b5ba8SSam Ravnborg if (err) 1249a88b5ba8SSam Ravnborg return err; 1250a88b5ba8SSam Ravnborg 1251a88b5ba8SSam Ravnborg err = request_irq(lp->cfg.tx_irq, ldc_tx, 1252a88b5ba8SSam Ravnborg IRQF_SAMPLE_RANDOM | IRQF_SHARED, 1253a88b5ba8SSam Ravnborg lp->tx_irq_name, lp); 1254a88b5ba8SSam Ravnborg if (err) { 1255a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1256a88b5ba8SSam Ravnborg return err; 1257a88b5ba8SSam Ravnborg } 1258a88b5ba8SSam Ravnborg 1259a88b5ba8SSam Ravnborg 1260a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1261a88b5ba8SSam Ravnborg 1262a88b5ba8SSam Ravnborg enable_irq(lp->cfg.rx_irq); 1263a88b5ba8SSam Ravnborg enable_irq(lp->cfg.tx_irq); 1264a88b5ba8SSam Ravnborg 1265a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_REGISTERED_IRQS; 1266a88b5ba8SSam Ravnborg 1267a88b5ba8SSam Ravnborg err = -ENODEV; 1268a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0); 1269a88b5ba8SSam Ravnborg if (hv_err) 1270a88b5ba8SSam Ravnborg goto out_free_irqs; 1271a88b5ba8SSam Ravnborg 1272a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); 1273a88b5ba8SSam Ravnborg if (hv_err) 1274a88b5ba8SSam Ravnborg goto out_free_irqs; 1275a88b5ba8SSam Ravnborg 1276a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0); 1277a88b5ba8SSam Ravnborg if (hv_err) 1278a88b5ba8SSam Ravnborg goto out_unmap_tx; 1279a88b5ba8SSam Ravnborg 1280a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); 1281a88b5ba8SSam Ravnborg if (hv_err) 1282a88b5ba8SSam Ravnborg goto out_unmap_tx; 1283a88b5ba8SSam Ravnborg 1284a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_REGISTERED_QUEUES; 1285a88b5ba8SSam Ravnborg 1286a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, 1287a88b5ba8SSam Ravnborg &lp->tx_head, 1288a88b5ba8SSam Ravnborg &lp->tx_tail, 1289a88b5ba8SSam Ravnborg &lp->chan_state); 1290a88b5ba8SSam Ravnborg err = -EBUSY; 1291a88b5ba8SSam Ravnborg if (hv_err) 1292a88b5ba8SSam Ravnborg goto out_unmap_rx; 1293a88b5ba8SSam Ravnborg 1294a88b5ba8SSam Ravnborg lp->tx_acked = lp->tx_head; 1295a88b5ba8SSam Ravnborg 1296a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN; 1297a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_BOUND); 1298a88b5ba8SSam Ravnborg 1299a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1300a88b5ba8SSam Ravnborg 1301a88b5ba8SSam Ravnborg return 0; 1302a88b5ba8SSam Ravnborg 1303a88b5ba8SSam Ravnborg out_unmap_rx: 1304a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES; 1305a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0); 1306a88b5ba8SSam Ravnborg 1307a88b5ba8SSam Ravnborg out_unmap_tx: 1308a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0); 1309a88b5ba8SSam Ravnborg 1310a88b5ba8SSam Ravnborg out_free_irqs: 1311a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_IRQS; 1312a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp); 1313a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1314a88b5ba8SSam Ravnborg 1315a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1316a88b5ba8SSam Ravnborg 1317a88b5ba8SSam Ravnborg return err; 1318a88b5ba8SSam Ravnborg } 1319a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_bind); 1320a88b5ba8SSam Ravnborg 1321a88b5ba8SSam Ravnborg int ldc_connect(struct ldc_channel *lp) 1322a88b5ba8SSam Ravnborg { 1323a88b5ba8SSam Ravnborg unsigned long flags; 1324a88b5ba8SSam Ravnborg int err; 1325a88b5ba8SSam Ravnborg 1326a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW) 1327a88b5ba8SSam Ravnborg return -EINVAL; 1328a88b5ba8SSam Ravnborg 1329a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1330a88b5ba8SSam Ravnborg 1331a88b5ba8SSam Ravnborg if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || 1332a88b5ba8SSam Ravnborg !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) || 1333a88b5ba8SSam Ravnborg lp->hs_state != LDC_HS_OPEN) 1334a88b5ba8SSam Ravnborg err = -EINVAL; 1335a88b5ba8SSam Ravnborg else 1336a88b5ba8SSam Ravnborg err = start_handshake(lp); 1337a88b5ba8SSam Ravnborg 1338a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1339a88b5ba8SSam Ravnborg 1340a88b5ba8SSam Ravnborg return err; 1341a88b5ba8SSam Ravnborg } 1342a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_connect); 1343a88b5ba8SSam Ravnborg 1344a88b5ba8SSam Ravnborg int ldc_disconnect(struct ldc_channel *lp) 1345a88b5ba8SSam Ravnborg { 1346a88b5ba8SSam Ravnborg unsigned long hv_err, flags; 1347a88b5ba8SSam Ravnborg int err; 1348a88b5ba8SSam Ravnborg 1349a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW) 1350a88b5ba8SSam Ravnborg return -EINVAL; 1351a88b5ba8SSam Ravnborg 1352a88b5ba8SSam Ravnborg if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || 1353a88b5ba8SSam Ravnborg !(lp->flags & LDC_FLAG_REGISTERED_QUEUES)) 1354a88b5ba8SSam Ravnborg return -EINVAL; 1355a88b5ba8SSam Ravnborg 1356a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1357a88b5ba8SSam Ravnborg 1358a88b5ba8SSam Ravnborg err = -ENODEV; 1359a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0); 1360a88b5ba8SSam Ravnborg if (hv_err) 1361a88b5ba8SSam Ravnborg goto out_err; 1362a88b5ba8SSam Ravnborg 1363a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); 1364a88b5ba8SSam Ravnborg if (hv_err) 1365a88b5ba8SSam Ravnborg goto out_err; 1366a88b5ba8SSam Ravnborg 1367a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0); 1368a88b5ba8SSam Ravnborg if (hv_err) 1369a88b5ba8SSam Ravnborg goto out_err; 1370a88b5ba8SSam Ravnborg 1371a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); 1372a88b5ba8SSam Ravnborg if (hv_err) 1373a88b5ba8SSam Ravnborg goto out_err; 1374a88b5ba8SSam Ravnborg 1375a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_BOUND); 1376a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN; 1377a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_RESET; 1378a88b5ba8SSam Ravnborg 1379a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1380a88b5ba8SSam Ravnborg 1381a88b5ba8SSam Ravnborg return 0; 1382a88b5ba8SSam Ravnborg 1383a88b5ba8SSam Ravnborg out_err: 1384a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0); 1385a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0); 1386a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp); 1387a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1388a88b5ba8SSam Ravnborg lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS | 1389a88b5ba8SSam Ravnborg LDC_FLAG_REGISTERED_QUEUES); 1390a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_INIT); 1391a88b5ba8SSam Ravnborg 1392a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1393a88b5ba8SSam Ravnborg 1394a88b5ba8SSam Ravnborg return err; 1395a88b5ba8SSam Ravnborg } 1396a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_disconnect); 1397a88b5ba8SSam Ravnborg 1398a88b5ba8SSam Ravnborg int ldc_state(struct ldc_channel *lp) 1399a88b5ba8SSam Ravnborg { 1400a88b5ba8SSam Ravnborg return lp->state; 1401a88b5ba8SSam Ravnborg } 1402a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_state); 1403a88b5ba8SSam Ravnborg 1404a88b5ba8SSam Ravnborg static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size) 1405a88b5ba8SSam Ravnborg { 1406a88b5ba8SSam Ravnborg struct ldc_packet *p; 1407a88b5ba8SSam Ravnborg unsigned long new_tail; 1408a88b5ba8SSam Ravnborg int err; 1409a88b5ba8SSam Ravnborg 1410a88b5ba8SSam Ravnborg if (size > LDC_PACKET_SIZE) 1411a88b5ba8SSam Ravnborg return -EMSGSIZE; 1412a88b5ba8SSam Ravnborg 1413a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail); 1414a88b5ba8SSam Ravnborg if (!p) 1415a88b5ba8SSam Ravnborg return -EAGAIN; 1416a88b5ba8SSam Ravnborg 1417a88b5ba8SSam Ravnborg memcpy(p, buf, size); 1418a88b5ba8SSam Ravnborg 1419a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail); 1420a88b5ba8SSam Ravnborg if (!err) 1421a88b5ba8SSam Ravnborg err = size; 1422a88b5ba8SSam Ravnborg 1423a88b5ba8SSam Ravnborg return err; 1424a88b5ba8SSam Ravnborg } 1425a88b5ba8SSam Ravnborg 1426a88b5ba8SSam Ravnborg static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size) 1427a88b5ba8SSam Ravnborg { 1428a88b5ba8SSam Ravnborg struct ldc_packet *p; 1429a88b5ba8SSam Ravnborg unsigned long hv_err, new; 1430a88b5ba8SSam Ravnborg int err; 1431a88b5ba8SSam Ravnborg 1432a88b5ba8SSam Ravnborg if (size < LDC_PACKET_SIZE) 1433a88b5ba8SSam Ravnborg return -EINVAL; 1434a88b5ba8SSam Ravnborg 1435a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 1436a88b5ba8SSam Ravnborg &lp->rx_head, 1437a88b5ba8SSam Ravnborg &lp->rx_tail, 1438a88b5ba8SSam Ravnborg &lp->chan_state); 1439a88b5ba8SSam Ravnborg if (hv_err) 1440a88b5ba8SSam Ravnborg return ldc_abort(lp); 1441a88b5ba8SSam Ravnborg 1442a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN || 1443a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING) 1444a88b5ba8SSam Ravnborg return -ECONNRESET; 1445a88b5ba8SSam Ravnborg 1446a88b5ba8SSam Ravnborg if (lp->rx_head == lp->rx_tail) 1447a88b5ba8SSam Ravnborg return 0; 1448a88b5ba8SSam Ravnborg 1449a88b5ba8SSam Ravnborg p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE); 1450a88b5ba8SSam Ravnborg memcpy(buf, p, LDC_PACKET_SIZE); 1451a88b5ba8SSam Ravnborg 1452a88b5ba8SSam Ravnborg new = rx_advance(lp, lp->rx_head); 1453a88b5ba8SSam Ravnborg lp->rx_head = new; 1454a88b5ba8SSam Ravnborg 1455a88b5ba8SSam Ravnborg err = __set_rx_head(lp, new); 1456a88b5ba8SSam Ravnborg if (err < 0) 1457a88b5ba8SSam Ravnborg err = -ECONNRESET; 1458a88b5ba8SSam Ravnborg else 1459a88b5ba8SSam Ravnborg err = LDC_PACKET_SIZE; 1460a88b5ba8SSam Ravnborg 1461a88b5ba8SSam Ravnborg return err; 1462a88b5ba8SSam Ravnborg } 1463a88b5ba8SSam Ravnborg 1464a88b5ba8SSam Ravnborg static const struct ldc_mode_ops raw_ops = { 1465a88b5ba8SSam Ravnborg .write = write_raw, 1466a88b5ba8SSam Ravnborg .read = read_raw, 1467a88b5ba8SSam Ravnborg }; 1468a88b5ba8SSam Ravnborg 1469a88b5ba8SSam Ravnborg static int write_nonraw(struct ldc_channel *lp, const void *buf, 1470a88b5ba8SSam Ravnborg unsigned int size) 1471a88b5ba8SSam Ravnborg { 1472a88b5ba8SSam Ravnborg unsigned long hv_err, tail; 1473a88b5ba8SSam Ravnborg unsigned int copied; 1474a88b5ba8SSam Ravnborg u32 seq; 1475a88b5ba8SSam Ravnborg int err; 1476a88b5ba8SSam Ravnborg 1477a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail, 1478a88b5ba8SSam Ravnborg &lp->chan_state); 1479a88b5ba8SSam Ravnborg if (unlikely(hv_err)) 1480a88b5ba8SSam Ravnborg return -EBUSY; 1481a88b5ba8SSam Ravnborg 1482a88b5ba8SSam Ravnborg if (unlikely(lp->chan_state != LDC_CHANNEL_UP)) 1483a88b5ba8SSam Ravnborg return ldc_abort(lp); 1484a88b5ba8SSam Ravnborg 1485a88b5ba8SSam Ravnborg if (!tx_has_space_for(lp, size)) 1486a88b5ba8SSam Ravnborg return -EAGAIN; 1487a88b5ba8SSam Ravnborg 1488a88b5ba8SSam Ravnborg seq = lp->snd_nxt; 1489a88b5ba8SSam Ravnborg copied = 0; 1490a88b5ba8SSam Ravnborg tail = lp->tx_tail; 1491a88b5ba8SSam Ravnborg while (copied < size) { 1492a88b5ba8SSam Ravnborg struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE); 1493a88b5ba8SSam Ravnborg u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ? 1494a88b5ba8SSam Ravnborg p->u.u_data : 1495a88b5ba8SSam Ravnborg p->u.r.r_data); 1496a88b5ba8SSam Ravnborg int data_len; 1497a88b5ba8SSam Ravnborg 1498a88b5ba8SSam Ravnborg p->type = LDC_DATA; 1499a88b5ba8SSam Ravnborg p->stype = LDC_INFO; 1500a88b5ba8SSam Ravnborg p->ctrl = 0; 1501a88b5ba8SSam Ravnborg 1502a88b5ba8SSam Ravnborg data_len = size - copied; 1503a88b5ba8SSam Ravnborg if (data_len > lp->mss) 1504a88b5ba8SSam Ravnborg data_len = lp->mss; 1505a88b5ba8SSam Ravnborg 1506a88b5ba8SSam Ravnborg BUG_ON(data_len > LDC_LEN); 1507a88b5ba8SSam Ravnborg 1508a88b5ba8SSam Ravnborg p->env = (data_len | 1509a88b5ba8SSam Ravnborg (copied == 0 ? LDC_START : 0) | 1510a88b5ba8SSam Ravnborg (data_len == size - copied ? LDC_STOP : 0)); 1511a88b5ba8SSam Ravnborg 1512a88b5ba8SSam Ravnborg p->seqid = ++seq; 1513a88b5ba8SSam Ravnborg 1514a88b5ba8SSam Ravnborg ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n", 1515a88b5ba8SSam Ravnborg p->type, 1516a88b5ba8SSam Ravnborg p->stype, 1517a88b5ba8SSam Ravnborg p->ctrl, 1518a88b5ba8SSam Ravnborg p->env, 1519a88b5ba8SSam Ravnborg p->seqid); 1520a88b5ba8SSam Ravnborg 1521a88b5ba8SSam Ravnborg memcpy(data, buf, data_len); 1522a88b5ba8SSam Ravnborg buf += data_len; 1523a88b5ba8SSam Ravnborg copied += data_len; 1524a88b5ba8SSam Ravnborg 1525a88b5ba8SSam Ravnborg tail = tx_advance(lp, tail); 1526a88b5ba8SSam Ravnborg } 1527a88b5ba8SSam Ravnborg 1528a88b5ba8SSam Ravnborg err = set_tx_tail(lp, tail); 1529a88b5ba8SSam Ravnborg if (!err) { 1530a88b5ba8SSam Ravnborg lp->snd_nxt = seq; 1531a88b5ba8SSam Ravnborg err = size; 1532a88b5ba8SSam Ravnborg } 1533a88b5ba8SSam Ravnborg 1534a88b5ba8SSam Ravnborg return err; 1535a88b5ba8SSam Ravnborg } 1536a88b5ba8SSam Ravnborg 1537a88b5ba8SSam Ravnborg static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p, 1538a88b5ba8SSam Ravnborg struct ldc_packet *first_frag) 1539a88b5ba8SSam Ravnborg { 1540a88b5ba8SSam Ravnborg int err; 1541a88b5ba8SSam Ravnborg 1542a88b5ba8SSam Ravnborg if (first_frag) 1543a88b5ba8SSam Ravnborg lp->rcv_nxt = first_frag->seqid - 1; 1544a88b5ba8SSam Ravnborg 1545a88b5ba8SSam Ravnborg err = send_data_nack(lp, p); 1546a88b5ba8SSam Ravnborg if (err) 1547a88b5ba8SSam Ravnborg return err; 1548a88b5ba8SSam Ravnborg 1549a88b5ba8SSam Ravnborg err = __set_rx_head(lp, lp->rx_tail); 1550a88b5ba8SSam Ravnborg if (err < 0) 1551a88b5ba8SSam Ravnborg return ldc_abort(lp); 1552a88b5ba8SSam Ravnborg 1553a88b5ba8SSam Ravnborg return 0; 1554a88b5ba8SSam Ravnborg } 1555a88b5ba8SSam Ravnborg 1556a88b5ba8SSam Ravnborg static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p) 1557a88b5ba8SSam Ravnborg { 1558a88b5ba8SSam Ravnborg if (p->stype & LDC_ACK) { 1559a88b5ba8SSam Ravnborg int err = process_data_ack(lp, p); 1560a88b5ba8SSam Ravnborg if (err) 1561a88b5ba8SSam Ravnborg return err; 1562a88b5ba8SSam Ravnborg } 1563a88b5ba8SSam Ravnborg if (p->stype & LDC_NACK) 1564a88b5ba8SSam Ravnborg return ldc_abort(lp); 1565a88b5ba8SSam Ravnborg 1566a88b5ba8SSam Ravnborg return 0; 1567a88b5ba8SSam Ravnborg } 1568a88b5ba8SSam Ravnborg 1569a88b5ba8SSam Ravnborg static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head) 1570a88b5ba8SSam Ravnborg { 1571a88b5ba8SSam Ravnborg unsigned long dummy; 1572a88b5ba8SSam Ravnborg int limit = 1000; 1573a88b5ba8SSam Ravnborg 1574a88b5ba8SSam Ravnborg ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n", 1575a88b5ba8SSam Ravnborg cur_head, lp->rx_head, lp->rx_tail); 1576a88b5ba8SSam Ravnborg while (limit-- > 0) { 1577a88b5ba8SSam Ravnborg unsigned long hv_err; 1578a88b5ba8SSam Ravnborg 1579a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 1580a88b5ba8SSam Ravnborg &dummy, 1581a88b5ba8SSam Ravnborg &lp->rx_tail, 1582a88b5ba8SSam Ravnborg &lp->chan_state); 1583a88b5ba8SSam Ravnborg if (hv_err) 1584a88b5ba8SSam Ravnborg return ldc_abort(lp); 1585a88b5ba8SSam Ravnborg 1586a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN || 1587a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING) 1588a88b5ba8SSam Ravnborg return -ECONNRESET; 1589a88b5ba8SSam Ravnborg 1590a88b5ba8SSam Ravnborg if (cur_head != lp->rx_tail) { 1591a88b5ba8SSam Ravnborg ldcdbg(DATA, "DATA WAIT DONE " 1592a88b5ba8SSam Ravnborg "head[%lx] tail[%lx] chan_state[%lx]\n", 1593a88b5ba8SSam Ravnborg dummy, lp->rx_tail, lp->chan_state); 1594a88b5ba8SSam Ravnborg return 0; 1595a88b5ba8SSam Ravnborg } 1596a88b5ba8SSam Ravnborg 1597a88b5ba8SSam Ravnborg udelay(1); 1598a88b5ba8SSam Ravnborg } 1599a88b5ba8SSam Ravnborg return -EAGAIN; 1600a88b5ba8SSam Ravnborg } 1601a88b5ba8SSam Ravnborg 1602a88b5ba8SSam Ravnborg static int rx_set_head(struct ldc_channel *lp, unsigned long head) 1603a88b5ba8SSam Ravnborg { 1604a88b5ba8SSam Ravnborg int err = __set_rx_head(lp, head); 1605a88b5ba8SSam Ravnborg 1606a88b5ba8SSam Ravnborg if (err < 0) 1607a88b5ba8SSam Ravnborg return ldc_abort(lp); 1608a88b5ba8SSam Ravnborg 1609a88b5ba8SSam Ravnborg lp->rx_head = head; 1610a88b5ba8SSam Ravnborg return 0; 1611a88b5ba8SSam Ravnborg } 1612a88b5ba8SSam Ravnborg 1613a88b5ba8SSam Ravnborg static void send_data_ack(struct ldc_channel *lp) 1614a88b5ba8SSam Ravnborg { 1615a88b5ba8SSam Ravnborg unsigned long new_tail; 1616a88b5ba8SSam Ravnborg struct ldc_packet *p; 1617a88b5ba8SSam Ravnborg 1618a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail); 1619a88b5ba8SSam Ravnborg if (likely(p)) { 1620a88b5ba8SSam Ravnborg int err; 1621a88b5ba8SSam Ravnborg 1622a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p)); 1623a88b5ba8SSam Ravnborg p->type = LDC_DATA; 1624a88b5ba8SSam Ravnborg p->stype = LDC_ACK; 1625a88b5ba8SSam Ravnborg p->ctrl = 0; 1626a88b5ba8SSam Ravnborg p->seqid = lp->snd_nxt + 1; 1627a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt; 1628a88b5ba8SSam Ravnborg 1629a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail); 1630a88b5ba8SSam Ravnborg if (!err) 1631a88b5ba8SSam Ravnborg lp->snd_nxt++; 1632a88b5ba8SSam Ravnborg } 1633a88b5ba8SSam Ravnborg } 1634a88b5ba8SSam Ravnborg 1635a88b5ba8SSam Ravnborg static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size) 1636a88b5ba8SSam Ravnborg { 1637a88b5ba8SSam Ravnborg struct ldc_packet *first_frag; 1638a88b5ba8SSam Ravnborg unsigned long hv_err, new; 1639a88b5ba8SSam Ravnborg int err, copied; 1640a88b5ba8SSam Ravnborg 1641a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 1642a88b5ba8SSam Ravnborg &lp->rx_head, 1643a88b5ba8SSam Ravnborg &lp->rx_tail, 1644a88b5ba8SSam Ravnborg &lp->chan_state); 1645a88b5ba8SSam Ravnborg if (hv_err) 1646a88b5ba8SSam Ravnborg return ldc_abort(lp); 1647a88b5ba8SSam Ravnborg 1648a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN || 1649a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING) 1650a88b5ba8SSam Ravnborg return -ECONNRESET; 1651a88b5ba8SSam Ravnborg 1652a88b5ba8SSam Ravnborg if (lp->rx_head == lp->rx_tail) 1653a88b5ba8SSam Ravnborg return 0; 1654a88b5ba8SSam Ravnborg 1655a88b5ba8SSam Ravnborg first_frag = NULL; 1656a88b5ba8SSam Ravnborg copied = err = 0; 1657a88b5ba8SSam Ravnborg new = lp->rx_head; 1658a88b5ba8SSam Ravnborg while (1) { 1659a88b5ba8SSam Ravnborg struct ldc_packet *p; 1660a88b5ba8SSam Ravnborg int pkt_len; 1661a88b5ba8SSam Ravnborg 1662a88b5ba8SSam Ravnborg BUG_ON(new == lp->rx_tail); 1663a88b5ba8SSam Ravnborg p = lp->rx_base + (new / LDC_PACKET_SIZE); 1664a88b5ba8SSam Ravnborg 1665a88b5ba8SSam Ravnborg ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] " 1666a88b5ba8SSam Ravnborg "rcv_nxt[%08x]\n", 1667a88b5ba8SSam Ravnborg p->type, 1668a88b5ba8SSam Ravnborg p->stype, 1669a88b5ba8SSam Ravnborg p->ctrl, 1670a88b5ba8SSam Ravnborg p->env, 1671a88b5ba8SSam Ravnborg p->seqid, 1672a88b5ba8SSam Ravnborg p->u.r.ackid, 1673a88b5ba8SSam Ravnborg lp->rcv_nxt); 1674a88b5ba8SSam Ravnborg 1675a88b5ba8SSam Ravnborg if (unlikely(!rx_seq_ok(lp, p->seqid))) { 1676a88b5ba8SSam Ravnborg err = rx_bad_seq(lp, p, first_frag); 1677a88b5ba8SSam Ravnborg copied = 0; 1678a88b5ba8SSam Ravnborg break; 1679a88b5ba8SSam Ravnborg } 1680a88b5ba8SSam Ravnborg 1681a88b5ba8SSam Ravnborg if (p->type & LDC_CTRL) { 1682a88b5ba8SSam Ravnborg err = process_control_frame(lp, p); 1683a88b5ba8SSam Ravnborg if (err < 0) 1684a88b5ba8SSam Ravnborg break; 1685a88b5ba8SSam Ravnborg err = 0; 1686a88b5ba8SSam Ravnborg } 1687a88b5ba8SSam Ravnborg 1688a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid; 1689a88b5ba8SSam Ravnborg 1690a88b5ba8SSam Ravnborg if (!(p->type & LDC_DATA)) { 1691a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1692a88b5ba8SSam Ravnborg goto no_data; 1693a88b5ba8SSam Ravnborg } 1694a88b5ba8SSam Ravnborg if (p->stype & (LDC_ACK | LDC_NACK)) { 1695a88b5ba8SSam Ravnborg err = data_ack_nack(lp, p); 1696a88b5ba8SSam Ravnborg if (err) 1697a88b5ba8SSam Ravnborg break; 1698a88b5ba8SSam Ravnborg } 1699a88b5ba8SSam Ravnborg if (!(p->stype & LDC_INFO)) { 1700a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1701a88b5ba8SSam Ravnborg err = rx_set_head(lp, new); 1702a88b5ba8SSam Ravnborg if (err) 1703a88b5ba8SSam Ravnborg break; 1704a88b5ba8SSam Ravnborg goto no_data; 1705a88b5ba8SSam Ravnborg } 1706a88b5ba8SSam Ravnborg 1707a88b5ba8SSam Ravnborg pkt_len = p->env & LDC_LEN; 1708a88b5ba8SSam Ravnborg 1709a88b5ba8SSam Ravnborg /* Every initial packet starts with the START bit set. 1710a88b5ba8SSam Ravnborg * 1711a88b5ba8SSam Ravnborg * Singleton packets will have both START+STOP set. 1712a88b5ba8SSam Ravnborg * 1713a88b5ba8SSam Ravnborg * Fragments will have START set in the first frame, STOP 1714a88b5ba8SSam Ravnborg * set in the last frame, and neither bit set in middle 1715a88b5ba8SSam Ravnborg * frames of the packet. 1716a88b5ba8SSam Ravnborg * 1717a88b5ba8SSam Ravnborg * Therefore if we are at the beginning of a packet and 1718a88b5ba8SSam Ravnborg * we don't see START, or we are in the middle of a fragmented 1719a88b5ba8SSam Ravnborg * packet and do see START, we are unsynchronized and should 1720a88b5ba8SSam Ravnborg * flush the RX queue. 1721a88b5ba8SSam Ravnborg */ 1722a88b5ba8SSam Ravnborg if ((first_frag == NULL && !(p->env & LDC_START)) || 1723a88b5ba8SSam Ravnborg (first_frag != NULL && (p->env & LDC_START))) { 1724a88b5ba8SSam Ravnborg if (!first_frag) 1725a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1726a88b5ba8SSam Ravnborg 1727a88b5ba8SSam Ravnborg err = rx_set_head(lp, new); 1728a88b5ba8SSam Ravnborg if (err) 1729a88b5ba8SSam Ravnborg break; 1730a88b5ba8SSam Ravnborg 1731a88b5ba8SSam Ravnborg if (!first_frag) 1732a88b5ba8SSam Ravnborg goto no_data; 1733a88b5ba8SSam Ravnborg } 1734a88b5ba8SSam Ravnborg if (!first_frag) 1735a88b5ba8SSam Ravnborg first_frag = p; 1736a88b5ba8SSam Ravnborg 1737a88b5ba8SSam Ravnborg if (pkt_len > size - copied) { 1738a88b5ba8SSam Ravnborg /* User didn't give us a big enough buffer, 1739a88b5ba8SSam Ravnborg * what to do? This is a pretty serious error. 1740a88b5ba8SSam Ravnborg * 1741a88b5ba8SSam Ravnborg * Since we haven't updated the RX ring head to 1742a88b5ba8SSam Ravnborg * consume any of the packets, signal the error 1743a88b5ba8SSam Ravnborg * to the user and just leave the RX ring alone. 1744a88b5ba8SSam Ravnborg * 1745a88b5ba8SSam Ravnborg * This seems the best behavior because this allows 1746a88b5ba8SSam Ravnborg * a user of the LDC layer to start with a small 1747a88b5ba8SSam Ravnborg * RX buffer for ldc_read() calls and use -EMSGSIZE 1748a88b5ba8SSam Ravnborg * as a cue to enlarge it's read buffer. 1749a88b5ba8SSam Ravnborg */ 1750a88b5ba8SSam Ravnborg err = -EMSGSIZE; 1751a88b5ba8SSam Ravnborg break; 1752a88b5ba8SSam Ravnborg } 1753a88b5ba8SSam Ravnborg 1754a88b5ba8SSam Ravnborg /* Ok, we are gonna eat this one. */ 1755a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1756a88b5ba8SSam Ravnborg 1757a88b5ba8SSam Ravnborg memcpy(buf, 1758a88b5ba8SSam Ravnborg (lp->cfg.mode == LDC_MODE_UNRELIABLE ? 1759a88b5ba8SSam Ravnborg p->u.u_data : p->u.r.r_data), pkt_len); 1760a88b5ba8SSam Ravnborg buf += pkt_len; 1761a88b5ba8SSam Ravnborg copied += pkt_len; 1762a88b5ba8SSam Ravnborg 1763a88b5ba8SSam Ravnborg if (p->env & LDC_STOP) 1764a88b5ba8SSam Ravnborg break; 1765a88b5ba8SSam Ravnborg 1766a88b5ba8SSam Ravnborg no_data: 1767a88b5ba8SSam Ravnborg if (new == lp->rx_tail) { 1768a88b5ba8SSam Ravnborg err = rx_data_wait(lp, new); 1769a88b5ba8SSam Ravnborg if (err) 1770a88b5ba8SSam Ravnborg break; 1771a88b5ba8SSam Ravnborg } 1772a88b5ba8SSam Ravnborg } 1773a88b5ba8SSam Ravnborg 1774a88b5ba8SSam Ravnborg if (!err) 1775a88b5ba8SSam Ravnborg err = rx_set_head(lp, new); 1776a88b5ba8SSam Ravnborg 1777a88b5ba8SSam Ravnborg if (err && first_frag) 1778a88b5ba8SSam Ravnborg lp->rcv_nxt = first_frag->seqid - 1; 1779a88b5ba8SSam Ravnborg 1780a88b5ba8SSam Ravnborg if (!err) { 1781a88b5ba8SSam Ravnborg err = copied; 1782a88b5ba8SSam Ravnborg if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE) 1783a88b5ba8SSam Ravnborg send_data_ack(lp); 1784a88b5ba8SSam Ravnborg } 1785a88b5ba8SSam Ravnborg 1786a88b5ba8SSam Ravnborg return err; 1787a88b5ba8SSam Ravnborg } 1788a88b5ba8SSam Ravnborg 1789a88b5ba8SSam Ravnborg static const struct ldc_mode_ops nonraw_ops = { 1790a88b5ba8SSam Ravnborg .write = write_nonraw, 1791a88b5ba8SSam Ravnborg .read = read_nonraw, 1792a88b5ba8SSam Ravnborg }; 1793a88b5ba8SSam Ravnborg 1794a88b5ba8SSam Ravnborg static int write_stream(struct ldc_channel *lp, const void *buf, 1795a88b5ba8SSam Ravnborg unsigned int size) 1796a88b5ba8SSam Ravnborg { 1797a88b5ba8SSam Ravnborg if (size > lp->cfg.mtu) 1798a88b5ba8SSam Ravnborg size = lp->cfg.mtu; 1799a88b5ba8SSam Ravnborg return write_nonraw(lp, buf, size); 1800a88b5ba8SSam Ravnborg } 1801a88b5ba8SSam Ravnborg 1802a88b5ba8SSam Ravnborg static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size) 1803a88b5ba8SSam Ravnborg { 1804a88b5ba8SSam Ravnborg if (!lp->mssbuf_len) { 1805a88b5ba8SSam Ravnborg int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu); 1806a88b5ba8SSam Ravnborg if (err < 0) 1807a88b5ba8SSam Ravnborg return err; 1808a88b5ba8SSam Ravnborg 1809a88b5ba8SSam Ravnborg lp->mssbuf_len = err; 1810a88b5ba8SSam Ravnborg lp->mssbuf_off = 0; 1811a88b5ba8SSam Ravnborg } 1812a88b5ba8SSam Ravnborg 1813a88b5ba8SSam Ravnborg if (size > lp->mssbuf_len) 1814a88b5ba8SSam Ravnborg size = lp->mssbuf_len; 1815a88b5ba8SSam Ravnborg memcpy(buf, lp->mssbuf + lp->mssbuf_off, size); 1816a88b5ba8SSam Ravnborg 1817a88b5ba8SSam Ravnborg lp->mssbuf_off += size; 1818a88b5ba8SSam Ravnborg lp->mssbuf_len -= size; 1819a88b5ba8SSam Ravnborg 1820a88b5ba8SSam Ravnborg return size; 1821a88b5ba8SSam Ravnborg } 1822a88b5ba8SSam Ravnborg 1823a88b5ba8SSam Ravnborg static const struct ldc_mode_ops stream_ops = { 1824a88b5ba8SSam Ravnborg .write = write_stream, 1825a88b5ba8SSam Ravnborg .read = read_stream, 1826a88b5ba8SSam Ravnborg }; 1827a88b5ba8SSam Ravnborg 1828a88b5ba8SSam Ravnborg int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size) 1829a88b5ba8SSam Ravnborg { 1830a88b5ba8SSam Ravnborg unsigned long flags; 1831a88b5ba8SSam Ravnborg int err; 1832a88b5ba8SSam Ravnborg 1833a88b5ba8SSam Ravnborg if (!buf) 1834a88b5ba8SSam Ravnborg return -EINVAL; 1835a88b5ba8SSam Ravnborg 1836a88b5ba8SSam Ravnborg if (!size) 1837a88b5ba8SSam Ravnborg return 0; 1838a88b5ba8SSam Ravnborg 1839a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1840a88b5ba8SSam Ravnborg 1841a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE) 1842a88b5ba8SSam Ravnborg err = -ENOTCONN; 1843a88b5ba8SSam Ravnborg else 1844a88b5ba8SSam Ravnborg err = lp->mops->write(lp, buf, size); 1845a88b5ba8SSam Ravnborg 1846a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1847a88b5ba8SSam Ravnborg 1848a88b5ba8SSam Ravnborg return err; 1849a88b5ba8SSam Ravnborg } 1850a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_write); 1851a88b5ba8SSam Ravnborg 1852a88b5ba8SSam Ravnborg int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size) 1853a88b5ba8SSam Ravnborg { 1854a88b5ba8SSam Ravnborg unsigned long flags; 1855a88b5ba8SSam Ravnborg int err; 1856a88b5ba8SSam Ravnborg 1857a88b5ba8SSam Ravnborg if (!buf) 1858a88b5ba8SSam Ravnborg return -EINVAL; 1859a88b5ba8SSam Ravnborg 1860a88b5ba8SSam Ravnborg if (!size) 1861a88b5ba8SSam Ravnborg return 0; 1862a88b5ba8SSam Ravnborg 1863a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1864a88b5ba8SSam Ravnborg 1865a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE) 1866a88b5ba8SSam Ravnborg err = -ENOTCONN; 1867a88b5ba8SSam Ravnborg else 1868a88b5ba8SSam Ravnborg err = lp->mops->read(lp, buf, size); 1869a88b5ba8SSam Ravnborg 1870a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1871a88b5ba8SSam Ravnborg 1872a88b5ba8SSam Ravnborg return err; 1873a88b5ba8SSam Ravnborg } 1874a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_read); 1875a88b5ba8SSam Ravnborg 1876a88b5ba8SSam Ravnborg static long arena_alloc(struct ldc_iommu *iommu, unsigned long npages) 1877a88b5ba8SSam Ravnborg { 1878a88b5ba8SSam Ravnborg struct iommu_arena *arena = &iommu->arena; 1879a88b5ba8SSam Ravnborg unsigned long n, i, start, end, limit; 1880a88b5ba8SSam Ravnborg int pass; 1881a88b5ba8SSam Ravnborg 1882a88b5ba8SSam Ravnborg limit = arena->limit; 1883a88b5ba8SSam Ravnborg start = arena->hint; 1884a88b5ba8SSam Ravnborg pass = 0; 1885a88b5ba8SSam Ravnborg 1886a88b5ba8SSam Ravnborg again: 1887a88b5ba8SSam Ravnborg n = find_next_zero_bit(arena->map, limit, start); 1888a88b5ba8SSam Ravnborg end = n + npages; 1889a88b5ba8SSam Ravnborg if (unlikely(end >= limit)) { 1890a88b5ba8SSam Ravnborg if (likely(pass < 1)) { 1891a88b5ba8SSam Ravnborg limit = start; 1892a88b5ba8SSam Ravnborg start = 0; 1893a88b5ba8SSam Ravnborg pass++; 1894a88b5ba8SSam Ravnborg goto again; 1895a88b5ba8SSam Ravnborg } else { 1896a88b5ba8SSam Ravnborg /* Scanned the whole thing, give up. */ 1897a88b5ba8SSam Ravnborg return -1; 1898a88b5ba8SSam Ravnborg } 1899a88b5ba8SSam Ravnborg } 1900a88b5ba8SSam Ravnborg 1901a88b5ba8SSam Ravnborg for (i = n; i < end; i++) { 1902a88b5ba8SSam Ravnborg if (test_bit(i, arena->map)) { 1903a88b5ba8SSam Ravnborg start = i + 1; 1904a88b5ba8SSam Ravnborg goto again; 1905a88b5ba8SSam Ravnborg } 1906a88b5ba8SSam Ravnborg } 1907a88b5ba8SSam Ravnborg 1908a88b5ba8SSam Ravnborg for (i = n; i < end; i++) 1909a88b5ba8SSam Ravnborg __set_bit(i, arena->map); 1910a88b5ba8SSam Ravnborg 1911a88b5ba8SSam Ravnborg arena->hint = end; 1912a88b5ba8SSam Ravnborg 1913a88b5ba8SSam Ravnborg return n; 1914a88b5ba8SSam Ravnborg } 1915a88b5ba8SSam Ravnborg 1916a88b5ba8SSam Ravnborg #define COOKIE_PGSZ_CODE 0xf000000000000000ULL 1917a88b5ba8SSam Ravnborg #define COOKIE_PGSZ_CODE_SHIFT 60ULL 1918a88b5ba8SSam Ravnborg 1919a88b5ba8SSam Ravnborg static u64 pagesize_code(void) 1920a88b5ba8SSam Ravnborg { 1921a88b5ba8SSam Ravnborg switch (PAGE_SIZE) { 1922a88b5ba8SSam Ravnborg default: 1923a88b5ba8SSam Ravnborg case (8ULL * 1024ULL): 1924a88b5ba8SSam Ravnborg return 0; 1925a88b5ba8SSam Ravnborg case (64ULL * 1024ULL): 1926a88b5ba8SSam Ravnborg return 1; 1927a88b5ba8SSam Ravnborg case (512ULL * 1024ULL): 1928a88b5ba8SSam Ravnborg return 2; 1929a88b5ba8SSam Ravnborg case (4ULL * 1024ULL * 1024ULL): 1930a88b5ba8SSam Ravnborg return 3; 1931a88b5ba8SSam Ravnborg case (32ULL * 1024ULL * 1024ULL): 1932a88b5ba8SSam Ravnborg return 4; 1933a88b5ba8SSam Ravnborg case (256ULL * 1024ULL * 1024ULL): 1934a88b5ba8SSam Ravnborg return 5; 1935a88b5ba8SSam Ravnborg } 1936a88b5ba8SSam Ravnborg } 1937a88b5ba8SSam Ravnborg 1938a88b5ba8SSam Ravnborg static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset) 1939a88b5ba8SSam Ravnborg { 1940a88b5ba8SSam Ravnborg return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) | 1941a88b5ba8SSam Ravnborg (index << PAGE_SHIFT) | 1942a88b5ba8SSam Ravnborg page_offset); 1943a88b5ba8SSam Ravnborg } 1944a88b5ba8SSam Ravnborg 1945a88b5ba8SSam Ravnborg static u64 cookie_to_index(u64 cookie, unsigned long *shift) 1946a88b5ba8SSam Ravnborg { 1947a88b5ba8SSam Ravnborg u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT; 1948a88b5ba8SSam Ravnborg 1949a88b5ba8SSam Ravnborg cookie &= ~COOKIE_PGSZ_CODE; 1950a88b5ba8SSam Ravnborg 1951a88b5ba8SSam Ravnborg *shift = szcode * 3; 1952a88b5ba8SSam Ravnborg 1953a88b5ba8SSam Ravnborg return (cookie >> (13ULL + (szcode * 3ULL))); 1954a88b5ba8SSam Ravnborg } 1955a88b5ba8SSam Ravnborg 1956a88b5ba8SSam Ravnborg static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu, 1957a88b5ba8SSam Ravnborg unsigned long npages) 1958a88b5ba8SSam Ravnborg { 1959a88b5ba8SSam Ravnborg long entry; 1960a88b5ba8SSam Ravnborg 1961a88b5ba8SSam Ravnborg entry = arena_alloc(iommu, npages); 1962a88b5ba8SSam Ravnborg if (unlikely(entry < 0)) 1963a88b5ba8SSam Ravnborg return NULL; 1964a88b5ba8SSam Ravnborg 1965a88b5ba8SSam Ravnborg return iommu->page_table + entry; 1966a88b5ba8SSam Ravnborg } 1967a88b5ba8SSam Ravnborg 1968a88b5ba8SSam Ravnborg static u64 perm_to_mte(unsigned int map_perm) 1969a88b5ba8SSam Ravnborg { 1970a88b5ba8SSam Ravnborg u64 mte_base; 1971a88b5ba8SSam Ravnborg 1972a88b5ba8SSam Ravnborg mte_base = pagesize_code(); 1973a88b5ba8SSam Ravnborg 1974a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_SHADOW) { 1975a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R) 1976a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_COPY_R; 1977a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W) 1978a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_COPY_W; 1979a88b5ba8SSam Ravnborg } 1980a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_DIRECT) { 1981a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R) 1982a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_READ; 1983a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W) 1984a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_WRITE; 1985a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_X) 1986a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_EXEC; 1987a88b5ba8SSam Ravnborg } 1988a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_IO) { 1989a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R) 1990a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_IOMMU_R; 1991a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W) 1992a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_IOMMU_W; 1993a88b5ba8SSam Ravnborg } 1994a88b5ba8SSam Ravnborg 1995a88b5ba8SSam Ravnborg return mte_base; 1996a88b5ba8SSam Ravnborg } 1997a88b5ba8SSam Ravnborg 1998a88b5ba8SSam Ravnborg static int pages_in_region(unsigned long base, long len) 1999a88b5ba8SSam Ravnborg { 2000a88b5ba8SSam Ravnborg int count = 0; 2001a88b5ba8SSam Ravnborg 2002a88b5ba8SSam Ravnborg do { 2003a88b5ba8SSam Ravnborg unsigned long new = (base + PAGE_SIZE) & PAGE_MASK; 2004a88b5ba8SSam Ravnborg 2005a88b5ba8SSam Ravnborg len -= (new - base); 2006a88b5ba8SSam Ravnborg base = new; 2007a88b5ba8SSam Ravnborg count++; 2008a88b5ba8SSam Ravnborg } while (len > 0); 2009a88b5ba8SSam Ravnborg 2010a88b5ba8SSam Ravnborg return count; 2011a88b5ba8SSam Ravnborg } 2012a88b5ba8SSam Ravnborg 2013a88b5ba8SSam Ravnborg struct cookie_state { 2014a88b5ba8SSam Ravnborg struct ldc_mtable_entry *page_table; 2015a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies; 2016a88b5ba8SSam Ravnborg u64 mte_base; 2017a88b5ba8SSam Ravnborg u64 prev_cookie; 2018a88b5ba8SSam Ravnborg u32 pte_idx; 2019a88b5ba8SSam Ravnborg u32 nc; 2020a88b5ba8SSam Ravnborg }; 2021a88b5ba8SSam Ravnborg 2022a88b5ba8SSam Ravnborg static void fill_cookies(struct cookie_state *sp, unsigned long pa, 2023a88b5ba8SSam Ravnborg unsigned long off, unsigned long len) 2024a88b5ba8SSam Ravnborg { 2025a88b5ba8SSam Ravnborg do { 2026a88b5ba8SSam Ravnborg unsigned long tlen, new = pa + PAGE_SIZE; 2027a88b5ba8SSam Ravnborg u64 this_cookie; 2028a88b5ba8SSam Ravnborg 2029a88b5ba8SSam Ravnborg sp->page_table[sp->pte_idx].mte = sp->mte_base | pa; 2030a88b5ba8SSam Ravnborg 2031a88b5ba8SSam Ravnborg tlen = PAGE_SIZE; 2032a88b5ba8SSam Ravnborg if (off) 2033a88b5ba8SSam Ravnborg tlen = PAGE_SIZE - off; 2034a88b5ba8SSam Ravnborg if (tlen > len) 2035a88b5ba8SSam Ravnborg tlen = len; 2036a88b5ba8SSam Ravnborg 2037a88b5ba8SSam Ravnborg this_cookie = make_cookie(sp->pte_idx, 2038a88b5ba8SSam Ravnborg pagesize_code(), off); 2039a88b5ba8SSam Ravnborg 2040a88b5ba8SSam Ravnborg off = 0; 2041a88b5ba8SSam Ravnborg 2042a88b5ba8SSam Ravnborg if (this_cookie == sp->prev_cookie) { 2043a88b5ba8SSam Ravnborg sp->cookies[sp->nc - 1].cookie_size += tlen; 2044a88b5ba8SSam Ravnborg } else { 2045a88b5ba8SSam Ravnborg sp->cookies[sp->nc].cookie_addr = this_cookie; 2046a88b5ba8SSam Ravnborg sp->cookies[sp->nc].cookie_size = tlen; 2047a88b5ba8SSam Ravnborg sp->nc++; 2048a88b5ba8SSam Ravnborg } 2049a88b5ba8SSam Ravnborg sp->prev_cookie = this_cookie + tlen; 2050a88b5ba8SSam Ravnborg 2051a88b5ba8SSam Ravnborg sp->pte_idx++; 2052a88b5ba8SSam Ravnborg 2053a88b5ba8SSam Ravnborg len -= tlen; 2054a88b5ba8SSam Ravnborg pa = new; 2055a88b5ba8SSam Ravnborg } while (len > 0); 2056a88b5ba8SSam Ravnborg } 2057a88b5ba8SSam Ravnborg 2058a88b5ba8SSam Ravnborg static int sg_count_one(struct scatterlist *sg) 2059a88b5ba8SSam Ravnborg { 2060a88b5ba8SSam Ravnborg unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT; 2061a88b5ba8SSam Ravnborg long len = sg->length; 2062a88b5ba8SSam Ravnborg 2063a88b5ba8SSam Ravnborg if ((sg->offset | len) & (8UL - 1)) 2064a88b5ba8SSam Ravnborg return -EFAULT; 2065a88b5ba8SSam Ravnborg 2066a88b5ba8SSam Ravnborg return pages_in_region(base + sg->offset, len); 2067a88b5ba8SSam Ravnborg } 2068a88b5ba8SSam Ravnborg 2069a88b5ba8SSam Ravnborg static int sg_count_pages(struct scatterlist *sg, int num_sg) 2070a88b5ba8SSam Ravnborg { 2071a88b5ba8SSam Ravnborg int count; 2072a88b5ba8SSam Ravnborg int i; 2073a88b5ba8SSam Ravnborg 2074a88b5ba8SSam Ravnborg count = 0; 2075a88b5ba8SSam Ravnborg for (i = 0; i < num_sg; i++) { 2076a88b5ba8SSam Ravnborg int err = sg_count_one(sg + i); 2077a88b5ba8SSam Ravnborg if (err < 0) 2078a88b5ba8SSam Ravnborg return err; 2079a88b5ba8SSam Ravnborg count += err; 2080a88b5ba8SSam Ravnborg } 2081a88b5ba8SSam Ravnborg 2082a88b5ba8SSam Ravnborg return count; 2083a88b5ba8SSam Ravnborg } 2084a88b5ba8SSam Ravnborg 2085a88b5ba8SSam Ravnborg int ldc_map_sg(struct ldc_channel *lp, 2086a88b5ba8SSam Ravnborg struct scatterlist *sg, int num_sg, 2087a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies, 2088a88b5ba8SSam Ravnborg unsigned int map_perm) 2089a88b5ba8SSam Ravnborg { 2090a88b5ba8SSam Ravnborg unsigned long i, npages, flags; 2091a88b5ba8SSam Ravnborg struct ldc_mtable_entry *base; 2092a88b5ba8SSam Ravnborg struct cookie_state state; 2093a88b5ba8SSam Ravnborg struct ldc_iommu *iommu; 2094a88b5ba8SSam Ravnborg int err; 2095a88b5ba8SSam Ravnborg 2096a88b5ba8SSam Ravnborg if (map_perm & ~LDC_MAP_ALL) 2097a88b5ba8SSam Ravnborg return -EINVAL; 2098a88b5ba8SSam Ravnborg 2099a88b5ba8SSam Ravnborg err = sg_count_pages(sg, num_sg); 2100a88b5ba8SSam Ravnborg if (err < 0) 2101a88b5ba8SSam Ravnborg return err; 2102a88b5ba8SSam Ravnborg 2103a88b5ba8SSam Ravnborg npages = err; 2104a88b5ba8SSam Ravnborg if (err > ncookies) 2105a88b5ba8SSam Ravnborg return -EMSGSIZE; 2106a88b5ba8SSam Ravnborg 2107a88b5ba8SSam Ravnborg iommu = &lp->iommu; 2108a88b5ba8SSam Ravnborg 2109a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags); 2110a88b5ba8SSam Ravnborg base = alloc_npages(iommu, npages); 2111a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags); 2112a88b5ba8SSam Ravnborg 2113a88b5ba8SSam Ravnborg if (!base) 2114a88b5ba8SSam Ravnborg return -ENOMEM; 2115a88b5ba8SSam Ravnborg 2116a88b5ba8SSam Ravnborg state.page_table = iommu->page_table; 2117a88b5ba8SSam Ravnborg state.cookies = cookies; 2118a88b5ba8SSam Ravnborg state.mte_base = perm_to_mte(map_perm); 2119a88b5ba8SSam Ravnborg state.prev_cookie = ~(u64)0; 2120a88b5ba8SSam Ravnborg state.pte_idx = (base - iommu->page_table); 2121a88b5ba8SSam Ravnborg state.nc = 0; 2122a88b5ba8SSam Ravnborg 2123a88b5ba8SSam Ravnborg for (i = 0; i < num_sg; i++) 2124a88b5ba8SSam Ravnborg fill_cookies(&state, page_to_pfn(sg_page(&sg[i])) << PAGE_SHIFT, 2125a88b5ba8SSam Ravnborg sg[i].offset, sg[i].length); 2126a88b5ba8SSam Ravnborg 2127a88b5ba8SSam Ravnborg return state.nc; 2128a88b5ba8SSam Ravnborg } 2129a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_map_sg); 2130a88b5ba8SSam Ravnborg 2131a88b5ba8SSam Ravnborg int ldc_map_single(struct ldc_channel *lp, 2132a88b5ba8SSam Ravnborg void *buf, unsigned int len, 2133a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies, 2134a88b5ba8SSam Ravnborg unsigned int map_perm) 2135a88b5ba8SSam Ravnborg { 2136a88b5ba8SSam Ravnborg unsigned long npages, pa, flags; 2137a88b5ba8SSam Ravnborg struct ldc_mtable_entry *base; 2138a88b5ba8SSam Ravnborg struct cookie_state state; 2139a88b5ba8SSam Ravnborg struct ldc_iommu *iommu; 2140a88b5ba8SSam Ravnborg 2141a88b5ba8SSam Ravnborg if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1)) 2142a88b5ba8SSam Ravnborg return -EINVAL; 2143a88b5ba8SSam Ravnborg 2144a88b5ba8SSam Ravnborg pa = __pa(buf); 2145a88b5ba8SSam Ravnborg if ((pa | len) & (8UL - 1)) 2146a88b5ba8SSam Ravnborg return -EFAULT; 2147a88b5ba8SSam Ravnborg 2148a88b5ba8SSam Ravnborg npages = pages_in_region(pa, len); 2149a88b5ba8SSam Ravnborg 2150a88b5ba8SSam Ravnborg iommu = &lp->iommu; 2151a88b5ba8SSam Ravnborg 2152a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags); 2153a88b5ba8SSam Ravnborg base = alloc_npages(iommu, npages); 2154a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags); 2155a88b5ba8SSam Ravnborg 2156a88b5ba8SSam Ravnborg if (!base) 2157a88b5ba8SSam Ravnborg return -ENOMEM; 2158a88b5ba8SSam Ravnborg 2159a88b5ba8SSam Ravnborg state.page_table = iommu->page_table; 2160a88b5ba8SSam Ravnborg state.cookies = cookies; 2161a88b5ba8SSam Ravnborg state.mte_base = perm_to_mte(map_perm); 2162a88b5ba8SSam Ravnborg state.prev_cookie = ~(u64)0; 2163a88b5ba8SSam Ravnborg state.pte_idx = (base - iommu->page_table); 2164a88b5ba8SSam Ravnborg state.nc = 0; 2165a88b5ba8SSam Ravnborg fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len); 2166a88b5ba8SSam Ravnborg BUG_ON(state.nc != 1); 2167a88b5ba8SSam Ravnborg 2168a88b5ba8SSam Ravnborg return state.nc; 2169a88b5ba8SSam Ravnborg } 2170a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_map_single); 2171a88b5ba8SSam Ravnborg 2172a88b5ba8SSam Ravnborg static void free_npages(unsigned long id, struct ldc_iommu *iommu, 2173a88b5ba8SSam Ravnborg u64 cookie, u64 size) 2174a88b5ba8SSam Ravnborg { 2175a88b5ba8SSam Ravnborg struct iommu_arena *arena = &iommu->arena; 2176a88b5ba8SSam Ravnborg unsigned long i, shift, index, npages; 2177a88b5ba8SSam Ravnborg struct ldc_mtable_entry *base; 2178a88b5ba8SSam Ravnborg 2179a88b5ba8SSam Ravnborg npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT; 2180a88b5ba8SSam Ravnborg index = cookie_to_index(cookie, &shift); 2181a88b5ba8SSam Ravnborg base = iommu->page_table + index; 2182a88b5ba8SSam Ravnborg 2183a88b5ba8SSam Ravnborg BUG_ON(index > arena->limit || 2184a88b5ba8SSam Ravnborg (index + npages) > arena->limit); 2185a88b5ba8SSam Ravnborg 2186a88b5ba8SSam Ravnborg for (i = 0; i < npages; i++) { 2187a88b5ba8SSam Ravnborg if (base->cookie) 2188a88b5ba8SSam Ravnborg sun4v_ldc_revoke(id, cookie + (i << shift), 2189a88b5ba8SSam Ravnborg base->cookie); 2190a88b5ba8SSam Ravnborg base->mte = 0; 2191a88b5ba8SSam Ravnborg __clear_bit(index + i, arena->map); 2192a88b5ba8SSam Ravnborg } 2193a88b5ba8SSam Ravnborg } 2194a88b5ba8SSam Ravnborg 2195a88b5ba8SSam Ravnborg void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, 2196a88b5ba8SSam Ravnborg int ncookies) 2197a88b5ba8SSam Ravnborg { 2198a88b5ba8SSam Ravnborg struct ldc_iommu *iommu = &lp->iommu; 2199a88b5ba8SSam Ravnborg unsigned long flags; 2200a88b5ba8SSam Ravnborg int i; 2201a88b5ba8SSam Ravnborg 2202a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags); 2203a88b5ba8SSam Ravnborg for (i = 0; i < ncookies; i++) { 2204a88b5ba8SSam Ravnborg u64 addr = cookies[i].cookie_addr; 2205a88b5ba8SSam Ravnborg u64 size = cookies[i].cookie_size; 2206a88b5ba8SSam Ravnborg 2207a88b5ba8SSam Ravnborg free_npages(lp->id, iommu, addr, size); 2208a88b5ba8SSam Ravnborg } 2209a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags); 2210a88b5ba8SSam Ravnborg } 2211a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_unmap); 2212a88b5ba8SSam Ravnborg 2213a88b5ba8SSam Ravnborg int ldc_copy(struct ldc_channel *lp, int copy_dir, 2214a88b5ba8SSam Ravnborg void *buf, unsigned int len, unsigned long offset, 2215a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies) 2216a88b5ba8SSam Ravnborg { 2217a88b5ba8SSam Ravnborg unsigned int orig_len; 2218a88b5ba8SSam Ravnborg unsigned long ra; 2219a88b5ba8SSam Ravnborg int i; 2220a88b5ba8SSam Ravnborg 2221a88b5ba8SSam Ravnborg if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) { 2222a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n", 2223a88b5ba8SSam Ravnborg lp->id, copy_dir); 2224a88b5ba8SSam Ravnborg return -EINVAL; 2225a88b5ba8SSam Ravnborg } 2226a88b5ba8SSam Ravnborg 2227a88b5ba8SSam Ravnborg ra = __pa(buf); 2228a88b5ba8SSam Ravnborg if ((ra | len | offset) & (8UL - 1)) { 2229a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer " 2230a88b5ba8SSam Ravnborg "ra[%lx] len[%x] offset[%lx]\n", 2231a88b5ba8SSam Ravnborg lp->id, ra, len, offset); 2232a88b5ba8SSam Ravnborg return -EFAULT; 2233a88b5ba8SSam Ravnborg } 2234a88b5ba8SSam Ravnborg 2235a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE || 2236a88b5ba8SSam Ravnborg (lp->flags & LDC_FLAG_RESET)) { 2237a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] " 2238a88b5ba8SSam Ravnborg "flags[%x]\n", lp->id, lp->hs_state, lp->flags); 2239a88b5ba8SSam Ravnborg return -ECONNRESET; 2240a88b5ba8SSam Ravnborg } 2241a88b5ba8SSam Ravnborg 2242a88b5ba8SSam Ravnborg orig_len = len; 2243a88b5ba8SSam Ravnborg for (i = 0; i < ncookies; i++) { 2244a88b5ba8SSam Ravnborg unsigned long cookie_raddr = cookies[i].cookie_addr; 2245a88b5ba8SSam Ravnborg unsigned long this_len = cookies[i].cookie_size; 2246a88b5ba8SSam Ravnborg unsigned long actual_len; 2247a88b5ba8SSam Ravnborg 2248a88b5ba8SSam Ravnborg if (unlikely(offset)) { 2249a88b5ba8SSam Ravnborg unsigned long this_off = offset; 2250a88b5ba8SSam Ravnborg 2251a88b5ba8SSam Ravnborg if (this_off > this_len) 2252a88b5ba8SSam Ravnborg this_off = this_len; 2253a88b5ba8SSam Ravnborg 2254a88b5ba8SSam Ravnborg offset -= this_off; 2255a88b5ba8SSam Ravnborg this_len -= this_off; 2256a88b5ba8SSam Ravnborg if (!this_len) 2257a88b5ba8SSam Ravnborg continue; 2258a88b5ba8SSam Ravnborg cookie_raddr += this_off; 2259a88b5ba8SSam Ravnborg } 2260a88b5ba8SSam Ravnborg 2261a88b5ba8SSam Ravnborg if (this_len > len) 2262a88b5ba8SSam Ravnborg this_len = len; 2263a88b5ba8SSam Ravnborg 2264a88b5ba8SSam Ravnborg while (1) { 2265a88b5ba8SSam Ravnborg unsigned long hv_err; 2266a88b5ba8SSam Ravnborg 2267a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_copy(lp->id, copy_dir, 2268a88b5ba8SSam Ravnborg cookie_raddr, ra, 2269a88b5ba8SSam Ravnborg this_len, &actual_len); 2270a88b5ba8SSam Ravnborg if (unlikely(hv_err)) { 2271a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] " 2272a88b5ba8SSam Ravnborg "HV error %lu\n", 2273a88b5ba8SSam Ravnborg lp->id, hv_err); 2274a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE || 2275a88b5ba8SSam Ravnborg (lp->flags & LDC_FLAG_RESET)) 2276a88b5ba8SSam Ravnborg return -ECONNRESET; 2277a88b5ba8SSam Ravnborg else 2278a88b5ba8SSam Ravnborg return -EFAULT; 2279a88b5ba8SSam Ravnborg } 2280a88b5ba8SSam Ravnborg 2281a88b5ba8SSam Ravnborg cookie_raddr += actual_len; 2282a88b5ba8SSam Ravnborg ra += actual_len; 2283a88b5ba8SSam Ravnborg len -= actual_len; 2284a88b5ba8SSam Ravnborg if (actual_len == this_len) 2285a88b5ba8SSam Ravnborg break; 2286a88b5ba8SSam Ravnborg 2287a88b5ba8SSam Ravnborg this_len -= actual_len; 2288a88b5ba8SSam Ravnborg } 2289a88b5ba8SSam Ravnborg 2290a88b5ba8SSam Ravnborg if (!len) 2291a88b5ba8SSam Ravnborg break; 2292a88b5ba8SSam Ravnborg } 2293a88b5ba8SSam Ravnborg 2294a88b5ba8SSam Ravnborg /* It is caller policy what to do about short copies. 2295a88b5ba8SSam Ravnborg * For example, a networking driver can declare the 2296a88b5ba8SSam Ravnborg * packet a runt and drop it. 2297a88b5ba8SSam Ravnborg */ 2298a88b5ba8SSam Ravnborg 2299a88b5ba8SSam Ravnborg return orig_len - len; 2300a88b5ba8SSam Ravnborg } 2301a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_copy); 2302a88b5ba8SSam Ravnborg 2303a88b5ba8SSam Ravnborg void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len, 2304a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int *ncookies, 2305a88b5ba8SSam Ravnborg unsigned int map_perm) 2306a88b5ba8SSam Ravnborg { 2307a88b5ba8SSam Ravnborg void *buf; 2308a88b5ba8SSam Ravnborg int err; 2309a88b5ba8SSam Ravnborg 2310a88b5ba8SSam Ravnborg if (len & (8UL - 1)) 2311a88b5ba8SSam Ravnborg return ERR_PTR(-EINVAL); 2312a88b5ba8SSam Ravnborg 2313a88b5ba8SSam Ravnborg buf = kzalloc(len, GFP_KERNEL); 2314a88b5ba8SSam Ravnborg if (!buf) 2315a88b5ba8SSam Ravnborg return ERR_PTR(-ENOMEM); 2316a88b5ba8SSam Ravnborg 2317a88b5ba8SSam Ravnborg err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm); 2318a88b5ba8SSam Ravnborg if (err < 0) { 2319a88b5ba8SSam Ravnborg kfree(buf); 2320a88b5ba8SSam Ravnborg return ERR_PTR(err); 2321a88b5ba8SSam Ravnborg } 2322a88b5ba8SSam Ravnborg *ncookies = err; 2323a88b5ba8SSam Ravnborg 2324a88b5ba8SSam Ravnborg return buf; 2325a88b5ba8SSam Ravnborg } 2326a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_alloc_exp_dring); 2327a88b5ba8SSam Ravnborg 2328a88b5ba8SSam Ravnborg void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len, 2329a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies) 2330a88b5ba8SSam Ravnborg { 2331a88b5ba8SSam Ravnborg ldc_unmap(lp, cookies, ncookies); 2332a88b5ba8SSam Ravnborg kfree(buf); 2333a88b5ba8SSam Ravnborg } 2334a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_free_exp_dring); 2335a88b5ba8SSam Ravnborg 2336a88b5ba8SSam Ravnborg static int __init ldc_init(void) 2337a88b5ba8SSam Ravnborg { 2338a88b5ba8SSam Ravnborg unsigned long major, minor; 2339a88b5ba8SSam Ravnborg struct mdesc_handle *hp; 2340a88b5ba8SSam Ravnborg const u64 *v; 2341a88b5ba8SSam Ravnborg int err; 2342a88b5ba8SSam Ravnborg u64 mp; 2343a88b5ba8SSam Ravnborg 2344a88b5ba8SSam Ravnborg hp = mdesc_grab(); 2345a88b5ba8SSam Ravnborg if (!hp) 2346a88b5ba8SSam Ravnborg return -ENODEV; 2347a88b5ba8SSam Ravnborg 2348a88b5ba8SSam Ravnborg mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform"); 2349a88b5ba8SSam Ravnborg err = -ENODEV; 2350a88b5ba8SSam Ravnborg if (mp == MDESC_NODE_NULL) 2351a88b5ba8SSam Ravnborg goto out; 2352a88b5ba8SSam Ravnborg 2353a88b5ba8SSam Ravnborg v = mdesc_get_property(hp, mp, "domaining-enabled", NULL); 2354a88b5ba8SSam Ravnborg if (!v) 2355a88b5ba8SSam Ravnborg goto out; 2356a88b5ba8SSam Ravnborg 2357a88b5ba8SSam Ravnborg major = 1; 2358a88b5ba8SSam Ravnborg minor = 0; 2359a88b5ba8SSam Ravnborg if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) { 2360a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "Could not register LDOM hvapi.\n"); 2361a88b5ba8SSam Ravnborg goto out; 2362a88b5ba8SSam Ravnborg } 2363a88b5ba8SSam Ravnborg 2364a88b5ba8SSam Ravnborg printk(KERN_INFO "%s", version); 2365a88b5ba8SSam Ravnborg 2366a88b5ba8SSam Ravnborg if (!*v) { 2367a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "Domaining disabled.\n"); 2368a88b5ba8SSam Ravnborg goto out; 2369a88b5ba8SSam Ravnborg } 2370a88b5ba8SSam Ravnborg ldom_domaining_enabled = 1; 2371a88b5ba8SSam Ravnborg err = 0; 2372a88b5ba8SSam Ravnborg 2373a88b5ba8SSam Ravnborg out: 2374a88b5ba8SSam Ravnborg mdesc_release(hp); 2375a88b5ba8SSam Ravnborg return err; 2376a88b5ba8SSam Ravnborg } 2377a88b5ba8SSam Ravnborg 2378a88b5ba8SSam Ravnborg core_initcall(ldc_init); 2379