1a88b5ba8SSam Ravnborg /* ldc.c: Logical Domain Channel link-layer protocol driver. 2a88b5ba8SSam Ravnborg * 3a88b5ba8SSam Ravnborg * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> 4a88b5ba8SSam Ravnborg */ 5a88b5ba8SSam Ravnborg 6a88b5ba8SSam Ravnborg #include <linux/kernel.h> 7a88b5ba8SSam Ravnborg #include <linux/module.h> 8a88b5ba8SSam Ravnborg #include <linux/slab.h> 9a88b5ba8SSam Ravnborg #include <linux/spinlock.h> 10a88b5ba8SSam Ravnborg #include <linux/delay.h> 11a88b5ba8SSam Ravnborg #include <linux/errno.h> 12a88b5ba8SSam Ravnborg #include <linux/string.h> 13a88b5ba8SSam Ravnborg #include <linux/scatterlist.h> 14a88b5ba8SSam Ravnborg #include <linux/interrupt.h> 15a88b5ba8SSam Ravnborg #include <linux/list.h> 16a88b5ba8SSam Ravnborg #include <linux/init.h> 17a88b5ba8SSam Ravnborg 18a88b5ba8SSam Ravnborg #include <asm/hypervisor.h> 19a88b5ba8SSam Ravnborg #include <asm/iommu.h> 20a88b5ba8SSam Ravnborg #include <asm/page.h> 21a88b5ba8SSam Ravnborg #include <asm/ldc.h> 22a88b5ba8SSam Ravnborg #include <asm/mdesc.h> 23a88b5ba8SSam Ravnborg 24a88b5ba8SSam Ravnborg #define DRV_MODULE_NAME "ldc" 25a88b5ba8SSam Ravnborg #define PFX DRV_MODULE_NAME ": " 26a88b5ba8SSam Ravnborg #define DRV_MODULE_VERSION "1.1" 27a88b5ba8SSam Ravnborg #define DRV_MODULE_RELDATE "July 22, 2008" 28a88b5ba8SSam Ravnborg 29a88b5ba8SSam Ravnborg static char version[] __devinitdata = 30a88b5ba8SSam Ravnborg DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 31a88b5ba8SSam Ravnborg #define LDC_PACKET_SIZE 64 32a88b5ba8SSam Ravnborg 33a88b5ba8SSam Ravnborg /* Packet header layout for unreliable and reliable mode frames. 34a88b5ba8SSam Ravnborg * When in RAW mode, packets are simply straight 64-byte payloads 35a88b5ba8SSam Ravnborg * with no headers. 36a88b5ba8SSam Ravnborg */ 37a88b5ba8SSam Ravnborg struct ldc_packet { 38a88b5ba8SSam Ravnborg u8 type; 39a88b5ba8SSam Ravnborg #define LDC_CTRL 0x01 40a88b5ba8SSam Ravnborg #define LDC_DATA 0x02 41a88b5ba8SSam Ravnborg #define LDC_ERR 0x10 42a88b5ba8SSam Ravnborg 43a88b5ba8SSam Ravnborg u8 stype; 44a88b5ba8SSam Ravnborg #define LDC_INFO 0x01 45a88b5ba8SSam Ravnborg #define LDC_ACK 0x02 46a88b5ba8SSam Ravnborg #define LDC_NACK 0x04 47a88b5ba8SSam Ravnborg 48a88b5ba8SSam Ravnborg u8 ctrl; 49a88b5ba8SSam Ravnborg #define LDC_VERS 0x01 /* Link Version */ 50a88b5ba8SSam Ravnborg #define LDC_RTS 0x02 /* Request To Send */ 51a88b5ba8SSam Ravnborg #define LDC_RTR 0x03 /* Ready To Receive */ 52a88b5ba8SSam Ravnborg #define LDC_RDX 0x04 /* Ready for Data eXchange */ 53a88b5ba8SSam Ravnborg #define LDC_CTRL_MSK 0x0f 54a88b5ba8SSam Ravnborg 55a88b5ba8SSam Ravnborg u8 env; 56a88b5ba8SSam Ravnborg #define LDC_LEN 0x3f 57a88b5ba8SSam Ravnborg #define LDC_FRAG_MASK 0xc0 58a88b5ba8SSam Ravnborg #define LDC_START 0x40 59a88b5ba8SSam Ravnborg #define LDC_STOP 0x80 60a88b5ba8SSam Ravnborg 61a88b5ba8SSam Ravnborg u32 seqid; 62a88b5ba8SSam Ravnborg 63a88b5ba8SSam Ravnborg union { 64a88b5ba8SSam Ravnborg u8 u_data[LDC_PACKET_SIZE - 8]; 65a88b5ba8SSam Ravnborg struct { 66a88b5ba8SSam Ravnborg u32 pad; 67a88b5ba8SSam Ravnborg u32 ackid; 68a88b5ba8SSam Ravnborg u8 r_data[LDC_PACKET_SIZE - 8 - 8]; 69a88b5ba8SSam Ravnborg } r; 70a88b5ba8SSam Ravnborg } u; 71a88b5ba8SSam Ravnborg }; 72a88b5ba8SSam Ravnborg 73a88b5ba8SSam Ravnborg struct ldc_version { 74a88b5ba8SSam Ravnborg u16 major; 75a88b5ba8SSam Ravnborg u16 minor; 76a88b5ba8SSam Ravnborg }; 77a88b5ba8SSam Ravnborg 78a88b5ba8SSam Ravnborg /* Ordered from largest major to lowest. */ 79a88b5ba8SSam Ravnborg static struct ldc_version ver_arr[] = { 80a88b5ba8SSam Ravnborg { .major = 1, .minor = 0 }, 81a88b5ba8SSam Ravnborg }; 82a88b5ba8SSam Ravnborg 83a88b5ba8SSam Ravnborg #define LDC_DEFAULT_MTU (4 * LDC_PACKET_SIZE) 84a88b5ba8SSam Ravnborg #define LDC_DEFAULT_NUM_ENTRIES (PAGE_SIZE / LDC_PACKET_SIZE) 85a88b5ba8SSam Ravnborg 86a88b5ba8SSam Ravnborg struct ldc_channel; 87a88b5ba8SSam Ravnborg 88a88b5ba8SSam Ravnborg struct ldc_mode_ops { 89a88b5ba8SSam Ravnborg int (*write)(struct ldc_channel *, const void *, unsigned int); 90a88b5ba8SSam Ravnborg int (*read)(struct ldc_channel *, void *, unsigned int); 91a88b5ba8SSam Ravnborg }; 92a88b5ba8SSam Ravnborg 93a88b5ba8SSam Ravnborg static const struct ldc_mode_ops raw_ops; 94a88b5ba8SSam Ravnborg static const struct ldc_mode_ops nonraw_ops; 95a88b5ba8SSam Ravnborg static const struct ldc_mode_ops stream_ops; 96a88b5ba8SSam Ravnborg 97a88b5ba8SSam Ravnborg int ldom_domaining_enabled; 98a88b5ba8SSam Ravnborg 99a88b5ba8SSam Ravnborg struct ldc_iommu { 100a88b5ba8SSam Ravnborg /* Protects arena alloc/free. */ 101a88b5ba8SSam Ravnborg spinlock_t lock; 102a88b5ba8SSam Ravnborg struct iommu_arena arena; 103a88b5ba8SSam Ravnborg struct ldc_mtable_entry *page_table; 104a88b5ba8SSam Ravnborg }; 105a88b5ba8SSam Ravnborg 106a88b5ba8SSam Ravnborg struct ldc_channel { 107a88b5ba8SSam Ravnborg /* Protects all operations that depend upon channel state. */ 108a88b5ba8SSam Ravnborg spinlock_t lock; 109a88b5ba8SSam Ravnborg 110a88b5ba8SSam Ravnborg unsigned long id; 111a88b5ba8SSam Ravnborg 112a88b5ba8SSam Ravnborg u8 *mssbuf; 113a88b5ba8SSam Ravnborg u32 mssbuf_len; 114a88b5ba8SSam Ravnborg u32 mssbuf_off; 115a88b5ba8SSam Ravnborg 116a88b5ba8SSam Ravnborg struct ldc_packet *tx_base; 117a88b5ba8SSam Ravnborg unsigned long tx_head; 118a88b5ba8SSam Ravnborg unsigned long tx_tail; 119a88b5ba8SSam Ravnborg unsigned long tx_num_entries; 120a88b5ba8SSam Ravnborg unsigned long tx_ra; 121a88b5ba8SSam Ravnborg 122a88b5ba8SSam Ravnborg unsigned long tx_acked; 123a88b5ba8SSam Ravnborg 124a88b5ba8SSam Ravnborg struct ldc_packet *rx_base; 125a88b5ba8SSam Ravnborg unsigned long rx_head; 126a88b5ba8SSam Ravnborg unsigned long rx_tail; 127a88b5ba8SSam Ravnborg unsigned long rx_num_entries; 128a88b5ba8SSam Ravnborg unsigned long rx_ra; 129a88b5ba8SSam Ravnborg 130a88b5ba8SSam Ravnborg u32 rcv_nxt; 131a88b5ba8SSam Ravnborg u32 snd_nxt; 132a88b5ba8SSam Ravnborg 133a88b5ba8SSam Ravnborg unsigned long chan_state; 134a88b5ba8SSam Ravnborg 135a88b5ba8SSam Ravnborg struct ldc_channel_config cfg; 136a88b5ba8SSam Ravnborg void *event_arg; 137a88b5ba8SSam Ravnborg 138a88b5ba8SSam Ravnborg const struct ldc_mode_ops *mops; 139a88b5ba8SSam Ravnborg 140a88b5ba8SSam Ravnborg struct ldc_iommu iommu; 141a88b5ba8SSam Ravnborg 142a88b5ba8SSam Ravnborg struct ldc_version ver; 143a88b5ba8SSam Ravnborg 144a88b5ba8SSam Ravnborg u8 hs_state; 145a88b5ba8SSam Ravnborg #define LDC_HS_CLOSED 0x00 146a88b5ba8SSam Ravnborg #define LDC_HS_OPEN 0x01 147a88b5ba8SSam Ravnborg #define LDC_HS_GOTVERS 0x02 148a88b5ba8SSam Ravnborg #define LDC_HS_SENTRTR 0x03 149a88b5ba8SSam Ravnborg #define LDC_HS_GOTRTR 0x04 150a88b5ba8SSam Ravnborg #define LDC_HS_COMPLETE 0x10 151a88b5ba8SSam Ravnborg 152a88b5ba8SSam Ravnborg u8 flags; 153a88b5ba8SSam Ravnborg #define LDC_FLAG_ALLOCED_QUEUES 0x01 154a88b5ba8SSam Ravnborg #define LDC_FLAG_REGISTERED_QUEUES 0x02 155a88b5ba8SSam Ravnborg #define LDC_FLAG_REGISTERED_IRQS 0x04 156a88b5ba8SSam Ravnborg #define LDC_FLAG_RESET 0x10 157a88b5ba8SSam Ravnborg 158a88b5ba8SSam Ravnborg u8 mss; 159a88b5ba8SSam Ravnborg u8 state; 160a88b5ba8SSam Ravnborg 161a88b5ba8SSam Ravnborg #define LDC_IRQ_NAME_MAX 32 162a88b5ba8SSam Ravnborg char rx_irq_name[LDC_IRQ_NAME_MAX]; 163a88b5ba8SSam Ravnborg char tx_irq_name[LDC_IRQ_NAME_MAX]; 164a88b5ba8SSam Ravnborg 165a88b5ba8SSam Ravnborg struct hlist_head mh_list; 166a88b5ba8SSam Ravnborg 167a88b5ba8SSam Ravnborg struct hlist_node list; 168a88b5ba8SSam Ravnborg }; 169a88b5ba8SSam Ravnborg 170a88b5ba8SSam Ravnborg #define ldcdbg(TYPE, f, a...) \ 171a88b5ba8SSam Ravnborg do { if (lp->cfg.debug & LDC_DEBUG_##TYPE) \ 172a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \ 173a88b5ba8SSam Ravnborg } while (0) 174a88b5ba8SSam Ravnborg 175a88b5ba8SSam Ravnborg static const char *state_to_str(u8 state) 176a88b5ba8SSam Ravnborg { 177a88b5ba8SSam Ravnborg switch (state) { 178a88b5ba8SSam Ravnborg case LDC_STATE_INVALID: 179a88b5ba8SSam Ravnborg return "INVALID"; 180a88b5ba8SSam Ravnborg case LDC_STATE_INIT: 181a88b5ba8SSam Ravnborg return "INIT"; 182a88b5ba8SSam Ravnborg case LDC_STATE_BOUND: 183a88b5ba8SSam Ravnborg return "BOUND"; 184a88b5ba8SSam Ravnborg case LDC_STATE_READY: 185a88b5ba8SSam Ravnborg return "READY"; 186a88b5ba8SSam Ravnborg case LDC_STATE_CONNECTED: 187a88b5ba8SSam Ravnborg return "CONNECTED"; 188a88b5ba8SSam Ravnborg default: 189a88b5ba8SSam Ravnborg return "<UNKNOWN>"; 190a88b5ba8SSam Ravnborg } 191a88b5ba8SSam Ravnborg } 192a88b5ba8SSam Ravnborg 193a88b5ba8SSam Ravnborg static void ldc_set_state(struct ldc_channel *lp, u8 state) 194a88b5ba8SSam Ravnborg { 195a88b5ba8SSam Ravnborg ldcdbg(STATE, "STATE (%s) --> (%s)\n", 196a88b5ba8SSam Ravnborg state_to_str(lp->state), 197a88b5ba8SSam Ravnborg state_to_str(state)); 198a88b5ba8SSam Ravnborg 199a88b5ba8SSam Ravnborg lp->state = state; 200a88b5ba8SSam Ravnborg } 201a88b5ba8SSam Ravnborg 202a88b5ba8SSam Ravnborg static unsigned long __advance(unsigned long off, unsigned long num_entries) 203a88b5ba8SSam Ravnborg { 204a88b5ba8SSam Ravnborg off += LDC_PACKET_SIZE; 205a88b5ba8SSam Ravnborg if (off == (num_entries * LDC_PACKET_SIZE)) 206a88b5ba8SSam Ravnborg off = 0; 207a88b5ba8SSam Ravnborg 208a88b5ba8SSam Ravnborg return off; 209a88b5ba8SSam Ravnborg } 210a88b5ba8SSam Ravnborg 211a88b5ba8SSam Ravnborg static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off) 212a88b5ba8SSam Ravnborg { 213a88b5ba8SSam Ravnborg return __advance(off, lp->rx_num_entries); 214a88b5ba8SSam Ravnborg } 215a88b5ba8SSam Ravnborg 216a88b5ba8SSam Ravnborg static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off) 217a88b5ba8SSam Ravnborg { 218a88b5ba8SSam Ravnborg return __advance(off, lp->tx_num_entries); 219a88b5ba8SSam Ravnborg } 220a88b5ba8SSam Ravnborg 221a88b5ba8SSam Ravnborg static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp, 222a88b5ba8SSam Ravnborg unsigned long *new_tail) 223a88b5ba8SSam Ravnborg { 224a88b5ba8SSam Ravnborg struct ldc_packet *p; 225a88b5ba8SSam Ravnborg unsigned long t; 226a88b5ba8SSam Ravnborg 227a88b5ba8SSam Ravnborg t = tx_advance(lp, lp->tx_tail); 228a88b5ba8SSam Ravnborg if (t == lp->tx_head) 229a88b5ba8SSam Ravnborg return NULL; 230a88b5ba8SSam Ravnborg 231a88b5ba8SSam Ravnborg *new_tail = t; 232a88b5ba8SSam Ravnborg 233a88b5ba8SSam Ravnborg p = lp->tx_base; 234a88b5ba8SSam Ravnborg return p + (lp->tx_tail / LDC_PACKET_SIZE); 235a88b5ba8SSam Ravnborg } 236a88b5ba8SSam Ravnborg 237a88b5ba8SSam Ravnborg /* When we are in reliable or stream mode, have to track the next packet 238a88b5ba8SSam Ravnborg * we haven't gotten an ACK for in the TX queue using tx_acked. We have 239a88b5ba8SSam Ravnborg * to be careful not to stomp over the queue past that point. During 240a88b5ba8SSam Ravnborg * the handshake, we don't have TX data packets pending in the queue 241a88b5ba8SSam Ravnborg * and that's why handshake_get_tx_packet() need not be mindful of 242a88b5ba8SSam Ravnborg * lp->tx_acked. 243a88b5ba8SSam Ravnborg */ 244a88b5ba8SSam Ravnborg static unsigned long head_for_data(struct ldc_channel *lp) 245a88b5ba8SSam Ravnborg { 246a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_STREAM) 247a88b5ba8SSam Ravnborg return lp->tx_acked; 248a88b5ba8SSam Ravnborg return lp->tx_head; 249a88b5ba8SSam Ravnborg } 250a88b5ba8SSam Ravnborg 251a88b5ba8SSam Ravnborg static int tx_has_space_for(struct ldc_channel *lp, unsigned int size) 252a88b5ba8SSam Ravnborg { 253a88b5ba8SSam Ravnborg unsigned long limit, tail, new_tail, diff; 254a88b5ba8SSam Ravnborg unsigned int mss; 255a88b5ba8SSam Ravnborg 256a88b5ba8SSam Ravnborg limit = head_for_data(lp); 257a88b5ba8SSam Ravnborg tail = lp->tx_tail; 258a88b5ba8SSam Ravnborg new_tail = tx_advance(lp, tail); 259a88b5ba8SSam Ravnborg if (new_tail == limit) 260a88b5ba8SSam Ravnborg return 0; 261a88b5ba8SSam Ravnborg 262a88b5ba8SSam Ravnborg if (limit > new_tail) 263a88b5ba8SSam Ravnborg diff = limit - new_tail; 264a88b5ba8SSam Ravnborg else 265a88b5ba8SSam Ravnborg diff = (limit + 266a88b5ba8SSam Ravnborg ((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail)); 267a88b5ba8SSam Ravnborg diff /= LDC_PACKET_SIZE; 268a88b5ba8SSam Ravnborg mss = lp->mss; 269a88b5ba8SSam Ravnborg 270a88b5ba8SSam Ravnborg if (diff * mss < size) 271a88b5ba8SSam Ravnborg return 0; 272a88b5ba8SSam Ravnborg 273a88b5ba8SSam Ravnborg return 1; 274a88b5ba8SSam Ravnborg } 275a88b5ba8SSam Ravnborg 276a88b5ba8SSam Ravnborg static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp, 277a88b5ba8SSam Ravnborg unsigned long *new_tail) 278a88b5ba8SSam Ravnborg { 279a88b5ba8SSam Ravnborg struct ldc_packet *p; 280a88b5ba8SSam Ravnborg unsigned long h, t; 281a88b5ba8SSam Ravnborg 282a88b5ba8SSam Ravnborg h = head_for_data(lp); 283a88b5ba8SSam Ravnborg t = tx_advance(lp, lp->tx_tail); 284a88b5ba8SSam Ravnborg if (t == h) 285a88b5ba8SSam Ravnborg return NULL; 286a88b5ba8SSam Ravnborg 287a88b5ba8SSam Ravnborg *new_tail = t; 288a88b5ba8SSam Ravnborg 289a88b5ba8SSam Ravnborg p = lp->tx_base; 290a88b5ba8SSam Ravnborg return p + (lp->tx_tail / LDC_PACKET_SIZE); 291a88b5ba8SSam Ravnborg } 292a88b5ba8SSam Ravnborg 293a88b5ba8SSam Ravnborg static int set_tx_tail(struct ldc_channel *lp, unsigned long tail) 294a88b5ba8SSam Ravnborg { 295a88b5ba8SSam Ravnborg unsigned long orig_tail = lp->tx_tail; 296a88b5ba8SSam Ravnborg int limit = 1000; 297a88b5ba8SSam Ravnborg 298a88b5ba8SSam Ravnborg lp->tx_tail = tail; 299a88b5ba8SSam Ravnborg while (limit-- > 0) { 300a88b5ba8SSam Ravnborg unsigned long err; 301a88b5ba8SSam Ravnborg 302a88b5ba8SSam Ravnborg err = sun4v_ldc_tx_set_qtail(lp->id, tail); 303a88b5ba8SSam Ravnborg if (!err) 304a88b5ba8SSam Ravnborg return 0; 305a88b5ba8SSam Ravnborg 306a88b5ba8SSam Ravnborg if (err != HV_EWOULDBLOCK) { 307a88b5ba8SSam Ravnborg lp->tx_tail = orig_tail; 308a88b5ba8SSam Ravnborg return -EINVAL; 309a88b5ba8SSam Ravnborg } 310a88b5ba8SSam Ravnborg udelay(1); 311a88b5ba8SSam Ravnborg } 312a88b5ba8SSam Ravnborg 313a88b5ba8SSam Ravnborg lp->tx_tail = orig_tail; 314a88b5ba8SSam Ravnborg return -EBUSY; 315a88b5ba8SSam Ravnborg } 316a88b5ba8SSam Ravnborg 317a88b5ba8SSam Ravnborg /* This just updates the head value in the hypervisor using 318a88b5ba8SSam Ravnborg * a polling loop with a timeout. The caller takes care of 319a88b5ba8SSam Ravnborg * upating software state representing the head change, if any. 320a88b5ba8SSam Ravnborg */ 321a88b5ba8SSam Ravnborg static int __set_rx_head(struct ldc_channel *lp, unsigned long head) 322a88b5ba8SSam Ravnborg { 323a88b5ba8SSam Ravnborg int limit = 1000; 324a88b5ba8SSam Ravnborg 325a88b5ba8SSam Ravnborg while (limit-- > 0) { 326a88b5ba8SSam Ravnborg unsigned long err; 327a88b5ba8SSam Ravnborg 328a88b5ba8SSam Ravnborg err = sun4v_ldc_rx_set_qhead(lp->id, head); 329a88b5ba8SSam Ravnborg if (!err) 330a88b5ba8SSam Ravnborg return 0; 331a88b5ba8SSam Ravnborg 332a88b5ba8SSam Ravnborg if (err != HV_EWOULDBLOCK) 333a88b5ba8SSam Ravnborg return -EINVAL; 334a88b5ba8SSam Ravnborg 335a88b5ba8SSam Ravnborg udelay(1); 336a88b5ba8SSam Ravnborg } 337a88b5ba8SSam Ravnborg 338a88b5ba8SSam Ravnborg return -EBUSY; 339a88b5ba8SSam Ravnborg } 340a88b5ba8SSam Ravnborg 341a88b5ba8SSam Ravnborg static int send_tx_packet(struct ldc_channel *lp, 342a88b5ba8SSam Ravnborg struct ldc_packet *p, 343a88b5ba8SSam Ravnborg unsigned long new_tail) 344a88b5ba8SSam Ravnborg { 345a88b5ba8SSam Ravnborg BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE))); 346a88b5ba8SSam Ravnborg 347a88b5ba8SSam Ravnborg return set_tx_tail(lp, new_tail); 348a88b5ba8SSam Ravnborg } 349a88b5ba8SSam Ravnborg 350a88b5ba8SSam Ravnborg static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp, 351a88b5ba8SSam Ravnborg u8 stype, u8 ctrl, 352a88b5ba8SSam Ravnborg void *data, int dlen, 353a88b5ba8SSam Ravnborg unsigned long *new_tail) 354a88b5ba8SSam Ravnborg { 355a88b5ba8SSam Ravnborg struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail); 356a88b5ba8SSam Ravnborg 357a88b5ba8SSam Ravnborg if (p) { 358a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p)); 359a88b5ba8SSam Ravnborg p->type = LDC_CTRL; 360a88b5ba8SSam Ravnborg p->stype = stype; 361a88b5ba8SSam Ravnborg p->ctrl = ctrl; 362a88b5ba8SSam Ravnborg if (data) 363a88b5ba8SSam Ravnborg memcpy(p->u.u_data, data, dlen); 364a88b5ba8SSam Ravnborg } 365a88b5ba8SSam Ravnborg return p; 366a88b5ba8SSam Ravnborg } 367a88b5ba8SSam Ravnborg 368a88b5ba8SSam Ravnborg static int start_handshake(struct ldc_channel *lp) 369a88b5ba8SSam Ravnborg { 370a88b5ba8SSam Ravnborg struct ldc_packet *p; 371a88b5ba8SSam Ravnborg struct ldc_version *ver; 372a88b5ba8SSam Ravnborg unsigned long new_tail; 373a88b5ba8SSam Ravnborg 374a88b5ba8SSam Ravnborg ver = &ver_arr[0]; 375a88b5ba8SSam Ravnborg 376a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n", 377a88b5ba8SSam Ravnborg ver->major, ver->minor); 378a88b5ba8SSam Ravnborg 379a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS, 380a88b5ba8SSam Ravnborg ver, sizeof(*ver), &new_tail); 381a88b5ba8SSam Ravnborg if (p) { 382a88b5ba8SSam Ravnborg int err = send_tx_packet(lp, p, new_tail); 383a88b5ba8SSam Ravnborg if (!err) 384a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_RESET; 385a88b5ba8SSam Ravnborg return err; 386a88b5ba8SSam Ravnborg } 387a88b5ba8SSam Ravnborg return -EBUSY; 388a88b5ba8SSam Ravnborg } 389a88b5ba8SSam Ravnborg 390a88b5ba8SSam Ravnborg static int send_version_nack(struct ldc_channel *lp, 391a88b5ba8SSam Ravnborg u16 major, u16 minor) 392a88b5ba8SSam Ravnborg { 393a88b5ba8SSam Ravnborg struct ldc_packet *p; 394a88b5ba8SSam Ravnborg struct ldc_version ver; 395a88b5ba8SSam Ravnborg unsigned long new_tail; 396a88b5ba8SSam Ravnborg 397a88b5ba8SSam Ravnborg ver.major = major; 398a88b5ba8SSam Ravnborg ver.minor = minor; 399a88b5ba8SSam Ravnborg 400a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS, 401a88b5ba8SSam Ravnborg &ver, sizeof(ver), &new_tail); 402a88b5ba8SSam Ravnborg if (p) { 403a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n", 404a88b5ba8SSam Ravnborg ver.major, ver.minor); 405a88b5ba8SSam Ravnborg 406a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 407a88b5ba8SSam Ravnborg } 408a88b5ba8SSam Ravnborg return -EBUSY; 409a88b5ba8SSam Ravnborg } 410a88b5ba8SSam Ravnborg 411a88b5ba8SSam Ravnborg static int send_version_ack(struct ldc_channel *lp, 412a88b5ba8SSam Ravnborg struct ldc_version *vp) 413a88b5ba8SSam Ravnborg { 414a88b5ba8SSam Ravnborg struct ldc_packet *p; 415a88b5ba8SSam Ravnborg unsigned long new_tail; 416a88b5ba8SSam Ravnborg 417a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS, 418a88b5ba8SSam Ravnborg vp, sizeof(*vp), &new_tail); 419a88b5ba8SSam Ravnborg if (p) { 420a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n", 421a88b5ba8SSam Ravnborg vp->major, vp->minor); 422a88b5ba8SSam Ravnborg 423a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 424a88b5ba8SSam Ravnborg } 425a88b5ba8SSam Ravnborg return -EBUSY; 426a88b5ba8SSam Ravnborg } 427a88b5ba8SSam Ravnborg 428a88b5ba8SSam Ravnborg static int send_rts(struct ldc_channel *lp) 429a88b5ba8SSam Ravnborg { 430a88b5ba8SSam Ravnborg struct ldc_packet *p; 431a88b5ba8SSam Ravnborg unsigned long new_tail; 432a88b5ba8SSam Ravnborg 433a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0, 434a88b5ba8SSam Ravnborg &new_tail); 435a88b5ba8SSam Ravnborg if (p) { 436a88b5ba8SSam Ravnborg p->env = lp->cfg.mode; 437a88b5ba8SSam Ravnborg p->seqid = 0; 438a88b5ba8SSam Ravnborg lp->rcv_nxt = 0; 439a88b5ba8SSam Ravnborg 440a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n", 441a88b5ba8SSam Ravnborg p->env, p->seqid); 442a88b5ba8SSam Ravnborg 443a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 444a88b5ba8SSam Ravnborg } 445a88b5ba8SSam Ravnborg return -EBUSY; 446a88b5ba8SSam Ravnborg } 447a88b5ba8SSam Ravnborg 448a88b5ba8SSam Ravnborg static int send_rtr(struct ldc_channel *lp) 449a88b5ba8SSam Ravnborg { 450a88b5ba8SSam Ravnborg struct ldc_packet *p; 451a88b5ba8SSam Ravnborg unsigned long new_tail; 452a88b5ba8SSam Ravnborg 453a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0, 454a88b5ba8SSam Ravnborg &new_tail); 455a88b5ba8SSam Ravnborg if (p) { 456a88b5ba8SSam Ravnborg p->env = lp->cfg.mode; 457a88b5ba8SSam Ravnborg p->seqid = 0; 458a88b5ba8SSam Ravnborg 459a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n", 460a88b5ba8SSam Ravnborg p->env, p->seqid); 461a88b5ba8SSam Ravnborg 462a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 463a88b5ba8SSam Ravnborg } 464a88b5ba8SSam Ravnborg return -EBUSY; 465a88b5ba8SSam Ravnborg } 466a88b5ba8SSam Ravnborg 467a88b5ba8SSam Ravnborg static int send_rdx(struct ldc_channel *lp) 468a88b5ba8SSam Ravnborg { 469a88b5ba8SSam Ravnborg struct ldc_packet *p; 470a88b5ba8SSam Ravnborg unsigned long new_tail; 471a88b5ba8SSam Ravnborg 472a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0, 473a88b5ba8SSam Ravnborg &new_tail); 474a88b5ba8SSam Ravnborg if (p) { 475a88b5ba8SSam Ravnborg p->env = 0; 476a88b5ba8SSam Ravnborg p->seqid = ++lp->snd_nxt; 477a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt; 478a88b5ba8SSam Ravnborg 479a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n", 480a88b5ba8SSam Ravnborg p->env, p->seqid, p->u.r.ackid); 481a88b5ba8SSam Ravnborg 482a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 483a88b5ba8SSam Ravnborg } 484a88b5ba8SSam Ravnborg return -EBUSY; 485a88b5ba8SSam Ravnborg } 486a88b5ba8SSam Ravnborg 487a88b5ba8SSam Ravnborg static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt) 488a88b5ba8SSam Ravnborg { 489a88b5ba8SSam Ravnborg struct ldc_packet *p; 490a88b5ba8SSam Ravnborg unsigned long new_tail; 491a88b5ba8SSam Ravnborg int err; 492a88b5ba8SSam Ravnborg 493a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail); 494a88b5ba8SSam Ravnborg if (!p) 495a88b5ba8SSam Ravnborg return -EBUSY; 496a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p)); 497a88b5ba8SSam Ravnborg p->type = data_pkt->type; 498a88b5ba8SSam Ravnborg p->stype = LDC_NACK; 499a88b5ba8SSam Ravnborg p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK; 500a88b5ba8SSam Ravnborg p->seqid = lp->snd_nxt + 1; 501a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt; 502a88b5ba8SSam Ravnborg 503a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n", 504a88b5ba8SSam Ravnborg p->type, p->ctrl, p->seqid, p->u.r.ackid); 505a88b5ba8SSam Ravnborg 506a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail); 507a88b5ba8SSam Ravnborg if (!err) 508a88b5ba8SSam Ravnborg lp->snd_nxt++; 509a88b5ba8SSam Ravnborg 510a88b5ba8SSam Ravnborg return err; 511a88b5ba8SSam Ravnborg } 512a88b5ba8SSam Ravnborg 513a88b5ba8SSam Ravnborg static int ldc_abort(struct ldc_channel *lp) 514a88b5ba8SSam Ravnborg { 515a88b5ba8SSam Ravnborg unsigned long hv_err; 516a88b5ba8SSam Ravnborg 517a88b5ba8SSam Ravnborg ldcdbg(STATE, "ABORT\n"); 518a88b5ba8SSam Ravnborg 519a88b5ba8SSam Ravnborg /* We report but do not act upon the hypervisor errors because 520a88b5ba8SSam Ravnborg * there really isn't much we can do if they fail at this point. 521a88b5ba8SSam Ravnborg */ 522a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); 523a88b5ba8SSam Ravnborg if (hv_err) 524a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 525a88b5ba8SSam Ravnborg "sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n", 526a88b5ba8SSam Ravnborg lp->id, lp->tx_ra, lp->tx_num_entries, hv_err); 527a88b5ba8SSam Ravnborg 528a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, 529a88b5ba8SSam Ravnborg &lp->tx_head, 530a88b5ba8SSam Ravnborg &lp->tx_tail, 531a88b5ba8SSam Ravnborg &lp->chan_state); 532a88b5ba8SSam Ravnborg if (hv_err) 533a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 534a88b5ba8SSam Ravnborg "sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n", 535a88b5ba8SSam Ravnborg lp->id, hv_err); 536a88b5ba8SSam Ravnborg 537a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); 538a88b5ba8SSam Ravnborg if (hv_err) 539a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 540a88b5ba8SSam Ravnborg "sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n", 541a88b5ba8SSam Ravnborg lp->id, lp->rx_ra, lp->rx_num_entries, hv_err); 542a88b5ba8SSam Ravnborg 543a88b5ba8SSam Ravnborg /* Refetch the RX queue state as well, because we could be invoked 544a88b5ba8SSam Ravnborg * here in the queue processing context. 545a88b5ba8SSam Ravnborg */ 546a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 547a88b5ba8SSam Ravnborg &lp->rx_head, 548a88b5ba8SSam Ravnborg &lp->rx_tail, 549a88b5ba8SSam Ravnborg &lp->chan_state); 550a88b5ba8SSam Ravnborg if (hv_err) 551a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 552a88b5ba8SSam Ravnborg "sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n", 553a88b5ba8SSam Ravnborg lp->id, hv_err); 554a88b5ba8SSam Ravnborg 555a88b5ba8SSam Ravnborg return -ECONNRESET; 556a88b5ba8SSam Ravnborg } 557a88b5ba8SSam Ravnborg 558a88b5ba8SSam Ravnborg static struct ldc_version *find_by_major(u16 major) 559a88b5ba8SSam Ravnborg { 560a88b5ba8SSam Ravnborg struct ldc_version *ret = NULL; 561a88b5ba8SSam Ravnborg int i; 562a88b5ba8SSam Ravnborg 563a88b5ba8SSam Ravnborg for (i = 0; i < ARRAY_SIZE(ver_arr); i++) { 564a88b5ba8SSam Ravnborg struct ldc_version *v = &ver_arr[i]; 565a88b5ba8SSam Ravnborg if (v->major <= major) { 566a88b5ba8SSam Ravnborg ret = v; 567a88b5ba8SSam Ravnborg break; 568a88b5ba8SSam Ravnborg } 569a88b5ba8SSam Ravnborg } 570a88b5ba8SSam Ravnborg return ret; 571a88b5ba8SSam Ravnborg } 572a88b5ba8SSam Ravnborg 573a88b5ba8SSam Ravnborg static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp) 574a88b5ba8SSam Ravnborg { 575a88b5ba8SSam Ravnborg struct ldc_version *vap; 576a88b5ba8SSam Ravnborg int err; 577a88b5ba8SSam Ravnborg 578a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n", 579a88b5ba8SSam Ravnborg vp->major, vp->minor); 580a88b5ba8SSam Ravnborg 581a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_GOTVERS) { 582a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN; 583a88b5ba8SSam Ravnborg memset(&lp->ver, 0, sizeof(lp->ver)); 584a88b5ba8SSam Ravnborg } 585a88b5ba8SSam Ravnborg 586a88b5ba8SSam Ravnborg vap = find_by_major(vp->major); 587a88b5ba8SSam Ravnborg if (!vap) { 588a88b5ba8SSam Ravnborg err = send_version_nack(lp, 0, 0); 589a88b5ba8SSam Ravnborg } else if (vap->major != vp->major) { 590a88b5ba8SSam Ravnborg err = send_version_nack(lp, vap->major, vap->minor); 591a88b5ba8SSam Ravnborg } else { 592a88b5ba8SSam Ravnborg struct ldc_version ver = *vp; 593a88b5ba8SSam Ravnborg if (ver.minor > vap->minor) 594a88b5ba8SSam Ravnborg ver.minor = vap->minor; 595a88b5ba8SSam Ravnborg err = send_version_ack(lp, &ver); 596a88b5ba8SSam Ravnborg if (!err) { 597a88b5ba8SSam Ravnborg lp->ver = ver; 598a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_GOTVERS; 599a88b5ba8SSam Ravnborg } 600a88b5ba8SSam Ravnborg } 601a88b5ba8SSam Ravnborg if (err) 602a88b5ba8SSam Ravnborg return ldc_abort(lp); 603a88b5ba8SSam Ravnborg 604a88b5ba8SSam Ravnborg return 0; 605a88b5ba8SSam Ravnborg } 606a88b5ba8SSam Ravnborg 607a88b5ba8SSam Ravnborg static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp) 608a88b5ba8SSam Ravnborg { 609a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n", 610a88b5ba8SSam Ravnborg vp->major, vp->minor); 611a88b5ba8SSam Ravnborg 612a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_GOTVERS) { 613a88b5ba8SSam Ravnborg if (lp->ver.major != vp->major || 614a88b5ba8SSam Ravnborg lp->ver.minor != vp->minor) 615a88b5ba8SSam Ravnborg return ldc_abort(lp); 616a88b5ba8SSam Ravnborg } else { 617a88b5ba8SSam Ravnborg lp->ver = *vp; 618a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_GOTVERS; 619a88b5ba8SSam Ravnborg } 620a88b5ba8SSam Ravnborg if (send_rts(lp)) 621a88b5ba8SSam Ravnborg return ldc_abort(lp); 622a88b5ba8SSam Ravnborg return 0; 623a88b5ba8SSam Ravnborg } 624a88b5ba8SSam Ravnborg 625a88b5ba8SSam Ravnborg static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp) 626a88b5ba8SSam Ravnborg { 627a88b5ba8SSam Ravnborg struct ldc_version *vap; 628a88b5ba8SSam Ravnborg struct ldc_packet *p; 629a88b5ba8SSam Ravnborg unsigned long new_tail; 630a88b5ba8SSam Ravnborg 631b2c0805fSSteven Rostedt if (vp->major == 0 && vp->minor == 0) 632b2c0805fSSteven Rostedt return ldc_abort(lp); 633b2c0805fSSteven Rostedt 634b2c0805fSSteven Rostedt vap = find_by_major(vp->major); 635b2c0805fSSteven Rostedt if (!vap) 636b2c0805fSSteven Rostedt return ldc_abort(lp); 637b2c0805fSSteven Rostedt 638a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS, 639a88b5ba8SSam Ravnborg vap, sizeof(*vap), 640a88b5ba8SSam Ravnborg &new_tail); 641b2c0805fSSteven Rostedt if (!p) 642a88b5ba8SSam Ravnborg return ldc_abort(lp); 643b2c0805fSSteven Rostedt 644b2c0805fSSteven Rostedt return send_tx_packet(lp, p, new_tail); 645a88b5ba8SSam Ravnborg } 646a88b5ba8SSam Ravnborg 647a88b5ba8SSam Ravnborg static int process_version(struct ldc_channel *lp, 648a88b5ba8SSam Ravnborg struct ldc_packet *p) 649a88b5ba8SSam Ravnborg { 650a88b5ba8SSam Ravnborg struct ldc_version *vp; 651a88b5ba8SSam Ravnborg 652a88b5ba8SSam Ravnborg vp = (struct ldc_version *) p->u.u_data; 653a88b5ba8SSam Ravnborg 654a88b5ba8SSam Ravnborg switch (p->stype) { 655a88b5ba8SSam Ravnborg case LDC_INFO: 656a88b5ba8SSam Ravnborg return process_ver_info(lp, vp); 657a88b5ba8SSam Ravnborg 658a88b5ba8SSam Ravnborg case LDC_ACK: 659a88b5ba8SSam Ravnborg return process_ver_ack(lp, vp); 660a88b5ba8SSam Ravnborg 661a88b5ba8SSam Ravnborg case LDC_NACK: 662a88b5ba8SSam Ravnborg return process_ver_nack(lp, vp); 663a88b5ba8SSam Ravnborg 664a88b5ba8SSam Ravnborg default: 665a88b5ba8SSam Ravnborg return ldc_abort(lp); 666a88b5ba8SSam Ravnborg } 667a88b5ba8SSam Ravnborg } 668a88b5ba8SSam Ravnborg 669a88b5ba8SSam Ravnborg static int process_rts(struct ldc_channel *lp, 670a88b5ba8SSam Ravnborg struct ldc_packet *p) 671a88b5ba8SSam Ravnborg { 672a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n", 673a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env); 674a88b5ba8SSam Ravnborg 675a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO || 676a88b5ba8SSam Ravnborg lp->hs_state != LDC_HS_GOTVERS || 677a88b5ba8SSam Ravnborg p->env != lp->cfg.mode) 678a88b5ba8SSam Ravnborg return ldc_abort(lp); 679a88b5ba8SSam Ravnborg 680a88b5ba8SSam Ravnborg lp->snd_nxt = p->seqid; 681a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid; 682a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_SENTRTR; 683a88b5ba8SSam Ravnborg if (send_rtr(lp)) 684a88b5ba8SSam Ravnborg return ldc_abort(lp); 685a88b5ba8SSam Ravnborg 686a88b5ba8SSam Ravnborg return 0; 687a88b5ba8SSam Ravnborg } 688a88b5ba8SSam Ravnborg 689a88b5ba8SSam Ravnborg static int process_rtr(struct ldc_channel *lp, 690a88b5ba8SSam Ravnborg struct ldc_packet *p) 691a88b5ba8SSam Ravnborg { 692a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n", 693a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env); 694a88b5ba8SSam Ravnborg 695a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO || 696a88b5ba8SSam Ravnborg p->env != lp->cfg.mode) 697a88b5ba8SSam Ravnborg return ldc_abort(lp); 698a88b5ba8SSam Ravnborg 699a88b5ba8SSam Ravnborg lp->snd_nxt = p->seqid; 700a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 701a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 702a88b5ba8SSam Ravnborg send_rdx(lp); 703a88b5ba8SSam Ravnborg 704a88b5ba8SSam Ravnborg return LDC_EVENT_UP; 705a88b5ba8SSam Ravnborg } 706a88b5ba8SSam Ravnborg 707a88b5ba8SSam Ravnborg static int rx_seq_ok(struct ldc_channel *lp, u32 seqid) 708a88b5ba8SSam Ravnborg { 709a88b5ba8SSam Ravnborg return lp->rcv_nxt + 1 == seqid; 710a88b5ba8SSam Ravnborg } 711a88b5ba8SSam Ravnborg 712a88b5ba8SSam Ravnborg static int process_rdx(struct ldc_channel *lp, 713a88b5ba8SSam Ravnborg struct ldc_packet *p) 714a88b5ba8SSam Ravnborg { 715a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n", 716a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env, p->u.r.ackid); 717a88b5ba8SSam Ravnborg 718a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO || 719a88b5ba8SSam Ravnborg !(rx_seq_ok(lp, p->seqid))) 720a88b5ba8SSam Ravnborg return ldc_abort(lp); 721a88b5ba8SSam Ravnborg 722a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid; 723a88b5ba8SSam Ravnborg 724a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 725a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 726a88b5ba8SSam Ravnborg 727a88b5ba8SSam Ravnborg return LDC_EVENT_UP; 728a88b5ba8SSam Ravnborg } 729a88b5ba8SSam Ravnborg 730a88b5ba8SSam Ravnborg static int process_control_frame(struct ldc_channel *lp, 731a88b5ba8SSam Ravnborg struct ldc_packet *p) 732a88b5ba8SSam Ravnborg { 733a88b5ba8SSam Ravnborg switch (p->ctrl) { 734a88b5ba8SSam Ravnborg case LDC_VERS: 735a88b5ba8SSam Ravnborg return process_version(lp, p); 736a88b5ba8SSam Ravnborg 737a88b5ba8SSam Ravnborg case LDC_RTS: 738a88b5ba8SSam Ravnborg return process_rts(lp, p); 739a88b5ba8SSam Ravnborg 740a88b5ba8SSam Ravnborg case LDC_RTR: 741a88b5ba8SSam Ravnborg return process_rtr(lp, p); 742a88b5ba8SSam Ravnborg 743a88b5ba8SSam Ravnborg case LDC_RDX: 744a88b5ba8SSam Ravnborg return process_rdx(lp, p); 745a88b5ba8SSam Ravnborg 746a88b5ba8SSam Ravnborg default: 747a88b5ba8SSam Ravnborg return ldc_abort(lp); 748a88b5ba8SSam Ravnborg } 749a88b5ba8SSam Ravnborg } 750a88b5ba8SSam Ravnborg 751a88b5ba8SSam Ravnborg static int process_error_frame(struct ldc_channel *lp, 752a88b5ba8SSam Ravnborg struct ldc_packet *p) 753a88b5ba8SSam Ravnborg { 754a88b5ba8SSam Ravnborg return ldc_abort(lp); 755a88b5ba8SSam Ravnborg } 756a88b5ba8SSam Ravnborg 757a88b5ba8SSam Ravnborg static int process_data_ack(struct ldc_channel *lp, 758a88b5ba8SSam Ravnborg struct ldc_packet *ack) 759a88b5ba8SSam Ravnborg { 760a88b5ba8SSam Ravnborg unsigned long head = lp->tx_acked; 761a88b5ba8SSam Ravnborg u32 ackid = ack->u.r.ackid; 762a88b5ba8SSam Ravnborg 763a88b5ba8SSam Ravnborg while (1) { 764a88b5ba8SSam Ravnborg struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE); 765a88b5ba8SSam Ravnborg 766a88b5ba8SSam Ravnborg head = tx_advance(lp, head); 767a88b5ba8SSam Ravnborg 768a88b5ba8SSam Ravnborg if (p->seqid == ackid) { 769a88b5ba8SSam Ravnborg lp->tx_acked = head; 770a88b5ba8SSam Ravnborg return 0; 771a88b5ba8SSam Ravnborg } 772a88b5ba8SSam Ravnborg if (head == lp->tx_tail) 773a88b5ba8SSam Ravnborg return ldc_abort(lp); 774a88b5ba8SSam Ravnborg } 775a88b5ba8SSam Ravnborg 776a88b5ba8SSam Ravnborg return 0; 777a88b5ba8SSam Ravnborg } 778a88b5ba8SSam Ravnborg 779a88b5ba8SSam Ravnborg static void send_events(struct ldc_channel *lp, unsigned int event_mask) 780a88b5ba8SSam Ravnborg { 781a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_RESET) 782a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_RESET); 783a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_UP) 784a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_UP); 785a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_DATA_READY) 786a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY); 787a88b5ba8SSam Ravnborg } 788a88b5ba8SSam Ravnborg 789a88b5ba8SSam Ravnborg static irqreturn_t ldc_rx(int irq, void *dev_id) 790a88b5ba8SSam Ravnborg { 791a88b5ba8SSam Ravnborg struct ldc_channel *lp = dev_id; 792a88b5ba8SSam Ravnborg unsigned long orig_state, hv_err, flags; 793a88b5ba8SSam Ravnborg unsigned int event_mask; 794a88b5ba8SSam Ravnborg 795a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 796a88b5ba8SSam Ravnborg 797a88b5ba8SSam Ravnborg orig_state = lp->chan_state; 798a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 799a88b5ba8SSam Ravnborg &lp->rx_head, 800a88b5ba8SSam Ravnborg &lp->rx_tail, 801a88b5ba8SSam Ravnborg &lp->chan_state); 802a88b5ba8SSam Ravnborg 803a88b5ba8SSam Ravnborg ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n", 804a88b5ba8SSam Ravnborg orig_state, lp->chan_state, lp->rx_head, lp->rx_tail); 805a88b5ba8SSam Ravnborg 806a88b5ba8SSam Ravnborg event_mask = 0; 807a88b5ba8SSam Ravnborg 808a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW && 809a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_UP) { 810a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 811a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 812a88b5ba8SSam Ravnborg 813a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_UP; 814a88b5ba8SSam Ravnborg 815a88b5ba8SSam Ravnborg orig_state = lp->chan_state; 816a88b5ba8SSam Ravnborg } 817a88b5ba8SSam Ravnborg 818a88b5ba8SSam Ravnborg /* If we are in reset state, flush the RX queue and ignore 819a88b5ba8SSam Ravnborg * everything. 820a88b5ba8SSam Ravnborg */ 821a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_RESET) { 822a88b5ba8SSam Ravnborg (void) __set_rx_head(lp, lp->rx_tail); 823a88b5ba8SSam Ravnborg goto out; 824a88b5ba8SSam Ravnborg } 825a88b5ba8SSam Ravnborg 826a88b5ba8SSam Ravnborg /* Once we finish the handshake, we let the ldc_read() 827a88b5ba8SSam Ravnborg * paths do all of the control frame and state management. 828a88b5ba8SSam Ravnborg * Just trigger the callback. 829a88b5ba8SSam Ravnborg */ 830a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_COMPLETE) { 831a88b5ba8SSam Ravnborg handshake_complete: 832a88b5ba8SSam Ravnborg if (lp->chan_state != orig_state) { 833a88b5ba8SSam Ravnborg unsigned int event = LDC_EVENT_RESET; 834a88b5ba8SSam Ravnborg 835a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_UP) 836a88b5ba8SSam Ravnborg event = LDC_EVENT_UP; 837a88b5ba8SSam Ravnborg 838a88b5ba8SSam Ravnborg event_mask |= event; 839a88b5ba8SSam Ravnborg } 840a88b5ba8SSam Ravnborg if (lp->rx_head != lp->rx_tail) 841a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_DATA_READY; 842a88b5ba8SSam Ravnborg 843a88b5ba8SSam Ravnborg goto out; 844a88b5ba8SSam Ravnborg } 845a88b5ba8SSam Ravnborg 846a88b5ba8SSam Ravnborg if (lp->chan_state != orig_state) 847a88b5ba8SSam Ravnborg goto out; 848a88b5ba8SSam Ravnborg 849a88b5ba8SSam Ravnborg while (lp->rx_head != lp->rx_tail) { 850a88b5ba8SSam Ravnborg struct ldc_packet *p; 851a88b5ba8SSam Ravnborg unsigned long new; 852a88b5ba8SSam Ravnborg int err; 853a88b5ba8SSam Ravnborg 854a88b5ba8SSam Ravnborg p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE); 855a88b5ba8SSam Ravnborg 856a88b5ba8SSam Ravnborg switch (p->type) { 857a88b5ba8SSam Ravnborg case LDC_CTRL: 858a88b5ba8SSam Ravnborg err = process_control_frame(lp, p); 859a88b5ba8SSam Ravnborg if (err > 0) 860a88b5ba8SSam Ravnborg event_mask |= err; 861a88b5ba8SSam Ravnborg break; 862a88b5ba8SSam Ravnborg 863a88b5ba8SSam Ravnborg case LDC_DATA: 864a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_DATA_READY; 865a88b5ba8SSam Ravnborg err = 0; 866a88b5ba8SSam Ravnborg break; 867a88b5ba8SSam Ravnborg 868a88b5ba8SSam Ravnborg case LDC_ERR: 869a88b5ba8SSam Ravnborg err = process_error_frame(lp, p); 870a88b5ba8SSam Ravnborg break; 871a88b5ba8SSam Ravnborg 872a88b5ba8SSam Ravnborg default: 873a88b5ba8SSam Ravnborg err = ldc_abort(lp); 874a88b5ba8SSam Ravnborg break; 875a88b5ba8SSam Ravnborg } 876a88b5ba8SSam Ravnborg 877a88b5ba8SSam Ravnborg if (err < 0) 878a88b5ba8SSam Ravnborg break; 879a88b5ba8SSam Ravnborg 880a88b5ba8SSam Ravnborg new = lp->rx_head; 881a88b5ba8SSam Ravnborg new += LDC_PACKET_SIZE; 882a88b5ba8SSam Ravnborg if (new == (lp->rx_num_entries * LDC_PACKET_SIZE)) 883a88b5ba8SSam Ravnborg new = 0; 884a88b5ba8SSam Ravnborg lp->rx_head = new; 885a88b5ba8SSam Ravnborg 886a88b5ba8SSam Ravnborg err = __set_rx_head(lp, new); 887a88b5ba8SSam Ravnborg if (err < 0) { 888a88b5ba8SSam Ravnborg (void) ldc_abort(lp); 889a88b5ba8SSam Ravnborg break; 890a88b5ba8SSam Ravnborg } 891a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_COMPLETE) 892a88b5ba8SSam Ravnborg goto handshake_complete; 893a88b5ba8SSam Ravnborg } 894a88b5ba8SSam Ravnborg 895a88b5ba8SSam Ravnborg out: 896a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 897a88b5ba8SSam Ravnborg 898a88b5ba8SSam Ravnborg send_events(lp, event_mask); 899a88b5ba8SSam Ravnborg 900a88b5ba8SSam Ravnborg return IRQ_HANDLED; 901a88b5ba8SSam Ravnborg } 902a88b5ba8SSam Ravnborg 903a88b5ba8SSam Ravnborg static irqreturn_t ldc_tx(int irq, void *dev_id) 904a88b5ba8SSam Ravnborg { 905a88b5ba8SSam Ravnborg struct ldc_channel *lp = dev_id; 906a88b5ba8SSam Ravnborg unsigned long flags, hv_err, orig_state; 907a88b5ba8SSam Ravnborg unsigned int event_mask = 0; 908a88b5ba8SSam Ravnborg 909a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 910a88b5ba8SSam Ravnborg 911a88b5ba8SSam Ravnborg orig_state = lp->chan_state; 912a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, 913a88b5ba8SSam Ravnborg &lp->tx_head, 914a88b5ba8SSam Ravnborg &lp->tx_tail, 915a88b5ba8SSam Ravnborg &lp->chan_state); 916a88b5ba8SSam Ravnborg 917a88b5ba8SSam Ravnborg ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n", 918a88b5ba8SSam Ravnborg orig_state, lp->chan_state, lp->tx_head, lp->tx_tail); 919a88b5ba8SSam Ravnborg 920a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW && 921a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_UP) { 922a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 923a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 924a88b5ba8SSam Ravnborg 925a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_UP; 926a88b5ba8SSam Ravnborg } 927a88b5ba8SSam Ravnborg 928a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 929a88b5ba8SSam Ravnborg 930a88b5ba8SSam Ravnborg send_events(lp, event_mask); 931a88b5ba8SSam Ravnborg 932a88b5ba8SSam Ravnborg return IRQ_HANDLED; 933a88b5ba8SSam Ravnborg } 934a88b5ba8SSam Ravnborg 935a88b5ba8SSam Ravnborg /* XXX ldc_alloc() and ldc_free() needs to run under a mutex so 936a88b5ba8SSam Ravnborg * XXX that addition and removal from the ldc_channel_list has 937a88b5ba8SSam Ravnborg * XXX atomicity, otherwise the __ldc_channel_exists() check is 938a88b5ba8SSam Ravnborg * XXX totally pointless as another thread can slip into ldc_alloc() 939a88b5ba8SSam Ravnborg * XXX and add a channel with the same ID. There also needs to be 940a88b5ba8SSam Ravnborg * XXX a spinlock for ldc_channel_list. 941a88b5ba8SSam Ravnborg */ 942a88b5ba8SSam Ravnborg static HLIST_HEAD(ldc_channel_list); 943a88b5ba8SSam Ravnborg 944a88b5ba8SSam Ravnborg static int __ldc_channel_exists(unsigned long id) 945a88b5ba8SSam Ravnborg { 946a88b5ba8SSam Ravnborg struct ldc_channel *lp; 947a88b5ba8SSam Ravnborg struct hlist_node *n; 948a88b5ba8SSam Ravnborg 949a88b5ba8SSam Ravnborg hlist_for_each_entry(lp, n, &ldc_channel_list, list) { 950a88b5ba8SSam Ravnborg if (lp->id == id) 951a88b5ba8SSam Ravnborg return 1; 952a88b5ba8SSam Ravnborg } 953a88b5ba8SSam Ravnborg return 0; 954a88b5ba8SSam Ravnborg } 955a88b5ba8SSam Ravnborg 956a88b5ba8SSam Ravnborg static int alloc_queue(const char *name, unsigned long num_entries, 957a88b5ba8SSam Ravnborg struct ldc_packet **base, unsigned long *ra) 958a88b5ba8SSam Ravnborg { 959a88b5ba8SSam Ravnborg unsigned long size, order; 960a88b5ba8SSam Ravnborg void *q; 961a88b5ba8SSam Ravnborg 962a88b5ba8SSam Ravnborg size = num_entries * LDC_PACKET_SIZE; 963a88b5ba8SSam Ravnborg order = get_order(size); 964a88b5ba8SSam Ravnborg 965a88b5ba8SSam Ravnborg q = (void *) __get_free_pages(GFP_KERNEL, order); 966a88b5ba8SSam Ravnborg if (!q) { 967a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of %s queue failed with " 968a88b5ba8SSam Ravnborg "size=%lu order=%lu\n", name, size, order); 969a88b5ba8SSam Ravnborg return -ENOMEM; 970a88b5ba8SSam Ravnborg } 971a88b5ba8SSam Ravnborg 972a88b5ba8SSam Ravnborg memset(q, 0, PAGE_SIZE << order); 973a88b5ba8SSam Ravnborg 974a88b5ba8SSam Ravnborg *base = q; 975a88b5ba8SSam Ravnborg *ra = __pa(q); 976a88b5ba8SSam Ravnborg 977a88b5ba8SSam Ravnborg return 0; 978a88b5ba8SSam Ravnborg } 979a88b5ba8SSam Ravnborg 980a88b5ba8SSam Ravnborg static void free_queue(unsigned long num_entries, struct ldc_packet *q) 981a88b5ba8SSam Ravnborg { 982a88b5ba8SSam Ravnborg unsigned long size, order; 983a88b5ba8SSam Ravnborg 984a88b5ba8SSam Ravnborg if (!q) 985a88b5ba8SSam Ravnborg return; 986a88b5ba8SSam Ravnborg 987a88b5ba8SSam Ravnborg size = num_entries * LDC_PACKET_SIZE; 988a88b5ba8SSam Ravnborg order = get_order(size); 989a88b5ba8SSam Ravnborg 990a88b5ba8SSam Ravnborg free_pages((unsigned long)q, order); 991a88b5ba8SSam Ravnborg } 992a88b5ba8SSam Ravnborg 993a88b5ba8SSam Ravnborg /* XXX Make this configurable... XXX */ 994a88b5ba8SSam Ravnborg #define LDC_IOTABLE_SIZE (8 * 1024) 995a88b5ba8SSam Ravnborg 996a88b5ba8SSam Ravnborg static int ldc_iommu_init(struct ldc_channel *lp) 997a88b5ba8SSam Ravnborg { 998a88b5ba8SSam Ravnborg unsigned long sz, num_tsb_entries, tsbsize, order; 999a88b5ba8SSam Ravnborg struct ldc_iommu *iommu = &lp->iommu; 1000a88b5ba8SSam Ravnborg struct ldc_mtable_entry *table; 1001a88b5ba8SSam Ravnborg unsigned long hv_err; 1002a88b5ba8SSam Ravnborg int err; 1003a88b5ba8SSam Ravnborg 1004a88b5ba8SSam Ravnborg num_tsb_entries = LDC_IOTABLE_SIZE; 1005a88b5ba8SSam Ravnborg tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); 1006a88b5ba8SSam Ravnborg 1007a88b5ba8SSam Ravnborg spin_lock_init(&iommu->lock); 1008a88b5ba8SSam Ravnborg 1009a88b5ba8SSam Ravnborg sz = num_tsb_entries / 8; 1010a88b5ba8SSam Ravnborg sz = (sz + 7UL) & ~7UL; 1011a88b5ba8SSam Ravnborg iommu->arena.map = kzalloc(sz, GFP_KERNEL); 1012a88b5ba8SSam Ravnborg if (!iommu->arena.map) { 1013a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz); 1014a88b5ba8SSam Ravnborg return -ENOMEM; 1015a88b5ba8SSam Ravnborg } 1016a88b5ba8SSam Ravnborg 1017a88b5ba8SSam Ravnborg iommu->arena.limit = num_tsb_entries; 1018a88b5ba8SSam Ravnborg 1019a88b5ba8SSam Ravnborg order = get_order(tsbsize); 1020a88b5ba8SSam Ravnborg 1021a88b5ba8SSam Ravnborg table = (struct ldc_mtable_entry *) 1022a88b5ba8SSam Ravnborg __get_free_pages(GFP_KERNEL, order); 1023a88b5ba8SSam Ravnborg err = -ENOMEM; 1024a88b5ba8SSam Ravnborg if (!table) { 1025a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of MTE table failed, " 1026a88b5ba8SSam Ravnborg "size=%lu order=%lu\n", tsbsize, order); 1027a88b5ba8SSam Ravnborg goto out_free_map; 1028a88b5ba8SSam Ravnborg } 1029a88b5ba8SSam Ravnborg 1030a88b5ba8SSam Ravnborg memset(table, 0, PAGE_SIZE << order); 1031a88b5ba8SSam Ravnborg 1032a88b5ba8SSam Ravnborg iommu->page_table = table; 1033a88b5ba8SSam Ravnborg 1034a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table), 1035a88b5ba8SSam Ravnborg num_tsb_entries); 1036a88b5ba8SSam Ravnborg err = -EINVAL; 1037a88b5ba8SSam Ravnborg if (hv_err) 1038a88b5ba8SSam Ravnborg goto out_free_table; 1039a88b5ba8SSam Ravnborg 1040a88b5ba8SSam Ravnborg return 0; 1041a88b5ba8SSam Ravnborg 1042a88b5ba8SSam Ravnborg out_free_table: 1043a88b5ba8SSam Ravnborg free_pages((unsigned long) table, order); 1044a88b5ba8SSam Ravnborg iommu->page_table = NULL; 1045a88b5ba8SSam Ravnborg 1046a88b5ba8SSam Ravnborg out_free_map: 1047a88b5ba8SSam Ravnborg kfree(iommu->arena.map); 1048a88b5ba8SSam Ravnborg iommu->arena.map = NULL; 1049a88b5ba8SSam Ravnborg 1050a88b5ba8SSam Ravnborg return err; 1051a88b5ba8SSam Ravnborg } 1052a88b5ba8SSam Ravnborg 1053a88b5ba8SSam Ravnborg static void ldc_iommu_release(struct ldc_channel *lp) 1054a88b5ba8SSam Ravnborg { 1055a88b5ba8SSam Ravnborg struct ldc_iommu *iommu = &lp->iommu; 1056a88b5ba8SSam Ravnborg unsigned long num_tsb_entries, tsbsize, order; 1057a88b5ba8SSam Ravnborg 1058a88b5ba8SSam Ravnborg (void) sun4v_ldc_set_map_table(lp->id, 0, 0); 1059a88b5ba8SSam Ravnborg 1060a88b5ba8SSam Ravnborg num_tsb_entries = iommu->arena.limit; 1061a88b5ba8SSam Ravnborg tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); 1062a88b5ba8SSam Ravnborg order = get_order(tsbsize); 1063a88b5ba8SSam Ravnborg 1064a88b5ba8SSam Ravnborg free_pages((unsigned long) iommu->page_table, order); 1065a88b5ba8SSam Ravnborg iommu->page_table = NULL; 1066a88b5ba8SSam Ravnborg 1067a88b5ba8SSam Ravnborg kfree(iommu->arena.map); 1068a88b5ba8SSam Ravnborg iommu->arena.map = NULL; 1069a88b5ba8SSam Ravnborg } 1070a88b5ba8SSam Ravnborg 1071a88b5ba8SSam Ravnborg struct ldc_channel *ldc_alloc(unsigned long id, 1072a88b5ba8SSam Ravnborg const struct ldc_channel_config *cfgp, 1073a88b5ba8SSam Ravnborg void *event_arg) 1074a88b5ba8SSam Ravnborg { 1075a88b5ba8SSam Ravnborg struct ldc_channel *lp; 1076a88b5ba8SSam Ravnborg const struct ldc_mode_ops *mops; 1077a88b5ba8SSam Ravnborg unsigned long dummy1, dummy2, hv_err; 1078a88b5ba8SSam Ravnborg u8 mss, *mssbuf; 1079a88b5ba8SSam Ravnborg int err; 1080a88b5ba8SSam Ravnborg 1081a88b5ba8SSam Ravnborg err = -ENODEV; 1082a88b5ba8SSam Ravnborg if (!ldom_domaining_enabled) 1083a88b5ba8SSam Ravnborg goto out_err; 1084a88b5ba8SSam Ravnborg 1085a88b5ba8SSam Ravnborg err = -EINVAL; 1086a88b5ba8SSam Ravnborg if (!cfgp) 1087a88b5ba8SSam Ravnborg goto out_err; 1088a88b5ba8SSam Ravnborg 1089a88b5ba8SSam Ravnborg switch (cfgp->mode) { 1090a88b5ba8SSam Ravnborg case LDC_MODE_RAW: 1091a88b5ba8SSam Ravnborg mops = &raw_ops; 1092a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE; 1093a88b5ba8SSam Ravnborg break; 1094a88b5ba8SSam Ravnborg 1095a88b5ba8SSam Ravnborg case LDC_MODE_UNRELIABLE: 1096a88b5ba8SSam Ravnborg mops = &nonraw_ops; 1097a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE - 8; 1098a88b5ba8SSam Ravnborg break; 1099a88b5ba8SSam Ravnborg 1100a88b5ba8SSam Ravnborg case LDC_MODE_STREAM: 1101a88b5ba8SSam Ravnborg mops = &stream_ops; 1102a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE - 8 - 8; 1103a88b5ba8SSam Ravnborg break; 1104a88b5ba8SSam Ravnborg 1105a88b5ba8SSam Ravnborg default: 1106a88b5ba8SSam Ravnborg goto out_err; 1107a88b5ba8SSam Ravnborg } 1108a88b5ba8SSam Ravnborg 1109a88b5ba8SSam Ravnborg if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq) 1110a88b5ba8SSam Ravnborg goto out_err; 1111a88b5ba8SSam Ravnborg 1112a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2); 1113a88b5ba8SSam Ravnborg err = -ENODEV; 1114a88b5ba8SSam Ravnborg if (hv_err == HV_ECHANNEL) 1115a88b5ba8SSam Ravnborg goto out_err; 1116a88b5ba8SSam Ravnborg 1117a88b5ba8SSam Ravnborg err = -EEXIST; 1118a88b5ba8SSam Ravnborg if (__ldc_channel_exists(id)) 1119a88b5ba8SSam Ravnborg goto out_err; 1120a88b5ba8SSam Ravnborg 1121a88b5ba8SSam Ravnborg mssbuf = NULL; 1122a88b5ba8SSam Ravnborg 1123a88b5ba8SSam Ravnborg lp = kzalloc(sizeof(*lp), GFP_KERNEL); 1124a88b5ba8SSam Ravnborg err = -ENOMEM; 1125a88b5ba8SSam Ravnborg if (!lp) 1126a88b5ba8SSam Ravnborg goto out_err; 1127a88b5ba8SSam Ravnborg 1128a88b5ba8SSam Ravnborg spin_lock_init(&lp->lock); 1129a88b5ba8SSam Ravnborg 1130a88b5ba8SSam Ravnborg lp->id = id; 1131a88b5ba8SSam Ravnborg 1132a88b5ba8SSam Ravnborg err = ldc_iommu_init(lp); 1133a88b5ba8SSam Ravnborg if (err) 1134a88b5ba8SSam Ravnborg goto out_free_ldc; 1135a88b5ba8SSam Ravnborg 1136a88b5ba8SSam Ravnborg lp->mops = mops; 1137a88b5ba8SSam Ravnborg lp->mss = mss; 1138a88b5ba8SSam Ravnborg 1139a88b5ba8SSam Ravnborg lp->cfg = *cfgp; 1140a88b5ba8SSam Ravnborg if (!lp->cfg.mtu) 1141a88b5ba8SSam Ravnborg lp->cfg.mtu = LDC_DEFAULT_MTU; 1142a88b5ba8SSam Ravnborg 1143a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_STREAM) { 1144a88b5ba8SSam Ravnborg mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL); 1145a88b5ba8SSam Ravnborg if (!mssbuf) { 1146a88b5ba8SSam Ravnborg err = -ENOMEM; 1147a88b5ba8SSam Ravnborg goto out_free_iommu; 1148a88b5ba8SSam Ravnborg } 1149a88b5ba8SSam Ravnborg lp->mssbuf = mssbuf; 1150a88b5ba8SSam Ravnborg } 1151a88b5ba8SSam Ravnborg 1152a88b5ba8SSam Ravnborg lp->event_arg = event_arg; 1153a88b5ba8SSam Ravnborg 1154a88b5ba8SSam Ravnborg /* XXX allow setting via ldc_channel_config to override defaults 1155a88b5ba8SSam Ravnborg * XXX or use some formula based upon mtu 1156a88b5ba8SSam Ravnborg */ 1157a88b5ba8SSam Ravnborg lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES; 1158a88b5ba8SSam Ravnborg lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES; 1159a88b5ba8SSam Ravnborg 1160a88b5ba8SSam Ravnborg err = alloc_queue("TX", lp->tx_num_entries, 1161a88b5ba8SSam Ravnborg &lp->tx_base, &lp->tx_ra); 1162a88b5ba8SSam Ravnborg if (err) 1163a88b5ba8SSam Ravnborg goto out_free_mssbuf; 1164a88b5ba8SSam Ravnborg 1165a88b5ba8SSam Ravnborg err = alloc_queue("RX", lp->rx_num_entries, 1166a88b5ba8SSam Ravnborg &lp->rx_base, &lp->rx_ra); 1167a88b5ba8SSam Ravnborg if (err) 1168a88b5ba8SSam Ravnborg goto out_free_txq; 1169a88b5ba8SSam Ravnborg 1170a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_ALLOCED_QUEUES; 1171a88b5ba8SSam Ravnborg 1172a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_CLOSED; 1173a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_INIT); 1174a88b5ba8SSam Ravnborg 1175a88b5ba8SSam Ravnborg INIT_HLIST_NODE(&lp->list); 1176a88b5ba8SSam Ravnborg hlist_add_head(&lp->list, &ldc_channel_list); 1177a88b5ba8SSam Ravnborg 1178a88b5ba8SSam Ravnborg INIT_HLIST_HEAD(&lp->mh_list); 1179a88b5ba8SSam Ravnborg 1180a88b5ba8SSam Ravnborg return lp; 1181a88b5ba8SSam Ravnborg 1182a88b5ba8SSam Ravnborg out_free_txq: 1183a88b5ba8SSam Ravnborg free_queue(lp->tx_num_entries, lp->tx_base); 1184a88b5ba8SSam Ravnborg 1185a88b5ba8SSam Ravnborg out_free_mssbuf: 1186a88b5ba8SSam Ravnborg if (mssbuf) 1187a88b5ba8SSam Ravnborg kfree(mssbuf); 1188a88b5ba8SSam Ravnborg 1189a88b5ba8SSam Ravnborg out_free_iommu: 1190a88b5ba8SSam Ravnborg ldc_iommu_release(lp); 1191a88b5ba8SSam Ravnborg 1192a88b5ba8SSam Ravnborg out_free_ldc: 1193a88b5ba8SSam Ravnborg kfree(lp); 1194a88b5ba8SSam Ravnborg 1195a88b5ba8SSam Ravnborg out_err: 1196a88b5ba8SSam Ravnborg return ERR_PTR(err); 1197a88b5ba8SSam Ravnborg } 1198a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_alloc); 1199a88b5ba8SSam Ravnborg 1200a88b5ba8SSam Ravnborg void ldc_free(struct ldc_channel *lp) 1201a88b5ba8SSam Ravnborg { 1202a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_REGISTERED_IRQS) { 1203a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1204a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp); 1205a88b5ba8SSam Ravnborg } 1206a88b5ba8SSam Ravnborg 1207a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) { 1208a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0); 1209a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0); 1210a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES; 1211a88b5ba8SSam Ravnborg } 1212a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) { 1213a88b5ba8SSam Ravnborg free_queue(lp->tx_num_entries, lp->tx_base); 1214a88b5ba8SSam Ravnborg free_queue(lp->rx_num_entries, lp->rx_base); 1215a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES; 1216a88b5ba8SSam Ravnborg } 1217a88b5ba8SSam Ravnborg 1218a88b5ba8SSam Ravnborg hlist_del(&lp->list); 1219a88b5ba8SSam Ravnborg 1220a88b5ba8SSam Ravnborg if (lp->mssbuf) 1221a88b5ba8SSam Ravnborg kfree(lp->mssbuf); 1222a88b5ba8SSam Ravnborg 1223a88b5ba8SSam Ravnborg ldc_iommu_release(lp); 1224a88b5ba8SSam Ravnborg 1225a88b5ba8SSam Ravnborg kfree(lp); 1226a88b5ba8SSam Ravnborg } 1227a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_free); 1228a88b5ba8SSam Ravnborg 1229a88b5ba8SSam Ravnborg /* Bind the channel. This registers the LDC queues with 1230a88b5ba8SSam Ravnborg * the hypervisor and puts the channel into a pseudo-listening 1231a88b5ba8SSam Ravnborg * state. This does not initiate a handshake, ldc_connect() does 1232a88b5ba8SSam Ravnborg * that. 1233a88b5ba8SSam Ravnborg */ 1234a88b5ba8SSam Ravnborg int ldc_bind(struct ldc_channel *lp, const char *name) 1235a88b5ba8SSam Ravnborg { 1236a88b5ba8SSam Ravnborg unsigned long hv_err, flags; 1237a88b5ba8SSam Ravnborg int err = -EINVAL; 1238a88b5ba8SSam Ravnborg 1239a88b5ba8SSam Ravnborg if (!name || 1240a88b5ba8SSam Ravnborg (lp->state != LDC_STATE_INIT)) 1241a88b5ba8SSam Ravnborg return -EINVAL; 1242a88b5ba8SSam Ravnborg 1243a88b5ba8SSam Ravnborg snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name); 1244a88b5ba8SSam Ravnborg snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name); 1245a88b5ba8SSam Ravnborg 1246a88b5ba8SSam Ravnborg err = request_irq(lp->cfg.rx_irq, ldc_rx, 1247a88b5ba8SSam Ravnborg IRQF_SAMPLE_RANDOM | IRQF_SHARED, 1248a88b5ba8SSam Ravnborg lp->rx_irq_name, lp); 1249a88b5ba8SSam Ravnborg if (err) 1250a88b5ba8SSam Ravnborg return err; 1251a88b5ba8SSam Ravnborg 1252a88b5ba8SSam Ravnborg err = request_irq(lp->cfg.tx_irq, ldc_tx, 1253a88b5ba8SSam Ravnborg IRQF_SAMPLE_RANDOM | IRQF_SHARED, 1254a88b5ba8SSam Ravnborg lp->tx_irq_name, lp); 1255a88b5ba8SSam Ravnborg if (err) { 1256a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1257a88b5ba8SSam Ravnborg return err; 1258a88b5ba8SSam Ravnborg } 1259a88b5ba8SSam Ravnborg 1260a88b5ba8SSam Ravnborg 1261a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1262a88b5ba8SSam Ravnborg 1263a88b5ba8SSam Ravnborg enable_irq(lp->cfg.rx_irq); 1264a88b5ba8SSam Ravnborg enable_irq(lp->cfg.tx_irq); 1265a88b5ba8SSam Ravnborg 1266a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_REGISTERED_IRQS; 1267a88b5ba8SSam Ravnborg 1268a88b5ba8SSam Ravnborg err = -ENODEV; 1269a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0); 1270a88b5ba8SSam Ravnborg if (hv_err) 1271a88b5ba8SSam Ravnborg goto out_free_irqs; 1272a88b5ba8SSam Ravnborg 1273a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); 1274a88b5ba8SSam Ravnborg if (hv_err) 1275a88b5ba8SSam Ravnborg goto out_free_irqs; 1276a88b5ba8SSam Ravnborg 1277a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0); 1278a88b5ba8SSam Ravnborg if (hv_err) 1279a88b5ba8SSam Ravnborg goto out_unmap_tx; 1280a88b5ba8SSam Ravnborg 1281a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); 1282a88b5ba8SSam Ravnborg if (hv_err) 1283a88b5ba8SSam Ravnborg goto out_unmap_tx; 1284a88b5ba8SSam Ravnborg 1285a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_REGISTERED_QUEUES; 1286a88b5ba8SSam Ravnborg 1287a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, 1288a88b5ba8SSam Ravnborg &lp->tx_head, 1289a88b5ba8SSam Ravnborg &lp->tx_tail, 1290a88b5ba8SSam Ravnborg &lp->chan_state); 1291a88b5ba8SSam Ravnborg err = -EBUSY; 1292a88b5ba8SSam Ravnborg if (hv_err) 1293a88b5ba8SSam Ravnborg goto out_unmap_rx; 1294a88b5ba8SSam Ravnborg 1295a88b5ba8SSam Ravnborg lp->tx_acked = lp->tx_head; 1296a88b5ba8SSam Ravnborg 1297a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN; 1298a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_BOUND); 1299a88b5ba8SSam Ravnborg 1300a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1301a88b5ba8SSam Ravnborg 1302a88b5ba8SSam Ravnborg return 0; 1303a88b5ba8SSam Ravnborg 1304a88b5ba8SSam Ravnborg out_unmap_rx: 1305a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES; 1306a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0); 1307a88b5ba8SSam Ravnborg 1308a88b5ba8SSam Ravnborg out_unmap_tx: 1309a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0); 1310a88b5ba8SSam Ravnborg 1311a88b5ba8SSam Ravnborg out_free_irqs: 1312a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_IRQS; 1313a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp); 1314a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1315a88b5ba8SSam Ravnborg 1316a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1317a88b5ba8SSam Ravnborg 1318a88b5ba8SSam Ravnborg return err; 1319a88b5ba8SSam Ravnborg } 1320a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_bind); 1321a88b5ba8SSam Ravnborg 1322a88b5ba8SSam Ravnborg int ldc_connect(struct ldc_channel *lp) 1323a88b5ba8SSam Ravnborg { 1324a88b5ba8SSam Ravnborg unsigned long flags; 1325a88b5ba8SSam Ravnborg int err; 1326a88b5ba8SSam Ravnborg 1327a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW) 1328a88b5ba8SSam Ravnborg return -EINVAL; 1329a88b5ba8SSam Ravnborg 1330a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1331a88b5ba8SSam Ravnborg 1332a88b5ba8SSam Ravnborg if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || 1333a88b5ba8SSam Ravnborg !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) || 1334a88b5ba8SSam Ravnborg lp->hs_state != LDC_HS_OPEN) 1335a88b5ba8SSam Ravnborg err = -EINVAL; 1336a88b5ba8SSam Ravnborg else 1337a88b5ba8SSam Ravnborg err = start_handshake(lp); 1338a88b5ba8SSam Ravnborg 1339a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1340a88b5ba8SSam Ravnborg 1341a88b5ba8SSam Ravnborg return err; 1342a88b5ba8SSam Ravnborg } 1343a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_connect); 1344a88b5ba8SSam Ravnborg 1345a88b5ba8SSam Ravnborg int ldc_disconnect(struct ldc_channel *lp) 1346a88b5ba8SSam Ravnborg { 1347a88b5ba8SSam Ravnborg unsigned long hv_err, flags; 1348a88b5ba8SSam Ravnborg int err; 1349a88b5ba8SSam Ravnborg 1350a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW) 1351a88b5ba8SSam Ravnborg return -EINVAL; 1352a88b5ba8SSam Ravnborg 1353a88b5ba8SSam Ravnborg if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || 1354a88b5ba8SSam Ravnborg !(lp->flags & LDC_FLAG_REGISTERED_QUEUES)) 1355a88b5ba8SSam Ravnborg return -EINVAL; 1356a88b5ba8SSam Ravnborg 1357a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1358a88b5ba8SSam Ravnborg 1359a88b5ba8SSam Ravnborg err = -ENODEV; 1360a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0); 1361a88b5ba8SSam Ravnborg if (hv_err) 1362a88b5ba8SSam Ravnborg goto out_err; 1363a88b5ba8SSam Ravnborg 1364a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); 1365a88b5ba8SSam Ravnborg if (hv_err) 1366a88b5ba8SSam Ravnborg goto out_err; 1367a88b5ba8SSam Ravnborg 1368a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0); 1369a88b5ba8SSam Ravnborg if (hv_err) 1370a88b5ba8SSam Ravnborg goto out_err; 1371a88b5ba8SSam Ravnborg 1372a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); 1373a88b5ba8SSam Ravnborg if (hv_err) 1374a88b5ba8SSam Ravnborg goto out_err; 1375a88b5ba8SSam Ravnborg 1376a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_BOUND); 1377a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN; 1378a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_RESET; 1379a88b5ba8SSam Ravnborg 1380a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1381a88b5ba8SSam Ravnborg 1382a88b5ba8SSam Ravnborg return 0; 1383a88b5ba8SSam Ravnborg 1384a88b5ba8SSam Ravnborg out_err: 1385a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0); 1386a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0); 1387a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp); 1388a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1389a88b5ba8SSam Ravnborg lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS | 1390a88b5ba8SSam Ravnborg LDC_FLAG_REGISTERED_QUEUES); 1391a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_INIT); 1392a88b5ba8SSam Ravnborg 1393a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1394a88b5ba8SSam Ravnborg 1395a88b5ba8SSam Ravnborg return err; 1396a88b5ba8SSam Ravnborg } 1397a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_disconnect); 1398a88b5ba8SSam Ravnborg 1399a88b5ba8SSam Ravnborg int ldc_state(struct ldc_channel *lp) 1400a88b5ba8SSam Ravnborg { 1401a88b5ba8SSam Ravnborg return lp->state; 1402a88b5ba8SSam Ravnborg } 1403a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_state); 1404a88b5ba8SSam Ravnborg 1405a88b5ba8SSam Ravnborg static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size) 1406a88b5ba8SSam Ravnborg { 1407a88b5ba8SSam Ravnborg struct ldc_packet *p; 1408a88b5ba8SSam Ravnborg unsigned long new_tail; 1409a88b5ba8SSam Ravnborg int err; 1410a88b5ba8SSam Ravnborg 1411a88b5ba8SSam Ravnborg if (size > LDC_PACKET_SIZE) 1412a88b5ba8SSam Ravnborg return -EMSGSIZE; 1413a88b5ba8SSam Ravnborg 1414a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail); 1415a88b5ba8SSam Ravnborg if (!p) 1416a88b5ba8SSam Ravnborg return -EAGAIN; 1417a88b5ba8SSam Ravnborg 1418a88b5ba8SSam Ravnborg memcpy(p, buf, size); 1419a88b5ba8SSam Ravnborg 1420a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail); 1421a88b5ba8SSam Ravnborg if (!err) 1422a88b5ba8SSam Ravnborg err = size; 1423a88b5ba8SSam Ravnborg 1424a88b5ba8SSam Ravnborg return err; 1425a88b5ba8SSam Ravnborg } 1426a88b5ba8SSam Ravnborg 1427a88b5ba8SSam Ravnborg static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size) 1428a88b5ba8SSam Ravnborg { 1429a88b5ba8SSam Ravnborg struct ldc_packet *p; 1430a88b5ba8SSam Ravnborg unsigned long hv_err, new; 1431a88b5ba8SSam Ravnborg int err; 1432a88b5ba8SSam Ravnborg 1433a88b5ba8SSam Ravnborg if (size < LDC_PACKET_SIZE) 1434a88b5ba8SSam Ravnborg return -EINVAL; 1435a88b5ba8SSam Ravnborg 1436a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 1437a88b5ba8SSam Ravnborg &lp->rx_head, 1438a88b5ba8SSam Ravnborg &lp->rx_tail, 1439a88b5ba8SSam Ravnborg &lp->chan_state); 1440a88b5ba8SSam Ravnborg if (hv_err) 1441a88b5ba8SSam Ravnborg return ldc_abort(lp); 1442a88b5ba8SSam Ravnborg 1443a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN || 1444a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING) 1445a88b5ba8SSam Ravnborg return -ECONNRESET; 1446a88b5ba8SSam Ravnborg 1447a88b5ba8SSam Ravnborg if (lp->rx_head == lp->rx_tail) 1448a88b5ba8SSam Ravnborg return 0; 1449a88b5ba8SSam Ravnborg 1450a88b5ba8SSam Ravnborg p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE); 1451a88b5ba8SSam Ravnborg memcpy(buf, p, LDC_PACKET_SIZE); 1452a88b5ba8SSam Ravnborg 1453a88b5ba8SSam Ravnborg new = rx_advance(lp, lp->rx_head); 1454a88b5ba8SSam Ravnborg lp->rx_head = new; 1455a88b5ba8SSam Ravnborg 1456a88b5ba8SSam Ravnborg err = __set_rx_head(lp, new); 1457a88b5ba8SSam Ravnborg if (err < 0) 1458a88b5ba8SSam Ravnborg err = -ECONNRESET; 1459a88b5ba8SSam Ravnborg else 1460a88b5ba8SSam Ravnborg err = LDC_PACKET_SIZE; 1461a88b5ba8SSam Ravnborg 1462a88b5ba8SSam Ravnborg return err; 1463a88b5ba8SSam Ravnborg } 1464a88b5ba8SSam Ravnborg 1465a88b5ba8SSam Ravnborg static const struct ldc_mode_ops raw_ops = { 1466a88b5ba8SSam Ravnborg .write = write_raw, 1467a88b5ba8SSam Ravnborg .read = read_raw, 1468a88b5ba8SSam Ravnborg }; 1469a88b5ba8SSam Ravnborg 1470a88b5ba8SSam Ravnborg static int write_nonraw(struct ldc_channel *lp, const void *buf, 1471a88b5ba8SSam Ravnborg unsigned int size) 1472a88b5ba8SSam Ravnborg { 1473a88b5ba8SSam Ravnborg unsigned long hv_err, tail; 1474a88b5ba8SSam Ravnborg unsigned int copied; 1475a88b5ba8SSam Ravnborg u32 seq; 1476a88b5ba8SSam Ravnborg int err; 1477a88b5ba8SSam Ravnborg 1478a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail, 1479a88b5ba8SSam Ravnborg &lp->chan_state); 1480a88b5ba8SSam Ravnborg if (unlikely(hv_err)) 1481a88b5ba8SSam Ravnborg return -EBUSY; 1482a88b5ba8SSam Ravnborg 1483a88b5ba8SSam Ravnborg if (unlikely(lp->chan_state != LDC_CHANNEL_UP)) 1484a88b5ba8SSam Ravnborg return ldc_abort(lp); 1485a88b5ba8SSam Ravnborg 1486a88b5ba8SSam Ravnborg if (!tx_has_space_for(lp, size)) 1487a88b5ba8SSam Ravnborg return -EAGAIN; 1488a88b5ba8SSam Ravnborg 1489a88b5ba8SSam Ravnborg seq = lp->snd_nxt; 1490a88b5ba8SSam Ravnborg copied = 0; 1491a88b5ba8SSam Ravnborg tail = lp->tx_tail; 1492a88b5ba8SSam Ravnborg while (copied < size) { 1493a88b5ba8SSam Ravnborg struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE); 1494a88b5ba8SSam Ravnborg u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ? 1495a88b5ba8SSam Ravnborg p->u.u_data : 1496a88b5ba8SSam Ravnborg p->u.r.r_data); 1497a88b5ba8SSam Ravnborg int data_len; 1498a88b5ba8SSam Ravnborg 1499a88b5ba8SSam Ravnborg p->type = LDC_DATA; 1500a88b5ba8SSam Ravnborg p->stype = LDC_INFO; 1501a88b5ba8SSam Ravnborg p->ctrl = 0; 1502a88b5ba8SSam Ravnborg 1503a88b5ba8SSam Ravnborg data_len = size - copied; 1504a88b5ba8SSam Ravnborg if (data_len > lp->mss) 1505a88b5ba8SSam Ravnborg data_len = lp->mss; 1506a88b5ba8SSam Ravnborg 1507a88b5ba8SSam Ravnborg BUG_ON(data_len > LDC_LEN); 1508a88b5ba8SSam Ravnborg 1509a88b5ba8SSam Ravnborg p->env = (data_len | 1510a88b5ba8SSam Ravnborg (copied == 0 ? LDC_START : 0) | 1511a88b5ba8SSam Ravnborg (data_len == size - copied ? LDC_STOP : 0)); 1512a88b5ba8SSam Ravnborg 1513a88b5ba8SSam Ravnborg p->seqid = ++seq; 1514a88b5ba8SSam Ravnborg 1515a88b5ba8SSam Ravnborg ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n", 1516a88b5ba8SSam Ravnborg p->type, 1517a88b5ba8SSam Ravnborg p->stype, 1518a88b5ba8SSam Ravnborg p->ctrl, 1519a88b5ba8SSam Ravnborg p->env, 1520a88b5ba8SSam Ravnborg p->seqid); 1521a88b5ba8SSam Ravnborg 1522a88b5ba8SSam Ravnborg memcpy(data, buf, data_len); 1523a88b5ba8SSam Ravnborg buf += data_len; 1524a88b5ba8SSam Ravnborg copied += data_len; 1525a88b5ba8SSam Ravnborg 1526a88b5ba8SSam Ravnborg tail = tx_advance(lp, tail); 1527a88b5ba8SSam Ravnborg } 1528a88b5ba8SSam Ravnborg 1529a88b5ba8SSam Ravnborg err = set_tx_tail(lp, tail); 1530a88b5ba8SSam Ravnborg if (!err) { 1531a88b5ba8SSam Ravnborg lp->snd_nxt = seq; 1532a88b5ba8SSam Ravnborg err = size; 1533a88b5ba8SSam Ravnborg } 1534a88b5ba8SSam Ravnborg 1535a88b5ba8SSam Ravnborg return err; 1536a88b5ba8SSam Ravnborg } 1537a88b5ba8SSam Ravnborg 1538a88b5ba8SSam Ravnborg static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p, 1539a88b5ba8SSam Ravnborg struct ldc_packet *first_frag) 1540a88b5ba8SSam Ravnborg { 1541a88b5ba8SSam Ravnborg int err; 1542a88b5ba8SSam Ravnborg 1543a88b5ba8SSam Ravnborg if (first_frag) 1544a88b5ba8SSam Ravnborg lp->rcv_nxt = first_frag->seqid - 1; 1545a88b5ba8SSam Ravnborg 1546a88b5ba8SSam Ravnborg err = send_data_nack(lp, p); 1547a88b5ba8SSam Ravnborg if (err) 1548a88b5ba8SSam Ravnborg return err; 1549a88b5ba8SSam Ravnborg 1550a88b5ba8SSam Ravnborg err = __set_rx_head(lp, lp->rx_tail); 1551a88b5ba8SSam Ravnborg if (err < 0) 1552a88b5ba8SSam Ravnborg return ldc_abort(lp); 1553a88b5ba8SSam Ravnborg 1554a88b5ba8SSam Ravnborg return 0; 1555a88b5ba8SSam Ravnborg } 1556a88b5ba8SSam Ravnborg 1557a88b5ba8SSam Ravnborg static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p) 1558a88b5ba8SSam Ravnborg { 1559a88b5ba8SSam Ravnborg if (p->stype & LDC_ACK) { 1560a88b5ba8SSam Ravnborg int err = process_data_ack(lp, p); 1561a88b5ba8SSam Ravnborg if (err) 1562a88b5ba8SSam Ravnborg return err; 1563a88b5ba8SSam Ravnborg } 1564a88b5ba8SSam Ravnborg if (p->stype & LDC_NACK) 1565a88b5ba8SSam Ravnborg return ldc_abort(lp); 1566a88b5ba8SSam Ravnborg 1567a88b5ba8SSam Ravnborg return 0; 1568a88b5ba8SSam Ravnborg } 1569a88b5ba8SSam Ravnborg 1570a88b5ba8SSam Ravnborg static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head) 1571a88b5ba8SSam Ravnborg { 1572a88b5ba8SSam Ravnborg unsigned long dummy; 1573a88b5ba8SSam Ravnborg int limit = 1000; 1574a88b5ba8SSam Ravnborg 1575a88b5ba8SSam Ravnborg ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n", 1576a88b5ba8SSam Ravnborg cur_head, lp->rx_head, lp->rx_tail); 1577a88b5ba8SSam Ravnborg while (limit-- > 0) { 1578a88b5ba8SSam Ravnborg unsigned long hv_err; 1579a88b5ba8SSam Ravnborg 1580a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 1581a88b5ba8SSam Ravnborg &dummy, 1582a88b5ba8SSam Ravnborg &lp->rx_tail, 1583a88b5ba8SSam Ravnborg &lp->chan_state); 1584a88b5ba8SSam Ravnborg if (hv_err) 1585a88b5ba8SSam Ravnborg return ldc_abort(lp); 1586a88b5ba8SSam Ravnborg 1587a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN || 1588a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING) 1589a88b5ba8SSam Ravnborg return -ECONNRESET; 1590a88b5ba8SSam Ravnborg 1591a88b5ba8SSam Ravnborg if (cur_head != lp->rx_tail) { 1592a88b5ba8SSam Ravnborg ldcdbg(DATA, "DATA WAIT DONE " 1593a88b5ba8SSam Ravnborg "head[%lx] tail[%lx] chan_state[%lx]\n", 1594a88b5ba8SSam Ravnborg dummy, lp->rx_tail, lp->chan_state); 1595a88b5ba8SSam Ravnborg return 0; 1596a88b5ba8SSam Ravnborg } 1597a88b5ba8SSam Ravnborg 1598a88b5ba8SSam Ravnborg udelay(1); 1599a88b5ba8SSam Ravnborg } 1600a88b5ba8SSam Ravnborg return -EAGAIN; 1601a88b5ba8SSam Ravnborg } 1602a88b5ba8SSam Ravnborg 1603a88b5ba8SSam Ravnborg static int rx_set_head(struct ldc_channel *lp, unsigned long head) 1604a88b5ba8SSam Ravnborg { 1605a88b5ba8SSam Ravnborg int err = __set_rx_head(lp, head); 1606a88b5ba8SSam Ravnborg 1607a88b5ba8SSam Ravnborg if (err < 0) 1608a88b5ba8SSam Ravnborg return ldc_abort(lp); 1609a88b5ba8SSam Ravnborg 1610a88b5ba8SSam Ravnborg lp->rx_head = head; 1611a88b5ba8SSam Ravnborg return 0; 1612a88b5ba8SSam Ravnborg } 1613a88b5ba8SSam Ravnborg 1614a88b5ba8SSam Ravnborg static void send_data_ack(struct ldc_channel *lp) 1615a88b5ba8SSam Ravnborg { 1616a88b5ba8SSam Ravnborg unsigned long new_tail; 1617a88b5ba8SSam Ravnborg struct ldc_packet *p; 1618a88b5ba8SSam Ravnborg 1619a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail); 1620a88b5ba8SSam Ravnborg if (likely(p)) { 1621a88b5ba8SSam Ravnborg int err; 1622a88b5ba8SSam Ravnborg 1623a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p)); 1624a88b5ba8SSam Ravnborg p->type = LDC_DATA; 1625a88b5ba8SSam Ravnborg p->stype = LDC_ACK; 1626a88b5ba8SSam Ravnborg p->ctrl = 0; 1627a88b5ba8SSam Ravnborg p->seqid = lp->snd_nxt + 1; 1628a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt; 1629a88b5ba8SSam Ravnborg 1630a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail); 1631a88b5ba8SSam Ravnborg if (!err) 1632a88b5ba8SSam Ravnborg lp->snd_nxt++; 1633a88b5ba8SSam Ravnborg } 1634a88b5ba8SSam Ravnborg } 1635a88b5ba8SSam Ravnborg 1636a88b5ba8SSam Ravnborg static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size) 1637a88b5ba8SSam Ravnborg { 1638a88b5ba8SSam Ravnborg struct ldc_packet *first_frag; 1639a88b5ba8SSam Ravnborg unsigned long hv_err, new; 1640a88b5ba8SSam Ravnborg int err, copied; 1641a88b5ba8SSam Ravnborg 1642a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 1643a88b5ba8SSam Ravnborg &lp->rx_head, 1644a88b5ba8SSam Ravnborg &lp->rx_tail, 1645a88b5ba8SSam Ravnborg &lp->chan_state); 1646a88b5ba8SSam Ravnborg if (hv_err) 1647a88b5ba8SSam Ravnborg return ldc_abort(lp); 1648a88b5ba8SSam Ravnborg 1649a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN || 1650a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING) 1651a88b5ba8SSam Ravnborg return -ECONNRESET; 1652a88b5ba8SSam Ravnborg 1653a88b5ba8SSam Ravnborg if (lp->rx_head == lp->rx_tail) 1654a88b5ba8SSam Ravnborg return 0; 1655a88b5ba8SSam Ravnborg 1656a88b5ba8SSam Ravnborg first_frag = NULL; 1657a88b5ba8SSam Ravnborg copied = err = 0; 1658a88b5ba8SSam Ravnborg new = lp->rx_head; 1659a88b5ba8SSam Ravnborg while (1) { 1660a88b5ba8SSam Ravnborg struct ldc_packet *p; 1661a88b5ba8SSam Ravnborg int pkt_len; 1662a88b5ba8SSam Ravnborg 1663a88b5ba8SSam Ravnborg BUG_ON(new == lp->rx_tail); 1664a88b5ba8SSam Ravnborg p = lp->rx_base + (new / LDC_PACKET_SIZE); 1665a88b5ba8SSam Ravnborg 1666a88b5ba8SSam Ravnborg ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] " 1667a88b5ba8SSam Ravnborg "rcv_nxt[%08x]\n", 1668a88b5ba8SSam Ravnborg p->type, 1669a88b5ba8SSam Ravnborg p->stype, 1670a88b5ba8SSam Ravnborg p->ctrl, 1671a88b5ba8SSam Ravnborg p->env, 1672a88b5ba8SSam Ravnborg p->seqid, 1673a88b5ba8SSam Ravnborg p->u.r.ackid, 1674a88b5ba8SSam Ravnborg lp->rcv_nxt); 1675a88b5ba8SSam Ravnborg 1676a88b5ba8SSam Ravnborg if (unlikely(!rx_seq_ok(lp, p->seqid))) { 1677a88b5ba8SSam Ravnborg err = rx_bad_seq(lp, p, first_frag); 1678a88b5ba8SSam Ravnborg copied = 0; 1679a88b5ba8SSam Ravnborg break; 1680a88b5ba8SSam Ravnborg } 1681a88b5ba8SSam Ravnborg 1682a88b5ba8SSam Ravnborg if (p->type & LDC_CTRL) { 1683a88b5ba8SSam Ravnborg err = process_control_frame(lp, p); 1684a88b5ba8SSam Ravnborg if (err < 0) 1685a88b5ba8SSam Ravnborg break; 1686a88b5ba8SSam Ravnborg err = 0; 1687a88b5ba8SSam Ravnborg } 1688a88b5ba8SSam Ravnborg 1689a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid; 1690a88b5ba8SSam Ravnborg 1691a88b5ba8SSam Ravnborg if (!(p->type & LDC_DATA)) { 1692a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1693a88b5ba8SSam Ravnborg goto no_data; 1694a88b5ba8SSam Ravnborg } 1695a88b5ba8SSam Ravnborg if (p->stype & (LDC_ACK | LDC_NACK)) { 1696a88b5ba8SSam Ravnborg err = data_ack_nack(lp, p); 1697a88b5ba8SSam Ravnborg if (err) 1698a88b5ba8SSam Ravnborg break; 1699a88b5ba8SSam Ravnborg } 1700a88b5ba8SSam Ravnborg if (!(p->stype & LDC_INFO)) { 1701a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1702a88b5ba8SSam Ravnborg err = rx_set_head(lp, new); 1703a88b5ba8SSam Ravnborg if (err) 1704a88b5ba8SSam Ravnborg break; 1705a88b5ba8SSam Ravnborg goto no_data; 1706a88b5ba8SSam Ravnborg } 1707a88b5ba8SSam Ravnborg 1708a88b5ba8SSam Ravnborg pkt_len = p->env & LDC_LEN; 1709a88b5ba8SSam Ravnborg 1710a88b5ba8SSam Ravnborg /* Every initial packet starts with the START bit set. 1711a88b5ba8SSam Ravnborg * 1712a88b5ba8SSam Ravnborg * Singleton packets will have both START+STOP set. 1713a88b5ba8SSam Ravnborg * 1714a88b5ba8SSam Ravnborg * Fragments will have START set in the first frame, STOP 1715a88b5ba8SSam Ravnborg * set in the last frame, and neither bit set in middle 1716a88b5ba8SSam Ravnborg * frames of the packet. 1717a88b5ba8SSam Ravnborg * 1718a88b5ba8SSam Ravnborg * Therefore if we are at the beginning of a packet and 1719a88b5ba8SSam Ravnborg * we don't see START, or we are in the middle of a fragmented 1720a88b5ba8SSam Ravnborg * packet and do see START, we are unsynchronized and should 1721a88b5ba8SSam Ravnborg * flush the RX queue. 1722a88b5ba8SSam Ravnborg */ 1723a88b5ba8SSam Ravnborg if ((first_frag == NULL && !(p->env & LDC_START)) || 1724a88b5ba8SSam Ravnborg (first_frag != NULL && (p->env & LDC_START))) { 1725a88b5ba8SSam Ravnborg if (!first_frag) 1726a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1727a88b5ba8SSam Ravnborg 1728a88b5ba8SSam Ravnborg err = rx_set_head(lp, new); 1729a88b5ba8SSam Ravnborg if (err) 1730a88b5ba8SSam Ravnborg break; 1731a88b5ba8SSam Ravnborg 1732a88b5ba8SSam Ravnborg if (!first_frag) 1733a88b5ba8SSam Ravnborg goto no_data; 1734a88b5ba8SSam Ravnborg } 1735a88b5ba8SSam Ravnborg if (!first_frag) 1736a88b5ba8SSam Ravnborg first_frag = p; 1737a88b5ba8SSam Ravnborg 1738a88b5ba8SSam Ravnborg if (pkt_len > size - copied) { 1739a88b5ba8SSam Ravnborg /* User didn't give us a big enough buffer, 1740a88b5ba8SSam Ravnborg * what to do? This is a pretty serious error. 1741a88b5ba8SSam Ravnborg * 1742a88b5ba8SSam Ravnborg * Since we haven't updated the RX ring head to 1743a88b5ba8SSam Ravnborg * consume any of the packets, signal the error 1744a88b5ba8SSam Ravnborg * to the user and just leave the RX ring alone. 1745a88b5ba8SSam Ravnborg * 1746a88b5ba8SSam Ravnborg * This seems the best behavior because this allows 1747a88b5ba8SSam Ravnborg * a user of the LDC layer to start with a small 1748a88b5ba8SSam Ravnborg * RX buffer for ldc_read() calls and use -EMSGSIZE 1749a88b5ba8SSam Ravnborg * as a cue to enlarge it's read buffer. 1750a88b5ba8SSam Ravnborg */ 1751a88b5ba8SSam Ravnborg err = -EMSGSIZE; 1752a88b5ba8SSam Ravnborg break; 1753a88b5ba8SSam Ravnborg } 1754a88b5ba8SSam Ravnborg 1755a88b5ba8SSam Ravnborg /* Ok, we are gonna eat this one. */ 1756a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1757a88b5ba8SSam Ravnborg 1758a88b5ba8SSam Ravnborg memcpy(buf, 1759a88b5ba8SSam Ravnborg (lp->cfg.mode == LDC_MODE_UNRELIABLE ? 1760a88b5ba8SSam Ravnborg p->u.u_data : p->u.r.r_data), pkt_len); 1761a88b5ba8SSam Ravnborg buf += pkt_len; 1762a88b5ba8SSam Ravnborg copied += pkt_len; 1763a88b5ba8SSam Ravnborg 1764a88b5ba8SSam Ravnborg if (p->env & LDC_STOP) 1765a88b5ba8SSam Ravnborg break; 1766a88b5ba8SSam Ravnborg 1767a88b5ba8SSam Ravnborg no_data: 1768a88b5ba8SSam Ravnborg if (new == lp->rx_tail) { 1769a88b5ba8SSam Ravnborg err = rx_data_wait(lp, new); 1770a88b5ba8SSam Ravnborg if (err) 1771a88b5ba8SSam Ravnborg break; 1772a88b5ba8SSam Ravnborg } 1773a88b5ba8SSam Ravnborg } 1774a88b5ba8SSam Ravnborg 1775a88b5ba8SSam Ravnborg if (!err) 1776a88b5ba8SSam Ravnborg err = rx_set_head(lp, new); 1777a88b5ba8SSam Ravnborg 1778a88b5ba8SSam Ravnborg if (err && first_frag) 1779a88b5ba8SSam Ravnborg lp->rcv_nxt = first_frag->seqid - 1; 1780a88b5ba8SSam Ravnborg 1781a88b5ba8SSam Ravnborg if (!err) { 1782a88b5ba8SSam Ravnborg err = copied; 1783a88b5ba8SSam Ravnborg if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE) 1784a88b5ba8SSam Ravnborg send_data_ack(lp); 1785a88b5ba8SSam Ravnborg } 1786a88b5ba8SSam Ravnborg 1787a88b5ba8SSam Ravnborg return err; 1788a88b5ba8SSam Ravnborg } 1789a88b5ba8SSam Ravnborg 1790a88b5ba8SSam Ravnborg static const struct ldc_mode_ops nonraw_ops = { 1791a88b5ba8SSam Ravnborg .write = write_nonraw, 1792a88b5ba8SSam Ravnborg .read = read_nonraw, 1793a88b5ba8SSam Ravnborg }; 1794a88b5ba8SSam Ravnborg 1795a88b5ba8SSam Ravnborg static int write_stream(struct ldc_channel *lp, const void *buf, 1796a88b5ba8SSam Ravnborg unsigned int size) 1797a88b5ba8SSam Ravnborg { 1798a88b5ba8SSam Ravnborg if (size > lp->cfg.mtu) 1799a88b5ba8SSam Ravnborg size = lp->cfg.mtu; 1800a88b5ba8SSam Ravnborg return write_nonraw(lp, buf, size); 1801a88b5ba8SSam Ravnborg } 1802a88b5ba8SSam Ravnborg 1803a88b5ba8SSam Ravnborg static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size) 1804a88b5ba8SSam Ravnborg { 1805a88b5ba8SSam Ravnborg if (!lp->mssbuf_len) { 1806a88b5ba8SSam Ravnborg int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu); 1807a88b5ba8SSam Ravnborg if (err < 0) 1808a88b5ba8SSam Ravnborg return err; 1809a88b5ba8SSam Ravnborg 1810a88b5ba8SSam Ravnborg lp->mssbuf_len = err; 1811a88b5ba8SSam Ravnborg lp->mssbuf_off = 0; 1812a88b5ba8SSam Ravnborg } 1813a88b5ba8SSam Ravnborg 1814a88b5ba8SSam Ravnborg if (size > lp->mssbuf_len) 1815a88b5ba8SSam Ravnborg size = lp->mssbuf_len; 1816a88b5ba8SSam Ravnborg memcpy(buf, lp->mssbuf + lp->mssbuf_off, size); 1817a88b5ba8SSam Ravnborg 1818a88b5ba8SSam Ravnborg lp->mssbuf_off += size; 1819a88b5ba8SSam Ravnborg lp->mssbuf_len -= size; 1820a88b5ba8SSam Ravnborg 1821a88b5ba8SSam Ravnborg return size; 1822a88b5ba8SSam Ravnborg } 1823a88b5ba8SSam Ravnborg 1824a88b5ba8SSam Ravnborg static const struct ldc_mode_ops stream_ops = { 1825a88b5ba8SSam Ravnborg .write = write_stream, 1826a88b5ba8SSam Ravnborg .read = read_stream, 1827a88b5ba8SSam Ravnborg }; 1828a88b5ba8SSam Ravnborg 1829a88b5ba8SSam Ravnborg int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size) 1830a88b5ba8SSam Ravnborg { 1831a88b5ba8SSam Ravnborg unsigned long flags; 1832a88b5ba8SSam Ravnborg int err; 1833a88b5ba8SSam Ravnborg 1834a88b5ba8SSam Ravnborg if (!buf) 1835a88b5ba8SSam Ravnborg return -EINVAL; 1836a88b5ba8SSam Ravnborg 1837a88b5ba8SSam Ravnborg if (!size) 1838a88b5ba8SSam Ravnborg return 0; 1839a88b5ba8SSam Ravnborg 1840a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1841a88b5ba8SSam Ravnborg 1842a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE) 1843a88b5ba8SSam Ravnborg err = -ENOTCONN; 1844a88b5ba8SSam Ravnborg else 1845a88b5ba8SSam Ravnborg err = lp->mops->write(lp, buf, size); 1846a88b5ba8SSam Ravnborg 1847a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1848a88b5ba8SSam Ravnborg 1849a88b5ba8SSam Ravnborg return err; 1850a88b5ba8SSam Ravnborg } 1851a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_write); 1852a88b5ba8SSam Ravnborg 1853a88b5ba8SSam Ravnborg int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size) 1854a88b5ba8SSam Ravnborg { 1855a88b5ba8SSam Ravnborg unsigned long flags; 1856a88b5ba8SSam Ravnborg int err; 1857a88b5ba8SSam Ravnborg 1858a88b5ba8SSam Ravnborg if (!buf) 1859a88b5ba8SSam Ravnborg return -EINVAL; 1860a88b5ba8SSam Ravnborg 1861a88b5ba8SSam Ravnborg if (!size) 1862a88b5ba8SSam Ravnborg return 0; 1863a88b5ba8SSam Ravnborg 1864a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1865a88b5ba8SSam Ravnborg 1866a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE) 1867a88b5ba8SSam Ravnborg err = -ENOTCONN; 1868a88b5ba8SSam Ravnborg else 1869a88b5ba8SSam Ravnborg err = lp->mops->read(lp, buf, size); 1870a88b5ba8SSam Ravnborg 1871a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1872a88b5ba8SSam Ravnborg 1873a88b5ba8SSam Ravnborg return err; 1874a88b5ba8SSam Ravnborg } 1875a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_read); 1876a88b5ba8SSam Ravnborg 1877a88b5ba8SSam Ravnborg static long arena_alloc(struct ldc_iommu *iommu, unsigned long npages) 1878a88b5ba8SSam Ravnborg { 1879a88b5ba8SSam Ravnborg struct iommu_arena *arena = &iommu->arena; 1880a88b5ba8SSam Ravnborg unsigned long n, i, start, end, limit; 1881a88b5ba8SSam Ravnborg int pass; 1882a88b5ba8SSam Ravnborg 1883a88b5ba8SSam Ravnborg limit = arena->limit; 1884a88b5ba8SSam Ravnborg start = arena->hint; 1885a88b5ba8SSam Ravnborg pass = 0; 1886a88b5ba8SSam Ravnborg 1887a88b5ba8SSam Ravnborg again: 1888a88b5ba8SSam Ravnborg n = find_next_zero_bit(arena->map, limit, start); 1889a88b5ba8SSam Ravnborg end = n + npages; 1890a88b5ba8SSam Ravnborg if (unlikely(end >= limit)) { 1891a88b5ba8SSam Ravnborg if (likely(pass < 1)) { 1892a88b5ba8SSam Ravnborg limit = start; 1893a88b5ba8SSam Ravnborg start = 0; 1894a88b5ba8SSam Ravnborg pass++; 1895a88b5ba8SSam Ravnborg goto again; 1896a88b5ba8SSam Ravnborg } else { 1897a88b5ba8SSam Ravnborg /* Scanned the whole thing, give up. */ 1898a88b5ba8SSam Ravnborg return -1; 1899a88b5ba8SSam Ravnborg } 1900a88b5ba8SSam Ravnborg } 1901a88b5ba8SSam Ravnborg 1902a88b5ba8SSam Ravnborg for (i = n; i < end; i++) { 1903a88b5ba8SSam Ravnborg if (test_bit(i, arena->map)) { 1904a88b5ba8SSam Ravnborg start = i + 1; 1905a88b5ba8SSam Ravnborg goto again; 1906a88b5ba8SSam Ravnborg } 1907a88b5ba8SSam Ravnborg } 1908a88b5ba8SSam Ravnborg 1909a88b5ba8SSam Ravnborg for (i = n; i < end; i++) 1910a88b5ba8SSam Ravnborg __set_bit(i, arena->map); 1911a88b5ba8SSam Ravnborg 1912a88b5ba8SSam Ravnborg arena->hint = end; 1913a88b5ba8SSam Ravnborg 1914a88b5ba8SSam Ravnborg return n; 1915a88b5ba8SSam Ravnborg } 1916a88b5ba8SSam Ravnborg 1917a88b5ba8SSam Ravnborg #define COOKIE_PGSZ_CODE 0xf000000000000000ULL 1918a88b5ba8SSam Ravnborg #define COOKIE_PGSZ_CODE_SHIFT 60ULL 1919a88b5ba8SSam Ravnborg 1920a88b5ba8SSam Ravnborg static u64 pagesize_code(void) 1921a88b5ba8SSam Ravnborg { 1922a88b5ba8SSam Ravnborg switch (PAGE_SIZE) { 1923a88b5ba8SSam Ravnborg default: 1924a88b5ba8SSam Ravnborg case (8ULL * 1024ULL): 1925a88b5ba8SSam Ravnborg return 0; 1926a88b5ba8SSam Ravnborg case (64ULL * 1024ULL): 1927a88b5ba8SSam Ravnborg return 1; 1928a88b5ba8SSam Ravnborg case (512ULL * 1024ULL): 1929a88b5ba8SSam Ravnborg return 2; 1930a88b5ba8SSam Ravnborg case (4ULL * 1024ULL * 1024ULL): 1931a88b5ba8SSam Ravnborg return 3; 1932a88b5ba8SSam Ravnborg case (32ULL * 1024ULL * 1024ULL): 1933a88b5ba8SSam Ravnborg return 4; 1934a88b5ba8SSam Ravnborg case (256ULL * 1024ULL * 1024ULL): 1935a88b5ba8SSam Ravnborg return 5; 1936a88b5ba8SSam Ravnborg } 1937a88b5ba8SSam Ravnborg } 1938a88b5ba8SSam Ravnborg 1939a88b5ba8SSam Ravnborg static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset) 1940a88b5ba8SSam Ravnborg { 1941a88b5ba8SSam Ravnborg return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) | 1942a88b5ba8SSam Ravnborg (index << PAGE_SHIFT) | 1943a88b5ba8SSam Ravnborg page_offset); 1944a88b5ba8SSam Ravnborg } 1945a88b5ba8SSam Ravnborg 1946a88b5ba8SSam Ravnborg static u64 cookie_to_index(u64 cookie, unsigned long *shift) 1947a88b5ba8SSam Ravnborg { 1948a88b5ba8SSam Ravnborg u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT; 1949a88b5ba8SSam Ravnborg 1950a88b5ba8SSam Ravnborg cookie &= ~COOKIE_PGSZ_CODE; 1951a88b5ba8SSam Ravnborg 1952a88b5ba8SSam Ravnborg *shift = szcode * 3; 1953a88b5ba8SSam Ravnborg 1954a88b5ba8SSam Ravnborg return (cookie >> (13ULL + (szcode * 3ULL))); 1955a88b5ba8SSam Ravnborg } 1956a88b5ba8SSam Ravnborg 1957a88b5ba8SSam Ravnborg static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu, 1958a88b5ba8SSam Ravnborg unsigned long npages) 1959a88b5ba8SSam Ravnborg { 1960a88b5ba8SSam Ravnborg long entry; 1961a88b5ba8SSam Ravnborg 1962a88b5ba8SSam Ravnborg entry = arena_alloc(iommu, npages); 1963a88b5ba8SSam Ravnborg if (unlikely(entry < 0)) 1964a88b5ba8SSam Ravnborg return NULL; 1965a88b5ba8SSam Ravnborg 1966a88b5ba8SSam Ravnborg return iommu->page_table + entry; 1967a88b5ba8SSam Ravnborg } 1968a88b5ba8SSam Ravnborg 1969a88b5ba8SSam Ravnborg static u64 perm_to_mte(unsigned int map_perm) 1970a88b5ba8SSam Ravnborg { 1971a88b5ba8SSam Ravnborg u64 mte_base; 1972a88b5ba8SSam Ravnborg 1973a88b5ba8SSam Ravnborg mte_base = pagesize_code(); 1974a88b5ba8SSam Ravnborg 1975a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_SHADOW) { 1976a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R) 1977a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_COPY_R; 1978a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W) 1979a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_COPY_W; 1980a88b5ba8SSam Ravnborg } 1981a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_DIRECT) { 1982a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R) 1983a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_READ; 1984a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W) 1985a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_WRITE; 1986a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_X) 1987a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_EXEC; 1988a88b5ba8SSam Ravnborg } 1989a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_IO) { 1990a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R) 1991a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_IOMMU_R; 1992a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W) 1993a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_IOMMU_W; 1994a88b5ba8SSam Ravnborg } 1995a88b5ba8SSam Ravnborg 1996a88b5ba8SSam Ravnborg return mte_base; 1997a88b5ba8SSam Ravnborg } 1998a88b5ba8SSam Ravnborg 1999a88b5ba8SSam Ravnborg static int pages_in_region(unsigned long base, long len) 2000a88b5ba8SSam Ravnborg { 2001a88b5ba8SSam Ravnborg int count = 0; 2002a88b5ba8SSam Ravnborg 2003a88b5ba8SSam Ravnborg do { 2004a88b5ba8SSam Ravnborg unsigned long new = (base + PAGE_SIZE) & PAGE_MASK; 2005a88b5ba8SSam Ravnborg 2006a88b5ba8SSam Ravnborg len -= (new - base); 2007a88b5ba8SSam Ravnborg base = new; 2008a88b5ba8SSam Ravnborg count++; 2009a88b5ba8SSam Ravnborg } while (len > 0); 2010a88b5ba8SSam Ravnborg 2011a88b5ba8SSam Ravnborg return count; 2012a88b5ba8SSam Ravnborg } 2013a88b5ba8SSam Ravnborg 2014a88b5ba8SSam Ravnborg struct cookie_state { 2015a88b5ba8SSam Ravnborg struct ldc_mtable_entry *page_table; 2016a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies; 2017a88b5ba8SSam Ravnborg u64 mte_base; 2018a88b5ba8SSam Ravnborg u64 prev_cookie; 2019a88b5ba8SSam Ravnborg u32 pte_idx; 2020a88b5ba8SSam Ravnborg u32 nc; 2021a88b5ba8SSam Ravnborg }; 2022a88b5ba8SSam Ravnborg 2023a88b5ba8SSam Ravnborg static void fill_cookies(struct cookie_state *sp, unsigned long pa, 2024a88b5ba8SSam Ravnborg unsigned long off, unsigned long len) 2025a88b5ba8SSam Ravnborg { 2026a88b5ba8SSam Ravnborg do { 2027a88b5ba8SSam Ravnborg unsigned long tlen, new = pa + PAGE_SIZE; 2028a88b5ba8SSam Ravnborg u64 this_cookie; 2029a88b5ba8SSam Ravnborg 2030a88b5ba8SSam Ravnborg sp->page_table[sp->pte_idx].mte = sp->mte_base | pa; 2031a88b5ba8SSam Ravnborg 2032a88b5ba8SSam Ravnborg tlen = PAGE_SIZE; 2033a88b5ba8SSam Ravnborg if (off) 2034a88b5ba8SSam Ravnborg tlen = PAGE_SIZE - off; 2035a88b5ba8SSam Ravnborg if (tlen > len) 2036a88b5ba8SSam Ravnborg tlen = len; 2037a88b5ba8SSam Ravnborg 2038a88b5ba8SSam Ravnborg this_cookie = make_cookie(sp->pte_idx, 2039a88b5ba8SSam Ravnborg pagesize_code(), off); 2040a88b5ba8SSam Ravnborg 2041a88b5ba8SSam Ravnborg off = 0; 2042a88b5ba8SSam Ravnborg 2043a88b5ba8SSam Ravnborg if (this_cookie == sp->prev_cookie) { 2044a88b5ba8SSam Ravnborg sp->cookies[sp->nc - 1].cookie_size += tlen; 2045a88b5ba8SSam Ravnborg } else { 2046a88b5ba8SSam Ravnborg sp->cookies[sp->nc].cookie_addr = this_cookie; 2047a88b5ba8SSam Ravnborg sp->cookies[sp->nc].cookie_size = tlen; 2048a88b5ba8SSam Ravnborg sp->nc++; 2049a88b5ba8SSam Ravnborg } 2050a88b5ba8SSam Ravnborg sp->prev_cookie = this_cookie + tlen; 2051a88b5ba8SSam Ravnborg 2052a88b5ba8SSam Ravnborg sp->pte_idx++; 2053a88b5ba8SSam Ravnborg 2054a88b5ba8SSam Ravnborg len -= tlen; 2055a88b5ba8SSam Ravnborg pa = new; 2056a88b5ba8SSam Ravnborg } while (len > 0); 2057a88b5ba8SSam Ravnborg } 2058a88b5ba8SSam Ravnborg 2059a88b5ba8SSam Ravnborg static int sg_count_one(struct scatterlist *sg) 2060a88b5ba8SSam Ravnborg { 2061a88b5ba8SSam Ravnborg unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT; 2062a88b5ba8SSam Ravnborg long len = sg->length; 2063a88b5ba8SSam Ravnborg 2064a88b5ba8SSam Ravnborg if ((sg->offset | len) & (8UL - 1)) 2065a88b5ba8SSam Ravnborg return -EFAULT; 2066a88b5ba8SSam Ravnborg 2067a88b5ba8SSam Ravnborg return pages_in_region(base + sg->offset, len); 2068a88b5ba8SSam Ravnborg } 2069a88b5ba8SSam Ravnborg 2070a88b5ba8SSam Ravnborg static int sg_count_pages(struct scatterlist *sg, int num_sg) 2071a88b5ba8SSam Ravnborg { 2072a88b5ba8SSam Ravnborg int count; 2073a88b5ba8SSam Ravnborg int i; 2074a88b5ba8SSam Ravnborg 2075a88b5ba8SSam Ravnborg count = 0; 2076a88b5ba8SSam Ravnborg for (i = 0; i < num_sg; i++) { 2077a88b5ba8SSam Ravnborg int err = sg_count_one(sg + i); 2078a88b5ba8SSam Ravnborg if (err < 0) 2079a88b5ba8SSam Ravnborg return err; 2080a88b5ba8SSam Ravnborg count += err; 2081a88b5ba8SSam Ravnborg } 2082a88b5ba8SSam Ravnborg 2083a88b5ba8SSam Ravnborg return count; 2084a88b5ba8SSam Ravnborg } 2085a88b5ba8SSam Ravnborg 2086a88b5ba8SSam Ravnborg int ldc_map_sg(struct ldc_channel *lp, 2087a88b5ba8SSam Ravnborg struct scatterlist *sg, int num_sg, 2088a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies, 2089a88b5ba8SSam Ravnborg unsigned int map_perm) 2090a88b5ba8SSam Ravnborg { 2091a88b5ba8SSam Ravnborg unsigned long i, npages, flags; 2092a88b5ba8SSam Ravnborg struct ldc_mtable_entry *base; 2093a88b5ba8SSam Ravnborg struct cookie_state state; 2094a88b5ba8SSam Ravnborg struct ldc_iommu *iommu; 2095a88b5ba8SSam Ravnborg int err; 2096a88b5ba8SSam Ravnborg 2097a88b5ba8SSam Ravnborg if (map_perm & ~LDC_MAP_ALL) 2098a88b5ba8SSam Ravnborg return -EINVAL; 2099a88b5ba8SSam Ravnborg 2100a88b5ba8SSam Ravnborg err = sg_count_pages(sg, num_sg); 2101a88b5ba8SSam Ravnborg if (err < 0) 2102a88b5ba8SSam Ravnborg return err; 2103a88b5ba8SSam Ravnborg 2104a88b5ba8SSam Ravnborg npages = err; 2105a88b5ba8SSam Ravnborg if (err > ncookies) 2106a88b5ba8SSam Ravnborg return -EMSGSIZE; 2107a88b5ba8SSam Ravnborg 2108a88b5ba8SSam Ravnborg iommu = &lp->iommu; 2109a88b5ba8SSam Ravnborg 2110a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags); 2111a88b5ba8SSam Ravnborg base = alloc_npages(iommu, npages); 2112a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags); 2113a88b5ba8SSam Ravnborg 2114a88b5ba8SSam Ravnborg if (!base) 2115a88b5ba8SSam Ravnborg return -ENOMEM; 2116a88b5ba8SSam Ravnborg 2117a88b5ba8SSam Ravnborg state.page_table = iommu->page_table; 2118a88b5ba8SSam Ravnborg state.cookies = cookies; 2119a88b5ba8SSam Ravnborg state.mte_base = perm_to_mte(map_perm); 2120a88b5ba8SSam Ravnborg state.prev_cookie = ~(u64)0; 2121a88b5ba8SSam Ravnborg state.pte_idx = (base - iommu->page_table); 2122a88b5ba8SSam Ravnborg state.nc = 0; 2123a88b5ba8SSam Ravnborg 2124a88b5ba8SSam Ravnborg for (i = 0; i < num_sg; i++) 2125a88b5ba8SSam Ravnborg fill_cookies(&state, page_to_pfn(sg_page(&sg[i])) << PAGE_SHIFT, 2126a88b5ba8SSam Ravnborg sg[i].offset, sg[i].length); 2127a88b5ba8SSam Ravnborg 2128a88b5ba8SSam Ravnborg return state.nc; 2129a88b5ba8SSam Ravnborg } 2130a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_map_sg); 2131a88b5ba8SSam Ravnborg 2132a88b5ba8SSam Ravnborg int ldc_map_single(struct ldc_channel *lp, 2133a88b5ba8SSam Ravnborg void *buf, unsigned int len, 2134a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies, 2135a88b5ba8SSam Ravnborg unsigned int map_perm) 2136a88b5ba8SSam Ravnborg { 2137a88b5ba8SSam Ravnborg unsigned long npages, pa, flags; 2138a88b5ba8SSam Ravnborg struct ldc_mtable_entry *base; 2139a88b5ba8SSam Ravnborg struct cookie_state state; 2140a88b5ba8SSam Ravnborg struct ldc_iommu *iommu; 2141a88b5ba8SSam Ravnborg 2142a88b5ba8SSam Ravnborg if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1)) 2143a88b5ba8SSam Ravnborg return -EINVAL; 2144a88b5ba8SSam Ravnborg 2145a88b5ba8SSam Ravnborg pa = __pa(buf); 2146a88b5ba8SSam Ravnborg if ((pa | len) & (8UL - 1)) 2147a88b5ba8SSam Ravnborg return -EFAULT; 2148a88b5ba8SSam Ravnborg 2149a88b5ba8SSam Ravnborg npages = pages_in_region(pa, len); 2150a88b5ba8SSam Ravnborg 2151a88b5ba8SSam Ravnborg iommu = &lp->iommu; 2152a88b5ba8SSam Ravnborg 2153a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags); 2154a88b5ba8SSam Ravnborg base = alloc_npages(iommu, npages); 2155a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags); 2156a88b5ba8SSam Ravnborg 2157a88b5ba8SSam Ravnborg if (!base) 2158a88b5ba8SSam Ravnborg return -ENOMEM; 2159a88b5ba8SSam Ravnborg 2160a88b5ba8SSam Ravnborg state.page_table = iommu->page_table; 2161a88b5ba8SSam Ravnborg state.cookies = cookies; 2162a88b5ba8SSam Ravnborg state.mte_base = perm_to_mte(map_perm); 2163a88b5ba8SSam Ravnborg state.prev_cookie = ~(u64)0; 2164a88b5ba8SSam Ravnborg state.pte_idx = (base - iommu->page_table); 2165a88b5ba8SSam Ravnborg state.nc = 0; 2166a88b5ba8SSam Ravnborg fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len); 2167a88b5ba8SSam Ravnborg BUG_ON(state.nc != 1); 2168a88b5ba8SSam Ravnborg 2169a88b5ba8SSam Ravnborg return state.nc; 2170a88b5ba8SSam Ravnborg } 2171a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_map_single); 2172a88b5ba8SSam Ravnborg 2173a88b5ba8SSam Ravnborg static void free_npages(unsigned long id, struct ldc_iommu *iommu, 2174a88b5ba8SSam Ravnborg u64 cookie, u64 size) 2175a88b5ba8SSam Ravnborg { 2176a88b5ba8SSam Ravnborg struct iommu_arena *arena = &iommu->arena; 2177a88b5ba8SSam Ravnborg unsigned long i, shift, index, npages; 2178a88b5ba8SSam Ravnborg struct ldc_mtable_entry *base; 2179a88b5ba8SSam Ravnborg 2180a88b5ba8SSam Ravnborg npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT; 2181a88b5ba8SSam Ravnborg index = cookie_to_index(cookie, &shift); 2182a88b5ba8SSam Ravnborg base = iommu->page_table + index; 2183a88b5ba8SSam Ravnborg 2184a88b5ba8SSam Ravnborg BUG_ON(index > arena->limit || 2185a88b5ba8SSam Ravnborg (index + npages) > arena->limit); 2186a88b5ba8SSam Ravnborg 2187a88b5ba8SSam Ravnborg for (i = 0; i < npages; i++) { 2188a88b5ba8SSam Ravnborg if (base->cookie) 2189a88b5ba8SSam Ravnborg sun4v_ldc_revoke(id, cookie + (i << shift), 2190a88b5ba8SSam Ravnborg base->cookie); 2191a88b5ba8SSam Ravnborg base->mte = 0; 2192a88b5ba8SSam Ravnborg __clear_bit(index + i, arena->map); 2193a88b5ba8SSam Ravnborg } 2194a88b5ba8SSam Ravnborg } 2195a88b5ba8SSam Ravnborg 2196a88b5ba8SSam Ravnborg void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, 2197a88b5ba8SSam Ravnborg int ncookies) 2198a88b5ba8SSam Ravnborg { 2199a88b5ba8SSam Ravnborg struct ldc_iommu *iommu = &lp->iommu; 2200a88b5ba8SSam Ravnborg unsigned long flags; 2201a88b5ba8SSam Ravnborg int i; 2202a88b5ba8SSam Ravnborg 2203a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags); 2204a88b5ba8SSam Ravnborg for (i = 0; i < ncookies; i++) { 2205a88b5ba8SSam Ravnborg u64 addr = cookies[i].cookie_addr; 2206a88b5ba8SSam Ravnborg u64 size = cookies[i].cookie_size; 2207a88b5ba8SSam Ravnborg 2208a88b5ba8SSam Ravnborg free_npages(lp->id, iommu, addr, size); 2209a88b5ba8SSam Ravnborg } 2210a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags); 2211a88b5ba8SSam Ravnborg } 2212a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_unmap); 2213a88b5ba8SSam Ravnborg 2214a88b5ba8SSam Ravnborg int ldc_copy(struct ldc_channel *lp, int copy_dir, 2215a88b5ba8SSam Ravnborg void *buf, unsigned int len, unsigned long offset, 2216a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies) 2217a88b5ba8SSam Ravnborg { 2218a88b5ba8SSam Ravnborg unsigned int orig_len; 2219a88b5ba8SSam Ravnborg unsigned long ra; 2220a88b5ba8SSam Ravnborg int i; 2221a88b5ba8SSam Ravnborg 2222a88b5ba8SSam Ravnborg if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) { 2223a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n", 2224a88b5ba8SSam Ravnborg lp->id, copy_dir); 2225a88b5ba8SSam Ravnborg return -EINVAL; 2226a88b5ba8SSam Ravnborg } 2227a88b5ba8SSam Ravnborg 2228a88b5ba8SSam Ravnborg ra = __pa(buf); 2229a88b5ba8SSam Ravnborg if ((ra | len | offset) & (8UL - 1)) { 2230a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer " 2231a88b5ba8SSam Ravnborg "ra[%lx] len[%x] offset[%lx]\n", 2232a88b5ba8SSam Ravnborg lp->id, ra, len, offset); 2233a88b5ba8SSam Ravnborg return -EFAULT; 2234a88b5ba8SSam Ravnborg } 2235a88b5ba8SSam Ravnborg 2236a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE || 2237a88b5ba8SSam Ravnborg (lp->flags & LDC_FLAG_RESET)) { 2238a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] " 2239a88b5ba8SSam Ravnborg "flags[%x]\n", lp->id, lp->hs_state, lp->flags); 2240a88b5ba8SSam Ravnborg return -ECONNRESET; 2241a88b5ba8SSam Ravnborg } 2242a88b5ba8SSam Ravnborg 2243a88b5ba8SSam Ravnborg orig_len = len; 2244a88b5ba8SSam Ravnborg for (i = 0; i < ncookies; i++) { 2245a88b5ba8SSam Ravnborg unsigned long cookie_raddr = cookies[i].cookie_addr; 2246a88b5ba8SSam Ravnborg unsigned long this_len = cookies[i].cookie_size; 2247a88b5ba8SSam Ravnborg unsigned long actual_len; 2248a88b5ba8SSam Ravnborg 2249a88b5ba8SSam Ravnborg if (unlikely(offset)) { 2250a88b5ba8SSam Ravnborg unsigned long this_off = offset; 2251a88b5ba8SSam Ravnborg 2252a88b5ba8SSam Ravnborg if (this_off > this_len) 2253a88b5ba8SSam Ravnborg this_off = this_len; 2254a88b5ba8SSam Ravnborg 2255a88b5ba8SSam Ravnborg offset -= this_off; 2256a88b5ba8SSam Ravnborg this_len -= this_off; 2257a88b5ba8SSam Ravnborg if (!this_len) 2258a88b5ba8SSam Ravnborg continue; 2259a88b5ba8SSam Ravnborg cookie_raddr += this_off; 2260a88b5ba8SSam Ravnborg } 2261a88b5ba8SSam Ravnborg 2262a88b5ba8SSam Ravnborg if (this_len > len) 2263a88b5ba8SSam Ravnborg this_len = len; 2264a88b5ba8SSam Ravnborg 2265a88b5ba8SSam Ravnborg while (1) { 2266a88b5ba8SSam Ravnborg unsigned long hv_err; 2267a88b5ba8SSam Ravnborg 2268a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_copy(lp->id, copy_dir, 2269a88b5ba8SSam Ravnborg cookie_raddr, ra, 2270a88b5ba8SSam Ravnborg this_len, &actual_len); 2271a88b5ba8SSam Ravnborg if (unlikely(hv_err)) { 2272a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] " 2273a88b5ba8SSam Ravnborg "HV error %lu\n", 2274a88b5ba8SSam Ravnborg lp->id, hv_err); 2275a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE || 2276a88b5ba8SSam Ravnborg (lp->flags & LDC_FLAG_RESET)) 2277a88b5ba8SSam Ravnborg return -ECONNRESET; 2278a88b5ba8SSam Ravnborg else 2279a88b5ba8SSam Ravnborg return -EFAULT; 2280a88b5ba8SSam Ravnborg } 2281a88b5ba8SSam Ravnborg 2282a88b5ba8SSam Ravnborg cookie_raddr += actual_len; 2283a88b5ba8SSam Ravnborg ra += actual_len; 2284a88b5ba8SSam Ravnborg len -= actual_len; 2285a88b5ba8SSam Ravnborg if (actual_len == this_len) 2286a88b5ba8SSam Ravnborg break; 2287a88b5ba8SSam Ravnborg 2288a88b5ba8SSam Ravnborg this_len -= actual_len; 2289a88b5ba8SSam Ravnborg } 2290a88b5ba8SSam Ravnborg 2291a88b5ba8SSam Ravnborg if (!len) 2292a88b5ba8SSam Ravnborg break; 2293a88b5ba8SSam Ravnborg } 2294a88b5ba8SSam Ravnborg 2295a88b5ba8SSam Ravnborg /* It is caller policy what to do about short copies. 2296a88b5ba8SSam Ravnborg * For example, a networking driver can declare the 2297a88b5ba8SSam Ravnborg * packet a runt and drop it. 2298a88b5ba8SSam Ravnborg */ 2299a88b5ba8SSam Ravnborg 2300a88b5ba8SSam Ravnborg return orig_len - len; 2301a88b5ba8SSam Ravnborg } 2302a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_copy); 2303a88b5ba8SSam Ravnborg 2304a88b5ba8SSam Ravnborg void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len, 2305a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int *ncookies, 2306a88b5ba8SSam Ravnborg unsigned int map_perm) 2307a88b5ba8SSam Ravnborg { 2308a88b5ba8SSam Ravnborg void *buf; 2309a88b5ba8SSam Ravnborg int err; 2310a88b5ba8SSam Ravnborg 2311a88b5ba8SSam Ravnborg if (len & (8UL - 1)) 2312a88b5ba8SSam Ravnborg return ERR_PTR(-EINVAL); 2313a88b5ba8SSam Ravnborg 2314a88b5ba8SSam Ravnborg buf = kzalloc(len, GFP_KERNEL); 2315a88b5ba8SSam Ravnborg if (!buf) 2316a88b5ba8SSam Ravnborg return ERR_PTR(-ENOMEM); 2317a88b5ba8SSam Ravnborg 2318a88b5ba8SSam Ravnborg err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm); 2319a88b5ba8SSam Ravnborg if (err < 0) { 2320a88b5ba8SSam Ravnborg kfree(buf); 2321a88b5ba8SSam Ravnborg return ERR_PTR(err); 2322a88b5ba8SSam Ravnborg } 2323a88b5ba8SSam Ravnborg *ncookies = err; 2324a88b5ba8SSam Ravnborg 2325a88b5ba8SSam Ravnborg return buf; 2326a88b5ba8SSam Ravnborg } 2327a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_alloc_exp_dring); 2328a88b5ba8SSam Ravnborg 2329a88b5ba8SSam Ravnborg void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len, 2330a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies) 2331a88b5ba8SSam Ravnborg { 2332a88b5ba8SSam Ravnborg ldc_unmap(lp, cookies, ncookies); 2333a88b5ba8SSam Ravnborg kfree(buf); 2334a88b5ba8SSam Ravnborg } 2335a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_free_exp_dring); 2336a88b5ba8SSam Ravnborg 2337a88b5ba8SSam Ravnborg static int __init ldc_init(void) 2338a88b5ba8SSam Ravnborg { 2339a88b5ba8SSam Ravnborg unsigned long major, minor; 2340a88b5ba8SSam Ravnborg struct mdesc_handle *hp; 2341a88b5ba8SSam Ravnborg const u64 *v; 2342a88b5ba8SSam Ravnborg int err; 2343a88b5ba8SSam Ravnborg u64 mp; 2344a88b5ba8SSam Ravnborg 2345a88b5ba8SSam Ravnborg hp = mdesc_grab(); 2346a88b5ba8SSam Ravnborg if (!hp) 2347a88b5ba8SSam Ravnborg return -ENODEV; 2348a88b5ba8SSam Ravnborg 2349a88b5ba8SSam Ravnborg mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform"); 2350a88b5ba8SSam Ravnborg err = -ENODEV; 2351a88b5ba8SSam Ravnborg if (mp == MDESC_NODE_NULL) 2352a88b5ba8SSam Ravnborg goto out; 2353a88b5ba8SSam Ravnborg 2354a88b5ba8SSam Ravnborg v = mdesc_get_property(hp, mp, "domaining-enabled", NULL); 2355a88b5ba8SSam Ravnborg if (!v) 2356a88b5ba8SSam Ravnborg goto out; 2357a88b5ba8SSam Ravnborg 2358a88b5ba8SSam Ravnborg major = 1; 2359a88b5ba8SSam Ravnborg minor = 0; 2360a88b5ba8SSam Ravnborg if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) { 2361a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "Could not register LDOM hvapi.\n"); 2362a88b5ba8SSam Ravnborg goto out; 2363a88b5ba8SSam Ravnborg } 2364a88b5ba8SSam Ravnborg 2365a88b5ba8SSam Ravnborg printk(KERN_INFO "%s", version); 2366a88b5ba8SSam Ravnborg 2367a88b5ba8SSam Ravnborg if (!*v) { 2368a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "Domaining disabled.\n"); 2369a88b5ba8SSam Ravnborg goto out; 2370a88b5ba8SSam Ravnborg } 2371a88b5ba8SSam Ravnborg ldom_domaining_enabled = 1; 2372a88b5ba8SSam Ravnborg err = 0; 2373a88b5ba8SSam Ravnborg 2374a88b5ba8SSam Ravnborg out: 2375a88b5ba8SSam Ravnborg mdesc_release(hp); 2376a88b5ba8SSam Ravnborg return err; 2377a88b5ba8SSam Ravnborg } 2378a88b5ba8SSam Ravnborg 2379a88b5ba8SSam Ravnborg core_initcall(ldc_init); 2380