1a88b5ba8SSam Ravnborg /* ldc.c: Logical Domain Channel link-layer protocol driver. 2a88b5ba8SSam Ravnborg * 3a88b5ba8SSam Ravnborg * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> 4a88b5ba8SSam Ravnborg */ 5a88b5ba8SSam Ravnborg 6a88b5ba8SSam Ravnborg #include <linux/kernel.h> 7066bcacaSPaul Gortmaker #include <linux/export.h> 8a88b5ba8SSam Ravnborg #include <linux/slab.h> 9a88b5ba8SSam Ravnborg #include <linux/spinlock.h> 10a88b5ba8SSam Ravnborg #include <linux/delay.h> 11a88b5ba8SSam Ravnborg #include <linux/errno.h> 12a88b5ba8SSam Ravnborg #include <linux/string.h> 13a88b5ba8SSam Ravnborg #include <linux/scatterlist.h> 14a88b5ba8SSam Ravnborg #include <linux/interrupt.h> 15a88b5ba8SSam Ravnborg #include <linux/list.h> 16a88b5ba8SSam Ravnborg #include <linux/init.h> 17e756fd80SAkinobu Mita #include <linux/bitmap.h> 18a88b5ba8SSam Ravnborg 19a88b5ba8SSam Ravnborg #include <asm/hypervisor.h> 20a88b5ba8SSam Ravnborg #include <asm/iommu.h> 21a88b5ba8SSam Ravnborg #include <asm/page.h> 22a88b5ba8SSam Ravnborg #include <asm/ldc.h> 23a88b5ba8SSam Ravnborg #include <asm/mdesc.h> 24a88b5ba8SSam Ravnborg 25a88b5ba8SSam Ravnborg #define DRV_MODULE_NAME "ldc" 26a88b5ba8SSam Ravnborg #define PFX DRV_MODULE_NAME ": " 27a88b5ba8SSam Ravnborg #define DRV_MODULE_VERSION "1.1" 28a88b5ba8SSam Ravnborg #define DRV_MODULE_RELDATE "July 22, 2008" 29a88b5ba8SSam Ravnborg 307c9503b8SGreg Kroah-Hartman static char version[] = 31a88b5ba8SSam Ravnborg DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 32a88b5ba8SSam Ravnborg #define LDC_PACKET_SIZE 64 33a88b5ba8SSam Ravnborg 34a88b5ba8SSam Ravnborg /* Packet header layout for unreliable and reliable mode frames. 35a88b5ba8SSam Ravnborg * When in RAW mode, packets are simply straight 64-byte payloads 36a88b5ba8SSam Ravnborg * with no headers. 37a88b5ba8SSam Ravnborg */ 38a88b5ba8SSam Ravnborg struct ldc_packet { 39a88b5ba8SSam Ravnborg u8 type; 40a88b5ba8SSam Ravnborg #define LDC_CTRL 0x01 41a88b5ba8SSam Ravnborg #define LDC_DATA 0x02 42a88b5ba8SSam Ravnborg #define LDC_ERR 0x10 43a88b5ba8SSam Ravnborg 44a88b5ba8SSam Ravnborg u8 stype; 45a88b5ba8SSam Ravnborg #define LDC_INFO 0x01 46a88b5ba8SSam Ravnborg #define LDC_ACK 0x02 47a88b5ba8SSam Ravnborg #define LDC_NACK 0x04 48a88b5ba8SSam Ravnborg 49a88b5ba8SSam Ravnborg u8 ctrl; 50a88b5ba8SSam Ravnborg #define LDC_VERS 0x01 /* Link Version */ 51a88b5ba8SSam Ravnborg #define LDC_RTS 0x02 /* Request To Send */ 52a88b5ba8SSam Ravnborg #define LDC_RTR 0x03 /* Ready To Receive */ 53a88b5ba8SSam Ravnborg #define LDC_RDX 0x04 /* Ready for Data eXchange */ 54a88b5ba8SSam Ravnborg #define LDC_CTRL_MSK 0x0f 55a88b5ba8SSam Ravnborg 56a88b5ba8SSam Ravnborg u8 env; 57a88b5ba8SSam Ravnborg #define LDC_LEN 0x3f 58a88b5ba8SSam Ravnborg #define LDC_FRAG_MASK 0xc0 59a88b5ba8SSam Ravnborg #define LDC_START 0x40 60a88b5ba8SSam Ravnborg #define LDC_STOP 0x80 61a88b5ba8SSam Ravnborg 62a88b5ba8SSam Ravnborg u32 seqid; 63a88b5ba8SSam Ravnborg 64a88b5ba8SSam Ravnborg union { 65a88b5ba8SSam Ravnborg u8 u_data[LDC_PACKET_SIZE - 8]; 66a88b5ba8SSam Ravnborg struct { 67a88b5ba8SSam Ravnborg u32 pad; 68a88b5ba8SSam Ravnborg u32 ackid; 69a88b5ba8SSam Ravnborg u8 r_data[LDC_PACKET_SIZE - 8 - 8]; 70a88b5ba8SSam Ravnborg } r; 71a88b5ba8SSam Ravnborg } u; 72a88b5ba8SSam Ravnborg }; 73a88b5ba8SSam Ravnborg 74a88b5ba8SSam Ravnborg struct ldc_version { 75a88b5ba8SSam Ravnborg u16 major; 76a88b5ba8SSam Ravnborg u16 minor; 77a88b5ba8SSam Ravnborg }; 78a88b5ba8SSam Ravnborg 79a88b5ba8SSam Ravnborg /* Ordered from largest major to lowest. */ 80a88b5ba8SSam Ravnborg static struct ldc_version ver_arr[] = { 81a88b5ba8SSam Ravnborg { .major = 1, .minor = 0 }, 82a88b5ba8SSam Ravnborg }; 83a88b5ba8SSam Ravnborg 84a88b5ba8SSam Ravnborg #define LDC_DEFAULT_MTU (4 * LDC_PACKET_SIZE) 85a88b5ba8SSam Ravnborg #define LDC_DEFAULT_NUM_ENTRIES (PAGE_SIZE / LDC_PACKET_SIZE) 86a88b5ba8SSam Ravnborg 87a88b5ba8SSam Ravnborg struct ldc_channel; 88a88b5ba8SSam Ravnborg 89a88b5ba8SSam Ravnborg struct ldc_mode_ops { 90a88b5ba8SSam Ravnborg int (*write)(struct ldc_channel *, const void *, unsigned int); 91a88b5ba8SSam Ravnborg int (*read)(struct ldc_channel *, void *, unsigned int); 92a88b5ba8SSam Ravnborg }; 93a88b5ba8SSam Ravnborg 94a88b5ba8SSam Ravnborg static const struct ldc_mode_ops raw_ops; 95a88b5ba8SSam Ravnborg static const struct ldc_mode_ops nonraw_ops; 96a88b5ba8SSam Ravnborg static const struct ldc_mode_ops stream_ops; 97a88b5ba8SSam Ravnborg 98a88b5ba8SSam Ravnborg int ldom_domaining_enabled; 99a88b5ba8SSam Ravnborg 100a88b5ba8SSam Ravnborg struct ldc_iommu { 101a88b5ba8SSam Ravnborg /* Protects arena alloc/free. */ 102a88b5ba8SSam Ravnborg spinlock_t lock; 103a88b5ba8SSam Ravnborg struct iommu_arena arena; 104a88b5ba8SSam Ravnborg struct ldc_mtable_entry *page_table; 105a88b5ba8SSam Ravnborg }; 106a88b5ba8SSam Ravnborg 107a88b5ba8SSam Ravnborg struct ldc_channel { 108a88b5ba8SSam Ravnborg /* Protects all operations that depend upon channel state. */ 109a88b5ba8SSam Ravnborg spinlock_t lock; 110a88b5ba8SSam Ravnborg 111a88b5ba8SSam Ravnborg unsigned long id; 112a88b5ba8SSam Ravnborg 113a88b5ba8SSam Ravnborg u8 *mssbuf; 114a88b5ba8SSam Ravnborg u32 mssbuf_len; 115a88b5ba8SSam Ravnborg u32 mssbuf_off; 116a88b5ba8SSam Ravnborg 117a88b5ba8SSam Ravnborg struct ldc_packet *tx_base; 118a88b5ba8SSam Ravnborg unsigned long tx_head; 119a88b5ba8SSam Ravnborg unsigned long tx_tail; 120a88b5ba8SSam Ravnborg unsigned long tx_num_entries; 121a88b5ba8SSam Ravnborg unsigned long tx_ra; 122a88b5ba8SSam Ravnborg 123a88b5ba8SSam Ravnborg unsigned long tx_acked; 124a88b5ba8SSam Ravnborg 125a88b5ba8SSam Ravnborg struct ldc_packet *rx_base; 126a88b5ba8SSam Ravnborg unsigned long rx_head; 127a88b5ba8SSam Ravnborg unsigned long rx_tail; 128a88b5ba8SSam Ravnborg unsigned long rx_num_entries; 129a88b5ba8SSam Ravnborg unsigned long rx_ra; 130a88b5ba8SSam Ravnborg 131a88b5ba8SSam Ravnborg u32 rcv_nxt; 132a88b5ba8SSam Ravnborg u32 snd_nxt; 133a88b5ba8SSam Ravnborg 134a88b5ba8SSam Ravnborg unsigned long chan_state; 135a88b5ba8SSam Ravnborg 136a88b5ba8SSam Ravnborg struct ldc_channel_config cfg; 137a88b5ba8SSam Ravnborg void *event_arg; 138a88b5ba8SSam Ravnborg 139a88b5ba8SSam Ravnborg const struct ldc_mode_ops *mops; 140a88b5ba8SSam Ravnborg 141a88b5ba8SSam Ravnborg struct ldc_iommu iommu; 142a88b5ba8SSam Ravnborg 143a88b5ba8SSam Ravnborg struct ldc_version ver; 144a88b5ba8SSam Ravnborg 145a88b5ba8SSam Ravnborg u8 hs_state; 146a88b5ba8SSam Ravnborg #define LDC_HS_CLOSED 0x00 147a88b5ba8SSam Ravnborg #define LDC_HS_OPEN 0x01 148a88b5ba8SSam Ravnborg #define LDC_HS_GOTVERS 0x02 149a88b5ba8SSam Ravnborg #define LDC_HS_SENTRTR 0x03 150a88b5ba8SSam Ravnborg #define LDC_HS_GOTRTR 0x04 151a88b5ba8SSam Ravnborg #define LDC_HS_COMPLETE 0x10 152a88b5ba8SSam Ravnborg 153a88b5ba8SSam Ravnborg u8 flags; 154a88b5ba8SSam Ravnborg #define LDC_FLAG_ALLOCED_QUEUES 0x01 155a88b5ba8SSam Ravnborg #define LDC_FLAG_REGISTERED_QUEUES 0x02 156a88b5ba8SSam Ravnborg #define LDC_FLAG_REGISTERED_IRQS 0x04 157a88b5ba8SSam Ravnborg #define LDC_FLAG_RESET 0x10 158a88b5ba8SSam Ravnborg 159a88b5ba8SSam Ravnborg u8 mss; 160a88b5ba8SSam Ravnborg u8 state; 161a88b5ba8SSam Ravnborg 162a88b5ba8SSam Ravnborg #define LDC_IRQ_NAME_MAX 32 163a88b5ba8SSam Ravnborg char rx_irq_name[LDC_IRQ_NAME_MAX]; 164a88b5ba8SSam Ravnborg char tx_irq_name[LDC_IRQ_NAME_MAX]; 165a88b5ba8SSam Ravnborg 166a88b5ba8SSam Ravnborg struct hlist_head mh_list; 167a88b5ba8SSam Ravnborg 168a88b5ba8SSam Ravnborg struct hlist_node list; 169a88b5ba8SSam Ravnborg }; 170a88b5ba8SSam Ravnborg 171a88b5ba8SSam Ravnborg #define ldcdbg(TYPE, f, a...) \ 172a88b5ba8SSam Ravnborg do { if (lp->cfg.debug & LDC_DEBUG_##TYPE) \ 173a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \ 174a88b5ba8SSam Ravnborg } while (0) 175a88b5ba8SSam Ravnborg 176a88b5ba8SSam Ravnborg static const char *state_to_str(u8 state) 177a88b5ba8SSam Ravnborg { 178a88b5ba8SSam Ravnborg switch (state) { 179a88b5ba8SSam Ravnborg case LDC_STATE_INVALID: 180a88b5ba8SSam Ravnborg return "INVALID"; 181a88b5ba8SSam Ravnborg case LDC_STATE_INIT: 182a88b5ba8SSam Ravnborg return "INIT"; 183a88b5ba8SSam Ravnborg case LDC_STATE_BOUND: 184a88b5ba8SSam Ravnborg return "BOUND"; 185a88b5ba8SSam Ravnborg case LDC_STATE_READY: 186a88b5ba8SSam Ravnborg return "READY"; 187a88b5ba8SSam Ravnborg case LDC_STATE_CONNECTED: 188a88b5ba8SSam Ravnborg return "CONNECTED"; 189a88b5ba8SSam Ravnborg default: 190a88b5ba8SSam Ravnborg return "<UNKNOWN>"; 191a88b5ba8SSam Ravnborg } 192a88b5ba8SSam Ravnborg } 193a88b5ba8SSam Ravnborg 194a88b5ba8SSam Ravnborg static void ldc_set_state(struct ldc_channel *lp, u8 state) 195a88b5ba8SSam Ravnborg { 196a88b5ba8SSam Ravnborg ldcdbg(STATE, "STATE (%s) --> (%s)\n", 197a88b5ba8SSam Ravnborg state_to_str(lp->state), 198a88b5ba8SSam Ravnborg state_to_str(state)); 199a88b5ba8SSam Ravnborg 200a88b5ba8SSam Ravnborg lp->state = state; 201a88b5ba8SSam Ravnborg } 202a88b5ba8SSam Ravnborg 203a88b5ba8SSam Ravnborg static unsigned long __advance(unsigned long off, unsigned long num_entries) 204a88b5ba8SSam Ravnborg { 205a88b5ba8SSam Ravnborg off += LDC_PACKET_SIZE; 206a88b5ba8SSam Ravnborg if (off == (num_entries * LDC_PACKET_SIZE)) 207a88b5ba8SSam Ravnborg off = 0; 208a88b5ba8SSam Ravnborg 209a88b5ba8SSam Ravnborg return off; 210a88b5ba8SSam Ravnborg } 211a88b5ba8SSam Ravnborg 212a88b5ba8SSam Ravnborg static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off) 213a88b5ba8SSam Ravnborg { 214a88b5ba8SSam Ravnborg return __advance(off, lp->rx_num_entries); 215a88b5ba8SSam Ravnborg } 216a88b5ba8SSam Ravnborg 217a88b5ba8SSam Ravnborg static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off) 218a88b5ba8SSam Ravnborg { 219a88b5ba8SSam Ravnborg return __advance(off, lp->tx_num_entries); 220a88b5ba8SSam Ravnborg } 221a88b5ba8SSam Ravnborg 222a88b5ba8SSam Ravnborg static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp, 223a88b5ba8SSam Ravnborg unsigned long *new_tail) 224a88b5ba8SSam Ravnborg { 225a88b5ba8SSam Ravnborg struct ldc_packet *p; 226a88b5ba8SSam Ravnborg unsigned long t; 227a88b5ba8SSam Ravnborg 228a88b5ba8SSam Ravnborg t = tx_advance(lp, lp->tx_tail); 229a88b5ba8SSam Ravnborg if (t == lp->tx_head) 230a88b5ba8SSam Ravnborg return NULL; 231a88b5ba8SSam Ravnborg 232a88b5ba8SSam Ravnborg *new_tail = t; 233a88b5ba8SSam Ravnborg 234a88b5ba8SSam Ravnborg p = lp->tx_base; 235a88b5ba8SSam Ravnborg return p + (lp->tx_tail / LDC_PACKET_SIZE); 236a88b5ba8SSam Ravnborg } 237a88b5ba8SSam Ravnborg 238a88b5ba8SSam Ravnborg /* When we are in reliable or stream mode, have to track the next packet 239a88b5ba8SSam Ravnborg * we haven't gotten an ACK for in the TX queue using tx_acked. We have 240a88b5ba8SSam Ravnborg * to be careful not to stomp over the queue past that point. During 241a88b5ba8SSam Ravnborg * the handshake, we don't have TX data packets pending in the queue 242a88b5ba8SSam Ravnborg * and that's why handshake_get_tx_packet() need not be mindful of 243a88b5ba8SSam Ravnborg * lp->tx_acked. 244a88b5ba8SSam Ravnborg */ 245a88b5ba8SSam Ravnborg static unsigned long head_for_data(struct ldc_channel *lp) 246a88b5ba8SSam Ravnborg { 247a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_STREAM) 248a88b5ba8SSam Ravnborg return lp->tx_acked; 249a88b5ba8SSam Ravnborg return lp->tx_head; 250a88b5ba8SSam Ravnborg } 251a88b5ba8SSam Ravnborg 252a88b5ba8SSam Ravnborg static int tx_has_space_for(struct ldc_channel *lp, unsigned int size) 253a88b5ba8SSam Ravnborg { 254a88b5ba8SSam Ravnborg unsigned long limit, tail, new_tail, diff; 255a88b5ba8SSam Ravnborg unsigned int mss; 256a88b5ba8SSam Ravnborg 257a88b5ba8SSam Ravnborg limit = head_for_data(lp); 258a88b5ba8SSam Ravnborg tail = lp->tx_tail; 259a88b5ba8SSam Ravnborg new_tail = tx_advance(lp, tail); 260a88b5ba8SSam Ravnborg if (new_tail == limit) 261a88b5ba8SSam Ravnborg return 0; 262a88b5ba8SSam Ravnborg 263a88b5ba8SSam Ravnborg if (limit > new_tail) 264a88b5ba8SSam Ravnborg diff = limit - new_tail; 265a88b5ba8SSam Ravnborg else 266a88b5ba8SSam Ravnborg diff = (limit + 267a88b5ba8SSam Ravnborg ((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail)); 268a88b5ba8SSam Ravnborg diff /= LDC_PACKET_SIZE; 269a88b5ba8SSam Ravnborg mss = lp->mss; 270a88b5ba8SSam Ravnborg 271a88b5ba8SSam Ravnborg if (diff * mss < size) 272a88b5ba8SSam Ravnborg return 0; 273a88b5ba8SSam Ravnborg 274a88b5ba8SSam Ravnborg return 1; 275a88b5ba8SSam Ravnborg } 276a88b5ba8SSam Ravnborg 277a88b5ba8SSam Ravnborg static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp, 278a88b5ba8SSam Ravnborg unsigned long *new_tail) 279a88b5ba8SSam Ravnborg { 280a88b5ba8SSam Ravnborg struct ldc_packet *p; 281a88b5ba8SSam Ravnborg unsigned long h, t; 282a88b5ba8SSam Ravnborg 283a88b5ba8SSam Ravnborg h = head_for_data(lp); 284a88b5ba8SSam Ravnborg t = tx_advance(lp, lp->tx_tail); 285a88b5ba8SSam Ravnborg if (t == h) 286a88b5ba8SSam Ravnborg return NULL; 287a88b5ba8SSam Ravnborg 288a88b5ba8SSam Ravnborg *new_tail = t; 289a88b5ba8SSam Ravnborg 290a88b5ba8SSam Ravnborg p = lp->tx_base; 291a88b5ba8SSam Ravnborg return p + (lp->tx_tail / LDC_PACKET_SIZE); 292a88b5ba8SSam Ravnborg } 293a88b5ba8SSam Ravnborg 294a88b5ba8SSam Ravnborg static int set_tx_tail(struct ldc_channel *lp, unsigned long tail) 295a88b5ba8SSam Ravnborg { 296a88b5ba8SSam Ravnborg unsigned long orig_tail = lp->tx_tail; 297a88b5ba8SSam Ravnborg int limit = 1000; 298a88b5ba8SSam Ravnborg 299a88b5ba8SSam Ravnborg lp->tx_tail = tail; 300a88b5ba8SSam Ravnborg while (limit-- > 0) { 301a88b5ba8SSam Ravnborg unsigned long err; 302a88b5ba8SSam Ravnborg 303a88b5ba8SSam Ravnborg err = sun4v_ldc_tx_set_qtail(lp->id, tail); 304a88b5ba8SSam Ravnborg if (!err) 305a88b5ba8SSam Ravnborg return 0; 306a88b5ba8SSam Ravnborg 307a88b5ba8SSam Ravnborg if (err != HV_EWOULDBLOCK) { 308a88b5ba8SSam Ravnborg lp->tx_tail = orig_tail; 309a88b5ba8SSam Ravnborg return -EINVAL; 310a88b5ba8SSam Ravnborg } 311a88b5ba8SSam Ravnborg udelay(1); 312a88b5ba8SSam Ravnborg } 313a88b5ba8SSam Ravnborg 314a88b5ba8SSam Ravnborg lp->tx_tail = orig_tail; 315a88b5ba8SSam Ravnborg return -EBUSY; 316a88b5ba8SSam Ravnborg } 317a88b5ba8SSam Ravnborg 318a88b5ba8SSam Ravnborg /* This just updates the head value in the hypervisor using 319a88b5ba8SSam Ravnborg * a polling loop with a timeout. The caller takes care of 320a88b5ba8SSam Ravnborg * upating software state representing the head change, if any. 321a88b5ba8SSam Ravnborg */ 322a88b5ba8SSam Ravnborg static int __set_rx_head(struct ldc_channel *lp, unsigned long head) 323a88b5ba8SSam Ravnborg { 324a88b5ba8SSam Ravnborg int limit = 1000; 325a88b5ba8SSam Ravnborg 326a88b5ba8SSam Ravnborg while (limit-- > 0) { 327a88b5ba8SSam Ravnborg unsigned long err; 328a88b5ba8SSam Ravnborg 329a88b5ba8SSam Ravnborg err = sun4v_ldc_rx_set_qhead(lp->id, head); 330a88b5ba8SSam Ravnborg if (!err) 331a88b5ba8SSam Ravnborg return 0; 332a88b5ba8SSam Ravnborg 333a88b5ba8SSam Ravnborg if (err != HV_EWOULDBLOCK) 334a88b5ba8SSam Ravnborg return -EINVAL; 335a88b5ba8SSam Ravnborg 336a88b5ba8SSam Ravnborg udelay(1); 337a88b5ba8SSam Ravnborg } 338a88b5ba8SSam Ravnborg 339a88b5ba8SSam Ravnborg return -EBUSY; 340a88b5ba8SSam Ravnborg } 341a88b5ba8SSam Ravnborg 342a88b5ba8SSam Ravnborg static int send_tx_packet(struct ldc_channel *lp, 343a88b5ba8SSam Ravnborg struct ldc_packet *p, 344a88b5ba8SSam Ravnborg unsigned long new_tail) 345a88b5ba8SSam Ravnborg { 346a88b5ba8SSam Ravnborg BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE))); 347a88b5ba8SSam Ravnborg 348a88b5ba8SSam Ravnborg return set_tx_tail(lp, new_tail); 349a88b5ba8SSam Ravnborg } 350a88b5ba8SSam Ravnborg 351a88b5ba8SSam Ravnborg static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp, 352a88b5ba8SSam Ravnborg u8 stype, u8 ctrl, 353a88b5ba8SSam Ravnborg void *data, int dlen, 354a88b5ba8SSam Ravnborg unsigned long *new_tail) 355a88b5ba8SSam Ravnborg { 356a88b5ba8SSam Ravnborg struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail); 357a88b5ba8SSam Ravnborg 358a88b5ba8SSam Ravnborg if (p) { 359a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p)); 360a88b5ba8SSam Ravnborg p->type = LDC_CTRL; 361a88b5ba8SSam Ravnborg p->stype = stype; 362a88b5ba8SSam Ravnborg p->ctrl = ctrl; 363a88b5ba8SSam Ravnborg if (data) 364a88b5ba8SSam Ravnborg memcpy(p->u.u_data, data, dlen); 365a88b5ba8SSam Ravnborg } 366a88b5ba8SSam Ravnborg return p; 367a88b5ba8SSam Ravnborg } 368a88b5ba8SSam Ravnborg 369a88b5ba8SSam Ravnborg static int start_handshake(struct ldc_channel *lp) 370a88b5ba8SSam Ravnborg { 371a88b5ba8SSam Ravnborg struct ldc_packet *p; 372a88b5ba8SSam Ravnborg struct ldc_version *ver; 373a88b5ba8SSam Ravnborg unsigned long new_tail; 374a88b5ba8SSam Ravnborg 375a88b5ba8SSam Ravnborg ver = &ver_arr[0]; 376a88b5ba8SSam Ravnborg 377a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n", 378a88b5ba8SSam Ravnborg ver->major, ver->minor); 379a88b5ba8SSam Ravnborg 380a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS, 381a88b5ba8SSam Ravnborg ver, sizeof(*ver), &new_tail); 382a88b5ba8SSam Ravnborg if (p) { 383a88b5ba8SSam Ravnborg int err = send_tx_packet(lp, p, new_tail); 384a88b5ba8SSam Ravnborg if (!err) 385a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_RESET; 386a88b5ba8SSam Ravnborg return err; 387a88b5ba8SSam Ravnborg } 388a88b5ba8SSam Ravnborg return -EBUSY; 389a88b5ba8SSam Ravnborg } 390a88b5ba8SSam Ravnborg 391a88b5ba8SSam Ravnborg static int send_version_nack(struct ldc_channel *lp, 392a88b5ba8SSam Ravnborg u16 major, u16 minor) 393a88b5ba8SSam Ravnborg { 394a88b5ba8SSam Ravnborg struct ldc_packet *p; 395a88b5ba8SSam Ravnborg struct ldc_version ver; 396a88b5ba8SSam Ravnborg unsigned long new_tail; 397a88b5ba8SSam Ravnborg 398a88b5ba8SSam Ravnborg ver.major = major; 399a88b5ba8SSam Ravnborg ver.minor = minor; 400a88b5ba8SSam Ravnborg 401a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS, 402a88b5ba8SSam Ravnborg &ver, sizeof(ver), &new_tail); 403a88b5ba8SSam Ravnborg if (p) { 404a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n", 405a88b5ba8SSam Ravnborg ver.major, ver.minor); 406a88b5ba8SSam Ravnborg 407a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 408a88b5ba8SSam Ravnborg } 409a88b5ba8SSam Ravnborg return -EBUSY; 410a88b5ba8SSam Ravnborg } 411a88b5ba8SSam Ravnborg 412a88b5ba8SSam Ravnborg static int send_version_ack(struct ldc_channel *lp, 413a88b5ba8SSam Ravnborg struct ldc_version *vp) 414a88b5ba8SSam Ravnborg { 415a88b5ba8SSam Ravnborg struct ldc_packet *p; 416a88b5ba8SSam Ravnborg unsigned long new_tail; 417a88b5ba8SSam Ravnborg 418a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS, 419a88b5ba8SSam Ravnborg vp, sizeof(*vp), &new_tail); 420a88b5ba8SSam Ravnborg if (p) { 421a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n", 422a88b5ba8SSam Ravnborg vp->major, vp->minor); 423a88b5ba8SSam Ravnborg 424a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 425a88b5ba8SSam Ravnborg } 426a88b5ba8SSam Ravnborg return -EBUSY; 427a88b5ba8SSam Ravnborg } 428a88b5ba8SSam Ravnborg 429a88b5ba8SSam Ravnborg static int send_rts(struct ldc_channel *lp) 430a88b5ba8SSam Ravnborg { 431a88b5ba8SSam Ravnborg struct ldc_packet *p; 432a88b5ba8SSam Ravnborg unsigned long new_tail; 433a88b5ba8SSam Ravnborg 434a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0, 435a88b5ba8SSam Ravnborg &new_tail); 436a88b5ba8SSam Ravnborg if (p) { 437a88b5ba8SSam Ravnborg p->env = lp->cfg.mode; 438a88b5ba8SSam Ravnborg p->seqid = 0; 439a88b5ba8SSam Ravnborg lp->rcv_nxt = 0; 440a88b5ba8SSam Ravnborg 441a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n", 442a88b5ba8SSam Ravnborg p->env, p->seqid); 443a88b5ba8SSam Ravnborg 444a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 445a88b5ba8SSam Ravnborg } 446a88b5ba8SSam Ravnborg return -EBUSY; 447a88b5ba8SSam Ravnborg } 448a88b5ba8SSam Ravnborg 449a88b5ba8SSam Ravnborg static int send_rtr(struct ldc_channel *lp) 450a88b5ba8SSam Ravnborg { 451a88b5ba8SSam Ravnborg struct ldc_packet *p; 452a88b5ba8SSam Ravnborg unsigned long new_tail; 453a88b5ba8SSam Ravnborg 454a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0, 455a88b5ba8SSam Ravnborg &new_tail); 456a88b5ba8SSam Ravnborg if (p) { 457a88b5ba8SSam Ravnborg p->env = lp->cfg.mode; 458a88b5ba8SSam Ravnborg p->seqid = 0; 459a88b5ba8SSam Ravnborg 460a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n", 461a88b5ba8SSam Ravnborg p->env, p->seqid); 462a88b5ba8SSam Ravnborg 463a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 464a88b5ba8SSam Ravnborg } 465a88b5ba8SSam Ravnborg return -EBUSY; 466a88b5ba8SSam Ravnborg } 467a88b5ba8SSam Ravnborg 468a88b5ba8SSam Ravnborg static int send_rdx(struct ldc_channel *lp) 469a88b5ba8SSam Ravnborg { 470a88b5ba8SSam Ravnborg struct ldc_packet *p; 471a88b5ba8SSam Ravnborg unsigned long new_tail; 472a88b5ba8SSam Ravnborg 473a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0, 474a88b5ba8SSam Ravnborg &new_tail); 475a88b5ba8SSam Ravnborg if (p) { 476a88b5ba8SSam Ravnborg p->env = 0; 477a88b5ba8SSam Ravnborg p->seqid = ++lp->snd_nxt; 478a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt; 479a88b5ba8SSam Ravnborg 480a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n", 481a88b5ba8SSam Ravnborg p->env, p->seqid, p->u.r.ackid); 482a88b5ba8SSam Ravnborg 483a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 484a88b5ba8SSam Ravnborg } 485a88b5ba8SSam Ravnborg return -EBUSY; 486a88b5ba8SSam Ravnborg } 487a88b5ba8SSam Ravnborg 488a88b5ba8SSam Ravnborg static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt) 489a88b5ba8SSam Ravnborg { 490a88b5ba8SSam Ravnborg struct ldc_packet *p; 491a88b5ba8SSam Ravnborg unsigned long new_tail; 492a88b5ba8SSam Ravnborg int err; 493a88b5ba8SSam Ravnborg 494a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail); 495a88b5ba8SSam Ravnborg if (!p) 496a88b5ba8SSam Ravnborg return -EBUSY; 497a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p)); 498a88b5ba8SSam Ravnborg p->type = data_pkt->type; 499a88b5ba8SSam Ravnborg p->stype = LDC_NACK; 500a88b5ba8SSam Ravnborg p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK; 501a88b5ba8SSam Ravnborg p->seqid = lp->snd_nxt + 1; 502a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt; 503a88b5ba8SSam Ravnborg 504a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n", 505a88b5ba8SSam Ravnborg p->type, p->ctrl, p->seqid, p->u.r.ackid); 506a88b5ba8SSam Ravnborg 507a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail); 508a88b5ba8SSam Ravnborg if (!err) 509a88b5ba8SSam Ravnborg lp->snd_nxt++; 510a88b5ba8SSam Ravnborg 511a88b5ba8SSam Ravnborg return err; 512a88b5ba8SSam Ravnborg } 513a88b5ba8SSam Ravnborg 514a88b5ba8SSam Ravnborg static int ldc_abort(struct ldc_channel *lp) 515a88b5ba8SSam Ravnborg { 516a88b5ba8SSam Ravnborg unsigned long hv_err; 517a88b5ba8SSam Ravnborg 518a88b5ba8SSam Ravnborg ldcdbg(STATE, "ABORT\n"); 519a88b5ba8SSam Ravnborg 520a88b5ba8SSam Ravnborg /* We report but do not act upon the hypervisor errors because 521a88b5ba8SSam Ravnborg * there really isn't much we can do if they fail at this point. 522a88b5ba8SSam Ravnborg */ 523a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); 524a88b5ba8SSam Ravnborg if (hv_err) 525a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 526a88b5ba8SSam Ravnborg "sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n", 527a88b5ba8SSam Ravnborg lp->id, lp->tx_ra, lp->tx_num_entries, hv_err); 528a88b5ba8SSam Ravnborg 529a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, 530a88b5ba8SSam Ravnborg &lp->tx_head, 531a88b5ba8SSam Ravnborg &lp->tx_tail, 532a88b5ba8SSam Ravnborg &lp->chan_state); 533a88b5ba8SSam Ravnborg if (hv_err) 534a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 535a88b5ba8SSam Ravnborg "sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n", 536a88b5ba8SSam Ravnborg lp->id, hv_err); 537a88b5ba8SSam Ravnborg 538a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); 539a88b5ba8SSam Ravnborg if (hv_err) 540a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 541a88b5ba8SSam Ravnborg "sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n", 542a88b5ba8SSam Ravnborg lp->id, lp->rx_ra, lp->rx_num_entries, hv_err); 543a88b5ba8SSam Ravnborg 544a88b5ba8SSam Ravnborg /* Refetch the RX queue state as well, because we could be invoked 545a88b5ba8SSam Ravnborg * here in the queue processing context. 546a88b5ba8SSam Ravnborg */ 547a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 548a88b5ba8SSam Ravnborg &lp->rx_head, 549a88b5ba8SSam Ravnborg &lp->rx_tail, 550a88b5ba8SSam Ravnborg &lp->chan_state); 551a88b5ba8SSam Ravnborg if (hv_err) 552a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 553a88b5ba8SSam Ravnborg "sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n", 554a88b5ba8SSam Ravnborg lp->id, hv_err); 555a88b5ba8SSam Ravnborg 556a88b5ba8SSam Ravnborg return -ECONNRESET; 557a88b5ba8SSam Ravnborg } 558a88b5ba8SSam Ravnborg 559a88b5ba8SSam Ravnborg static struct ldc_version *find_by_major(u16 major) 560a88b5ba8SSam Ravnborg { 561a88b5ba8SSam Ravnborg struct ldc_version *ret = NULL; 562a88b5ba8SSam Ravnborg int i; 563a88b5ba8SSam Ravnborg 564a88b5ba8SSam Ravnborg for (i = 0; i < ARRAY_SIZE(ver_arr); i++) { 565a88b5ba8SSam Ravnborg struct ldc_version *v = &ver_arr[i]; 566a88b5ba8SSam Ravnborg if (v->major <= major) { 567a88b5ba8SSam Ravnborg ret = v; 568a88b5ba8SSam Ravnborg break; 569a88b5ba8SSam Ravnborg } 570a88b5ba8SSam Ravnborg } 571a88b5ba8SSam Ravnborg return ret; 572a88b5ba8SSam Ravnborg } 573a88b5ba8SSam Ravnborg 574a88b5ba8SSam Ravnborg static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp) 575a88b5ba8SSam Ravnborg { 576a88b5ba8SSam Ravnborg struct ldc_version *vap; 577a88b5ba8SSam Ravnborg int err; 578a88b5ba8SSam Ravnborg 579a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n", 580a88b5ba8SSam Ravnborg vp->major, vp->minor); 581a88b5ba8SSam Ravnborg 582a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_GOTVERS) { 583a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN; 584a88b5ba8SSam Ravnborg memset(&lp->ver, 0, sizeof(lp->ver)); 585a88b5ba8SSam Ravnborg } 586a88b5ba8SSam Ravnborg 587a88b5ba8SSam Ravnborg vap = find_by_major(vp->major); 588a88b5ba8SSam Ravnborg if (!vap) { 589a88b5ba8SSam Ravnborg err = send_version_nack(lp, 0, 0); 590a88b5ba8SSam Ravnborg } else if (vap->major != vp->major) { 591a88b5ba8SSam Ravnborg err = send_version_nack(lp, vap->major, vap->minor); 592a88b5ba8SSam Ravnborg } else { 593a88b5ba8SSam Ravnborg struct ldc_version ver = *vp; 594a88b5ba8SSam Ravnborg if (ver.minor > vap->minor) 595a88b5ba8SSam Ravnborg ver.minor = vap->minor; 596a88b5ba8SSam Ravnborg err = send_version_ack(lp, &ver); 597a88b5ba8SSam Ravnborg if (!err) { 598a88b5ba8SSam Ravnborg lp->ver = ver; 599a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_GOTVERS; 600a88b5ba8SSam Ravnborg } 601a88b5ba8SSam Ravnborg } 602a88b5ba8SSam Ravnborg if (err) 603a88b5ba8SSam Ravnborg return ldc_abort(lp); 604a88b5ba8SSam Ravnborg 605a88b5ba8SSam Ravnborg return 0; 606a88b5ba8SSam Ravnborg } 607a88b5ba8SSam Ravnborg 608a88b5ba8SSam Ravnborg static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp) 609a88b5ba8SSam Ravnborg { 610a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n", 611a88b5ba8SSam Ravnborg vp->major, vp->minor); 612a88b5ba8SSam Ravnborg 613a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_GOTVERS) { 614a88b5ba8SSam Ravnborg if (lp->ver.major != vp->major || 615a88b5ba8SSam Ravnborg lp->ver.minor != vp->minor) 616a88b5ba8SSam Ravnborg return ldc_abort(lp); 617a88b5ba8SSam Ravnborg } else { 618a88b5ba8SSam Ravnborg lp->ver = *vp; 619a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_GOTVERS; 620a88b5ba8SSam Ravnborg } 621a88b5ba8SSam Ravnborg if (send_rts(lp)) 622a88b5ba8SSam Ravnborg return ldc_abort(lp); 623a88b5ba8SSam Ravnborg return 0; 624a88b5ba8SSam Ravnborg } 625a88b5ba8SSam Ravnborg 626a88b5ba8SSam Ravnborg static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp) 627a88b5ba8SSam Ravnborg { 628a88b5ba8SSam Ravnborg struct ldc_version *vap; 629a88b5ba8SSam Ravnborg struct ldc_packet *p; 630a88b5ba8SSam Ravnborg unsigned long new_tail; 631a88b5ba8SSam Ravnborg 632b2c0805fSSteven Rostedt if (vp->major == 0 && vp->minor == 0) 633b2c0805fSSteven Rostedt return ldc_abort(lp); 634b2c0805fSSteven Rostedt 635b2c0805fSSteven Rostedt vap = find_by_major(vp->major); 636b2c0805fSSteven Rostedt if (!vap) 637b2c0805fSSteven Rostedt return ldc_abort(lp); 638b2c0805fSSteven Rostedt 639a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS, 640a88b5ba8SSam Ravnborg vap, sizeof(*vap), 641a88b5ba8SSam Ravnborg &new_tail); 642b2c0805fSSteven Rostedt if (!p) 643a88b5ba8SSam Ravnborg return ldc_abort(lp); 644b2c0805fSSteven Rostedt 645b2c0805fSSteven Rostedt return send_tx_packet(lp, p, new_tail); 646a88b5ba8SSam Ravnborg } 647a88b5ba8SSam Ravnborg 648a88b5ba8SSam Ravnborg static int process_version(struct ldc_channel *lp, 649a88b5ba8SSam Ravnborg struct ldc_packet *p) 650a88b5ba8SSam Ravnborg { 651a88b5ba8SSam Ravnborg struct ldc_version *vp; 652a88b5ba8SSam Ravnborg 653a88b5ba8SSam Ravnborg vp = (struct ldc_version *) p->u.u_data; 654a88b5ba8SSam Ravnborg 655a88b5ba8SSam Ravnborg switch (p->stype) { 656a88b5ba8SSam Ravnborg case LDC_INFO: 657a88b5ba8SSam Ravnborg return process_ver_info(lp, vp); 658a88b5ba8SSam Ravnborg 659a88b5ba8SSam Ravnborg case LDC_ACK: 660a88b5ba8SSam Ravnborg return process_ver_ack(lp, vp); 661a88b5ba8SSam Ravnborg 662a88b5ba8SSam Ravnborg case LDC_NACK: 663a88b5ba8SSam Ravnborg return process_ver_nack(lp, vp); 664a88b5ba8SSam Ravnborg 665a88b5ba8SSam Ravnborg default: 666a88b5ba8SSam Ravnborg return ldc_abort(lp); 667a88b5ba8SSam Ravnborg } 668a88b5ba8SSam Ravnborg } 669a88b5ba8SSam Ravnborg 670a88b5ba8SSam Ravnborg static int process_rts(struct ldc_channel *lp, 671a88b5ba8SSam Ravnborg struct ldc_packet *p) 672a88b5ba8SSam Ravnborg { 673a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n", 674a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env); 675a88b5ba8SSam Ravnborg 676a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO || 677a88b5ba8SSam Ravnborg lp->hs_state != LDC_HS_GOTVERS || 678a88b5ba8SSam Ravnborg p->env != lp->cfg.mode) 679a88b5ba8SSam Ravnborg return ldc_abort(lp); 680a88b5ba8SSam Ravnborg 681a88b5ba8SSam Ravnborg lp->snd_nxt = p->seqid; 682a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid; 683a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_SENTRTR; 684a88b5ba8SSam Ravnborg if (send_rtr(lp)) 685a88b5ba8SSam Ravnborg return ldc_abort(lp); 686a88b5ba8SSam Ravnborg 687a88b5ba8SSam Ravnborg return 0; 688a88b5ba8SSam Ravnborg } 689a88b5ba8SSam Ravnborg 690a88b5ba8SSam Ravnborg static int process_rtr(struct ldc_channel *lp, 691a88b5ba8SSam Ravnborg struct ldc_packet *p) 692a88b5ba8SSam Ravnborg { 693a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n", 694a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env); 695a88b5ba8SSam Ravnborg 696a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO || 697a88b5ba8SSam Ravnborg p->env != lp->cfg.mode) 698a88b5ba8SSam Ravnborg return ldc_abort(lp); 699a88b5ba8SSam Ravnborg 700a88b5ba8SSam Ravnborg lp->snd_nxt = p->seqid; 701a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 702a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 703a88b5ba8SSam Ravnborg send_rdx(lp); 704a88b5ba8SSam Ravnborg 705a88b5ba8SSam Ravnborg return LDC_EVENT_UP; 706a88b5ba8SSam Ravnborg } 707a88b5ba8SSam Ravnborg 708a88b5ba8SSam Ravnborg static int rx_seq_ok(struct ldc_channel *lp, u32 seqid) 709a88b5ba8SSam Ravnborg { 710a88b5ba8SSam Ravnborg return lp->rcv_nxt + 1 == seqid; 711a88b5ba8SSam Ravnborg } 712a88b5ba8SSam Ravnborg 713a88b5ba8SSam Ravnborg static int process_rdx(struct ldc_channel *lp, 714a88b5ba8SSam Ravnborg struct ldc_packet *p) 715a88b5ba8SSam Ravnborg { 716a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n", 717a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env, p->u.r.ackid); 718a88b5ba8SSam Ravnborg 719a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO || 720a88b5ba8SSam Ravnborg !(rx_seq_ok(lp, p->seqid))) 721a88b5ba8SSam Ravnborg return ldc_abort(lp); 722a88b5ba8SSam Ravnborg 723a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid; 724a88b5ba8SSam Ravnborg 725a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 726a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 727a88b5ba8SSam Ravnborg 728a88b5ba8SSam Ravnborg return LDC_EVENT_UP; 729a88b5ba8SSam Ravnborg } 730a88b5ba8SSam Ravnborg 731a88b5ba8SSam Ravnborg static int process_control_frame(struct ldc_channel *lp, 732a88b5ba8SSam Ravnborg struct ldc_packet *p) 733a88b5ba8SSam Ravnborg { 734a88b5ba8SSam Ravnborg switch (p->ctrl) { 735a88b5ba8SSam Ravnborg case LDC_VERS: 736a88b5ba8SSam Ravnborg return process_version(lp, p); 737a88b5ba8SSam Ravnborg 738a88b5ba8SSam Ravnborg case LDC_RTS: 739a88b5ba8SSam Ravnborg return process_rts(lp, p); 740a88b5ba8SSam Ravnborg 741a88b5ba8SSam Ravnborg case LDC_RTR: 742a88b5ba8SSam Ravnborg return process_rtr(lp, p); 743a88b5ba8SSam Ravnborg 744a88b5ba8SSam Ravnborg case LDC_RDX: 745a88b5ba8SSam Ravnborg return process_rdx(lp, p); 746a88b5ba8SSam Ravnborg 747a88b5ba8SSam Ravnborg default: 748a88b5ba8SSam Ravnborg return ldc_abort(lp); 749a88b5ba8SSam Ravnborg } 750a88b5ba8SSam Ravnborg } 751a88b5ba8SSam Ravnborg 752a88b5ba8SSam Ravnborg static int process_error_frame(struct ldc_channel *lp, 753a88b5ba8SSam Ravnborg struct ldc_packet *p) 754a88b5ba8SSam Ravnborg { 755a88b5ba8SSam Ravnborg return ldc_abort(lp); 756a88b5ba8SSam Ravnborg } 757a88b5ba8SSam Ravnborg 758a88b5ba8SSam Ravnborg static int process_data_ack(struct ldc_channel *lp, 759a88b5ba8SSam Ravnborg struct ldc_packet *ack) 760a88b5ba8SSam Ravnborg { 761a88b5ba8SSam Ravnborg unsigned long head = lp->tx_acked; 762a88b5ba8SSam Ravnborg u32 ackid = ack->u.r.ackid; 763a88b5ba8SSam Ravnborg 764a88b5ba8SSam Ravnborg while (1) { 765a88b5ba8SSam Ravnborg struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE); 766a88b5ba8SSam Ravnborg 767a88b5ba8SSam Ravnborg head = tx_advance(lp, head); 768a88b5ba8SSam Ravnborg 769a88b5ba8SSam Ravnborg if (p->seqid == ackid) { 770a88b5ba8SSam Ravnborg lp->tx_acked = head; 771a88b5ba8SSam Ravnborg return 0; 772a88b5ba8SSam Ravnborg } 773a88b5ba8SSam Ravnborg if (head == lp->tx_tail) 774a88b5ba8SSam Ravnborg return ldc_abort(lp); 775a88b5ba8SSam Ravnborg } 776a88b5ba8SSam Ravnborg 777a88b5ba8SSam Ravnborg return 0; 778a88b5ba8SSam Ravnborg } 779a88b5ba8SSam Ravnborg 780a88b5ba8SSam Ravnborg static void send_events(struct ldc_channel *lp, unsigned int event_mask) 781a88b5ba8SSam Ravnborg { 782a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_RESET) 783a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_RESET); 784a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_UP) 785a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_UP); 786a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_DATA_READY) 787a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY); 788a88b5ba8SSam Ravnborg } 789a88b5ba8SSam Ravnborg 790a88b5ba8SSam Ravnborg static irqreturn_t ldc_rx(int irq, void *dev_id) 791a88b5ba8SSam Ravnborg { 792a88b5ba8SSam Ravnborg struct ldc_channel *lp = dev_id; 793c6fee081SDavid S. Miller unsigned long orig_state, flags; 794a88b5ba8SSam Ravnborg unsigned int event_mask; 795a88b5ba8SSam Ravnborg 796a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 797a88b5ba8SSam Ravnborg 798a88b5ba8SSam Ravnborg orig_state = lp->chan_state; 799c6fee081SDavid S. Miller 800c6fee081SDavid S. Miller /* We should probably check for hypervisor errors here and 801c6fee081SDavid S. Miller * reset the LDC channel if we get one. 802c6fee081SDavid S. Miller */ 803c6fee081SDavid S. Miller sun4v_ldc_rx_get_state(lp->id, 804a88b5ba8SSam Ravnborg &lp->rx_head, 805a88b5ba8SSam Ravnborg &lp->rx_tail, 806a88b5ba8SSam Ravnborg &lp->chan_state); 807a88b5ba8SSam Ravnborg 808a88b5ba8SSam Ravnborg ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n", 809a88b5ba8SSam Ravnborg orig_state, lp->chan_state, lp->rx_head, lp->rx_tail); 810a88b5ba8SSam Ravnborg 811a88b5ba8SSam Ravnborg event_mask = 0; 812a88b5ba8SSam Ravnborg 813a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW && 814a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_UP) { 815a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 816a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 817a88b5ba8SSam Ravnborg 818a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_UP; 819a88b5ba8SSam Ravnborg 820a88b5ba8SSam Ravnborg orig_state = lp->chan_state; 821a88b5ba8SSam Ravnborg } 822a88b5ba8SSam Ravnborg 823a88b5ba8SSam Ravnborg /* If we are in reset state, flush the RX queue and ignore 824a88b5ba8SSam Ravnborg * everything. 825a88b5ba8SSam Ravnborg */ 826a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_RESET) { 827a88b5ba8SSam Ravnborg (void) __set_rx_head(lp, lp->rx_tail); 828a88b5ba8SSam Ravnborg goto out; 829a88b5ba8SSam Ravnborg } 830a88b5ba8SSam Ravnborg 831a88b5ba8SSam Ravnborg /* Once we finish the handshake, we let the ldc_read() 832a88b5ba8SSam Ravnborg * paths do all of the control frame and state management. 833a88b5ba8SSam Ravnborg * Just trigger the callback. 834a88b5ba8SSam Ravnborg */ 835a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_COMPLETE) { 836a88b5ba8SSam Ravnborg handshake_complete: 837a88b5ba8SSam Ravnborg if (lp->chan_state != orig_state) { 838a88b5ba8SSam Ravnborg unsigned int event = LDC_EVENT_RESET; 839a88b5ba8SSam Ravnborg 840a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_UP) 841a88b5ba8SSam Ravnborg event = LDC_EVENT_UP; 842a88b5ba8SSam Ravnborg 843a88b5ba8SSam Ravnborg event_mask |= event; 844a88b5ba8SSam Ravnborg } 845a88b5ba8SSam Ravnborg if (lp->rx_head != lp->rx_tail) 846a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_DATA_READY; 847a88b5ba8SSam Ravnborg 848a88b5ba8SSam Ravnborg goto out; 849a88b5ba8SSam Ravnborg } 850a88b5ba8SSam Ravnborg 851a88b5ba8SSam Ravnborg if (lp->chan_state != orig_state) 852a88b5ba8SSam Ravnborg goto out; 853a88b5ba8SSam Ravnborg 854a88b5ba8SSam Ravnborg while (lp->rx_head != lp->rx_tail) { 855a88b5ba8SSam Ravnborg struct ldc_packet *p; 856a88b5ba8SSam Ravnborg unsigned long new; 857a88b5ba8SSam Ravnborg int err; 858a88b5ba8SSam Ravnborg 859a88b5ba8SSam Ravnborg p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE); 860a88b5ba8SSam Ravnborg 861a88b5ba8SSam Ravnborg switch (p->type) { 862a88b5ba8SSam Ravnborg case LDC_CTRL: 863a88b5ba8SSam Ravnborg err = process_control_frame(lp, p); 864a88b5ba8SSam Ravnborg if (err > 0) 865a88b5ba8SSam Ravnborg event_mask |= err; 866a88b5ba8SSam Ravnborg break; 867a88b5ba8SSam Ravnborg 868a88b5ba8SSam Ravnborg case LDC_DATA: 869a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_DATA_READY; 870a88b5ba8SSam Ravnborg err = 0; 871a88b5ba8SSam Ravnborg break; 872a88b5ba8SSam Ravnborg 873a88b5ba8SSam Ravnborg case LDC_ERR: 874a88b5ba8SSam Ravnborg err = process_error_frame(lp, p); 875a88b5ba8SSam Ravnborg break; 876a88b5ba8SSam Ravnborg 877a88b5ba8SSam Ravnborg default: 878a88b5ba8SSam Ravnborg err = ldc_abort(lp); 879a88b5ba8SSam Ravnborg break; 880a88b5ba8SSam Ravnborg } 881a88b5ba8SSam Ravnborg 882a88b5ba8SSam Ravnborg if (err < 0) 883a88b5ba8SSam Ravnborg break; 884a88b5ba8SSam Ravnborg 885a88b5ba8SSam Ravnborg new = lp->rx_head; 886a88b5ba8SSam Ravnborg new += LDC_PACKET_SIZE; 887a88b5ba8SSam Ravnborg if (new == (lp->rx_num_entries * LDC_PACKET_SIZE)) 888a88b5ba8SSam Ravnborg new = 0; 889a88b5ba8SSam Ravnborg lp->rx_head = new; 890a88b5ba8SSam Ravnborg 891a88b5ba8SSam Ravnborg err = __set_rx_head(lp, new); 892a88b5ba8SSam Ravnborg if (err < 0) { 893a88b5ba8SSam Ravnborg (void) ldc_abort(lp); 894a88b5ba8SSam Ravnborg break; 895a88b5ba8SSam Ravnborg } 896a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_COMPLETE) 897a88b5ba8SSam Ravnborg goto handshake_complete; 898a88b5ba8SSam Ravnborg } 899a88b5ba8SSam Ravnborg 900a88b5ba8SSam Ravnborg out: 901a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 902a88b5ba8SSam Ravnborg 903a88b5ba8SSam Ravnborg send_events(lp, event_mask); 904a88b5ba8SSam Ravnborg 905a88b5ba8SSam Ravnborg return IRQ_HANDLED; 906a88b5ba8SSam Ravnborg } 907a88b5ba8SSam Ravnborg 908a88b5ba8SSam Ravnborg static irqreturn_t ldc_tx(int irq, void *dev_id) 909a88b5ba8SSam Ravnborg { 910a88b5ba8SSam Ravnborg struct ldc_channel *lp = dev_id; 911c6fee081SDavid S. Miller unsigned long flags, orig_state; 912a88b5ba8SSam Ravnborg unsigned int event_mask = 0; 913a88b5ba8SSam Ravnborg 914a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 915a88b5ba8SSam Ravnborg 916a88b5ba8SSam Ravnborg orig_state = lp->chan_state; 917c6fee081SDavid S. Miller 918c6fee081SDavid S. Miller /* We should probably check for hypervisor errors here and 919c6fee081SDavid S. Miller * reset the LDC channel if we get one. 920c6fee081SDavid S. Miller */ 921c6fee081SDavid S. Miller sun4v_ldc_tx_get_state(lp->id, 922a88b5ba8SSam Ravnborg &lp->tx_head, 923a88b5ba8SSam Ravnborg &lp->tx_tail, 924a88b5ba8SSam Ravnborg &lp->chan_state); 925a88b5ba8SSam Ravnborg 926a88b5ba8SSam Ravnborg ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n", 927a88b5ba8SSam Ravnborg orig_state, lp->chan_state, lp->tx_head, lp->tx_tail); 928a88b5ba8SSam Ravnborg 929a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW && 930a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_UP) { 931a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 932a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 933a88b5ba8SSam Ravnborg 934a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_UP; 935a88b5ba8SSam Ravnborg } 936a88b5ba8SSam Ravnborg 937a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 938a88b5ba8SSam Ravnborg 939a88b5ba8SSam Ravnborg send_events(lp, event_mask); 940a88b5ba8SSam Ravnborg 941a88b5ba8SSam Ravnborg return IRQ_HANDLED; 942a88b5ba8SSam Ravnborg } 943a88b5ba8SSam Ravnborg 944a88b5ba8SSam Ravnborg /* XXX ldc_alloc() and ldc_free() needs to run under a mutex so 945a88b5ba8SSam Ravnborg * XXX that addition and removal from the ldc_channel_list has 946a88b5ba8SSam Ravnborg * XXX atomicity, otherwise the __ldc_channel_exists() check is 947a88b5ba8SSam Ravnborg * XXX totally pointless as another thread can slip into ldc_alloc() 948a88b5ba8SSam Ravnborg * XXX and add a channel with the same ID. There also needs to be 949a88b5ba8SSam Ravnborg * XXX a spinlock for ldc_channel_list. 950a88b5ba8SSam Ravnborg */ 951a88b5ba8SSam Ravnborg static HLIST_HEAD(ldc_channel_list); 952a88b5ba8SSam Ravnborg 953a88b5ba8SSam Ravnborg static int __ldc_channel_exists(unsigned long id) 954a88b5ba8SSam Ravnborg { 955a88b5ba8SSam Ravnborg struct ldc_channel *lp; 956a88b5ba8SSam Ravnborg 957b67bfe0dSSasha Levin hlist_for_each_entry(lp, &ldc_channel_list, list) { 958a88b5ba8SSam Ravnborg if (lp->id == id) 959a88b5ba8SSam Ravnborg return 1; 960a88b5ba8SSam Ravnborg } 961a88b5ba8SSam Ravnborg return 0; 962a88b5ba8SSam Ravnborg } 963a88b5ba8SSam Ravnborg 964a88b5ba8SSam Ravnborg static int alloc_queue(const char *name, unsigned long num_entries, 965a88b5ba8SSam Ravnborg struct ldc_packet **base, unsigned long *ra) 966a88b5ba8SSam Ravnborg { 967a88b5ba8SSam Ravnborg unsigned long size, order; 968a88b5ba8SSam Ravnborg void *q; 969a88b5ba8SSam Ravnborg 970a88b5ba8SSam Ravnborg size = num_entries * LDC_PACKET_SIZE; 971a88b5ba8SSam Ravnborg order = get_order(size); 972a88b5ba8SSam Ravnborg 973a88b5ba8SSam Ravnborg q = (void *) __get_free_pages(GFP_KERNEL, order); 974a88b5ba8SSam Ravnborg if (!q) { 975a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of %s queue failed with " 976a88b5ba8SSam Ravnborg "size=%lu order=%lu\n", name, size, order); 977a88b5ba8SSam Ravnborg return -ENOMEM; 978a88b5ba8SSam Ravnborg } 979a88b5ba8SSam Ravnborg 980a88b5ba8SSam Ravnborg memset(q, 0, PAGE_SIZE << order); 981a88b5ba8SSam Ravnborg 982a88b5ba8SSam Ravnborg *base = q; 983a88b5ba8SSam Ravnborg *ra = __pa(q); 984a88b5ba8SSam Ravnborg 985a88b5ba8SSam Ravnborg return 0; 986a88b5ba8SSam Ravnborg } 987a88b5ba8SSam Ravnborg 988a88b5ba8SSam Ravnborg static void free_queue(unsigned long num_entries, struct ldc_packet *q) 989a88b5ba8SSam Ravnborg { 990a88b5ba8SSam Ravnborg unsigned long size, order; 991a88b5ba8SSam Ravnborg 992a88b5ba8SSam Ravnborg if (!q) 993a88b5ba8SSam Ravnborg return; 994a88b5ba8SSam Ravnborg 995a88b5ba8SSam Ravnborg size = num_entries * LDC_PACKET_SIZE; 996a88b5ba8SSam Ravnborg order = get_order(size); 997a88b5ba8SSam Ravnborg 998a88b5ba8SSam Ravnborg free_pages((unsigned long)q, order); 999a88b5ba8SSam Ravnborg } 1000a88b5ba8SSam Ravnborg 1001a88b5ba8SSam Ravnborg /* XXX Make this configurable... XXX */ 1002a88b5ba8SSam Ravnborg #define LDC_IOTABLE_SIZE (8 * 1024) 1003a88b5ba8SSam Ravnborg 1004a88b5ba8SSam Ravnborg static int ldc_iommu_init(struct ldc_channel *lp) 1005a88b5ba8SSam Ravnborg { 1006a88b5ba8SSam Ravnborg unsigned long sz, num_tsb_entries, tsbsize, order; 1007a88b5ba8SSam Ravnborg struct ldc_iommu *iommu = &lp->iommu; 1008a88b5ba8SSam Ravnborg struct ldc_mtable_entry *table; 1009a88b5ba8SSam Ravnborg unsigned long hv_err; 1010a88b5ba8SSam Ravnborg int err; 1011a88b5ba8SSam Ravnborg 1012a88b5ba8SSam Ravnborg num_tsb_entries = LDC_IOTABLE_SIZE; 1013a88b5ba8SSam Ravnborg tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); 1014a88b5ba8SSam Ravnborg 1015a88b5ba8SSam Ravnborg spin_lock_init(&iommu->lock); 1016a88b5ba8SSam Ravnborg 1017a88b5ba8SSam Ravnborg sz = num_tsb_entries / 8; 1018a88b5ba8SSam Ravnborg sz = (sz + 7UL) & ~7UL; 1019a88b5ba8SSam Ravnborg iommu->arena.map = kzalloc(sz, GFP_KERNEL); 1020a88b5ba8SSam Ravnborg if (!iommu->arena.map) { 1021a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz); 1022a88b5ba8SSam Ravnborg return -ENOMEM; 1023a88b5ba8SSam Ravnborg } 1024a88b5ba8SSam Ravnborg 1025a88b5ba8SSam Ravnborg iommu->arena.limit = num_tsb_entries; 1026a88b5ba8SSam Ravnborg 1027a88b5ba8SSam Ravnborg order = get_order(tsbsize); 1028a88b5ba8SSam Ravnborg 1029a88b5ba8SSam Ravnborg table = (struct ldc_mtable_entry *) 1030a88b5ba8SSam Ravnborg __get_free_pages(GFP_KERNEL, order); 1031a88b5ba8SSam Ravnborg err = -ENOMEM; 1032a88b5ba8SSam Ravnborg if (!table) { 1033a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of MTE table failed, " 1034a88b5ba8SSam Ravnborg "size=%lu order=%lu\n", tsbsize, order); 1035a88b5ba8SSam Ravnborg goto out_free_map; 1036a88b5ba8SSam Ravnborg } 1037a88b5ba8SSam Ravnborg 1038a88b5ba8SSam Ravnborg memset(table, 0, PAGE_SIZE << order); 1039a88b5ba8SSam Ravnborg 1040a88b5ba8SSam Ravnborg iommu->page_table = table; 1041a88b5ba8SSam Ravnborg 1042a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table), 1043a88b5ba8SSam Ravnborg num_tsb_entries); 1044a88b5ba8SSam Ravnborg err = -EINVAL; 1045a88b5ba8SSam Ravnborg if (hv_err) 1046a88b5ba8SSam Ravnborg goto out_free_table; 1047a88b5ba8SSam Ravnborg 1048a88b5ba8SSam Ravnborg return 0; 1049a88b5ba8SSam Ravnborg 1050a88b5ba8SSam Ravnborg out_free_table: 1051a88b5ba8SSam Ravnborg free_pages((unsigned long) table, order); 1052a88b5ba8SSam Ravnborg iommu->page_table = NULL; 1053a88b5ba8SSam Ravnborg 1054a88b5ba8SSam Ravnborg out_free_map: 1055a88b5ba8SSam Ravnborg kfree(iommu->arena.map); 1056a88b5ba8SSam Ravnborg iommu->arena.map = NULL; 1057a88b5ba8SSam Ravnborg 1058a88b5ba8SSam Ravnborg return err; 1059a88b5ba8SSam Ravnborg } 1060a88b5ba8SSam Ravnborg 1061a88b5ba8SSam Ravnborg static void ldc_iommu_release(struct ldc_channel *lp) 1062a88b5ba8SSam Ravnborg { 1063a88b5ba8SSam Ravnborg struct ldc_iommu *iommu = &lp->iommu; 1064a88b5ba8SSam Ravnborg unsigned long num_tsb_entries, tsbsize, order; 1065a88b5ba8SSam Ravnborg 1066a88b5ba8SSam Ravnborg (void) sun4v_ldc_set_map_table(lp->id, 0, 0); 1067a88b5ba8SSam Ravnborg 1068a88b5ba8SSam Ravnborg num_tsb_entries = iommu->arena.limit; 1069a88b5ba8SSam Ravnborg tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); 1070a88b5ba8SSam Ravnborg order = get_order(tsbsize); 1071a88b5ba8SSam Ravnborg 1072a88b5ba8SSam Ravnborg free_pages((unsigned long) iommu->page_table, order); 1073a88b5ba8SSam Ravnborg iommu->page_table = NULL; 1074a88b5ba8SSam Ravnborg 1075a88b5ba8SSam Ravnborg kfree(iommu->arena.map); 1076a88b5ba8SSam Ravnborg iommu->arena.map = NULL; 1077a88b5ba8SSam Ravnborg } 1078a88b5ba8SSam Ravnborg 1079a88b5ba8SSam Ravnborg struct ldc_channel *ldc_alloc(unsigned long id, 1080a88b5ba8SSam Ravnborg const struct ldc_channel_config *cfgp, 1081a88b5ba8SSam Ravnborg void *event_arg) 1082a88b5ba8SSam Ravnborg { 1083a88b5ba8SSam Ravnborg struct ldc_channel *lp; 1084a88b5ba8SSam Ravnborg const struct ldc_mode_ops *mops; 1085a88b5ba8SSam Ravnborg unsigned long dummy1, dummy2, hv_err; 1086a88b5ba8SSam Ravnborg u8 mss, *mssbuf; 1087a88b5ba8SSam Ravnborg int err; 1088a88b5ba8SSam Ravnborg 1089a88b5ba8SSam Ravnborg err = -ENODEV; 1090a88b5ba8SSam Ravnborg if (!ldom_domaining_enabled) 1091a88b5ba8SSam Ravnborg goto out_err; 1092a88b5ba8SSam Ravnborg 1093a88b5ba8SSam Ravnborg err = -EINVAL; 1094a88b5ba8SSam Ravnborg if (!cfgp) 1095a88b5ba8SSam Ravnborg goto out_err; 1096a88b5ba8SSam Ravnborg 1097a88b5ba8SSam Ravnborg switch (cfgp->mode) { 1098a88b5ba8SSam Ravnborg case LDC_MODE_RAW: 1099a88b5ba8SSam Ravnborg mops = &raw_ops; 1100a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE; 1101a88b5ba8SSam Ravnborg break; 1102a88b5ba8SSam Ravnborg 1103a88b5ba8SSam Ravnborg case LDC_MODE_UNRELIABLE: 1104a88b5ba8SSam Ravnborg mops = &nonraw_ops; 1105a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE - 8; 1106a88b5ba8SSam Ravnborg break; 1107a88b5ba8SSam Ravnborg 1108a88b5ba8SSam Ravnborg case LDC_MODE_STREAM: 1109a88b5ba8SSam Ravnborg mops = &stream_ops; 1110a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE - 8 - 8; 1111a88b5ba8SSam Ravnborg break; 1112a88b5ba8SSam Ravnborg 1113a88b5ba8SSam Ravnborg default: 1114a88b5ba8SSam Ravnborg goto out_err; 1115a88b5ba8SSam Ravnborg } 1116a88b5ba8SSam Ravnborg 1117a88b5ba8SSam Ravnborg if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq) 1118a88b5ba8SSam Ravnborg goto out_err; 1119a88b5ba8SSam Ravnborg 1120a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2); 1121a88b5ba8SSam Ravnborg err = -ENODEV; 1122a88b5ba8SSam Ravnborg if (hv_err == HV_ECHANNEL) 1123a88b5ba8SSam Ravnborg goto out_err; 1124a88b5ba8SSam Ravnborg 1125a88b5ba8SSam Ravnborg err = -EEXIST; 1126a88b5ba8SSam Ravnborg if (__ldc_channel_exists(id)) 1127a88b5ba8SSam Ravnborg goto out_err; 1128a88b5ba8SSam Ravnborg 1129a88b5ba8SSam Ravnborg mssbuf = NULL; 1130a88b5ba8SSam Ravnborg 1131a88b5ba8SSam Ravnborg lp = kzalloc(sizeof(*lp), GFP_KERNEL); 1132a88b5ba8SSam Ravnborg err = -ENOMEM; 1133a88b5ba8SSam Ravnborg if (!lp) 1134a88b5ba8SSam Ravnborg goto out_err; 1135a88b5ba8SSam Ravnborg 1136a88b5ba8SSam Ravnborg spin_lock_init(&lp->lock); 1137a88b5ba8SSam Ravnborg 1138a88b5ba8SSam Ravnborg lp->id = id; 1139a88b5ba8SSam Ravnborg 1140a88b5ba8SSam Ravnborg err = ldc_iommu_init(lp); 1141a88b5ba8SSam Ravnborg if (err) 1142a88b5ba8SSam Ravnborg goto out_free_ldc; 1143a88b5ba8SSam Ravnborg 1144a88b5ba8SSam Ravnborg lp->mops = mops; 1145a88b5ba8SSam Ravnborg lp->mss = mss; 1146a88b5ba8SSam Ravnborg 1147a88b5ba8SSam Ravnborg lp->cfg = *cfgp; 1148a88b5ba8SSam Ravnborg if (!lp->cfg.mtu) 1149a88b5ba8SSam Ravnborg lp->cfg.mtu = LDC_DEFAULT_MTU; 1150a88b5ba8SSam Ravnborg 1151a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_STREAM) { 1152a88b5ba8SSam Ravnborg mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL); 1153a88b5ba8SSam Ravnborg if (!mssbuf) { 1154a88b5ba8SSam Ravnborg err = -ENOMEM; 1155a88b5ba8SSam Ravnborg goto out_free_iommu; 1156a88b5ba8SSam Ravnborg } 1157a88b5ba8SSam Ravnborg lp->mssbuf = mssbuf; 1158a88b5ba8SSam Ravnborg } 1159a88b5ba8SSam Ravnborg 1160a88b5ba8SSam Ravnborg lp->event_arg = event_arg; 1161a88b5ba8SSam Ravnborg 1162a88b5ba8SSam Ravnborg /* XXX allow setting via ldc_channel_config to override defaults 1163a88b5ba8SSam Ravnborg * XXX or use some formula based upon mtu 1164a88b5ba8SSam Ravnborg */ 1165a88b5ba8SSam Ravnborg lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES; 1166a88b5ba8SSam Ravnborg lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES; 1167a88b5ba8SSam Ravnborg 1168a88b5ba8SSam Ravnborg err = alloc_queue("TX", lp->tx_num_entries, 1169a88b5ba8SSam Ravnborg &lp->tx_base, &lp->tx_ra); 1170a88b5ba8SSam Ravnborg if (err) 1171a88b5ba8SSam Ravnborg goto out_free_mssbuf; 1172a88b5ba8SSam Ravnborg 1173a88b5ba8SSam Ravnborg err = alloc_queue("RX", lp->rx_num_entries, 1174a88b5ba8SSam Ravnborg &lp->rx_base, &lp->rx_ra); 1175a88b5ba8SSam Ravnborg if (err) 1176a88b5ba8SSam Ravnborg goto out_free_txq; 1177a88b5ba8SSam Ravnborg 1178a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_ALLOCED_QUEUES; 1179a88b5ba8SSam Ravnborg 1180a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_CLOSED; 1181a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_INIT); 1182a88b5ba8SSam Ravnborg 1183a88b5ba8SSam Ravnborg INIT_HLIST_NODE(&lp->list); 1184a88b5ba8SSam Ravnborg hlist_add_head(&lp->list, &ldc_channel_list); 1185a88b5ba8SSam Ravnborg 1186a88b5ba8SSam Ravnborg INIT_HLIST_HEAD(&lp->mh_list); 1187a88b5ba8SSam Ravnborg 1188a88b5ba8SSam Ravnborg return lp; 1189a88b5ba8SSam Ravnborg 1190a88b5ba8SSam Ravnborg out_free_txq: 1191a88b5ba8SSam Ravnborg free_queue(lp->tx_num_entries, lp->tx_base); 1192a88b5ba8SSam Ravnborg 1193a88b5ba8SSam Ravnborg out_free_mssbuf: 1194a88b5ba8SSam Ravnborg kfree(mssbuf); 1195a88b5ba8SSam Ravnborg 1196a88b5ba8SSam Ravnborg out_free_iommu: 1197a88b5ba8SSam Ravnborg ldc_iommu_release(lp); 1198a88b5ba8SSam Ravnborg 1199a88b5ba8SSam Ravnborg out_free_ldc: 1200a88b5ba8SSam Ravnborg kfree(lp); 1201a88b5ba8SSam Ravnborg 1202a88b5ba8SSam Ravnborg out_err: 1203a88b5ba8SSam Ravnborg return ERR_PTR(err); 1204a88b5ba8SSam Ravnborg } 1205a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_alloc); 1206a88b5ba8SSam Ravnborg 1207a88b5ba8SSam Ravnborg void ldc_free(struct ldc_channel *lp) 1208a88b5ba8SSam Ravnborg { 1209a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_REGISTERED_IRQS) { 1210a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1211a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp); 1212a88b5ba8SSam Ravnborg } 1213a88b5ba8SSam Ravnborg 1214a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) { 1215a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0); 1216a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0); 1217a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES; 1218a88b5ba8SSam Ravnborg } 1219a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) { 1220a88b5ba8SSam Ravnborg free_queue(lp->tx_num_entries, lp->tx_base); 1221a88b5ba8SSam Ravnborg free_queue(lp->rx_num_entries, lp->rx_base); 1222a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES; 1223a88b5ba8SSam Ravnborg } 1224a88b5ba8SSam Ravnborg 1225a88b5ba8SSam Ravnborg hlist_del(&lp->list); 1226a88b5ba8SSam Ravnborg 1227a88b5ba8SSam Ravnborg kfree(lp->mssbuf); 1228a88b5ba8SSam Ravnborg 1229a88b5ba8SSam Ravnborg ldc_iommu_release(lp); 1230a88b5ba8SSam Ravnborg 1231a88b5ba8SSam Ravnborg kfree(lp); 1232a88b5ba8SSam Ravnborg } 1233a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_free); 1234a88b5ba8SSam Ravnborg 1235a88b5ba8SSam Ravnborg /* Bind the channel. This registers the LDC queues with 1236a88b5ba8SSam Ravnborg * the hypervisor and puts the channel into a pseudo-listening 1237a88b5ba8SSam Ravnborg * state. This does not initiate a handshake, ldc_connect() does 1238a88b5ba8SSam Ravnborg * that. 1239a88b5ba8SSam Ravnborg */ 1240a88b5ba8SSam Ravnborg int ldc_bind(struct ldc_channel *lp, const char *name) 1241a88b5ba8SSam Ravnborg { 1242a88b5ba8SSam Ravnborg unsigned long hv_err, flags; 1243a88b5ba8SSam Ravnborg int err = -EINVAL; 1244a88b5ba8SSam Ravnborg 1245a88b5ba8SSam Ravnborg if (!name || 1246a88b5ba8SSam Ravnborg (lp->state != LDC_STATE_INIT)) 1247a88b5ba8SSam Ravnborg return -EINVAL; 1248a88b5ba8SSam Ravnborg 1249a88b5ba8SSam Ravnborg snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name); 1250a88b5ba8SSam Ravnborg snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name); 1251a88b5ba8SSam Ravnborg 125271ba8d44STheodore Ts'o err = request_irq(lp->cfg.rx_irq, ldc_rx, IRQF_DISABLED, 1253a88b5ba8SSam Ravnborg lp->rx_irq_name, lp); 1254a88b5ba8SSam Ravnborg if (err) 1255a88b5ba8SSam Ravnborg return err; 1256a88b5ba8SSam Ravnborg 125771ba8d44STheodore Ts'o err = request_irq(lp->cfg.tx_irq, ldc_tx, IRQF_DISABLED, 1258a88b5ba8SSam Ravnborg lp->tx_irq_name, lp); 1259a88b5ba8SSam Ravnborg if (err) { 1260a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1261a88b5ba8SSam Ravnborg return err; 1262a88b5ba8SSam Ravnborg } 1263a88b5ba8SSam Ravnborg 1264a88b5ba8SSam Ravnborg 1265a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1266a88b5ba8SSam Ravnborg 1267a88b5ba8SSam Ravnborg enable_irq(lp->cfg.rx_irq); 1268a88b5ba8SSam Ravnborg enable_irq(lp->cfg.tx_irq); 1269a88b5ba8SSam Ravnborg 1270a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_REGISTERED_IRQS; 1271a88b5ba8SSam Ravnborg 1272a88b5ba8SSam Ravnborg err = -ENODEV; 1273a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0); 1274a88b5ba8SSam Ravnborg if (hv_err) 1275a88b5ba8SSam Ravnborg goto out_free_irqs; 1276a88b5ba8SSam Ravnborg 1277a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); 1278a88b5ba8SSam Ravnborg if (hv_err) 1279a88b5ba8SSam Ravnborg goto out_free_irqs; 1280a88b5ba8SSam Ravnborg 1281a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0); 1282a88b5ba8SSam Ravnborg if (hv_err) 1283a88b5ba8SSam Ravnborg goto out_unmap_tx; 1284a88b5ba8SSam Ravnborg 1285a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); 1286a88b5ba8SSam Ravnborg if (hv_err) 1287a88b5ba8SSam Ravnborg goto out_unmap_tx; 1288a88b5ba8SSam Ravnborg 1289a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_REGISTERED_QUEUES; 1290a88b5ba8SSam Ravnborg 1291a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, 1292a88b5ba8SSam Ravnborg &lp->tx_head, 1293a88b5ba8SSam Ravnborg &lp->tx_tail, 1294a88b5ba8SSam Ravnborg &lp->chan_state); 1295a88b5ba8SSam Ravnborg err = -EBUSY; 1296a88b5ba8SSam Ravnborg if (hv_err) 1297a88b5ba8SSam Ravnborg goto out_unmap_rx; 1298a88b5ba8SSam Ravnborg 1299a88b5ba8SSam Ravnborg lp->tx_acked = lp->tx_head; 1300a88b5ba8SSam Ravnborg 1301a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN; 1302a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_BOUND); 1303a88b5ba8SSam Ravnborg 1304a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1305a88b5ba8SSam Ravnborg 1306a88b5ba8SSam Ravnborg return 0; 1307a88b5ba8SSam Ravnborg 1308a88b5ba8SSam Ravnborg out_unmap_rx: 1309a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES; 1310a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0); 1311a88b5ba8SSam Ravnborg 1312a88b5ba8SSam Ravnborg out_unmap_tx: 1313a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0); 1314a88b5ba8SSam Ravnborg 1315a88b5ba8SSam Ravnborg out_free_irqs: 1316a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_IRQS; 1317a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp); 1318a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1319a88b5ba8SSam Ravnborg 1320a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1321a88b5ba8SSam Ravnborg 1322a88b5ba8SSam Ravnborg return err; 1323a88b5ba8SSam Ravnborg } 1324a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_bind); 1325a88b5ba8SSam Ravnborg 1326a88b5ba8SSam Ravnborg int ldc_connect(struct ldc_channel *lp) 1327a88b5ba8SSam Ravnborg { 1328a88b5ba8SSam Ravnborg unsigned long flags; 1329a88b5ba8SSam Ravnborg int err; 1330a88b5ba8SSam Ravnborg 1331a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW) 1332a88b5ba8SSam Ravnborg return -EINVAL; 1333a88b5ba8SSam Ravnborg 1334a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1335a88b5ba8SSam Ravnborg 1336a88b5ba8SSam Ravnborg if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || 1337a88b5ba8SSam Ravnborg !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) || 1338a88b5ba8SSam Ravnborg lp->hs_state != LDC_HS_OPEN) 1339a88b5ba8SSam Ravnborg err = -EINVAL; 1340a88b5ba8SSam Ravnborg else 1341a88b5ba8SSam Ravnborg err = start_handshake(lp); 1342a88b5ba8SSam Ravnborg 1343a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1344a88b5ba8SSam Ravnborg 1345a88b5ba8SSam Ravnborg return err; 1346a88b5ba8SSam Ravnborg } 1347a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_connect); 1348a88b5ba8SSam Ravnborg 1349a88b5ba8SSam Ravnborg int ldc_disconnect(struct ldc_channel *lp) 1350a88b5ba8SSam Ravnborg { 1351a88b5ba8SSam Ravnborg unsigned long hv_err, flags; 1352a88b5ba8SSam Ravnborg int err; 1353a88b5ba8SSam Ravnborg 1354a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW) 1355a88b5ba8SSam Ravnborg return -EINVAL; 1356a88b5ba8SSam Ravnborg 1357a88b5ba8SSam Ravnborg if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || 1358a88b5ba8SSam Ravnborg !(lp->flags & LDC_FLAG_REGISTERED_QUEUES)) 1359a88b5ba8SSam Ravnborg return -EINVAL; 1360a88b5ba8SSam Ravnborg 1361a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1362a88b5ba8SSam Ravnborg 1363a88b5ba8SSam Ravnborg err = -ENODEV; 1364a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0); 1365a88b5ba8SSam Ravnborg if (hv_err) 1366a88b5ba8SSam Ravnborg goto out_err; 1367a88b5ba8SSam Ravnborg 1368a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); 1369a88b5ba8SSam Ravnborg if (hv_err) 1370a88b5ba8SSam Ravnborg goto out_err; 1371a88b5ba8SSam Ravnborg 1372a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0); 1373a88b5ba8SSam Ravnborg if (hv_err) 1374a88b5ba8SSam Ravnborg goto out_err; 1375a88b5ba8SSam Ravnborg 1376a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); 1377a88b5ba8SSam Ravnborg if (hv_err) 1378a88b5ba8SSam Ravnborg goto out_err; 1379a88b5ba8SSam Ravnborg 1380a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_BOUND); 1381a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN; 1382a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_RESET; 1383a88b5ba8SSam Ravnborg 1384a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1385a88b5ba8SSam Ravnborg 1386a88b5ba8SSam Ravnborg return 0; 1387a88b5ba8SSam Ravnborg 1388a88b5ba8SSam Ravnborg out_err: 1389a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0); 1390a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0); 1391a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp); 1392a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1393a88b5ba8SSam Ravnborg lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS | 1394a88b5ba8SSam Ravnborg LDC_FLAG_REGISTERED_QUEUES); 1395a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_INIT); 1396a88b5ba8SSam Ravnborg 1397a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1398a88b5ba8SSam Ravnborg 1399a88b5ba8SSam Ravnborg return err; 1400a88b5ba8SSam Ravnborg } 1401a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_disconnect); 1402a88b5ba8SSam Ravnborg 1403a88b5ba8SSam Ravnborg int ldc_state(struct ldc_channel *lp) 1404a88b5ba8SSam Ravnborg { 1405a88b5ba8SSam Ravnborg return lp->state; 1406a88b5ba8SSam Ravnborg } 1407a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_state); 1408a88b5ba8SSam Ravnborg 1409a88b5ba8SSam Ravnborg static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size) 1410a88b5ba8SSam Ravnborg { 1411a88b5ba8SSam Ravnborg struct ldc_packet *p; 1412a88b5ba8SSam Ravnborg unsigned long new_tail; 1413a88b5ba8SSam Ravnborg int err; 1414a88b5ba8SSam Ravnborg 1415a88b5ba8SSam Ravnborg if (size > LDC_PACKET_SIZE) 1416a88b5ba8SSam Ravnborg return -EMSGSIZE; 1417a88b5ba8SSam Ravnborg 1418a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail); 1419a88b5ba8SSam Ravnborg if (!p) 1420a88b5ba8SSam Ravnborg return -EAGAIN; 1421a88b5ba8SSam Ravnborg 1422a88b5ba8SSam Ravnborg memcpy(p, buf, size); 1423a88b5ba8SSam Ravnborg 1424a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail); 1425a88b5ba8SSam Ravnborg if (!err) 1426a88b5ba8SSam Ravnborg err = size; 1427a88b5ba8SSam Ravnborg 1428a88b5ba8SSam Ravnborg return err; 1429a88b5ba8SSam Ravnborg } 1430a88b5ba8SSam Ravnborg 1431a88b5ba8SSam Ravnborg static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size) 1432a88b5ba8SSam Ravnborg { 1433a88b5ba8SSam Ravnborg struct ldc_packet *p; 1434a88b5ba8SSam Ravnborg unsigned long hv_err, new; 1435a88b5ba8SSam Ravnborg int err; 1436a88b5ba8SSam Ravnborg 1437a88b5ba8SSam Ravnborg if (size < LDC_PACKET_SIZE) 1438a88b5ba8SSam Ravnborg return -EINVAL; 1439a88b5ba8SSam Ravnborg 1440a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 1441a88b5ba8SSam Ravnborg &lp->rx_head, 1442a88b5ba8SSam Ravnborg &lp->rx_tail, 1443a88b5ba8SSam Ravnborg &lp->chan_state); 1444a88b5ba8SSam Ravnborg if (hv_err) 1445a88b5ba8SSam Ravnborg return ldc_abort(lp); 1446a88b5ba8SSam Ravnborg 1447a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN || 1448a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING) 1449a88b5ba8SSam Ravnborg return -ECONNRESET; 1450a88b5ba8SSam Ravnborg 1451a88b5ba8SSam Ravnborg if (lp->rx_head == lp->rx_tail) 1452a88b5ba8SSam Ravnborg return 0; 1453a88b5ba8SSam Ravnborg 1454a88b5ba8SSam Ravnborg p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE); 1455a88b5ba8SSam Ravnborg memcpy(buf, p, LDC_PACKET_SIZE); 1456a88b5ba8SSam Ravnborg 1457a88b5ba8SSam Ravnborg new = rx_advance(lp, lp->rx_head); 1458a88b5ba8SSam Ravnborg lp->rx_head = new; 1459a88b5ba8SSam Ravnborg 1460a88b5ba8SSam Ravnborg err = __set_rx_head(lp, new); 1461a88b5ba8SSam Ravnborg if (err < 0) 1462a88b5ba8SSam Ravnborg err = -ECONNRESET; 1463a88b5ba8SSam Ravnborg else 1464a88b5ba8SSam Ravnborg err = LDC_PACKET_SIZE; 1465a88b5ba8SSam Ravnborg 1466a88b5ba8SSam Ravnborg return err; 1467a88b5ba8SSam Ravnborg } 1468a88b5ba8SSam Ravnborg 1469a88b5ba8SSam Ravnborg static const struct ldc_mode_ops raw_ops = { 1470a88b5ba8SSam Ravnborg .write = write_raw, 1471a88b5ba8SSam Ravnborg .read = read_raw, 1472a88b5ba8SSam Ravnborg }; 1473a88b5ba8SSam Ravnborg 1474a88b5ba8SSam Ravnborg static int write_nonraw(struct ldc_channel *lp, const void *buf, 1475a88b5ba8SSam Ravnborg unsigned int size) 1476a88b5ba8SSam Ravnborg { 1477a88b5ba8SSam Ravnborg unsigned long hv_err, tail; 1478a88b5ba8SSam Ravnborg unsigned int copied; 1479a88b5ba8SSam Ravnborg u32 seq; 1480a88b5ba8SSam Ravnborg int err; 1481a88b5ba8SSam Ravnborg 1482a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail, 1483a88b5ba8SSam Ravnborg &lp->chan_state); 1484a88b5ba8SSam Ravnborg if (unlikely(hv_err)) 1485a88b5ba8SSam Ravnborg return -EBUSY; 1486a88b5ba8SSam Ravnborg 1487a88b5ba8SSam Ravnborg if (unlikely(lp->chan_state != LDC_CHANNEL_UP)) 1488a88b5ba8SSam Ravnborg return ldc_abort(lp); 1489a88b5ba8SSam Ravnborg 1490a88b5ba8SSam Ravnborg if (!tx_has_space_for(lp, size)) 1491a88b5ba8SSam Ravnborg return -EAGAIN; 1492a88b5ba8SSam Ravnborg 1493a88b5ba8SSam Ravnborg seq = lp->snd_nxt; 1494a88b5ba8SSam Ravnborg copied = 0; 1495a88b5ba8SSam Ravnborg tail = lp->tx_tail; 1496a88b5ba8SSam Ravnborg while (copied < size) { 1497a88b5ba8SSam Ravnborg struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE); 1498a88b5ba8SSam Ravnborg u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ? 1499a88b5ba8SSam Ravnborg p->u.u_data : 1500a88b5ba8SSam Ravnborg p->u.r.r_data); 1501a88b5ba8SSam Ravnborg int data_len; 1502a88b5ba8SSam Ravnborg 1503a88b5ba8SSam Ravnborg p->type = LDC_DATA; 1504a88b5ba8SSam Ravnborg p->stype = LDC_INFO; 1505a88b5ba8SSam Ravnborg p->ctrl = 0; 1506a88b5ba8SSam Ravnborg 1507a88b5ba8SSam Ravnborg data_len = size - copied; 1508a88b5ba8SSam Ravnborg if (data_len > lp->mss) 1509a88b5ba8SSam Ravnborg data_len = lp->mss; 1510a88b5ba8SSam Ravnborg 1511a88b5ba8SSam Ravnborg BUG_ON(data_len > LDC_LEN); 1512a88b5ba8SSam Ravnborg 1513a88b5ba8SSam Ravnborg p->env = (data_len | 1514a88b5ba8SSam Ravnborg (copied == 0 ? LDC_START : 0) | 1515a88b5ba8SSam Ravnborg (data_len == size - copied ? LDC_STOP : 0)); 1516a88b5ba8SSam Ravnborg 1517a88b5ba8SSam Ravnborg p->seqid = ++seq; 1518a88b5ba8SSam Ravnborg 1519a88b5ba8SSam Ravnborg ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n", 1520a88b5ba8SSam Ravnborg p->type, 1521a88b5ba8SSam Ravnborg p->stype, 1522a88b5ba8SSam Ravnborg p->ctrl, 1523a88b5ba8SSam Ravnborg p->env, 1524a88b5ba8SSam Ravnborg p->seqid); 1525a88b5ba8SSam Ravnborg 1526a88b5ba8SSam Ravnborg memcpy(data, buf, data_len); 1527a88b5ba8SSam Ravnborg buf += data_len; 1528a88b5ba8SSam Ravnborg copied += data_len; 1529a88b5ba8SSam Ravnborg 1530a88b5ba8SSam Ravnborg tail = tx_advance(lp, tail); 1531a88b5ba8SSam Ravnborg } 1532a88b5ba8SSam Ravnborg 1533a88b5ba8SSam Ravnborg err = set_tx_tail(lp, tail); 1534a88b5ba8SSam Ravnborg if (!err) { 1535a88b5ba8SSam Ravnborg lp->snd_nxt = seq; 1536a88b5ba8SSam Ravnborg err = size; 1537a88b5ba8SSam Ravnborg } 1538a88b5ba8SSam Ravnborg 1539a88b5ba8SSam Ravnborg return err; 1540a88b5ba8SSam Ravnborg } 1541a88b5ba8SSam Ravnborg 1542a88b5ba8SSam Ravnborg static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p, 1543a88b5ba8SSam Ravnborg struct ldc_packet *first_frag) 1544a88b5ba8SSam Ravnborg { 1545a88b5ba8SSam Ravnborg int err; 1546a88b5ba8SSam Ravnborg 1547a88b5ba8SSam Ravnborg if (first_frag) 1548a88b5ba8SSam Ravnborg lp->rcv_nxt = first_frag->seqid - 1; 1549a88b5ba8SSam Ravnborg 1550a88b5ba8SSam Ravnborg err = send_data_nack(lp, p); 1551a88b5ba8SSam Ravnborg if (err) 1552a88b5ba8SSam Ravnborg return err; 1553a88b5ba8SSam Ravnborg 1554a88b5ba8SSam Ravnborg err = __set_rx_head(lp, lp->rx_tail); 1555a88b5ba8SSam Ravnborg if (err < 0) 1556a88b5ba8SSam Ravnborg return ldc_abort(lp); 1557a88b5ba8SSam Ravnborg 1558a88b5ba8SSam Ravnborg return 0; 1559a88b5ba8SSam Ravnborg } 1560a88b5ba8SSam Ravnborg 1561a88b5ba8SSam Ravnborg static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p) 1562a88b5ba8SSam Ravnborg { 1563a88b5ba8SSam Ravnborg if (p->stype & LDC_ACK) { 1564a88b5ba8SSam Ravnborg int err = process_data_ack(lp, p); 1565a88b5ba8SSam Ravnborg if (err) 1566a88b5ba8SSam Ravnborg return err; 1567a88b5ba8SSam Ravnborg } 1568a88b5ba8SSam Ravnborg if (p->stype & LDC_NACK) 1569a88b5ba8SSam Ravnborg return ldc_abort(lp); 1570a88b5ba8SSam Ravnborg 1571a88b5ba8SSam Ravnborg return 0; 1572a88b5ba8SSam Ravnborg } 1573a88b5ba8SSam Ravnborg 1574a88b5ba8SSam Ravnborg static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head) 1575a88b5ba8SSam Ravnborg { 1576a88b5ba8SSam Ravnborg unsigned long dummy; 1577a88b5ba8SSam Ravnborg int limit = 1000; 1578a88b5ba8SSam Ravnborg 1579a88b5ba8SSam Ravnborg ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n", 1580a88b5ba8SSam Ravnborg cur_head, lp->rx_head, lp->rx_tail); 1581a88b5ba8SSam Ravnborg while (limit-- > 0) { 1582a88b5ba8SSam Ravnborg unsigned long hv_err; 1583a88b5ba8SSam Ravnborg 1584a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 1585a88b5ba8SSam Ravnborg &dummy, 1586a88b5ba8SSam Ravnborg &lp->rx_tail, 1587a88b5ba8SSam Ravnborg &lp->chan_state); 1588a88b5ba8SSam Ravnborg if (hv_err) 1589a88b5ba8SSam Ravnborg return ldc_abort(lp); 1590a88b5ba8SSam Ravnborg 1591a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN || 1592a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING) 1593a88b5ba8SSam Ravnborg return -ECONNRESET; 1594a88b5ba8SSam Ravnborg 1595a88b5ba8SSam Ravnborg if (cur_head != lp->rx_tail) { 1596a88b5ba8SSam Ravnborg ldcdbg(DATA, "DATA WAIT DONE " 1597a88b5ba8SSam Ravnborg "head[%lx] tail[%lx] chan_state[%lx]\n", 1598a88b5ba8SSam Ravnborg dummy, lp->rx_tail, lp->chan_state); 1599a88b5ba8SSam Ravnborg return 0; 1600a88b5ba8SSam Ravnborg } 1601a88b5ba8SSam Ravnborg 1602a88b5ba8SSam Ravnborg udelay(1); 1603a88b5ba8SSam Ravnborg } 1604a88b5ba8SSam Ravnborg return -EAGAIN; 1605a88b5ba8SSam Ravnborg } 1606a88b5ba8SSam Ravnborg 1607a88b5ba8SSam Ravnborg static int rx_set_head(struct ldc_channel *lp, unsigned long head) 1608a88b5ba8SSam Ravnborg { 1609a88b5ba8SSam Ravnborg int err = __set_rx_head(lp, head); 1610a88b5ba8SSam Ravnborg 1611a88b5ba8SSam Ravnborg if (err < 0) 1612a88b5ba8SSam Ravnborg return ldc_abort(lp); 1613a88b5ba8SSam Ravnborg 1614a88b5ba8SSam Ravnborg lp->rx_head = head; 1615a88b5ba8SSam Ravnborg return 0; 1616a88b5ba8SSam Ravnborg } 1617a88b5ba8SSam Ravnborg 1618a88b5ba8SSam Ravnborg static void send_data_ack(struct ldc_channel *lp) 1619a88b5ba8SSam Ravnborg { 1620a88b5ba8SSam Ravnborg unsigned long new_tail; 1621a88b5ba8SSam Ravnborg struct ldc_packet *p; 1622a88b5ba8SSam Ravnborg 1623a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail); 1624a88b5ba8SSam Ravnborg if (likely(p)) { 1625a88b5ba8SSam Ravnborg int err; 1626a88b5ba8SSam Ravnborg 1627a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p)); 1628a88b5ba8SSam Ravnborg p->type = LDC_DATA; 1629a88b5ba8SSam Ravnborg p->stype = LDC_ACK; 1630a88b5ba8SSam Ravnborg p->ctrl = 0; 1631a88b5ba8SSam Ravnborg p->seqid = lp->snd_nxt + 1; 1632a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt; 1633a88b5ba8SSam Ravnborg 1634a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail); 1635a88b5ba8SSam Ravnborg if (!err) 1636a88b5ba8SSam Ravnborg lp->snd_nxt++; 1637a88b5ba8SSam Ravnborg } 1638a88b5ba8SSam Ravnborg } 1639a88b5ba8SSam Ravnborg 1640a88b5ba8SSam Ravnborg static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size) 1641a88b5ba8SSam Ravnborg { 1642a88b5ba8SSam Ravnborg struct ldc_packet *first_frag; 1643a88b5ba8SSam Ravnborg unsigned long hv_err, new; 1644a88b5ba8SSam Ravnborg int err, copied; 1645a88b5ba8SSam Ravnborg 1646a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 1647a88b5ba8SSam Ravnborg &lp->rx_head, 1648a88b5ba8SSam Ravnborg &lp->rx_tail, 1649a88b5ba8SSam Ravnborg &lp->chan_state); 1650a88b5ba8SSam Ravnborg if (hv_err) 1651a88b5ba8SSam Ravnborg return ldc_abort(lp); 1652a88b5ba8SSam Ravnborg 1653a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN || 1654a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING) 1655a88b5ba8SSam Ravnborg return -ECONNRESET; 1656a88b5ba8SSam Ravnborg 1657a88b5ba8SSam Ravnborg if (lp->rx_head == lp->rx_tail) 1658a88b5ba8SSam Ravnborg return 0; 1659a88b5ba8SSam Ravnborg 1660a88b5ba8SSam Ravnborg first_frag = NULL; 1661a88b5ba8SSam Ravnborg copied = err = 0; 1662a88b5ba8SSam Ravnborg new = lp->rx_head; 1663a88b5ba8SSam Ravnborg while (1) { 1664a88b5ba8SSam Ravnborg struct ldc_packet *p; 1665a88b5ba8SSam Ravnborg int pkt_len; 1666a88b5ba8SSam Ravnborg 1667a88b5ba8SSam Ravnborg BUG_ON(new == lp->rx_tail); 1668a88b5ba8SSam Ravnborg p = lp->rx_base + (new / LDC_PACKET_SIZE); 1669a88b5ba8SSam Ravnborg 1670a88b5ba8SSam Ravnborg ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] " 1671a88b5ba8SSam Ravnborg "rcv_nxt[%08x]\n", 1672a88b5ba8SSam Ravnborg p->type, 1673a88b5ba8SSam Ravnborg p->stype, 1674a88b5ba8SSam Ravnborg p->ctrl, 1675a88b5ba8SSam Ravnborg p->env, 1676a88b5ba8SSam Ravnborg p->seqid, 1677a88b5ba8SSam Ravnborg p->u.r.ackid, 1678a88b5ba8SSam Ravnborg lp->rcv_nxt); 1679a88b5ba8SSam Ravnborg 1680a88b5ba8SSam Ravnborg if (unlikely(!rx_seq_ok(lp, p->seqid))) { 1681a88b5ba8SSam Ravnborg err = rx_bad_seq(lp, p, first_frag); 1682a88b5ba8SSam Ravnborg copied = 0; 1683a88b5ba8SSam Ravnborg break; 1684a88b5ba8SSam Ravnborg } 1685a88b5ba8SSam Ravnborg 1686a88b5ba8SSam Ravnborg if (p->type & LDC_CTRL) { 1687a88b5ba8SSam Ravnborg err = process_control_frame(lp, p); 1688a88b5ba8SSam Ravnborg if (err < 0) 1689a88b5ba8SSam Ravnborg break; 1690a88b5ba8SSam Ravnborg err = 0; 1691a88b5ba8SSam Ravnborg } 1692a88b5ba8SSam Ravnborg 1693a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid; 1694a88b5ba8SSam Ravnborg 1695a88b5ba8SSam Ravnborg if (!(p->type & LDC_DATA)) { 1696a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1697a88b5ba8SSam Ravnborg goto no_data; 1698a88b5ba8SSam Ravnborg } 1699a88b5ba8SSam Ravnborg if (p->stype & (LDC_ACK | LDC_NACK)) { 1700a88b5ba8SSam Ravnborg err = data_ack_nack(lp, p); 1701a88b5ba8SSam Ravnborg if (err) 1702a88b5ba8SSam Ravnborg break; 1703a88b5ba8SSam Ravnborg } 1704a88b5ba8SSam Ravnborg if (!(p->stype & LDC_INFO)) { 1705a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1706a88b5ba8SSam Ravnborg err = rx_set_head(lp, new); 1707a88b5ba8SSam Ravnborg if (err) 1708a88b5ba8SSam Ravnborg break; 1709a88b5ba8SSam Ravnborg goto no_data; 1710a88b5ba8SSam Ravnborg } 1711a88b5ba8SSam Ravnborg 1712a88b5ba8SSam Ravnborg pkt_len = p->env & LDC_LEN; 1713a88b5ba8SSam Ravnborg 1714a88b5ba8SSam Ravnborg /* Every initial packet starts with the START bit set. 1715a88b5ba8SSam Ravnborg * 1716a88b5ba8SSam Ravnborg * Singleton packets will have both START+STOP set. 1717a88b5ba8SSam Ravnborg * 1718a88b5ba8SSam Ravnborg * Fragments will have START set in the first frame, STOP 1719a88b5ba8SSam Ravnborg * set in the last frame, and neither bit set in middle 1720a88b5ba8SSam Ravnborg * frames of the packet. 1721a88b5ba8SSam Ravnborg * 1722a88b5ba8SSam Ravnborg * Therefore if we are at the beginning of a packet and 1723a88b5ba8SSam Ravnborg * we don't see START, or we are in the middle of a fragmented 1724a88b5ba8SSam Ravnborg * packet and do see START, we are unsynchronized and should 1725a88b5ba8SSam Ravnborg * flush the RX queue. 1726a88b5ba8SSam Ravnborg */ 1727a88b5ba8SSam Ravnborg if ((first_frag == NULL && !(p->env & LDC_START)) || 1728a88b5ba8SSam Ravnborg (first_frag != NULL && (p->env & LDC_START))) { 1729a88b5ba8SSam Ravnborg if (!first_frag) 1730a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1731a88b5ba8SSam Ravnborg 1732a88b5ba8SSam Ravnborg err = rx_set_head(lp, new); 1733a88b5ba8SSam Ravnborg if (err) 1734a88b5ba8SSam Ravnborg break; 1735a88b5ba8SSam Ravnborg 1736a88b5ba8SSam Ravnborg if (!first_frag) 1737a88b5ba8SSam Ravnborg goto no_data; 1738a88b5ba8SSam Ravnborg } 1739a88b5ba8SSam Ravnborg if (!first_frag) 1740a88b5ba8SSam Ravnborg first_frag = p; 1741a88b5ba8SSam Ravnborg 1742a88b5ba8SSam Ravnborg if (pkt_len > size - copied) { 1743a88b5ba8SSam Ravnborg /* User didn't give us a big enough buffer, 1744a88b5ba8SSam Ravnborg * what to do? This is a pretty serious error. 1745a88b5ba8SSam Ravnborg * 1746a88b5ba8SSam Ravnborg * Since we haven't updated the RX ring head to 1747a88b5ba8SSam Ravnborg * consume any of the packets, signal the error 1748a88b5ba8SSam Ravnborg * to the user and just leave the RX ring alone. 1749a88b5ba8SSam Ravnborg * 1750a88b5ba8SSam Ravnborg * This seems the best behavior because this allows 1751a88b5ba8SSam Ravnborg * a user of the LDC layer to start with a small 1752a88b5ba8SSam Ravnborg * RX buffer for ldc_read() calls and use -EMSGSIZE 1753a88b5ba8SSam Ravnborg * as a cue to enlarge it's read buffer. 1754a88b5ba8SSam Ravnborg */ 1755a88b5ba8SSam Ravnborg err = -EMSGSIZE; 1756a88b5ba8SSam Ravnborg break; 1757a88b5ba8SSam Ravnborg } 1758a88b5ba8SSam Ravnborg 1759a88b5ba8SSam Ravnborg /* Ok, we are gonna eat this one. */ 1760a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1761a88b5ba8SSam Ravnborg 1762a88b5ba8SSam Ravnborg memcpy(buf, 1763a88b5ba8SSam Ravnborg (lp->cfg.mode == LDC_MODE_UNRELIABLE ? 1764a88b5ba8SSam Ravnborg p->u.u_data : p->u.r.r_data), pkt_len); 1765a88b5ba8SSam Ravnborg buf += pkt_len; 1766a88b5ba8SSam Ravnborg copied += pkt_len; 1767a88b5ba8SSam Ravnborg 1768a88b5ba8SSam Ravnborg if (p->env & LDC_STOP) 1769a88b5ba8SSam Ravnborg break; 1770a88b5ba8SSam Ravnborg 1771a88b5ba8SSam Ravnborg no_data: 1772a88b5ba8SSam Ravnborg if (new == lp->rx_tail) { 1773a88b5ba8SSam Ravnborg err = rx_data_wait(lp, new); 1774a88b5ba8SSam Ravnborg if (err) 1775a88b5ba8SSam Ravnborg break; 1776a88b5ba8SSam Ravnborg } 1777a88b5ba8SSam Ravnborg } 1778a88b5ba8SSam Ravnborg 1779a88b5ba8SSam Ravnborg if (!err) 1780a88b5ba8SSam Ravnborg err = rx_set_head(lp, new); 1781a88b5ba8SSam Ravnborg 1782a88b5ba8SSam Ravnborg if (err && first_frag) 1783a88b5ba8SSam Ravnborg lp->rcv_nxt = first_frag->seqid - 1; 1784a88b5ba8SSam Ravnborg 1785a88b5ba8SSam Ravnborg if (!err) { 1786a88b5ba8SSam Ravnborg err = copied; 1787a88b5ba8SSam Ravnborg if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE) 1788a88b5ba8SSam Ravnborg send_data_ack(lp); 1789a88b5ba8SSam Ravnborg } 1790a88b5ba8SSam Ravnborg 1791a88b5ba8SSam Ravnborg return err; 1792a88b5ba8SSam Ravnborg } 1793a88b5ba8SSam Ravnborg 1794a88b5ba8SSam Ravnborg static const struct ldc_mode_ops nonraw_ops = { 1795a88b5ba8SSam Ravnborg .write = write_nonraw, 1796a88b5ba8SSam Ravnborg .read = read_nonraw, 1797a88b5ba8SSam Ravnborg }; 1798a88b5ba8SSam Ravnborg 1799a88b5ba8SSam Ravnborg static int write_stream(struct ldc_channel *lp, const void *buf, 1800a88b5ba8SSam Ravnborg unsigned int size) 1801a88b5ba8SSam Ravnborg { 1802a88b5ba8SSam Ravnborg if (size > lp->cfg.mtu) 1803a88b5ba8SSam Ravnborg size = lp->cfg.mtu; 1804a88b5ba8SSam Ravnborg return write_nonraw(lp, buf, size); 1805a88b5ba8SSam Ravnborg } 1806a88b5ba8SSam Ravnborg 1807a88b5ba8SSam Ravnborg static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size) 1808a88b5ba8SSam Ravnborg { 1809a88b5ba8SSam Ravnborg if (!lp->mssbuf_len) { 1810a88b5ba8SSam Ravnborg int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu); 1811a88b5ba8SSam Ravnborg if (err < 0) 1812a88b5ba8SSam Ravnborg return err; 1813a88b5ba8SSam Ravnborg 1814a88b5ba8SSam Ravnborg lp->mssbuf_len = err; 1815a88b5ba8SSam Ravnborg lp->mssbuf_off = 0; 1816a88b5ba8SSam Ravnborg } 1817a88b5ba8SSam Ravnborg 1818a88b5ba8SSam Ravnborg if (size > lp->mssbuf_len) 1819a88b5ba8SSam Ravnborg size = lp->mssbuf_len; 1820a88b5ba8SSam Ravnborg memcpy(buf, lp->mssbuf + lp->mssbuf_off, size); 1821a88b5ba8SSam Ravnborg 1822a88b5ba8SSam Ravnborg lp->mssbuf_off += size; 1823a88b5ba8SSam Ravnborg lp->mssbuf_len -= size; 1824a88b5ba8SSam Ravnborg 1825a88b5ba8SSam Ravnborg return size; 1826a88b5ba8SSam Ravnborg } 1827a88b5ba8SSam Ravnborg 1828a88b5ba8SSam Ravnborg static const struct ldc_mode_ops stream_ops = { 1829a88b5ba8SSam Ravnborg .write = write_stream, 1830a88b5ba8SSam Ravnborg .read = read_stream, 1831a88b5ba8SSam Ravnborg }; 1832a88b5ba8SSam Ravnborg 1833a88b5ba8SSam Ravnborg int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size) 1834a88b5ba8SSam Ravnborg { 1835a88b5ba8SSam Ravnborg unsigned long flags; 1836a88b5ba8SSam Ravnborg int err; 1837a88b5ba8SSam Ravnborg 1838a88b5ba8SSam Ravnborg if (!buf) 1839a88b5ba8SSam Ravnborg return -EINVAL; 1840a88b5ba8SSam Ravnborg 1841a88b5ba8SSam Ravnborg if (!size) 1842a88b5ba8SSam Ravnborg return 0; 1843a88b5ba8SSam Ravnborg 1844a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1845a88b5ba8SSam Ravnborg 1846a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE) 1847a88b5ba8SSam Ravnborg err = -ENOTCONN; 1848a88b5ba8SSam Ravnborg else 1849a88b5ba8SSam Ravnborg err = lp->mops->write(lp, buf, size); 1850a88b5ba8SSam Ravnborg 1851a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1852a88b5ba8SSam Ravnborg 1853a88b5ba8SSam Ravnborg return err; 1854a88b5ba8SSam Ravnborg } 1855a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_write); 1856a88b5ba8SSam Ravnborg 1857a88b5ba8SSam Ravnborg int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size) 1858a88b5ba8SSam Ravnborg { 1859a88b5ba8SSam Ravnborg unsigned long flags; 1860a88b5ba8SSam Ravnborg int err; 1861a88b5ba8SSam Ravnborg 1862a88b5ba8SSam Ravnborg if (!buf) 1863a88b5ba8SSam Ravnborg return -EINVAL; 1864a88b5ba8SSam Ravnborg 1865a88b5ba8SSam Ravnborg if (!size) 1866a88b5ba8SSam Ravnborg return 0; 1867a88b5ba8SSam Ravnborg 1868a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1869a88b5ba8SSam Ravnborg 1870a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE) 1871a88b5ba8SSam Ravnborg err = -ENOTCONN; 1872a88b5ba8SSam Ravnborg else 1873a88b5ba8SSam Ravnborg err = lp->mops->read(lp, buf, size); 1874a88b5ba8SSam Ravnborg 1875a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1876a88b5ba8SSam Ravnborg 1877a88b5ba8SSam Ravnborg return err; 1878a88b5ba8SSam Ravnborg } 1879a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_read); 1880a88b5ba8SSam Ravnborg 1881a88b5ba8SSam Ravnborg static long arena_alloc(struct ldc_iommu *iommu, unsigned long npages) 1882a88b5ba8SSam Ravnborg { 1883a88b5ba8SSam Ravnborg struct iommu_arena *arena = &iommu->arena; 1884e756fd80SAkinobu Mita unsigned long n, start, end, limit; 1885a88b5ba8SSam Ravnborg int pass; 1886a88b5ba8SSam Ravnborg 1887a88b5ba8SSam Ravnborg limit = arena->limit; 1888a88b5ba8SSam Ravnborg start = arena->hint; 1889a88b5ba8SSam Ravnborg pass = 0; 1890a88b5ba8SSam Ravnborg 1891a88b5ba8SSam Ravnborg again: 1892e756fd80SAkinobu Mita n = bitmap_find_next_zero_area(arena->map, limit, start, npages, 0); 1893a88b5ba8SSam Ravnborg end = n + npages; 1894a88b5ba8SSam Ravnborg if (unlikely(end >= limit)) { 1895a88b5ba8SSam Ravnborg if (likely(pass < 1)) { 1896a88b5ba8SSam Ravnborg limit = start; 1897a88b5ba8SSam Ravnborg start = 0; 1898a88b5ba8SSam Ravnborg pass++; 1899a88b5ba8SSam Ravnborg goto again; 1900a88b5ba8SSam Ravnborg } else { 1901a88b5ba8SSam Ravnborg /* Scanned the whole thing, give up. */ 1902a88b5ba8SSam Ravnborg return -1; 1903a88b5ba8SSam Ravnborg } 1904a88b5ba8SSam Ravnborg } 1905e756fd80SAkinobu Mita bitmap_set(arena->map, n, npages); 1906a88b5ba8SSam Ravnborg 1907a88b5ba8SSam Ravnborg arena->hint = end; 1908a88b5ba8SSam Ravnborg 1909a88b5ba8SSam Ravnborg return n; 1910a88b5ba8SSam Ravnborg } 1911a88b5ba8SSam Ravnborg 1912a88b5ba8SSam Ravnborg #define COOKIE_PGSZ_CODE 0xf000000000000000ULL 1913a88b5ba8SSam Ravnborg #define COOKIE_PGSZ_CODE_SHIFT 60ULL 1914a88b5ba8SSam Ravnborg 1915a88b5ba8SSam Ravnborg static u64 pagesize_code(void) 1916a88b5ba8SSam Ravnborg { 1917a88b5ba8SSam Ravnborg switch (PAGE_SIZE) { 1918a88b5ba8SSam Ravnborg default: 1919a88b5ba8SSam Ravnborg case (8ULL * 1024ULL): 1920a88b5ba8SSam Ravnborg return 0; 1921a88b5ba8SSam Ravnborg case (64ULL * 1024ULL): 1922a88b5ba8SSam Ravnborg return 1; 1923a88b5ba8SSam Ravnborg case (512ULL * 1024ULL): 1924a88b5ba8SSam Ravnborg return 2; 1925a88b5ba8SSam Ravnborg case (4ULL * 1024ULL * 1024ULL): 1926a88b5ba8SSam Ravnborg return 3; 1927a88b5ba8SSam Ravnborg case (32ULL * 1024ULL * 1024ULL): 1928a88b5ba8SSam Ravnborg return 4; 1929a88b5ba8SSam Ravnborg case (256ULL * 1024ULL * 1024ULL): 1930a88b5ba8SSam Ravnborg return 5; 1931a88b5ba8SSam Ravnborg } 1932a88b5ba8SSam Ravnborg } 1933a88b5ba8SSam Ravnborg 1934a88b5ba8SSam Ravnborg static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset) 1935a88b5ba8SSam Ravnborg { 1936a88b5ba8SSam Ravnborg return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) | 1937a88b5ba8SSam Ravnborg (index << PAGE_SHIFT) | 1938a88b5ba8SSam Ravnborg page_offset); 1939a88b5ba8SSam Ravnborg } 1940a88b5ba8SSam Ravnborg 1941a88b5ba8SSam Ravnborg static u64 cookie_to_index(u64 cookie, unsigned long *shift) 1942a88b5ba8SSam Ravnborg { 1943a88b5ba8SSam Ravnborg u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT; 1944a88b5ba8SSam Ravnborg 1945a88b5ba8SSam Ravnborg cookie &= ~COOKIE_PGSZ_CODE; 1946a88b5ba8SSam Ravnborg 1947a88b5ba8SSam Ravnborg *shift = szcode * 3; 1948a88b5ba8SSam Ravnborg 1949a88b5ba8SSam Ravnborg return (cookie >> (13ULL + (szcode * 3ULL))); 1950a88b5ba8SSam Ravnborg } 1951a88b5ba8SSam Ravnborg 1952a88b5ba8SSam Ravnborg static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu, 1953a88b5ba8SSam Ravnborg unsigned long npages) 1954a88b5ba8SSam Ravnborg { 1955a88b5ba8SSam Ravnborg long entry; 1956a88b5ba8SSam Ravnborg 1957a88b5ba8SSam Ravnborg entry = arena_alloc(iommu, npages); 1958a88b5ba8SSam Ravnborg if (unlikely(entry < 0)) 1959a88b5ba8SSam Ravnborg return NULL; 1960a88b5ba8SSam Ravnborg 1961a88b5ba8SSam Ravnborg return iommu->page_table + entry; 1962a88b5ba8SSam Ravnborg } 1963a88b5ba8SSam Ravnborg 1964a88b5ba8SSam Ravnborg static u64 perm_to_mte(unsigned int map_perm) 1965a88b5ba8SSam Ravnborg { 1966a88b5ba8SSam Ravnborg u64 mte_base; 1967a88b5ba8SSam Ravnborg 1968a88b5ba8SSam Ravnborg mte_base = pagesize_code(); 1969a88b5ba8SSam Ravnborg 1970a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_SHADOW) { 1971a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R) 1972a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_COPY_R; 1973a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W) 1974a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_COPY_W; 1975a88b5ba8SSam Ravnborg } 1976a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_DIRECT) { 1977a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R) 1978a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_READ; 1979a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W) 1980a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_WRITE; 1981a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_X) 1982a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_EXEC; 1983a88b5ba8SSam Ravnborg } 1984a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_IO) { 1985a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R) 1986a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_IOMMU_R; 1987a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W) 1988a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_IOMMU_W; 1989a88b5ba8SSam Ravnborg } 1990a88b5ba8SSam Ravnborg 1991a88b5ba8SSam Ravnborg return mte_base; 1992a88b5ba8SSam Ravnborg } 1993a88b5ba8SSam Ravnborg 1994a88b5ba8SSam Ravnborg static int pages_in_region(unsigned long base, long len) 1995a88b5ba8SSam Ravnborg { 1996a88b5ba8SSam Ravnborg int count = 0; 1997a88b5ba8SSam Ravnborg 1998a88b5ba8SSam Ravnborg do { 1999a88b5ba8SSam Ravnborg unsigned long new = (base + PAGE_SIZE) & PAGE_MASK; 2000a88b5ba8SSam Ravnborg 2001a88b5ba8SSam Ravnborg len -= (new - base); 2002a88b5ba8SSam Ravnborg base = new; 2003a88b5ba8SSam Ravnborg count++; 2004a88b5ba8SSam Ravnborg } while (len > 0); 2005a88b5ba8SSam Ravnborg 2006a88b5ba8SSam Ravnborg return count; 2007a88b5ba8SSam Ravnborg } 2008a88b5ba8SSam Ravnborg 2009a88b5ba8SSam Ravnborg struct cookie_state { 2010a88b5ba8SSam Ravnborg struct ldc_mtable_entry *page_table; 2011a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies; 2012a88b5ba8SSam Ravnborg u64 mte_base; 2013a88b5ba8SSam Ravnborg u64 prev_cookie; 2014a88b5ba8SSam Ravnborg u32 pte_idx; 2015a88b5ba8SSam Ravnborg u32 nc; 2016a88b5ba8SSam Ravnborg }; 2017a88b5ba8SSam Ravnborg 2018a88b5ba8SSam Ravnborg static void fill_cookies(struct cookie_state *sp, unsigned long pa, 2019a88b5ba8SSam Ravnborg unsigned long off, unsigned long len) 2020a88b5ba8SSam Ravnborg { 2021a88b5ba8SSam Ravnborg do { 2022a88b5ba8SSam Ravnborg unsigned long tlen, new = pa + PAGE_SIZE; 2023a88b5ba8SSam Ravnborg u64 this_cookie; 2024a88b5ba8SSam Ravnborg 2025a88b5ba8SSam Ravnborg sp->page_table[sp->pte_idx].mte = sp->mte_base | pa; 2026a88b5ba8SSam Ravnborg 2027a88b5ba8SSam Ravnborg tlen = PAGE_SIZE; 2028a88b5ba8SSam Ravnborg if (off) 2029a88b5ba8SSam Ravnborg tlen = PAGE_SIZE - off; 2030a88b5ba8SSam Ravnborg if (tlen > len) 2031a88b5ba8SSam Ravnborg tlen = len; 2032a88b5ba8SSam Ravnborg 2033a88b5ba8SSam Ravnborg this_cookie = make_cookie(sp->pte_idx, 2034a88b5ba8SSam Ravnborg pagesize_code(), off); 2035a88b5ba8SSam Ravnborg 2036a88b5ba8SSam Ravnborg off = 0; 2037a88b5ba8SSam Ravnborg 2038a88b5ba8SSam Ravnborg if (this_cookie == sp->prev_cookie) { 2039a88b5ba8SSam Ravnborg sp->cookies[sp->nc - 1].cookie_size += tlen; 2040a88b5ba8SSam Ravnborg } else { 2041a88b5ba8SSam Ravnborg sp->cookies[sp->nc].cookie_addr = this_cookie; 2042a88b5ba8SSam Ravnborg sp->cookies[sp->nc].cookie_size = tlen; 2043a88b5ba8SSam Ravnborg sp->nc++; 2044a88b5ba8SSam Ravnborg } 2045a88b5ba8SSam Ravnborg sp->prev_cookie = this_cookie + tlen; 2046a88b5ba8SSam Ravnborg 2047a88b5ba8SSam Ravnborg sp->pte_idx++; 2048a88b5ba8SSam Ravnborg 2049a88b5ba8SSam Ravnborg len -= tlen; 2050a88b5ba8SSam Ravnborg pa = new; 2051a88b5ba8SSam Ravnborg } while (len > 0); 2052a88b5ba8SSam Ravnborg } 2053a88b5ba8SSam Ravnborg 2054a88b5ba8SSam Ravnborg static int sg_count_one(struct scatterlist *sg) 2055a88b5ba8SSam Ravnborg { 2056a88b5ba8SSam Ravnborg unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT; 2057a88b5ba8SSam Ravnborg long len = sg->length; 2058a88b5ba8SSam Ravnborg 2059a88b5ba8SSam Ravnborg if ((sg->offset | len) & (8UL - 1)) 2060a88b5ba8SSam Ravnborg return -EFAULT; 2061a88b5ba8SSam Ravnborg 2062a88b5ba8SSam Ravnborg return pages_in_region(base + sg->offset, len); 2063a88b5ba8SSam Ravnborg } 2064a88b5ba8SSam Ravnborg 2065a88b5ba8SSam Ravnborg static int sg_count_pages(struct scatterlist *sg, int num_sg) 2066a88b5ba8SSam Ravnborg { 2067a88b5ba8SSam Ravnborg int count; 2068a88b5ba8SSam Ravnborg int i; 2069a88b5ba8SSam Ravnborg 2070a88b5ba8SSam Ravnborg count = 0; 2071a88b5ba8SSam Ravnborg for (i = 0; i < num_sg; i++) { 2072a88b5ba8SSam Ravnborg int err = sg_count_one(sg + i); 2073a88b5ba8SSam Ravnborg if (err < 0) 2074a88b5ba8SSam Ravnborg return err; 2075a88b5ba8SSam Ravnborg count += err; 2076a88b5ba8SSam Ravnborg } 2077a88b5ba8SSam Ravnborg 2078a88b5ba8SSam Ravnborg return count; 2079a88b5ba8SSam Ravnborg } 2080a88b5ba8SSam Ravnborg 2081a88b5ba8SSam Ravnborg int ldc_map_sg(struct ldc_channel *lp, 2082a88b5ba8SSam Ravnborg struct scatterlist *sg, int num_sg, 2083a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies, 2084a88b5ba8SSam Ravnborg unsigned int map_perm) 2085a88b5ba8SSam Ravnborg { 2086a88b5ba8SSam Ravnborg unsigned long i, npages, flags; 2087a88b5ba8SSam Ravnborg struct ldc_mtable_entry *base; 2088a88b5ba8SSam Ravnborg struct cookie_state state; 2089a88b5ba8SSam Ravnborg struct ldc_iommu *iommu; 2090a88b5ba8SSam Ravnborg int err; 2091a88b5ba8SSam Ravnborg 2092a88b5ba8SSam Ravnborg if (map_perm & ~LDC_MAP_ALL) 2093a88b5ba8SSam Ravnborg return -EINVAL; 2094a88b5ba8SSam Ravnborg 2095a88b5ba8SSam Ravnborg err = sg_count_pages(sg, num_sg); 2096a88b5ba8SSam Ravnborg if (err < 0) 2097a88b5ba8SSam Ravnborg return err; 2098a88b5ba8SSam Ravnborg 2099a88b5ba8SSam Ravnborg npages = err; 2100a88b5ba8SSam Ravnborg if (err > ncookies) 2101a88b5ba8SSam Ravnborg return -EMSGSIZE; 2102a88b5ba8SSam Ravnborg 2103a88b5ba8SSam Ravnborg iommu = &lp->iommu; 2104a88b5ba8SSam Ravnborg 2105a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags); 2106a88b5ba8SSam Ravnborg base = alloc_npages(iommu, npages); 2107a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags); 2108a88b5ba8SSam Ravnborg 2109a88b5ba8SSam Ravnborg if (!base) 2110a88b5ba8SSam Ravnborg return -ENOMEM; 2111a88b5ba8SSam Ravnborg 2112a88b5ba8SSam Ravnborg state.page_table = iommu->page_table; 2113a88b5ba8SSam Ravnborg state.cookies = cookies; 2114a88b5ba8SSam Ravnborg state.mte_base = perm_to_mte(map_perm); 2115a88b5ba8SSam Ravnborg state.prev_cookie = ~(u64)0; 2116a88b5ba8SSam Ravnborg state.pte_idx = (base - iommu->page_table); 2117a88b5ba8SSam Ravnborg state.nc = 0; 2118a88b5ba8SSam Ravnborg 2119a88b5ba8SSam Ravnborg for (i = 0; i < num_sg; i++) 2120a88b5ba8SSam Ravnborg fill_cookies(&state, page_to_pfn(sg_page(&sg[i])) << PAGE_SHIFT, 2121a88b5ba8SSam Ravnborg sg[i].offset, sg[i].length); 2122a88b5ba8SSam Ravnborg 2123a88b5ba8SSam Ravnborg return state.nc; 2124a88b5ba8SSam Ravnborg } 2125a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_map_sg); 2126a88b5ba8SSam Ravnborg 2127a88b5ba8SSam Ravnborg int ldc_map_single(struct ldc_channel *lp, 2128a88b5ba8SSam Ravnborg void *buf, unsigned int len, 2129a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies, 2130a88b5ba8SSam Ravnborg unsigned int map_perm) 2131a88b5ba8SSam Ravnborg { 2132a88b5ba8SSam Ravnborg unsigned long npages, pa, flags; 2133a88b5ba8SSam Ravnborg struct ldc_mtable_entry *base; 2134a88b5ba8SSam Ravnborg struct cookie_state state; 2135a88b5ba8SSam Ravnborg struct ldc_iommu *iommu; 2136a88b5ba8SSam Ravnborg 2137a88b5ba8SSam Ravnborg if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1)) 2138a88b5ba8SSam Ravnborg return -EINVAL; 2139a88b5ba8SSam Ravnborg 2140a88b5ba8SSam Ravnborg pa = __pa(buf); 2141a88b5ba8SSam Ravnborg if ((pa | len) & (8UL - 1)) 2142a88b5ba8SSam Ravnborg return -EFAULT; 2143a88b5ba8SSam Ravnborg 2144a88b5ba8SSam Ravnborg npages = pages_in_region(pa, len); 2145a88b5ba8SSam Ravnborg 2146a88b5ba8SSam Ravnborg iommu = &lp->iommu; 2147a88b5ba8SSam Ravnborg 2148a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags); 2149a88b5ba8SSam Ravnborg base = alloc_npages(iommu, npages); 2150a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags); 2151a88b5ba8SSam Ravnborg 2152a88b5ba8SSam Ravnborg if (!base) 2153a88b5ba8SSam Ravnborg return -ENOMEM; 2154a88b5ba8SSam Ravnborg 2155a88b5ba8SSam Ravnborg state.page_table = iommu->page_table; 2156a88b5ba8SSam Ravnborg state.cookies = cookies; 2157a88b5ba8SSam Ravnborg state.mte_base = perm_to_mte(map_perm); 2158a88b5ba8SSam Ravnborg state.prev_cookie = ~(u64)0; 2159a88b5ba8SSam Ravnborg state.pte_idx = (base - iommu->page_table); 2160a88b5ba8SSam Ravnborg state.nc = 0; 2161a88b5ba8SSam Ravnborg fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len); 2162a88b5ba8SSam Ravnborg BUG_ON(state.nc != 1); 2163a88b5ba8SSam Ravnborg 2164a88b5ba8SSam Ravnborg return state.nc; 2165a88b5ba8SSam Ravnborg } 2166a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_map_single); 2167a88b5ba8SSam Ravnborg 2168a88b5ba8SSam Ravnborg static void free_npages(unsigned long id, struct ldc_iommu *iommu, 2169a88b5ba8SSam Ravnborg u64 cookie, u64 size) 2170a88b5ba8SSam Ravnborg { 2171a88b5ba8SSam Ravnborg struct iommu_arena *arena = &iommu->arena; 2172a88b5ba8SSam Ravnborg unsigned long i, shift, index, npages; 2173a88b5ba8SSam Ravnborg struct ldc_mtable_entry *base; 2174a88b5ba8SSam Ravnborg 2175a88b5ba8SSam Ravnborg npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT; 2176a88b5ba8SSam Ravnborg index = cookie_to_index(cookie, &shift); 2177a88b5ba8SSam Ravnborg base = iommu->page_table + index; 2178a88b5ba8SSam Ravnborg 2179a88b5ba8SSam Ravnborg BUG_ON(index > arena->limit || 2180a88b5ba8SSam Ravnborg (index + npages) > arena->limit); 2181a88b5ba8SSam Ravnborg 2182a88b5ba8SSam Ravnborg for (i = 0; i < npages; i++) { 2183a88b5ba8SSam Ravnborg if (base->cookie) 2184a88b5ba8SSam Ravnborg sun4v_ldc_revoke(id, cookie + (i << shift), 2185a88b5ba8SSam Ravnborg base->cookie); 2186a88b5ba8SSam Ravnborg base->mte = 0; 2187a88b5ba8SSam Ravnborg __clear_bit(index + i, arena->map); 2188a88b5ba8SSam Ravnborg } 2189a88b5ba8SSam Ravnborg } 2190a88b5ba8SSam Ravnborg 2191a88b5ba8SSam Ravnborg void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, 2192a88b5ba8SSam Ravnborg int ncookies) 2193a88b5ba8SSam Ravnborg { 2194a88b5ba8SSam Ravnborg struct ldc_iommu *iommu = &lp->iommu; 2195a88b5ba8SSam Ravnborg unsigned long flags; 2196a88b5ba8SSam Ravnborg int i; 2197a88b5ba8SSam Ravnborg 2198a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags); 2199a88b5ba8SSam Ravnborg for (i = 0; i < ncookies; i++) { 2200a88b5ba8SSam Ravnborg u64 addr = cookies[i].cookie_addr; 2201a88b5ba8SSam Ravnborg u64 size = cookies[i].cookie_size; 2202a88b5ba8SSam Ravnborg 2203a88b5ba8SSam Ravnborg free_npages(lp->id, iommu, addr, size); 2204a88b5ba8SSam Ravnborg } 2205a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags); 2206a88b5ba8SSam Ravnborg } 2207a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_unmap); 2208a88b5ba8SSam Ravnborg 2209a88b5ba8SSam Ravnborg int ldc_copy(struct ldc_channel *lp, int copy_dir, 2210a88b5ba8SSam Ravnborg void *buf, unsigned int len, unsigned long offset, 2211a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies) 2212a88b5ba8SSam Ravnborg { 2213a88b5ba8SSam Ravnborg unsigned int orig_len; 2214a88b5ba8SSam Ravnborg unsigned long ra; 2215a88b5ba8SSam Ravnborg int i; 2216a88b5ba8SSam Ravnborg 2217a88b5ba8SSam Ravnborg if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) { 2218a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n", 2219a88b5ba8SSam Ravnborg lp->id, copy_dir); 2220a88b5ba8SSam Ravnborg return -EINVAL; 2221a88b5ba8SSam Ravnborg } 2222a88b5ba8SSam Ravnborg 2223a88b5ba8SSam Ravnborg ra = __pa(buf); 2224a88b5ba8SSam Ravnborg if ((ra | len | offset) & (8UL - 1)) { 2225a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer " 2226a88b5ba8SSam Ravnborg "ra[%lx] len[%x] offset[%lx]\n", 2227a88b5ba8SSam Ravnborg lp->id, ra, len, offset); 2228a88b5ba8SSam Ravnborg return -EFAULT; 2229a88b5ba8SSam Ravnborg } 2230a88b5ba8SSam Ravnborg 2231a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE || 2232a88b5ba8SSam Ravnborg (lp->flags & LDC_FLAG_RESET)) { 2233a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] " 2234a88b5ba8SSam Ravnborg "flags[%x]\n", lp->id, lp->hs_state, lp->flags); 2235a88b5ba8SSam Ravnborg return -ECONNRESET; 2236a88b5ba8SSam Ravnborg } 2237a88b5ba8SSam Ravnborg 2238a88b5ba8SSam Ravnborg orig_len = len; 2239a88b5ba8SSam Ravnborg for (i = 0; i < ncookies; i++) { 2240a88b5ba8SSam Ravnborg unsigned long cookie_raddr = cookies[i].cookie_addr; 2241a88b5ba8SSam Ravnborg unsigned long this_len = cookies[i].cookie_size; 2242a88b5ba8SSam Ravnborg unsigned long actual_len; 2243a88b5ba8SSam Ravnborg 2244a88b5ba8SSam Ravnborg if (unlikely(offset)) { 2245a88b5ba8SSam Ravnborg unsigned long this_off = offset; 2246a88b5ba8SSam Ravnborg 2247a88b5ba8SSam Ravnborg if (this_off > this_len) 2248a88b5ba8SSam Ravnborg this_off = this_len; 2249a88b5ba8SSam Ravnborg 2250a88b5ba8SSam Ravnborg offset -= this_off; 2251a88b5ba8SSam Ravnborg this_len -= this_off; 2252a88b5ba8SSam Ravnborg if (!this_len) 2253a88b5ba8SSam Ravnborg continue; 2254a88b5ba8SSam Ravnborg cookie_raddr += this_off; 2255a88b5ba8SSam Ravnborg } 2256a88b5ba8SSam Ravnborg 2257a88b5ba8SSam Ravnborg if (this_len > len) 2258a88b5ba8SSam Ravnborg this_len = len; 2259a88b5ba8SSam Ravnborg 2260a88b5ba8SSam Ravnborg while (1) { 2261a88b5ba8SSam Ravnborg unsigned long hv_err; 2262a88b5ba8SSam Ravnborg 2263a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_copy(lp->id, copy_dir, 2264a88b5ba8SSam Ravnborg cookie_raddr, ra, 2265a88b5ba8SSam Ravnborg this_len, &actual_len); 2266a88b5ba8SSam Ravnborg if (unlikely(hv_err)) { 2267a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] " 2268a88b5ba8SSam Ravnborg "HV error %lu\n", 2269a88b5ba8SSam Ravnborg lp->id, hv_err); 2270a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE || 2271a88b5ba8SSam Ravnborg (lp->flags & LDC_FLAG_RESET)) 2272a88b5ba8SSam Ravnborg return -ECONNRESET; 2273a88b5ba8SSam Ravnborg else 2274a88b5ba8SSam Ravnborg return -EFAULT; 2275a88b5ba8SSam Ravnborg } 2276a88b5ba8SSam Ravnborg 2277a88b5ba8SSam Ravnborg cookie_raddr += actual_len; 2278a88b5ba8SSam Ravnborg ra += actual_len; 2279a88b5ba8SSam Ravnborg len -= actual_len; 2280a88b5ba8SSam Ravnborg if (actual_len == this_len) 2281a88b5ba8SSam Ravnborg break; 2282a88b5ba8SSam Ravnborg 2283a88b5ba8SSam Ravnborg this_len -= actual_len; 2284a88b5ba8SSam Ravnborg } 2285a88b5ba8SSam Ravnborg 2286a88b5ba8SSam Ravnborg if (!len) 2287a88b5ba8SSam Ravnborg break; 2288a88b5ba8SSam Ravnborg } 2289a88b5ba8SSam Ravnborg 2290a88b5ba8SSam Ravnborg /* It is caller policy what to do about short copies. 2291a88b5ba8SSam Ravnborg * For example, a networking driver can declare the 2292a88b5ba8SSam Ravnborg * packet a runt and drop it. 2293a88b5ba8SSam Ravnborg */ 2294a88b5ba8SSam Ravnborg 2295a88b5ba8SSam Ravnborg return orig_len - len; 2296a88b5ba8SSam Ravnborg } 2297a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_copy); 2298a88b5ba8SSam Ravnborg 2299a88b5ba8SSam Ravnborg void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len, 2300a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int *ncookies, 2301a88b5ba8SSam Ravnborg unsigned int map_perm) 2302a88b5ba8SSam Ravnborg { 2303a88b5ba8SSam Ravnborg void *buf; 2304a88b5ba8SSam Ravnborg int err; 2305a88b5ba8SSam Ravnborg 2306a88b5ba8SSam Ravnborg if (len & (8UL - 1)) 2307a88b5ba8SSam Ravnborg return ERR_PTR(-EINVAL); 2308a88b5ba8SSam Ravnborg 2309a88b5ba8SSam Ravnborg buf = kzalloc(len, GFP_KERNEL); 2310a88b5ba8SSam Ravnborg if (!buf) 2311a88b5ba8SSam Ravnborg return ERR_PTR(-ENOMEM); 2312a88b5ba8SSam Ravnborg 2313a88b5ba8SSam Ravnborg err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm); 2314a88b5ba8SSam Ravnborg if (err < 0) { 2315a88b5ba8SSam Ravnborg kfree(buf); 2316a88b5ba8SSam Ravnborg return ERR_PTR(err); 2317a88b5ba8SSam Ravnborg } 2318a88b5ba8SSam Ravnborg *ncookies = err; 2319a88b5ba8SSam Ravnborg 2320a88b5ba8SSam Ravnborg return buf; 2321a88b5ba8SSam Ravnborg } 2322a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_alloc_exp_dring); 2323a88b5ba8SSam Ravnborg 2324a88b5ba8SSam Ravnborg void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len, 2325a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies) 2326a88b5ba8SSam Ravnborg { 2327a88b5ba8SSam Ravnborg ldc_unmap(lp, cookies, ncookies); 2328a88b5ba8SSam Ravnborg kfree(buf); 2329a88b5ba8SSam Ravnborg } 2330a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_free_exp_dring); 2331a88b5ba8SSam Ravnborg 2332a88b5ba8SSam Ravnborg static int __init ldc_init(void) 2333a88b5ba8SSam Ravnborg { 2334a88b5ba8SSam Ravnborg unsigned long major, minor; 2335a88b5ba8SSam Ravnborg struct mdesc_handle *hp; 2336a88b5ba8SSam Ravnborg const u64 *v; 2337a88b5ba8SSam Ravnborg int err; 2338a88b5ba8SSam Ravnborg u64 mp; 2339a88b5ba8SSam Ravnborg 2340a88b5ba8SSam Ravnborg hp = mdesc_grab(); 2341a88b5ba8SSam Ravnborg if (!hp) 2342a88b5ba8SSam Ravnborg return -ENODEV; 2343a88b5ba8SSam Ravnborg 2344a88b5ba8SSam Ravnborg mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform"); 2345a88b5ba8SSam Ravnborg err = -ENODEV; 2346a88b5ba8SSam Ravnborg if (mp == MDESC_NODE_NULL) 2347a88b5ba8SSam Ravnborg goto out; 2348a88b5ba8SSam Ravnborg 2349a88b5ba8SSam Ravnborg v = mdesc_get_property(hp, mp, "domaining-enabled", NULL); 2350a88b5ba8SSam Ravnborg if (!v) 2351a88b5ba8SSam Ravnborg goto out; 2352a88b5ba8SSam Ravnborg 2353a88b5ba8SSam Ravnborg major = 1; 2354a88b5ba8SSam Ravnborg minor = 0; 2355a88b5ba8SSam Ravnborg if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) { 2356a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "Could not register LDOM hvapi.\n"); 2357a88b5ba8SSam Ravnborg goto out; 2358a88b5ba8SSam Ravnborg } 2359a88b5ba8SSam Ravnborg 2360a88b5ba8SSam Ravnborg printk(KERN_INFO "%s", version); 2361a88b5ba8SSam Ravnborg 2362a88b5ba8SSam Ravnborg if (!*v) { 2363a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "Domaining disabled.\n"); 2364a88b5ba8SSam Ravnborg goto out; 2365a88b5ba8SSam Ravnborg } 2366a88b5ba8SSam Ravnborg ldom_domaining_enabled = 1; 2367a88b5ba8SSam Ravnborg err = 0; 2368a88b5ba8SSam Ravnborg 2369a88b5ba8SSam Ravnborg out: 2370a88b5ba8SSam Ravnborg mdesc_release(hp); 2371a88b5ba8SSam Ravnborg return err; 2372a88b5ba8SSam Ravnborg } 2373a88b5ba8SSam Ravnborg 2374a88b5ba8SSam Ravnborg core_initcall(ldc_init); 2375