1a88b5ba8SSam Ravnborg /* ldc.c: Logical Domain Channel link-layer protocol driver. 2a88b5ba8SSam Ravnborg * 3a88b5ba8SSam Ravnborg * Copyright (C) 2007, 2008 David S. Miller <davem@davemloft.net> 4a88b5ba8SSam Ravnborg */ 5a88b5ba8SSam Ravnborg 6a88b5ba8SSam Ravnborg #include <linux/kernel.h> 7066bcacaSPaul Gortmaker #include <linux/export.h> 8a88b5ba8SSam Ravnborg #include <linux/slab.h> 9a88b5ba8SSam Ravnborg #include <linux/spinlock.h> 10a88b5ba8SSam Ravnborg #include <linux/delay.h> 11a88b5ba8SSam Ravnborg #include <linux/errno.h> 12a88b5ba8SSam Ravnborg #include <linux/string.h> 13a88b5ba8SSam Ravnborg #include <linux/scatterlist.h> 14a88b5ba8SSam Ravnborg #include <linux/interrupt.h> 15a88b5ba8SSam Ravnborg #include <linux/list.h> 16a88b5ba8SSam Ravnborg #include <linux/init.h> 17e756fd80SAkinobu Mita #include <linux/bitmap.h> 180ae53ed1SSowmini Varadhan #include <linux/iommu-common.h> 19a88b5ba8SSam Ravnborg 20a88b5ba8SSam Ravnborg #include <asm/hypervisor.h> 21a88b5ba8SSam Ravnborg #include <asm/iommu.h> 22a88b5ba8SSam Ravnborg #include <asm/page.h> 23a88b5ba8SSam Ravnborg #include <asm/ldc.h> 24a88b5ba8SSam Ravnborg #include <asm/mdesc.h> 25a88b5ba8SSam Ravnborg 26a88b5ba8SSam Ravnborg #define DRV_MODULE_NAME "ldc" 27a88b5ba8SSam Ravnborg #define PFX DRV_MODULE_NAME ": " 28a88b5ba8SSam Ravnborg #define DRV_MODULE_VERSION "1.1" 29a88b5ba8SSam Ravnborg #define DRV_MODULE_RELDATE "July 22, 2008" 30a88b5ba8SSam Ravnborg 310ae53ed1SSowmini Varadhan #define COOKIE_PGSZ_CODE 0xf000000000000000ULL 320ae53ed1SSowmini Varadhan #define COOKIE_PGSZ_CODE_SHIFT 60ULL 330ae53ed1SSowmini Varadhan 340ae53ed1SSowmini Varadhan 357c9503b8SGreg Kroah-Hartman static char version[] = 36a88b5ba8SSam Ravnborg DRV_MODULE_NAME ".c:v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 37a88b5ba8SSam Ravnborg 38a88b5ba8SSam Ravnborg /* Packet header layout for unreliable and reliable mode frames. 39a88b5ba8SSam Ravnborg * When in RAW mode, packets are simply straight 64-byte payloads 40a88b5ba8SSam Ravnborg * with no headers. 41a88b5ba8SSam Ravnborg */ 42a88b5ba8SSam Ravnborg struct ldc_packet { 43a88b5ba8SSam Ravnborg u8 type; 44a88b5ba8SSam Ravnborg #define LDC_CTRL 0x01 45a88b5ba8SSam Ravnborg #define LDC_DATA 0x02 46a88b5ba8SSam Ravnborg #define LDC_ERR 0x10 47a88b5ba8SSam Ravnborg 48a88b5ba8SSam Ravnborg u8 stype; 49a88b5ba8SSam Ravnborg #define LDC_INFO 0x01 50a88b5ba8SSam Ravnborg #define LDC_ACK 0x02 51a88b5ba8SSam Ravnborg #define LDC_NACK 0x04 52a88b5ba8SSam Ravnborg 53a88b5ba8SSam Ravnborg u8 ctrl; 54a88b5ba8SSam Ravnborg #define LDC_VERS 0x01 /* Link Version */ 55a88b5ba8SSam Ravnborg #define LDC_RTS 0x02 /* Request To Send */ 56a88b5ba8SSam Ravnborg #define LDC_RTR 0x03 /* Ready To Receive */ 57a88b5ba8SSam Ravnborg #define LDC_RDX 0x04 /* Ready for Data eXchange */ 58a88b5ba8SSam Ravnborg #define LDC_CTRL_MSK 0x0f 59a88b5ba8SSam Ravnborg 60a88b5ba8SSam Ravnborg u8 env; 61a88b5ba8SSam Ravnborg #define LDC_LEN 0x3f 62a88b5ba8SSam Ravnborg #define LDC_FRAG_MASK 0xc0 63a88b5ba8SSam Ravnborg #define LDC_START 0x40 64a88b5ba8SSam Ravnborg #define LDC_STOP 0x80 65a88b5ba8SSam Ravnborg 66a88b5ba8SSam Ravnborg u32 seqid; 67a88b5ba8SSam Ravnborg 68a88b5ba8SSam Ravnborg union { 69a88b5ba8SSam Ravnborg u8 u_data[LDC_PACKET_SIZE - 8]; 70a88b5ba8SSam Ravnborg struct { 71a88b5ba8SSam Ravnborg u32 pad; 72a88b5ba8SSam Ravnborg u32 ackid; 73a88b5ba8SSam Ravnborg u8 r_data[LDC_PACKET_SIZE - 8 - 8]; 74a88b5ba8SSam Ravnborg } r; 75a88b5ba8SSam Ravnborg } u; 76a88b5ba8SSam Ravnborg }; 77a88b5ba8SSam Ravnborg 78a88b5ba8SSam Ravnborg struct ldc_version { 79a88b5ba8SSam Ravnborg u16 major; 80a88b5ba8SSam Ravnborg u16 minor; 81a88b5ba8SSam Ravnborg }; 82a88b5ba8SSam Ravnborg 83a88b5ba8SSam Ravnborg /* Ordered from largest major to lowest. */ 84a88b5ba8SSam Ravnborg static struct ldc_version ver_arr[] = { 85a88b5ba8SSam Ravnborg { .major = 1, .minor = 0 }, 86a88b5ba8SSam Ravnborg }; 87a88b5ba8SSam Ravnborg 88a88b5ba8SSam Ravnborg #define LDC_DEFAULT_MTU (4 * LDC_PACKET_SIZE) 89a88b5ba8SSam Ravnborg #define LDC_DEFAULT_NUM_ENTRIES (PAGE_SIZE / LDC_PACKET_SIZE) 90a88b5ba8SSam Ravnborg 91a88b5ba8SSam Ravnborg struct ldc_channel; 92a88b5ba8SSam Ravnborg 93a88b5ba8SSam Ravnborg struct ldc_mode_ops { 94a88b5ba8SSam Ravnborg int (*write)(struct ldc_channel *, const void *, unsigned int); 95a88b5ba8SSam Ravnborg int (*read)(struct ldc_channel *, void *, unsigned int); 96a88b5ba8SSam Ravnborg }; 97a88b5ba8SSam Ravnborg 98a88b5ba8SSam Ravnborg static const struct ldc_mode_ops raw_ops; 99a88b5ba8SSam Ravnborg static const struct ldc_mode_ops nonraw_ops; 100a88b5ba8SSam Ravnborg static const struct ldc_mode_ops stream_ops; 101a88b5ba8SSam Ravnborg 102a88b5ba8SSam Ravnborg int ldom_domaining_enabled; 103a88b5ba8SSam Ravnborg 104a88b5ba8SSam Ravnborg struct ldc_iommu { 1050ae53ed1SSowmini Varadhan /* Protects ldc_unmap. */ 106a88b5ba8SSam Ravnborg spinlock_t lock; 107a88b5ba8SSam Ravnborg struct ldc_mtable_entry *page_table; 1080ae53ed1SSowmini Varadhan struct iommu_map_table iommu_map_table; 109a88b5ba8SSam Ravnborg }; 110a88b5ba8SSam Ravnborg 111a88b5ba8SSam Ravnborg struct ldc_channel { 112a88b5ba8SSam Ravnborg /* Protects all operations that depend upon channel state. */ 113a88b5ba8SSam Ravnborg spinlock_t lock; 114a88b5ba8SSam Ravnborg 115a88b5ba8SSam Ravnborg unsigned long id; 116a88b5ba8SSam Ravnborg 117a88b5ba8SSam Ravnborg u8 *mssbuf; 118a88b5ba8SSam Ravnborg u32 mssbuf_len; 119a88b5ba8SSam Ravnborg u32 mssbuf_off; 120a88b5ba8SSam Ravnborg 121a88b5ba8SSam Ravnborg struct ldc_packet *tx_base; 122a88b5ba8SSam Ravnborg unsigned long tx_head; 123a88b5ba8SSam Ravnborg unsigned long tx_tail; 124a88b5ba8SSam Ravnborg unsigned long tx_num_entries; 125a88b5ba8SSam Ravnborg unsigned long tx_ra; 126a88b5ba8SSam Ravnborg 127a88b5ba8SSam Ravnborg unsigned long tx_acked; 128a88b5ba8SSam Ravnborg 129a88b5ba8SSam Ravnborg struct ldc_packet *rx_base; 130a88b5ba8SSam Ravnborg unsigned long rx_head; 131a88b5ba8SSam Ravnborg unsigned long rx_tail; 132a88b5ba8SSam Ravnborg unsigned long rx_num_entries; 133a88b5ba8SSam Ravnborg unsigned long rx_ra; 134a88b5ba8SSam Ravnborg 135a88b5ba8SSam Ravnborg u32 rcv_nxt; 136a88b5ba8SSam Ravnborg u32 snd_nxt; 137a88b5ba8SSam Ravnborg 138a88b5ba8SSam Ravnborg unsigned long chan_state; 139a88b5ba8SSam Ravnborg 140a88b5ba8SSam Ravnborg struct ldc_channel_config cfg; 141a88b5ba8SSam Ravnborg void *event_arg; 142a88b5ba8SSam Ravnborg 143a88b5ba8SSam Ravnborg const struct ldc_mode_ops *mops; 144a88b5ba8SSam Ravnborg 145a88b5ba8SSam Ravnborg struct ldc_iommu iommu; 146a88b5ba8SSam Ravnborg 147a88b5ba8SSam Ravnborg struct ldc_version ver; 148a88b5ba8SSam Ravnborg 149a88b5ba8SSam Ravnborg u8 hs_state; 150a88b5ba8SSam Ravnborg #define LDC_HS_CLOSED 0x00 151a88b5ba8SSam Ravnborg #define LDC_HS_OPEN 0x01 152a88b5ba8SSam Ravnborg #define LDC_HS_GOTVERS 0x02 153a88b5ba8SSam Ravnborg #define LDC_HS_SENTRTR 0x03 154a88b5ba8SSam Ravnborg #define LDC_HS_GOTRTR 0x04 155a88b5ba8SSam Ravnborg #define LDC_HS_COMPLETE 0x10 156a88b5ba8SSam Ravnborg 157a88b5ba8SSam Ravnborg u8 flags; 158a88b5ba8SSam Ravnborg #define LDC_FLAG_ALLOCED_QUEUES 0x01 159a88b5ba8SSam Ravnborg #define LDC_FLAG_REGISTERED_QUEUES 0x02 160a88b5ba8SSam Ravnborg #define LDC_FLAG_REGISTERED_IRQS 0x04 161a88b5ba8SSam Ravnborg #define LDC_FLAG_RESET 0x10 162a88b5ba8SSam Ravnborg 163a88b5ba8SSam Ravnborg u8 mss; 164a88b5ba8SSam Ravnborg u8 state; 165a88b5ba8SSam Ravnborg 166a88b5ba8SSam Ravnborg #define LDC_IRQ_NAME_MAX 32 167a88b5ba8SSam Ravnborg char rx_irq_name[LDC_IRQ_NAME_MAX]; 168a88b5ba8SSam Ravnborg char tx_irq_name[LDC_IRQ_NAME_MAX]; 169a88b5ba8SSam Ravnborg 170a88b5ba8SSam Ravnborg struct hlist_head mh_list; 171a88b5ba8SSam Ravnborg 172a88b5ba8SSam Ravnborg struct hlist_node list; 173a88b5ba8SSam Ravnborg }; 174a88b5ba8SSam Ravnborg 175a88b5ba8SSam Ravnborg #define ldcdbg(TYPE, f, a...) \ 176a88b5ba8SSam Ravnborg do { if (lp->cfg.debug & LDC_DEBUG_##TYPE) \ 177a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "ID[%lu] " f, lp->id, ## a); \ 178a88b5ba8SSam Ravnborg } while (0) 179a88b5ba8SSam Ravnborg 18029693e75SJag Raman #define LDC_ABORT(lp) ldc_abort((lp), __func__) 18129693e75SJag Raman 182a88b5ba8SSam Ravnborg static const char *state_to_str(u8 state) 183a88b5ba8SSam Ravnborg { 184a88b5ba8SSam Ravnborg switch (state) { 185a88b5ba8SSam Ravnborg case LDC_STATE_INVALID: 186a88b5ba8SSam Ravnborg return "INVALID"; 187a88b5ba8SSam Ravnborg case LDC_STATE_INIT: 188a88b5ba8SSam Ravnborg return "INIT"; 189a88b5ba8SSam Ravnborg case LDC_STATE_BOUND: 190a88b5ba8SSam Ravnborg return "BOUND"; 191a88b5ba8SSam Ravnborg case LDC_STATE_READY: 192a88b5ba8SSam Ravnborg return "READY"; 193a88b5ba8SSam Ravnborg case LDC_STATE_CONNECTED: 194a88b5ba8SSam Ravnborg return "CONNECTED"; 195a88b5ba8SSam Ravnborg default: 196a88b5ba8SSam Ravnborg return "<UNKNOWN>"; 197a88b5ba8SSam Ravnborg } 198a88b5ba8SSam Ravnborg } 199a88b5ba8SSam Ravnborg 200a88b5ba8SSam Ravnborg static unsigned long __advance(unsigned long off, unsigned long num_entries) 201a88b5ba8SSam Ravnborg { 202a88b5ba8SSam Ravnborg off += LDC_PACKET_SIZE; 203a88b5ba8SSam Ravnborg if (off == (num_entries * LDC_PACKET_SIZE)) 204a88b5ba8SSam Ravnborg off = 0; 205a88b5ba8SSam Ravnborg 206a88b5ba8SSam Ravnborg return off; 207a88b5ba8SSam Ravnborg } 208a88b5ba8SSam Ravnborg 209a88b5ba8SSam Ravnborg static unsigned long rx_advance(struct ldc_channel *lp, unsigned long off) 210a88b5ba8SSam Ravnborg { 211a88b5ba8SSam Ravnborg return __advance(off, lp->rx_num_entries); 212a88b5ba8SSam Ravnborg } 213a88b5ba8SSam Ravnborg 214a88b5ba8SSam Ravnborg static unsigned long tx_advance(struct ldc_channel *lp, unsigned long off) 215a88b5ba8SSam Ravnborg { 216a88b5ba8SSam Ravnborg return __advance(off, lp->tx_num_entries); 217a88b5ba8SSam Ravnborg } 218a88b5ba8SSam Ravnborg 219a88b5ba8SSam Ravnborg static struct ldc_packet *handshake_get_tx_packet(struct ldc_channel *lp, 220a88b5ba8SSam Ravnborg unsigned long *new_tail) 221a88b5ba8SSam Ravnborg { 222a88b5ba8SSam Ravnborg struct ldc_packet *p; 223a88b5ba8SSam Ravnborg unsigned long t; 224a88b5ba8SSam Ravnborg 225a88b5ba8SSam Ravnborg t = tx_advance(lp, lp->tx_tail); 226a88b5ba8SSam Ravnborg if (t == lp->tx_head) 227a88b5ba8SSam Ravnborg return NULL; 228a88b5ba8SSam Ravnborg 229a88b5ba8SSam Ravnborg *new_tail = t; 230a88b5ba8SSam Ravnborg 231a88b5ba8SSam Ravnborg p = lp->tx_base; 232a88b5ba8SSam Ravnborg return p + (lp->tx_tail / LDC_PACKET_SIZE); 233a88b5ba8SSam Ravnborg } 234a88b5ba8SSam Ravnborg 235a88b5ba8SSam Ravnborg /* When we are in reliable or stream mode, have to track the next packet 236a88b5ba8SSam Ravnborg * we haven't gotten an ACK for in the TX queue using tx_acked. We have 237a88b5ba8SSam Ravnborg * to be careful not to stomp over the queue past that point. During 238a88b5ba8SSam Ravnborg * the handshake, we don't have TX data packets pending in the queue 239a88b5ba8SSam Ravnborg * and that's why handshake_get_tx_packet() need not be mindful of 240a88b5ba8SSam Ravnborg * lp->tx_acked. 241a88b5ba8SSam Ravnborg */ 242a88b5ba8SSam Ravnborg static unsigned long head_for_data(struct ldc_channel *lp) 243a88b5ba8SSam Ravnborg { 244a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_STREAM) 245a88b5ba8SSam Ravnborg return lp->tx_acked; 246a88b5ba8SSam Ravnborg return lp->tx_head; 247a88b5ba8SSam Ravnborg } 248a88b5ba8SSam Ravnborg 249a88b5ba8SSam Ravnborg static int tx_has_space_for(struct ldc_channel *lp, unsigned int size) 250a88b5ba8SSam Ravnborg { 251a88b5ba8SSam Ravnborg unsigned long limit, tail, new_tail, diff; 252a88b5ba8SSam Ravnborg unsigned int mss; 253a88b5ba8SSam Ravnborg 254a88b5ba8SSam Ravnborg limit = head_for_data(lp); 255a88b5ba8SSam Ravnborg tail = lp->tx_tail; 256a88b5ba8SSam Ravnborg new_tail = tx_advance(lp, tail); 257a88b5ba8SSam Ravnborg if (new_tail == limit) 258a88b5ba8SSam Ravnborg return 0; 259a88b5ba8SSam Ravnborg 260a88b5ba8SSam Ravnborg if (limit > new_tail) 261a88b5ba8SSam Ravnborg diff = limit - new_tail; 262a88b5ba8SSam Ravnborg else 263a88b5ba8SSam Ravnborg diff = (limit + 264a88b5ba8SSam Ravnborg ((lp->tx_num_entries * LDC_PACKET_SIZE) - new_tail)); 265a88b5ba8SSam Ravnborg diff /= LDC_PACKET_SIZE; 266a88b5ba8SSam Ravnborg mss = lp->mss; 267a88b5ba8SSam Ravnborg 268a88b5ba8SSam Ravnborg if (diff * mss < size) 269a88b5ba8SSam Ravnborg return 0; 270a88b5ba8SSam Ravnborg 271a88b5ba8SSam Ravnborg return 1; 272a88b5ba8SSam Ravnborg } 273a88b5ba8SSam Ravnborg 274a88b5ba8SSam Ravnborg static struct ldc_packet *data_get_tx_packet(struct ldc_channel *lp, 275a88b5ba8SSam Ravnborg unsigned long *new_tail) 276a88b5ba8SSam Ravnborg { 277a88b5ba8SSam Ravnborg struct ldc_packet *p; 278a88b5ba8SSam Ravnborg unsigned long h, t; 279a88b5ba8SSam Ravnborg 280a88b5ba8SSam Ravnborg h = head_for_data(lp); 281a88b5ba8SSam Ravnborg t = tx_advance(lp, lp->tx_tail); 282a88b5ba8SSam Ravnborg if (t == h) 283a88b5ba8SSam Ravnborg return NULL; 284a88b5ba8SSam Ravnborg 285a88b5ba8SSam Ravnborg *new_tail = t; 286a88b5ba8SSam Ravnborg 287a88b5ba8SSam Ravnborg p = lp->tx_base; 288a88b5ba8SSam Ravnborg return p + (lp->tx_tail / LDC_PACKET_SIZE); 289a88b5ba8SSam Ravnborg } 290a88b5ba8SSam Ravnborg 291a88b5ba8SSam Ravnborg static int set_tx_tail(struct ldc_channel *lp, unsigned long tail) 292a88b5ba8SSam Ravnborg { 293a88b5ba8SSam Ravnborg unsigned long orig_tail = lp->tx_tail; 294a88b5ba8SSam Ravnborg int limit = 1000; 295a88b5ba8SSam Ravnborg 296a88b5ba8SSam Ravnborg lp->tx_tail = tail; 297a88b5ba8SSam Ravnborg while (limit-- > 0) { 298a88b5ba8SSam Ravnborg unsigned long err; 299a88b5ba8SSam Ravnborg 300a88b5ba8SSam Ravnborg err = sun4v_ldc_tx_set_qtail(lp->id, tail); 301a88b5ba8SSam Ravnborg if (!err) 302a88b5ba8SSam Ravnborg return 0; 303a88b5ba8SSam Ravnborg 304a88b5ba8SSam Ravnborg if (err != HV_EWOULDBLOCK) { 305a88b5ba8SSam Ravnborg lp->tx_tail = orig_tail; 306a88b5ba8SSam Ravnborg return -EINVAL; 307a88b5ba8SSam Ravnborg } 308a88b5ba8SSam Ravnborg udelay(1); 309a88b5ba8SSam Ravnborg } 310a88b5ba8SSam Ravnborg 311a88b5ba8SSam Ravnborg lp->tx_tail = orig_tail; 312a88b5ba8SSam Ravnborg return -EBUSY; 313a88b5ba8SSam Ravnborg } 314a88b5ba8SSam Ravnborg 315a88b5ba8SSam Ravnborg /* This just updates the head value in the hypervisor using 316a88b5ba8SSam Ravnborg * a polling loop with a timeout. The caller takes care of 317a88b5ba8SSam Ravnborg * upating software state representing the head change, if any. 318a88b5ba8SSam Ravnborg */ 319a88b5ba8SSam Ravnborg static int __set_rx_head(struct ldc_channel *lp, unsigned long head) 320a88b5ba8SSam Ravnborg { 321a88b5ba8SSam Ravnborg int limit = 1000; 322a88b5ba8SSam Ravnborg 323a88b5ba8SSam Ravnborg while (limit-- > 0) { 324a88b5ba8SSam Ravnborg unsigned long err; 325a88b5ba8SSam Ravnborg 326a88b5ba8SSam Ravnborg err = sun4v_ldc_rx_set_qhead(lp->id, head); 327a88b5ba8SSam Ravnborg if (!err) 328a88b5ba8SSam Ravnborg return 0; 329a88b5ba8SSam Ravnborg 330a88b5ba8SSam Ravnborg if (err != HV_EWOULDBLOCK) 331a88b5ba8SSam Ravnborg return -EINVAL; 332a88b5ba8SSam Ravnborg 333a88b5ba8SSam Ravnborg udelay(1); 334a88b5ba8SSam Ravnborg } 335a88b5ba8SSam Ravnborg 336a88b5ba8SSam Ravnborg return -EBUSY; 337a88b5ba8SSam Ravnborg } 338a88b5ba8SSam Ravnborg 339a88b5ba8SSam Ravnborg static int send_tx_packet(struct ldc_channel *lp, 340a88b5ba8SSam Ravnborg struct ldc_packet *p, 341a88b5ba8SSam Ravnborg unsigned long new_tail) 342a88b5ba8SSam Ravnborg { 343a88b5ba8SSam Ravnborg BUG_ON(p != (lp->tx_base + (lp->tx_tail / LDC_PACKET_SIZE))); 344a88b5ba8SSam Ravnborg 345a88b5ba8SSam Ravnborg return set_tx_tail(lp, new_tail); 346a88b5ba8SSam Ravnborg } 347a88b5ba8SSam Ravnborg 348a88b5ba8SSam Ravnborg static struct ldc_packet *handshake_compose_ctrl(struct ldc_channel *lp, 349a88b5ba8SSam Ravnborg u8 stype, u8 ctrl, 350a88b5ba8SSam Ravnborg void *data, int dlen, 351a88b5ba8SSam Ravnborg unsigned long *new_tail) 352a88b5ba8SSam Ravnborg { 353a88b5ba8SSam Ravnborg struct ldc_packet *p = handshake_get_tx_packet(lp, new_tail); 354a88b5ba8SSam Ravnborg 355a88b5ba8SSam Ravnborg if (p) { 356a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p)); 357a88b5ba8SSam Ravnborg p->type = LDC_CTRL; 358a88b5ba8SSam Ravnborg p->stype = stype; 359a88b5ba8SSam Ravnborg p->ctrl = ctrl; 360a88b5ba8SSam Ravnborg if (data) 361a88b5ba8SSam Ravnborg memcpy(p->u.u_data, data, dlen); 362a88b5ba8SSam Ravnborg } 363a88b5ba8SSam Ravnborg return p; 364a88b5ba8SSam Ravnborg } 365a88b5ba8SSam Ravnborg 366a88b5ba8SSam Ravnborg static int start_handshake(struct ldc_channel *lp) 367a88b5ba8SSam Ravnborg { 368a88b5ba8SSam Ravnborg struct ldc_packet *p; 369a88b5ba8SSam Ravnborg struct ldc_version *ver; 370a88b5ba8SSam Ravnborg unsigned long new_tail; 371a88b5ba8SSam Ravnborg 372a88b5ba8SSam Ravnborg ver = &ver_arr[0]; 373a88b5ba8SSam Ravnborg 374a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER INFO maj[%u] min[%u]\n", 375a88b5ba8SSam Ravnborg ver->major, ver->minor); 376a88b5ba8SSam Ravnborg 377a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS, 378a88b5ba8SSam Ravnborg ver, sizeof(*ver), &new_tail); 379a88b5ba8SSam Ravnborg if (p) { 380a88b5ba8SSam Ravnborg int err = send_tx_packet(lp, p, new_tail); 381a88b5ba8SSam Ravnborg if (!err) 382a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_RESET; 383a88b5ba8SSam Ravnborg return err; 384a88b5ba8SSam Ravnborg } 385a88b5ba8SSam Ravnborg return -EBUSY; 386a88b5ba8SSam Ravnborg } 387a88b5ba8SSam Ravnborg 388a88b5ba8SSam Ravnborg static int send_version_nack(struct ldc_channel *lp, 389a88b5ba8SSam Ravnborg u16 major, u16 minor) 390a88b5ba8SSam Ravnborg { 391a88b5ba8SSam Ravnborg struct ldc_packet *p; 392a88b5ba8SSam Ravnborg struct ldc_version ver; 393a88b5ba8SSam Ravnborg unsigned long new_tail; 394a88b5ba8SSam Ravnborg 395a88b5ba8SSam Ravnborg ver.major = major; 396a88b5ba8SSam Ravnborg ver.minor = minor; 397a88b5ba8SSam Ravnborg 398a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_NACK, LDC_VERS, 399a88b5ba8SSam Ravnborg &ver, sizeof(ver), &new_tail); 400a88b5ba8SSam Ravnborg if (p) { 401a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER NACK maj[%u] min[%u]\n", 402a88b5ba8SSam Ravnborg ver.major, ver.minor); 403a88b5ba8SSam Ravnborg 404a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 405a88b5ba8SSam Ravnborg } 406a88b5ba8SSam Ravnborg return -EBUSY; 407a88b5ba8SSam Ravnborg } 408a88b5ba8SSam Ravnborg 409a88b5ba8SSam Ravnborg static int send_version_ack(struct ldc_channel *lp, 410a88b5ba8SSam Ravnborg struct ldc_version *vp) 411a88b5ba8SSam Ravnborg { 412a88b5ba8SSam Ravnborg struct ldc_packet *p; 413a88b5ba8SSam Ravnborg unsigned long new_tail; 414a88b5ba8SSam Ravnborg 415a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_ACK, LDC_VERS, 416a88b5ba8SSam Ravnborg vp, sizeof(*vp), &new_tail); 417a88b5ba8SSam Ravnborg if (p) { 418a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND VER ACK maj[%u] min[%u]\n", 419a88b5ba8SSam Ravnborg vp->major, vp->minor); 420a88b5ba8SSam Ravnborg 421a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 422a88b5ba8SSam Ravnborg } 423a88b5ba8SSam Ravnborg return -EBUSY; 424a88b5ba8SSam Ravnborg } 425a88b5ba8SSam Ravnborg 426a88b5ba8SSam Ravnborg static int send_rts(struct ldc_channel *lp) 427a88b5ba8SSam Ravnborg { 428a88b5ba8SSam Ravnborg struct ldc_packet *p; 429a88b5ba8SSam Ravnborg unsigned long new_tail; 430a88b5ba8SSam Ravnborg 431a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTS, NULL, 0, 432a88b5ba8SSam Ravnborg &new_tail); 433a88b5ba8SSam Ravnborg if (p) { 434a88b5ba8SSam Ravnborg p->env = lp->cfg.mode; 435a88b5ba8SSam Ravnborg p->seqid = 0; 436a88b5ba8SSam Ravnborg lp->rcv_nxt = 0; 437a88b5ba8SSam Ravnborg 438a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RTS env[0x%x] seqid[0x%x]\n", 439a88b5ba8SSam Ravnborg p->env, p->seqid); 440a88b5ba8SSam Ravnborg 441a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 442a88b5ba8SSam Ravnborg } 443a88b5ba8SSam Ravnborg return -EBUSY; 444a88b5ba8SSam Ravnborg } 445a88b5ba8SSam Ravnborg 446a88b5ba8SSam Ravnborg static int send_rtr(struct ldc_channel *lp) 447a88b5ba8SSam Ravnborg { 448a88b5ba8SSam Ravnborg struct ldc_packet *p; 449a88b5ba8SSam Ravnborg unsigned long new_tail; 450a88b5ba8SSam Ravnborg 451a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RTR, NULL, 0, 452a88b5ba8SSam Ravnborg &new_tail); 453a88b5ba8SSam Ravnborg if (p) { 454a88b5ba8SSam Ravnborg p->env = lp->cfg.mode; 455a88b5ba8SSam Ravnborg p->seqid = 0; 456a88b5ba8SSam Ravnborg 457a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RTR env[0x%x] seqid[0x%x]\n", 458a88b5ba8SSam Ravnborg p->env, p->seqid); 459a88b5ba8SSam Ravnborg 460a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 461a88b5ba8SSam Ravnborg } 462a88b5ba8SSam Ravnborg return -EBUSY; 463a88b5ba8SSam Ravnborg } 464a88b5ba8SSam Ravnborg 465a88b5ba8SSam Ravnborg static int send_rdx(struct ldc_channel *lp) 466a88b5ba8SSam Ravnborg { 467a88b5ba8SSam Ravnborg struct ldc_packet *p; 468a88b5ba8SSam Ravnborg unsigned long new_tail; 469a88b5ba8SSam Ravnborg 470a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_RDX, NULL, 0, 471a88b5ba8SSam Ravnborg &new_tail); 472a88b5ba8SSam Ravnborg if (p) { 473a88b5ba8SSam Ravnborg p->env = 0; 474a88b5ba8SSam Ravnborg p->seqid = ++lp->snd_nxt; 475a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt; 476a88b5ba8SSam Ravnborg 477a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND RDX env[0x%x] seqid[0x%x] ackid[0x%x]\n", 478a88b5ba8SSam Ravnborg p->env, p->seqid, p->u.r.ackid); 479a88b5ba8SSam Ravnborg 480a88b5ba8SSam Ravnborg return send_tx_packet(lp, p, new_tail); 481a88b5ba8SSam Ravnborg } 482a88b5ba8SSam Ravnborg return -EBUSY; 483a88b5ba8SSam Ravnborg } 484a88b5ba8SSam Ravnborg 485a88b5ba8SSam Ravnborg static int send_data_nack(struct ldc_channel *lp, struct ldc_packet *data_pkt) 486a88b5ba8SSam Ravnborg { 487a88b5ba8SSam Ravnborg struct ldc_packet *p; 488a88b5ba8SSam Ravnborg unsigned long new_tail; 489a88b5ba8SSam Ravnborg int err; 490a88b5ba8SSam Ravnborg 491a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail); 492a88b5ba8SSam Ravnborg if (!p) 493a88b5ba8SSam Ravnborg return -EBUSY; 494a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p)); 495a88b5ba8SSam Ravnborg p->type = data_pkt->type; 496a88b5ba8SSam Ravnborg p->stype = LDC_NACK; 497a88b5ba8SSam Ravnborg p->ctrl = data_pkt->ctrl & LDC_CTRL_MSK; 498a88b5ba8SSam Ravnborg p->seqid = lp->snd_nxt + 1; 499a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt; 500a88b5ba8SSam Ravnborg 501a88b5ba8SSam Ravnborg ldcdbg(HS, "SEND DATA NACK type[0x%x] ctl[0x%x] seq[0x%x] ack[0x%x]\n", 502a88b5ba8SSam Ravnborg p->type, p->ctrl, p->seqid, p->u.r.ackid); 503a88b5ba8SSam Ravnborg 504a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail); 505a88b5ba8SSam Ravnborg if (!err) 506a88b5ba8SSam Ravnborg lp->snd_nxt++; 507a88b5ba8SSam Ravnborg 508a88b5ba8SSam Ravnborg return err; 509a88b5ba8SSam Ravnborg } 510a88b5ba8SSam Ravnborg 51129693e75SJag Raman static int ldc_abort(struct ldc_channel *lp, const char *msg) 512a88b5ba8SSam Ravnborg { 513a88b5ba8SSam Ravnborg unsigned long hv_err; 514a88b5ba8SSam Ravnborg 51529693e75SJag Raman ldcdbg(STATE, "ABORT[%s]\n", msg); 51629693e75SJag Raman ldc_print(lp); 517a88b5ba8SSam Ravnborg 518a88b5ba8SSam Ravnborg /* We report but do not act upon the hypervisor errors because 519a88b5ba8SSam Ravnborg * there really isn't much we can do if they fail at this point. 520a88b5ba8SSam Ravnborg */ 521a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); 522a88b5ba8SSam Ravnborg if (hv_err) 523a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 524a88b5ba8SSam Ravnborg "sun4v_ldc_tx_qconf(%lx,%lx,%lx) failed, err=%lu\n", 525a88b5ba8SSam Ravnborg lp->id, lp->tx_ra, lp->tx_num_entries, hv_err); 526a88b5ba8SSam Ravnborg 527a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, 528a88b5ba8SSam Ravnborg &lp->tx_head, 529a88b5ba8SSam Ravnborg &lp->tx_tail, 530a88b5ba8SSam Ravnborg &lp->chan_state); 531a88b5ba8SSam Ravnborg if (hv_err) 532a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 533a88b5ba8SSam Ravnborg "sun4v_ldc_tx_get_state(%lx,...) failed, err=%lu\n", 534a88b5ba8SSam Ravnborg lp->id, hv_err); 535a88b5ba8SSam Ravnborg 536a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); 537a88b5ba8SSam Ravnborg if (hv_err) 538a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 539a88b5ba8SSam Ravnborg "sun4v_ldc_rx_qconf(%lx,%lx,%lx) failed, err=%lu\n", 540a88b5ba8SSam Ravnborg lp->id, lp->rx_ra, lp->rx_num_entries, hv_err); 541a88b5ba8SSam Ravnborg 542a88b5ba8SSam Ravnborg /* Refetch the RX queue state as well, because we could be invoked 543a88b5ba8SSam Ravnborg * here in the queue processing context. 544a88b5ba8SSam Ravnborg */ 545a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 546a88b5ba8SSam Ravnborg &lp->rx_head, 547a88b5ba8SSam Ravnborg &lp->rx_tail, 548a88b5ba8SSam Ravnborg &lp->chan_state); 549a88b5ba8SSam Ravnborg if (hv_err) 550a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_abort: " 551a88b5ba8SSam Ravnborg "sun4v_ldc_rx_get_state(%lx,...) failed, err=%lu\n", 552a88b5ba8SSam Ravnborg lp->id, hv_err); 553a88b5ba8SSam Ravnborg 554a88b5ba8SSam Ravnborg return -ECONNRESET; 555a88b5ba8SSam Ravnborg } 556a88b5ba8SSam Ravnborg 557a88b5ba8SSam Ravnborg static struct ldc_version *find_by_major(u16 major) 558a88b5ba8SSam Ravnborg { 559a88b5ba8SSam Ravnborg struct ldc_version *ret = NULL; 560a88b5ba8SSam Ravnborg int i; 561a88b5ba8SSam Ravnborg 562a88b5ba8SSam Ravnborg for (i = 0; i < ARRAY_SIZE(ver_arr); i++) { 563a88b5ba8SSam Ravnborg struct ldc_version *v = &ver_arr[i]; 564a88b5ba8SSam Ravnborg if (v->major <= major) { 565a88b5ba8SSam Ravnborg ret = v; 566a88b5ba8SSam Ravnborg break; 567a88b5ba8SSam Ravnborg } 568a88b5ba8SSam Ravnborg } 569a88b5ba8SSam Ravnborg return ret; 570a88b5ba8SSam Ravnborg } 571a88b5ba8SSam Ravnborg 572a88b5ba8SSam Ravnborg static int process_ver_info(struct ldc_channel *lp, struct ldc_version *vp) 573a88b5ba8SSam Ravnborg { 574a88b5ba8SSam Ravnborg struct ldc_version *vap; 575a88b5ba8SSam Ravnborg int err; 576a88b5ba8SSam Ravnborg 577a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT VERSION INFO major[%x] minor[%x]\n", 578a88b5ba8SSam Ravnborg vp->major, vp->minor); 579a88b5ba8SSam Ravnborg 580a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_GOTVERS) { 581a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN; 582a88b5ba8SSam Ravnborg memset(&lp->ver, 0, sizeof(lp->ver)); 583a88b5ba8SSam Ravnborg } 584a88b5ba8SSam Ravnborg 585a88b5ba8SSam Ravnborg vap = find_by_major(vp->major); 586a88b5ba8SSam Ravnborg if (!vap) { 587a88b5ba8SSam Ravnborg err = send_version_nack(lp, 0, 0); 588a88b5ba8SSam Ravnborg } else if (vap->major != vp->major) { 589a88b5ba8SSam Ravnborg err = send_version_nack(lp, vap->major, vap->minor); 590a88b5ba8SSam Ravnborg } else { 591a88b5ba8SSam Ravnborg struct ldc_version ver = *vp; 592a88b5ba8SSam Ravnborg if (ver.minor > vap->minor) 593a88b5ba8SSam Ravnborg ver.minor = vap->minor; 594a88b5ba8SSam Ravnborg err = send_version_ack(lp, &ver); 595a88b5ba8SSam Ravnborg if (!err) { 596a88b5ba8SSam Ravnborg lp->ver = ver; 597a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_GOTVERS; 598a88b5ba8SSam Ravnborg } 599a88b5ba8SSam Ravnborg } 600a88b5ba8SSam Ravnborg if (err) 60129693e75SJag Raman return LDC_ABORT(lp); 602a88b5ba8SSam Ravnborg 603a88b5ba8SSam Ravnborg return 0; 604a88b5ba8SSam Ravnborg } 605a88b5ba8SSam Ravnborg 606a88b5ba8SSam Ravnborg static int process_ver_ack(struct ldc_channel *lp, struct ldc_version *vp) 607a88b5ba8SSam Ravnborg { 608a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT VERSION ACK major[%x] minor[%x]\n", 609a88b5ba8SSam Ravnborg vp->major, vp->minor); 610a88b5ba8SSam Ravnborg 611a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_GOTVERS) { 612a88b5ba8SSam Ravnborg if (lp->ver.major != vp->major || 613a88b5ba8SSam Ravnborg lp->ver.minor != vp->minor) 61429693e75SJag Raman return LDC_ABORT(lp); 615a88b5ba8SSam Ravnborg } else { 616a88b5ba8SSam Ravnborg lp->ver = *vp; 617a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_GOTVERS; 618a88b5ba8SSam Ravnborg } 619a88b5ba8SSam Ravnborg if (send_rts(lp)) 62029693e75SJag Raman return LDC_ABORT(lp); 621a88b5ba8SSam Ravnborg return 0; 622a88b5ba8SSam Ravnborg } 623a88b5ba8SSam Ravnborg 624a88b5ba8SSam Ravnborg static int process_ver_nack(struct ldc_channel *lp, struct ldc_version *vp) 625a88b5ba8SSam Ravnborg { 626a88b5ba8SSam Ravnborg struct ldc_version *vap; 627a88b5ba8SSam Ravnborg struct ldc_packet *p; 628a88b5ba8SSam Ravnborg unsigned long new_tail; 629a88b5ba8SSam Ravnborg 630b2c0805fSSteven Rostedt if (vp->major == 0 && vp->minor == 0) 63129693e75SJag Raman return LDC_ABORT(lp); 632b2c0805fSSteven Rostedt 633b2c0805fSSteven Rostedt vap = find_by_major(vp->major); 634b2c0805fSSteven Rostedt if (!vap) 63529693e75SJag Raman return LDC_ABORT(lp); 636b2c0805fSSteven Rostedt 637a88b5ba8SSam Ravnborg p = handshake_compose_ctrl(lp, LDC_INFO, LDC_VERS, 638a88b5ba8SSam Ravnborg vap, sizeof(*vap), 639a88b5ba8SSam Ravnborg &new_tail); 640b2c0805fSSteven Rostedt if (!p) 64129693e75SJag Raman return LDC_ABORT(lp); 642b2c0805fSSteven Rostedt 643b2c0805fSSteven Rostedt return send_tx_packet(lp, p, new_tail); 644a88b5ba8SSam Ravnborg } 645a88b5ba8SSam Ravnborg 646a88b5ba8SSam Ravnborg static int process_version(struct ldc_channel *lp, 647a88b5ba8SSam Ravnborg struct ldc_packet *p) 648a88b5ba8SSam Ravnborg { 649a88b5ba8SSam Ravnborg struct ldc_version *vp; 650a88b5ba8SSam Ravnborg 651a88b5ba8SSam Ravnborg vp = (struct ldc_version *) p->u.u_data; 652a88b5ba8SSam Ravnborg 653a88b5ba8SSam Ravnborg switch (p->stype) { 654a88b5ba8SSam Ravnborg case LDC_INFO: 655a88b5ba8SSam Ravnborg return process_ver_info(lp, vp); 656a88b5ba8SSam Ravnborg 657a88b5ba8SSam Ravnborg case LDC_ACK: 658a88b5ba8SSam Ravnborg return process_ver_ack(lp, vp); 659a88b5ba8SSam Ravnborg 660a88b5ba8SSam Ravnborg case LDC_NACK: 661a88b5ba8SSam Ravnborg return process_ver_nack(lp, vp); 662a88b5ba8SSam Ravnborg 663a88b5ba8SSam Ravnborg default: 66429693e75SJag Raman return LDC_ABORT(lp); 665a88b5ba8SSam Ravnborg } 666a88b5ba8SSam Ravnborg } 667a88b5ba8SSam Ravnborg 668a88b5ba8SSam Ravnborg static int process_rts(struct ldc_channel *lp, 669a88b5ba8SSam Ravnborg struct ldc_packet *p) 670a88b5ba8SSam Ravnborg { 671a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RTS stype[%x] seqid[%x] env[%x]\n", 672a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env); 673a88b5ba8SSam Ravnborg 674a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO || 675a88b5ba8SSam Ravnborg lp->hs_state != LDC_HS_GOTVERS || 676a88b5ba8SSam Ravnborg p->env != lp->cfg.mode) 67729693e75SJag Raman return LDC_ABORT(lp); 678a88b5ba8SSam Ravnborg 679a88b5ba8SSam Ravnborg lp->snd_nxt = p->seqid; 680a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid; 681a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_SENTRTR; 682a88b5ba8SSam Ravnborg if (send_rtr(lp)) 68329693e75SJag Raman return LDC_ABORT(lp); 684a88b5ba8SSam Ravnborg 685a88b5ba8SSam Ravnborg return 0; 686a88b5ba8SSam Ravnborg } 687a88b5ba8SSam Ravnborg 688a88b5ba8SSam Ravnborg static int process_rtr(struct ldc_channel *lp, 689a88b5ba8SSam Ravnborg struct ldc_packet *p) 690a88b5ba8SSam Ravnborg { 691a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RTR stype[%x] seqid[%x] env[%x]\n", 692a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env); 693a88b5ba8SSam Ravnborg 694a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO || 695a88b5ba8SSam Ravnborg p->env != lp->cfg.mode) 69629693e75SJag Raman return LDC_ABORT(lp); 697a88b5ba8SSam Ravnborg 698a88b5ba8SSam Ravnborg lp->snd_nxt = p->seqid; 699a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 700a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 701a88b5ba8SSam Ravnborg send_rdx(lp); 702a88b5ba8SSam Ravnborg 703a88b5ba8SSam Ravnborg return LDC_EVENT_UP; 704a88b5ba8SSam Ravnborg } 705a88b5ba8SSam Ravnborg 706a88b5ba8SSam Ravnborg static int rx_seq_ok(struct ldc_channel *lp, u32 seqid) 707a88b5ba8SSam Ravnborg { 708a88b5ba8SSam Ravnborg return lp->rcv_nxt + 1 == seqid; 709a88b5ba8SSam Ravnborg } 710a88b5ba8SSam Ravnborg 711a88b5ba8SSam Ravnborg static int process_rdx(struct ldc_channel *lp, 712a88b5ba8SSam Ravnborg struct ldc_packet *p) 713a88b5ba8SSam Ravnborg { 714a88b5ba8SSam Ravnborg ldcdbg(HS, "GOT RDX stype[%x] seqid[%x] env[%x] ackid[%x]\n", 715a88b5ba8SSam Ravnborg p->stype, p->seqid, p->env, p->u.r.ackid); 716a88b5ba8SSam Ravnborg 717a88b5ba8SSam Ravnborg if (p->stype != LDC_INFO || 718a88b5ba8SSam Ravnborg !(rx_seq_ok(lp, p->seqid))) 71929693e75SJag Raman return LDC_ABORT(lp); 720a88b5ba8SSam Ravnborg 721a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid; 722a88b5ba8SSam Ravnborg 723a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 724a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 725a88b5ba8SSam Ravnborg 726a88b5ba8SSam Ravnborg return LDC_EVENT_UP; 727a88b5ba8SSam Ravnborg } 728a88b5ba8SSam Ravnborg 729a88b5ba8SSam Ravnborg static int process_control_frame(struct ldc_channel *lp, 730a88b5ba8SSam Ravnborg struct ldc_packet *p) 731a88b5ba8SSam Ravnborg { 732a88b5ba8SSam Ravnborg switch (p->ctrl) { 733a88b5ba8SSam Ravnborg case LDC_VERS: 734a88b5ba8SSam Ravnborg return process_version(lp, p); 735a88b5ba8SSam Ravnborg 736a88b5ba8SSam Ravnborg case LDC_RTS: 737a88b5ba8SSam Ravnborg return process_rts(lp, p); 738a88b5ba8SSam Ravnborg 739a88b5ba8SSam Ravnborg case LDC_RTR: 740a88b5ba8SSam Ravnborg return process_rtr(lp, p); 741a88b5ba8SSam Ravnborg 742a88b5ba8SSam Ravnborg case LDC_RDX: 743a88b5ba8SSam Ravnborg return process_rdx(lp, p); 744a88b5ba8SSam Ravnborg 745a88b5ba8SSam Ravnborg default: 74629693e75SJag Raman return LDC_ABORT(lp); 747a88b5ba8SSam Ravnborg } 748a88b5ba8SSam Ravnborg } 749a88b5ba8SSam Ravnborg 750a88b5ba8SSam Ravnborg static int process_error_frame(struct ldc_channel *lp, 751a88b5ba8SSam Ravnborg struct ldc_packet *p) 752a88b5ba8SSam Ravnborg { 75329693e75SJag Raman return LDC_ABORT(lp); 754a88b5ba8SSam Ravnborg } 755a88b5ba8SSam Ravnborg 756a88b5ba8SSam Ravnborg static int process_data_ack(struct ldc_channel *lp, 757a88b5ba8SSam Ravnborg struct ldc_packet *ack) 758a88b5ba8SSam Ravnborg { 759a88b5ba8SSam Ravnborg unsigned long head = lp->tx_acked; 760a88b5ba8SSam Ravnborg u32 ackid = ack->u.r.ackid; 761a88b5ba8SSam Ravnborg 762a88b5ba8SSam Ravnborg while (1) { 763a88b5ba8SSam Ravnborg struct ldc_packet *p = lp->tx_base + (head / LDC_PACKET_SIZE); 764a88b5ba8SSam Ravnborg 765a88b5ba8SSam Ravnborg head = tx_advance(lp, head); 766a88b5ba8SSam Ravnborg 767a88b5ba8SSam Ravnborg if (p->seqid == ackid) { 768a88b5ba8SSam Ravnborg lp->tx_acked = head; 769a88b5ba8SSam Ravnborg return 0; 770a88b5ba8SSam Ravnborg } 771a88b5ba8SSam Ravnborg if (head == lp->tx_tail) 77229693e75SJag Raman return LDC_ABORT(lp); 773a88b5ba8SSam Ravnborg } 774a88b5ba8SSam Ravnborg 775a88b5ba8SSam Ravnborg return 0; 776a88b5ba8SSam Ravnborg } 777a88b5ba8SSam Ravnborg 778a88b5ba8SSam Ravnborg static void send_events(struct ldc_channel *lp, unsigned int event_mask) 779a88b5ba8SSam Ravnborg { 780a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_RESET) 781a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_RESET); 782a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_UP) 783a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_UP); 784a88b5ba8SSam Ravnborg if (event_mask & LDC_EVENT_DATA_READY) 785a88b5ba8SSam Ravnborg lp->cfg.event(lp->event_arg, LDC_EVENT_DATA_READY); 786a88b5ba8SSam Ravnborg } 787a88b5ba8SSam Ravnborg 788a88b5ba8SSam Ravnborg static irqreturn_t ldc_rx(int irq, void *dev_id) 789a88b5ba8SSam Ravnborg { 790a88b5ba8SSam Ravnborg struct ldc_channel *lp = dev_id; 791c6fee081SDavid S. Miller unsigned long orig_state, flags; 792a88b5ba8SSam Ravnborg unsigned int event_mask; 793a88b5ba8SSam Ravnborg 794a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 795a88b5ba8SSam Ravnborg 796a88b5ba8SSam Ravnborg orig_state = lp->chan_state; 797c6fee081SDavid S. Miller 798c6fee081SDavid S. Miller /* We should probably check for hypervisor errors here and 799c6fee081SDavid S. Miller * reset the LDC channel if we get one. 800c6fee081SDavid S. Miller */ 801c6fee081SDavid S. Miller sun4v_ldc_rx_get_state(lp->id, 802a88b5ba8SSam Ravnborg &lp->rx_head, 803a88b5ba8SSam Ravnborg &lp->rx_tail, 804a88b5ba8SSam Ravnborg &lp->chan_state); 805a88b5ba8SSam Ravnborg 806a88b5ba8SSam Ravnborg ldcdbg(RX, "RX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n", 807a88b5ba8SSam Ravnborg orig_state, lp->chan_state, lp->rx_head, lp->rx_tail); 808a88b5ba8SSam Ravnborg 809a88b5ba8SSam Ravnborg event_mask = 0; 810a88b5ba8SSam Ravnborg 811a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW && 812a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_UP) { 813a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 814a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 815a88b5ba8SSam Ravnborg 816fc43b978SJag Raman /* 817fc43b978SJag Raman * Generate an LDC_EVENT_UP event if the channel 818fc43b978SJag Raman * was not already up. 819fc43b978SJag Raman */ 820fc43b978SJag Raman if (orig_state != LDC_CHANNEL_UP) { 821a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_UP; 822a88b5ba8SSam Ravnborg orig_state = lp->chan_state; 823a88b5ba8SSam Ravnborg } 824fc43b978SJag Raman } 825a88b5ba8SSam Ravnborg 826a88b5ba8SSam Ravnborg /* If we are in reset state, flush the RX queue and ignore 827a88b5ba8SSam Ravnborg * everything. 828a88b5ba8SSam Ravnborg */ 829a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_RESET) { 83085762a65SJag Raman (void) ldc_rx_reset(lp); 831a88b5ba8SSam Ravnborg goto out; 832a88b5ba8SSam Ravnborg } 833a88b5ba8SSam Ravnborg 834a88b5ba8SSam Ravnborg /* Once we finish the handshake, we let the ldc_read() 835a88b5ba8SSam Ravnborg * paths do all of the control frame and state management. 836a88b5ba8SSam Ravnborg * Just trigger the callback. 837a88b5ba8SSam Ravnborg */ 838a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_COMPLETE) { 839a88b5ba8SSam Ravnborg handshake_complete: 840a88b5ba8SSam Ravnborg if (lp->chan_state != orig_state) { 841a88b5ba8SSam Ravnborg unsigned int event = LDC_EVENT_RESET; 842a88b5ba8SSam Ravnborg 843a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_UP) 844a88b5ba8SSam Ravnborg event = LDC_EVENT_UP; 845a88b5ba8SSam Ravnborg 846a88b5ba8SSam Ravnborg event_mask |= event; 847a88b5ba8SSam Ravnborg } 848a88b5ba8SSam Ravnborg if (lp->rx_head != lp->rx_tail) 849a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_DATA_READY; 850a88b5ba8SSam Ravnborg 851a88b5ba8SSam Ravnborg goto out; 852a88b5ba8SSam Ravnborg } 853a88b5ba8SSam Ravnborg 854a88b5ba8SSam Ravnborg if (lp->chan_state != orig_state) 855a88b5ba8SSam Ravnborg goto out; 856a88b5ba8SSam Ravnborg 857a88b5ba8SSam Ravnborg while (lp->rx_head != lp->rx_tail) { 858a88b5ba8SSam Ravnborg struct ldc_packet *p; 859a88b5ba8SSam Ravnborg unsigned long new; 860a88b5ba8SSam Ravnborg int err; 861a88b5ba8SSam Ravnborg 862a88b5ba8SSam Ravnborg p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE); 863a88b5ba8SSam Ravnborg 864a88b5ba8SSam Ravnborg switch (p->type) { 865a88b5ba8SSam Ravnborg case LDC_CTRL: 866a88b5ba8SSam Ravnborg err = process_control_frame(lp, p); 867a88b5ba8SSam Ravnborg if (err > 0) 868a88b5ba8SSam Ravnborg event_mask |= err; 869a88b5ba8SSam Ravnborg break; 870a88b5ba8SSam Ravnborg 871a88b5ba8SSam Ravnborg case LDC_DATA: 872a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_DATA_READY; 873a88b5ba8SSam Ravnborg err = 0; 874a88b5ba8SSam Ravnborg break; 875a88b5ba8SSam Ravnborg 876a88b5ba8SSam Ravnborg case LDC_ERR: 877a88b5ba8SSam Ravnborg err = process_error_frame(lp, p); 878a88b5ba8SSam Ravnborg break; 879a88b5ba8SSam Ravnborg 880a88b5ba8SSam Ravnborg default: 88129693e75SJag Raman err = LDC_ABORT(lp); 882a88b5ba8SSam Ravnborg break; 883a88b5ba8SSam Ravnborg } 884a88b5ba8SSam Ravnborg 885a88b5ba8SSam Ravnborg if (err < 0) 886a88b5ba8SSam Ravnborg break; 887a88b5ba8SSam Ravnborg 888a88b5ba8SSam Ravnborg new = lp->rx_head; 889a88b5ba8SSam Ravnborg new += LDC_PACKET_SIZE; 890a88b5ba8SSam Ravnborg if (new == (lp->rx_num_entries * LDC_PACKET_SIZE)) 891a88b5ba8SSam Ravnborg new = 0; 892a88b5ba8SSam Ravnborg lp->rx_head = new; 893a88b5ba8SSam Ravnborg 894a88b5ba8SSam Ravnborg err = __set_rx_head(lp, new); 895a88b5ba8SSam Ravnborg if (err < 0) { 89629693e75SJag Raman (void) LDC_ABORT(lp); 897a88b5ba8SSam Ravnborg break; 898a88b5ba8SSam Ravnborg } 899a88b5ba8SSam Ravnborg if (lp->hs_state == LDC_HS_COMPLETE) 900a88b5ba8SSam Ravnborg goto handshake_complete; 901a88b5ba8SSam Ravnborg } 902a88b5ba8SSam Ravnborg 903a88b5ba8SSam Ravnborg out: 904a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 905a88b5ba8SSam Ravnborg 906a88b5ba8SSam Ravnborg send_events(lp, event_mask); 907a88b5ba8SSam Ravnborg 908a88b5ba8SSam Ravnborg return IRQ_HANDLED; 909a88b5ba8SSam Ravnborg } 910a88b5ba8SSam Ravnborg 911a88b5ba8SSam Ravnborg static irqreturn_t ldc_tx(int irq, void *dev_id) 912a88b5ba8SSam Ravnborg { 913a88b5ba8SSam Ravnborg struct ldc_channel *lp = dev_id; 914c6fee081SDavid S. Miller unsigned long flags, orig_state; 915a88b5ba8SSam Ravnborg unsigned int event_mask = 0; 916a88b5ba8SSam Ravnborg 917a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 918a88b5ba8SSam Ravnborg 919a88b5ba8SSam Ravnborg orig_state = lp->chan_state; 920c6fee081SDavid S. Miller 921c6fee081SDavid S. Miller /* We should probably check for hypervisor errors here and 922c6fee081SDavid S. Miller * reset the LDC channel if we get one. 923c6fee081SDavid S. Miller */ 924c6fee081SDavid S. Miller sun4v_ldc_tx_get_state(lp->id, 925a88b5ba8SSam Ravnborg &lp->tx_head, 926a88b5ba8SSam Ravnborg &lp->tx_tail, 927a88b5ba8SSam Ravnborg &lp->chan_state); 928a88b5ba8SSam Ravnborg 929a88b5ba8SSam Ravnborg ldcdbg(TX, " TX state[0x%02lx:0x%02lx] head[0x%04lx] tail[0x%04lx]\n", 930a88b5ba8SSam Ravnborg orig_state, lp->chan_state, lp->tx_head, lp->tx_tail); 931a88b5ba8SSam Ravnborg 932a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW && 933a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_UP) { 934a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_COMPLETE; 935a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_CONNECTED); 936a88b5ba8SSam Ravnborg 937fc43b978SJag Raman /* 938fc43b978SJag Raman * Generate an LDC_EVENT_UP event if the channel 939fc43b978SJag Raman * was not already up. 940fc43b978SJag Raman */ 941fc43b978SJag Raman if (orig_state != LDC_CHANNEL_UP) { 942a88b5ba8SSam Ravnborg event_mask |= LDC_EVENT_UP; 943fc43b978SJag Raman orig_state = lp->chan_state; 944fc43b978SJag Raman } 945a88b5ba8SSam Ravnborg } 946a88b5ba8SSam Ravnborg 947a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 948a88b5ba8SSam Ravnborg 949a88b5ba8SSam Ravnborg send_events(lp, event_mask); 950a88b5ba8SSam Ravnborg 951a88b5ba8SSam Ravnborg return IRQ_HANDLED; 952a88b5ba8SSam Ravnborg } 953a88b5ba8SSam Ravnborg 954a88b5ba8SSam Ravnborg /* XXX ldc_alloc() and ldc_free() needs to run under a mutex so 955a88b5ba8SSam Ravnborg * XXX that addition and removal from the ldc_channel_list has 956a88b5ba8SSam Ravnborg * XXX atomicity, otherwise the __ldc_channel_exists() check is 957a88b5ba8SSam Ravnborg * XXX totally pointless as another thread can slip into ldc_alloc() 958a88b5ba8SSam Ravnborg * XXX and add a channel with the same ID. There also needs to be 959a88b5ba8SSam Ravnborg * XXX a spinlock for ldc_channel_list. 960a88b5ba8SSam Ravnborg */ 961a88b5ba8SSam Ravnborg static HLIST_HEAD(ldc_channel_list); 962a88b5ba8SSam Ravnborg 963a88b5ba8SSam Ravnborg static int __ldc_channel_exists(unsigned long id) 964a88b5ba8SSam Ravnborg { 965a88b5ba8SSam Ravnborg struct ldc_channel *lp; 966a88b5ba8SSam Ravnborg 967b67bfe0dSSasha Levin hlist_for_each_entry(lp, &ldc_channel_list, list) { 968a88b5ba8SSam Ravnborg if (lp->id == id) 969a88b5ba8SSam Ravnborg return 1; 970a88b5ba8SSam Ravnborg } 971a88b5ba8SSam Ravnborg return 0; 972a88b5ba8SSam Ravnborg } 973a88b5ba8SSam Ravnborg 974a88b5ba8SSam Ravnborg static int alloc_queue(const char *name, unsigned long num_entries, 975a88b5ba8SSam Ravnborg struct ldc_packet **base, unsigned long *ra) 976a88b5ba8SSam Ravnborg { 977a88b5ba8SSam Ravnborg unsigned long size, order; 978a88b5ba8SSam Ravnborg void *q; 979a88b5ba8SSam Ravnborg 980a88b5ba8SSam Ravnborg size = num_entries * LDC_PACKET_SIZE; 981a88b5ba8SSam Ravnborg order = get_order(size); 982a88b5ba8SSam Ravnborg 983a88b5ba8SSam Ravnborg q = (void *) __get_free_pages(GFP_KERNEL, order); 984a88b5ba8SSam Ravnborg if (!q) { 985a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of %s queue failed with " 986a88b5ba8SSam Ravnborg "size=%lu order=%lu\n", name, size, order); 987a88b5ba8SSam Ravnborg return -ENOMEM; 988a88b5ba8SSam Ravnborg } 989a88b5ba8SSam Ravnborg 990a88b5ba8SSam Ravnborg memset(q, 0, PAGE_SIZE << order); 991a88b5ba8SSam Ravnborg 992a88b5ba8SSam Ravnborg *base = q; 993a88b5ba8SSam Ravnborg *ra = __pa(q); 994a88b5ba8SSam Ravnborg 995a88b5ba8SSam Ravnborg return 0; 996a88b5ba8SSam Ravnborg } 997a88b5ba8SSam Ravnborg 998a88b5ba8SSam Ravnborg static void free_queue(unsigned long num_entries, struct ldc_packet *q) 999a88b5ba8SSam Ravnborg { 1000a88b5ba8SSam Ravnborg unsigned long size, order; 1001a88b5ba8SSam Ravnborg 1002a88b5ba8SSam Ravnborg if (!q) 1003a88b5ba8SSam Ravnborg return; 1004a88b5ba8SSam Ravnborg 1005a88b5ba8SSam Ravnborg size = num_entries * LDC_PACKET_SIZE; 1006a88b5ba8SSam Ravnborg order = get_order(size); 1007a88b5ba8SSam Ravnborg 1008a88b5ba8SSam Ravnborg free_pages((unsigned long)q, order); 1009a88b5ba8SSam Ravnborg } 1010a88b5ba8SSam Ravnborg 10110ae53ed1SSowmini Varadhan static unsigned long ldc_cookie_to_index(u64 cookie, void *arg) 10120ae53ed1SSowmini Varadhan { 10130ae53ed1SSowmini Varadhan u64 szcode = cookie >> COOKIE_PGSZ_CODE_SHIFT; 10140ae53ed1SSowmini Varadhan /* struct ldc_iommu *ldc_iommu = (struct ldc_iommu *)arg; */ 10150ae53ed1SSowmini Varadhan 10160ae53ed1SSowmini Varadhan cookie &= ~COOKIE_PGSZ_CODE; 10170ae53ed1SSowmini Varadhan 10180ae53ed1SSowmini Varadhan return (cookie >> (13ULL + (szcode * 3ULL))); 10190ae53ed1SSowmini Varadhan } 10200ae53ed1SSowmini Varadhan 10210ae53ed1SSowmini Varadhan static void ldc_demap(struct ldc_iommu *iommu, unsigned long id, u64 cookie, 10220ae53ed1SSowmini Varadhan unsigned long entry, unsigned long npages) 10230ae53ed1SSowmini Varadhan { 10240ae53ed1SSowmini Varadhan struct ldc_mtable_entry *base; 10250ae53ed1SSowmini Varadhan unsigned long i, shift; 10260ae53ed1SSowmini Varadhan 10270ae53ed1SSowmini Varadhan shift = (cookie >> COOKIE_PGSZ_CODE_SHIFT) * 3; 10280ae53ed1SSowmini Varadhan base = iommu->page_table + entry; 10290ae53ed1SSowmini Varadhan for (i = 0; i < npages; i++) { 10300ae53ed1SSowmini Varadhan if (base->cookie) 10310ae53ed1SSowmini Varadhan sun4v_ldc_revoke(id, cookie + (i << shift), 10320ae53ed1SSowmini Varadhan base->cookie); 10330ae53ed1SSowmini Varadhan base->mte = 0; 10340ae53ed1SSowmini Varadhan } 10350ae53ed1SSowmini Varadhan } 10360ae53ed1SSowmini Varadhan 1037a88b5ba8SSam Ravnborg /* XXX Make this configurable... XXX */ 1038a88b5ba8SSam Ravnborg #define LDC_IOTABLE_SIZE (8 * 1024) 1039a88b5ba8SSam Ravnborg 10400ae53ed1SSowmini Varadhan static int ldc_iommu_init(const char *name, struct ldc_channel *lp) 1041a88b5ba8SSam Ravnborg { 1042a88b5ba8SSam Ravnborg unsigned long sz, num_tsb_entries, tsbsize, order; 10430ae53ed1SSowmini Varadhan struct ldc_iommu *ldc_iommu = &lp->iommu; 10440ae53ed1SSowmini Varadhan struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table; 1045a88b5ba8SSam Ravnborg struct ldc_mtable_entry *table; 1046a88b5ba8SSam Ravnborg unsigned long hv_err; 1047a88b5ba8SSam Ravnborg int err; 1048a88b5ba8SSam Ravnborg 1049a88b5ba8SSam Ravnborg num_tsb_entries = LDC_IOTABLE_SIZE; 1050a88b5ba8SSam Ravnborg tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); 10510ae53ed1SSowmini Varadhan spin_lock_init(&ldc_iommu->lock); 1052a88b5ba8SSam Ravnborg 1053a88b5ba8SSam Ravnborg sz = num_tsb_entries / 8; 1054a88b5ba8SSam Ravnborg sz = (sz + 7UL) & ~7UL; 10550ae53ed1SSowmini Varadhan iommu->map = kzalloc(sz, GFP_KERNEL); 10560ae53ed1SSowmini Varadhan if (!iommu->map) { 1057a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of arena map failed, sz=%lu\n", sz); 1058a88b5ba8SSam Ravnborg return -ENOMEM; 1059a88b5ba8SSam Ravnborg } 10600ae53ed1SSowmini Varadhan iommu_tbl_pool_init(iommu, num_tsb_entries, PAGE_SHIFT, 10610ae53ed1SSowmini Varadhan NULL, false /* no large pool */, 10620ae53ed1SSowmini Varadhan 1 /* npools */, 10630ae53ed1SSowmini Varadhan true /* skip span boundary check */); 1064a88b5ba8SSam Ravnborg 1065a88b5ba8SSam Ravnborg order = get_order(tsbsize); 1066a88b5ba8SSam Ravnborg 1067a88b5ba8SSam Ravnborg table = (struct ldc_mtable_entry *) 1068a88b5ba8SSam Ravnborg __get_free_pages(GFP_KERNEL, order); 1069a88b5ba8SSam Ravnborg err = -ENOMEM; 1070a88b5ba8SSam Ravnborg if (!table) { 1071a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "Alloc of MTE table failed, " 1072a88b5ba8SSam Ravnborg "size=%lu order=%lu\n", tsbsize, order); 1073a88b5ba8SSam Ravnborg goto out_free_map; 1074a88b5ba8SSam Ravnborg } 1075a88b5ba8SSam Ravnborg 1076a88b5ba8SSam Ravnborg memset(table, 0, PAGE_SIZE << order); 1077a88b5ba8SSam Ravnborg 10780ae53ed1SSowmini Varadhan ldc_iommu->page_table = table; 1079a88b5ba8SSam Ravnborg 1080a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_set_map_table(lp->id, __pa(table), 1081a88b5ba8SSam Ravnborg num_tsb_entries); 1082a88b5ba8SSam Ravnborg err = -EINVAL; 1083a88b5ba8SSam Ravnborg if (hv_err) 1084a88b5ba8SSam Ravnborg goto out_free_table; 1085a88b5ba8SSam Ravnborg 1086a88b5ba8SSam Ravnborg return 0; 1087a88b5ba8SSam Ravnborg 1088a88b5ba8SSam Ravnborg out_free_table: 1089a88b5ba8SSam Ravnborg free_pages((unsigned long) table, order); 10900ae53ed1SSowmini Varadhan ldc_iommu->page_table = NULL; 1091a88b5ba8SSam Ravnborg 1092a88b5ba8SSam Ravnborg out_free_map: 10930ae53ed1SSowmini Varadhan kfree(iommu->map); 10940ae53ed1SSowmini Varadhan iommu->map = NULL; 1095a88b5ba8SSam Ravnborg 1096a88b5ba8SSam Ravnborg return err; 1097a88b5ba8SSam Ravnborg } 1098a88b5ba8SSam Ravnborg 1099a88b5ba8SSam Ravnborg static void ldc_iommu_release(struct ldc_channel *lp) 1100a88b5ba8SSam Ravnborg { 11010ae53ed1SSowmini Varadhan struct ldc_iommu *ldc_iommu = &lp->iommu; 11020ae53ed1SSowmini Varadhan struct iommu_map_table *iommu = &ldc_iommu->iommu_map_table; 1103a88b5ba8SSam Ravnborg unsigned long num_tsb_entries, tsbsize, order; 1104a88b5ba8SSam Ravnborg 1105a88b5ba8SSam Ravnborg (void) sun4v_ldc_set_map_table(lp->id, 0, 0); 1106a88b5ba8SSam Ravnborg 11070ae53ed1SSowmini Varadhan num_tsb_entries = iommu->poolsize * iommu->nr_pools; 1108a88b5ba8SSam Ravnborg tsbsize = num_tsb_entries * sizeof(struct ldc_mtable_entry); 1109a88b5ba8SSam Ravnborg order = get_order(tsbsize); 1110a88b5ba8SSam Ravnborg 11110ae53ed1SSowmini Varadhan free_pages((unsigned long) ldc_iommu->page_table, order); 11120ae53ed1SSowmini Varadhan ldc_iommu->page_table = NULL; 1113a88b5ba8SSam Ravnborg 11140ae53ed1SSowmini Varadhan kfree(iommu->map); 11150ae53ed1SSowmini Varadhan iommu->map = NULL; 1116a88b5ba8SSam Ravnborg } 1117a88b5ba8SSam Ravnborg 1118a88b5ba8SSam Ravnborg struct ldc_channel *ldc_alloc(unsigned long id, 1119a88b5ba8SSam Ravnborg const struct ldc_channel_config *cfgp, 1120c21c4ab0SSowmini Varadhan void *event_arg, 1121c21c4ab0SSowmini Varadhan const char *name) 1122a88b5ba8SSam Ravnborg { 1123a88b5ba8SSam Ravnborg struct ldc_channel *lp; 1124a88b5ba8SSam Ravnborg const struct ldc_mode_ops *mops; 1125a88b5ba8SSam Ravnborg unsigned long dummy1, dummy2, hv_err; 1126a88b5ba8SSam Ravnborg u8 mss, *mssbuf; 1127a88b5ba8SSam Ravnborg int err; 1128a88b5ba8SSam Ravnborg 1129a88b5ba8SSam Ravnborg err = -ENODEV; 1130a88b5ba8SSam Ravnborg if (!ldom_domaining_enabled) 1131a88b5ba8SSam Ravnborg goto out_err; 1132a88b5ba8SSam Ravnborg 1133a88b5ba8SSam Ravnborg err = -EINVAL; 1134a88b5ba8SSam Ravnborg if (!cfgp) 1135a88b5ba8SSam Ravnborg goto out_err; 1136c21c4ab0SSowmini Varadhan if (!name) 1137c21c4ab0SSowmini Varadhan goto out_err; 1138a88b5ba8SSam Ravnborg 1139a88b5ba8SSam Ravnborg switch (cfgp->mode) { 1140a88b5ba8SSam Ravnborg case LDC_MODE_RAW: 1141a88b5ba8SSam Ravnborg mops = &raw_ops; 1142a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE; 1143a88b5ba8SSam Ravnborg break; 1144a88b5ba8SSam Ravnborg 1145a88b5ba8SSam Ravnborg case LDC_MODE_UNRELIABLE: 1146a88b5ba8SSam Ravnborg mops = &nonraw_ops; 1147a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE - 8; 1148a88b5ba8SSam Ravnborg break; 1149a88b5ba8SSam Ravnborg 1150a88b5ba8SSam Ravnborg case LDC_MODE_STREAM: 1151a88b5ba8SSam Ravnborg mops = &stream_ops; 1152a88b5ba8SSam Ravnborg mss = LDC_PACKET_SIZE - 8 - 8; 1153a88b5ba8SSam Ravnborg break; 1154a88b5ba8SSam Ravnborg 1155a88b5ba8SSam Ravnborg default: 1156a88b5ba8SSam Ravnborg goto out_err; 1157a88b5ba8SSam Ravnborg } 1158a88b5ba8SSam Ravnborg 1159a88b5ba8SSam Ravnborg if (!cfgp->event || !event_arg || !cfgp->rx_irq || !cfgp->tx_irq) 1160a88b5ba8SSam Ravnborg goto out_err; 1161a88b5ba8SSam Ravnborg 1162a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qinfo(id, &dummy1, &dummy2); 1163a88b5ba8SSam Ravnborg err = -ENODEV; 1164a88b5ba8SSam Ravnborg if (hv_err == HV_ECHANNEL) 1165a88b5ba8SSam Ravnborg goto out_err; 1166a88b5ba8SSam Ravnborg 1167a88b5ba8SSam Ravnborg err = -EEXIST; 1168a88b5ba8SSam Ravnborg if (__ldc_channel_exists(id)) 1169a88b5ba8SSam Ravnborg goto out_err; 1170a88b5ba8SSam Ravnborg 1171a88b5ba8SSam Ravnborg mssbuf = NULL; 1172a88b5ba8SSam Ravnborg 1173a88b5ba8SSam Ravnborg lp = kzalloc(sizeof(*lp), GFP_KERNEL); 1174a88b5ba8SSam Ravnborg err = -ENOMEM; 1175a88b5ba8SSam Ravnborg if (!lp) 1176a88b5ba8SSam Ravnborg goto out_err; 1177a88b5ba8SSam Ravnborg 1178a88b5ba8SSam Ravnborg spin_lock_init(&lp->lock); 1179a88b5ba8SSam Ravnborg 1180a88b5ba8SSam Ravnborg lp->id = id; 1181a88b5ba8SSam Ravnborg 11820ae53ed1SSowmini Varadhan err = ldc_iommu_init(name, lp); 1183a88b5ba8SSam Ravnborg if (err) 1184a88b5ba8SSam Ravnborg goto out_free_ldc; 1185a88b5ba8SSam Ravnborg 1186a88b5ba8SSam Ravnborg lp->mops = mops; 1187a88b5ba8SSam Ravnborg lp->mss = mss; 1188a88b5ba8SSam Ravnborg 1189a88b5ba8SSam Ravnborg lp->cfg = *cfgp; 1190a88b5ba8SSam Ravnborg if (!lp->cfg.mtu) 1191a88b5ba8SSam Ravnborg lp->cfg.mtu = LDC_DEFAULT_MTU; 1192a88b5ba8SSam Ravnborg 1193a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_STREAM) { 1194a88b5ba8SSam Ravnborg mssbuf = kzalloc(lp->cfg.mtu, GFP_KERNEL); 1195a88b5ba8SSam Ravnborg if (!mssbuf) { 1196a88b5ba8SSam Ravnborg err = -ENOMEM; 1197a88b5ba8SSam Ravnborg goto out_free_iommu; 1198a88b5ba8SSam Ravnborg } 1199a88b5ba8SSam Ravnborg lp->mssbuf = mssbuf; 1200a88b5ba8SSam Ravnborg } 1201a88b5ba8SSam Ravnborg 1202a88b5ba8SSam Ravnborg lp->event_arg = event_arg; 1203a88b5ba8SSam Ravnborg 1204a88b5ba8SSam Ravnborg /* XXX allow setting via ldc_channel_config to override defaults 1205a88b5ba8SSam Ravnborg * XXX or use some formula based upon mtu 1206a88b5ba8SSam Ravnborg */ 1207a88b5ba8SSam Ravnborg lp->tx_num_entries = LDC_DEFAULT_NUM_ENTRIES; 1208a88b5ba8SSam Ravnborg lp->rx_num_entries = LDC_DEFAULT_NUM_ENTRIES; 1209a88b5ba8SSam Ravnborg 1210a88b5ba8SSam Ravnborg err = alloc_queue("TX", lp->tx_num_entries, 1211a88b5ba8SSam Ravnborg &lp->tx_base, &lp->tx_ra); 1212a88b5ba8SSam Ravnborg if (err) 1213a88b5ba8SSam Ravnborg goto out_free_mssbuf; 1214a88b5ba8SSam Ravnborg 1215a88b5ba8SSam Ravnborg err = alloc_queue("RX", lp->rx_num_entries, 1216a88b5ba8SSam Ravnborg &lp->rx_base, &lp->rx_ra); 1217a88b5ba8SSam Ravnborg if (err) 1218a88b5ba8SSam Ravnborg goto out_free_txq; 1219a88b5ba8SSam Ravnborg 1220a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_ALLOCED_QUEUES; 1221a88b5ba8SSam Ravnborg 1222a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_CLOSED; 1223a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_INIT); 1224a88b5ba8SSam Ravnborg 1225a88b5ba8SSam Ravnborg INIT_HLIST_NODE(&lp->list); 1226a88b5ba8SSam Ravnborg hlist_add_head(&lp->list, &ldc_channel_list); 1227a88b5ba8SSam Ravnborg 1228a88b5ba8SSam Ravnborg INIT_HLIST_HEAD(&lp->mh_list); 1229a88b5ba8SSam Ravnborg 1230c21c4ab0SSowmini Varadhan snprintf(lp->rx_irq_name, LDC_IRQ_NAME_MAX, "%s RX", name); 1231c21c4ab0SSowmini Varadhan snprintf(lp->tx_irq_name, LDC_IRQ_NAME_MAX, "%s TX", name); 1232c21c4ab0SSowmini Varadhan 1233c21c4ab0SSowmini Varadhan err = request_irq(lp->cfg.rx_irq, ldc_rx, 0, 1234c21c4ab0SSowmini Varadhan lp->rx_irq_name, lp); 1235c21c4ab0SSowmini Varadhan if (err) 1236c21c4ab0SSowmini Varadhan goto out_free_txq; 1237c21c4ab0SSowmini Varadhan 1238c21c4ab0SSowmini Varadhan err = request_irq(lp->cfg.tx_irq, ldc_tx, 0, 1239c21c4ab0SSowmini Varadhan lp->tx_irq_name, lp); 1240c21c4ab0SSowmini Varadhan if (err) { 1241c21c4ab0SSowmini Varadhan free_irq(lp->cfg.rx_irq, lp); 1242c21c4ab0SSowmini Varadhan goto out_free_txq; 1243c21c4ab0SSowmini Varadhan } 1244c21c4ab0SSowmini Varadhan 1245a88b5ba8SSam Ravnborg return lp; 1246a88b5ba8SSam Ravnborg 1247a88b5ba8SSam Ravnborg out_free_txq: 1248a88b5ba8SSam Ravnborg free_queue(lp->tx_num_entries, lp->tx_base); 1249a88b5ba8SSam Ravnborg 1250a88b5ba8SSam Ravnborg out_free_mssbuf: 1251a88b5ba8SSam Ravnborg kfree(mssbuf); 1252a88b5ba8SSam Ravnborg 1253a88b5ba8SSam Ravnborg out_free_iommu: 1254a88b5ba8SSam Ravnborg ldc_iommu_release(lp); 1255a88b5ba8SSam Ravnborg 1256a88b5ba8SSam Ravnborg out_free_ldc: 1257a88b5ba8SSam Ravnborg kfree(lp); 1258a88b5ba8SSam Ravnborg 1259a88b5ba8SSam Ravnborg out_err: 1260a88b5ba8SSam Ravnborg return ERR_PTR(err); 1261a88b5ba8SSam Ravnborg } 1262a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_alloc); 1263a88b5ba8SSam Ravnborg 12641678c2bdSDwight Engen void ldc_unbind(struct ldc_channel *lp) 1265a88b5ba8SSam Ravnborg { 1266a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_REGISTERED_IRQS) { 1267a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1268a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp); 12691678c2bdSDwight Engen lp->flags &= ~LDC_FLAG_REGISTERED_IRQS; 1270a88b5ba8SSam Ravnborg } 1271a88b5ba8SSam Ravnborg 1272a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_REGISTERED_QUEUES) { 1273a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0); 1274a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0); 1275a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES; 1276a88b5ba8SSam Ravnborg } 1277a88b5ba8SSam Ravnborg if (lp->flags & LDC_FLAG_ALLOCED_QUEUES) { 1278a88b5ba8SSam Ravnborg free_queue(lp->tx_num_entries, lp->tx_base); 1279a88b5ba8SSam Ravnborg free_queue(lp->rx_num_entries, lp->rx_base); 1280a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_ALLOCED_QUEUES; 1281a88b5ba8SSam Ravnborg } 1282a88b5ba8SSam Ravnborg 12831678c2bdSDwight Engen ldc_set_state(lp, LDC_STATE_INIT); 12841678c2bdSDwight Engen } 12851678c2bdSDwight Engen EXPORT_SYMBOL(ldc_unbind); 12861678c2bdSDwight Engen 12871678c2bdSDwight Engen void ldc_free(struct ldc_channel *lp) 12881678c2bdSDwight Engen { 12891678c2bdSDwight Engen ldc_unbind(lp); 1290a88b5ba8SSam Ravnborg hlist_del(&lp->list); 1291a88b5ba8SSam Ravnborg kfree(lp->mssbuf); 1292a88b5ba8SSam Ravnborg ldc_iommu_release(lp); 1293a88b5ba8SSam Ravnborg 1294a88b5ba8SSam Ravnborg kfree(lp); 1295a88b5ba8SSam Ravnborg } 1296a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_free); 1297a88b5ba8SSam Ravnborg 1298a88b5ba8SSam Ravnborg /* Bind the channel. This registers the LDC queues with 1299a88b5ba8SSam Ravnborg * the hypervisor and puts the channel into a pseudo-listening 1300a88b5ba8SSam Ravnborg * state. This does not initiate a handshake, ldc_connect() does 1301a88b5ba8SSam Ravnborg * that. 1302a88b5ba8SSam Ravnborg */ 1303c21c4ab0SSowmini Varadhan int ldc_bind(struct ldc_channel *lp) 1304a88b5ba8SSam Ravnborg { 1305a88b5ba8SSam Ravnborg unsigned long hv_err, flags; 1306a88b5ba8SSam Ravnborg int err = -EINVAL; 1307a88b5ba8SSam Ravnborg 1308c21c4ab0SSowmini Varadhan if (lp->state != LDC_STATE_INIT) 1309a88b5ba8SSam Ravnborg return -EINVAL; 1310a88b5ba8SSam Ravnborg 1311a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1312a88b5ba8SSam Ravnborg 1313a88b5ba8SSam Ravnborg enable_irq(lp->cfg.rx_irq); 1314a88b5ba8SSam Ravnborg enable_irq(lp->cfg.tx_irq); 1315a88b5ba8SSam Ravnborg 1316a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_REGISTERED_IRQS; 1317a88b5ba8SSam Ravnborg 1318a88b5ba8SSam Ravnborg err = -ENODEV; 1319a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0); 1320a88b5ba8SSam Ravnborg if (hv_err) 1321a88b5ba8SSam Ravnborg goto out_free_irqs; 1322a88b5ba8SSam Ravnborg 1323a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); 1324a88b5ba8SSam Ravnborg if (hv_err) 1325a88b5ba8SSam Ravnborg goto out_free_irqs; 1326a88b5ba8SSam Ravnborg 1327a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0); 1328a88b5ba8SSam Ravnborg if (hv_err) 1329a88b5ba8SSam Ravnborg goto out_unmap_tx; 1330a88b5ba8SSam Ravnborg 1331a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); 1332a88b5ba8SSam Ravnborg if (hv_err) 1333a88b5ba8SSam Ravnborg goto out_unmap_tx; 1334a88b5ba8SSam Ravnborg 1335a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_REGISTERED_QUEUES; 1336a88b5ba8SSam Ravnborg 1337a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, 1338a88b5ba8SSam Ravnborg &lp->tx_head, 1339a88b5ba8SSam Ravnborg &lp->tx_tail, 1340a88b5ba8SSam Ravnborg &lp->chan_state); 1341a88b5ba8SSam Ravnborg err = -EBUSY; 1342a88b5ba8SSam Ravnborg if (hv_err) 1343a88b5ba8SSam Ravnborg goto out_unmap_rx; 1344a88b5ba8SSam Ravnborg 1345a88b5ba8SSam Ravnborg lp->tx_acked = lp->tx_head; 1346a88b5ba8SSam Ravnborg 1347a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN; 1348a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_BOUND); 1349a88b5ba8SSam Ravnborg 1350a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1351a88b5ba8SSam Ravnborg 1352a88b5ba8SSam Ravnborg return 0; 1353a88b5ba8SSam Ravnborg 1354a88b5ba8SSam Ravnborg out_unmap_rx: 1355a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_QUEUES; 1356a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0); 1357a88b5ba8SSam Ravnborg 1358a88b5ba8SSam Ravnborg out_unmap_tx: 1359a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0); 1360a88b5ba8SSam Ravnborg 1361a88b5ba8SSam Ravnborg out_free_irqs: 1362a88b5ba8SSam Ravnborg lp->flags &= ~LDC_FLAG_REGISTERED_IRQS; 1363a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp); 1364a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1365a88b5ba8SSam Ravnborg 1366a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1367a88b5ba8SSam Ravnborg 1368a88b5ba8SSam Ravnborg return err; 1369a88b5ba8SSam Ravnborg } 1370a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_bind); 1371a88b5ba8SSam Ravnborg 1372a88b5ba8SSam Ravnborg int ldc_connect(struct ldc_channel *lp) 1373a88b5ba8SSam Ravnborg { 1374a88b5ba8SSam Ravnborg unsigned long flags; 1375a88b5ba8SSam Ravnborg int err; 1376a88b5ba8SSam Ravnborg 1377a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW) 1378a88b5ba8SSam Ravnborg return -EINVAL; 1379a88b5ba8SSam Ravnborg 1380a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1381a88b5ba8SSam Ravnborg 1382a88b5ba8SSam Ravnborg if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || 1383a88b5ba8SSam Ravnborg !(lp->flags & LDC_FLAG_REGISTERED_QUEUES) || 1384a88b5ba8SSam Ravnborg lp->hs_state != LDC_HS_OPEN) 13854ec1b010SSowmini Varadhan err = ((lp->hs_state > LDC_HS_OPEN) ? 0 : -EINVAL); 1386a88b5ba8SSam Ravnborg else 1387a88b5ba8SSam Ravnborg err = start_handshake(lp); 1388a88b5ba8SSam Ravnborg 1389a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1390a88b5ba8SSam Ravnborg 1391a88b5ba8SSam Ravnborg return err; 1392a88b5ba8SSam Ravnborg } 1393a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_connect); 1394a88b5ba8SSam Ravnborg 1395a88b5ba8SSam Ravnborg int ldc_disconnect(struct ldc_channel *lp) 1396a88b5ba8SSam Ravnborg { 1397a88b5ba8SSam Ravnborg unsigned long hv_err, flags; 1398a88b5ba8SSam Ravnborg int err; 1399a88b5ba8SSam Ravnborg 1400a88b5ba8SSam Ravnborg if (lp->cfg.mode == LDC_MODE_RAW) 1401a88b5ba8SSam Ravnborg return -EINVAL; 1402a88b5ba8SSam Ravnborg 1403a88b5ba8SSam Ravnborg if (!(lp->flags & LDC_FLAG_ALLOCED_QUEUES) || 1404a88b5ba8SSam Ravnborg !(lp->flags & LDC_FLAG_REGISTERED_QUEUES)) 1405a88b5ba8SSam Ravnborg return -EINVAL; 1406a88b5ba8SSam Ravnborg 1407a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1408a88b5ba8SSam Ravnborg 1409a88b5ba8SSam Ravnborg err = -ENODEV; 1410a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, 0, 0); 1411a88b5ba8SSam Ravnborg if (hv_err) 1412a88b5ba8SSam Ravnborg goto out_err; 1413a88b5ba8SSam Ravnborg 1414a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_qconf(lp->id, lp->tx_ra, lp->tx_num_entries); 1415a88b5ba8SSam Ravnborg if (hv_err) 1416a88b5ba8SSam Ravnborg goto out_err; 1417a88b5ba8SSam Ravnborg 1418a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, 0, 0); 1419a88b5ba8SSam Ravnborg if (hv_err) 1420a88b5ba8SSam Ravnborg goto out_err; 1421a88b5ba8SSam Ravnborg 1422a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_qconf(lp->id, lp->rx_ra, lp->rx_num_entries); 1423a88b5ba8SSam Ravnborg if (hv_err) 1424a88b5ba8SSam Ravnborg goto out_err; 1425a88b5ba8SSam Ravnborg 1426a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_BOUND); 1427a88b5ba8SSam Ravnborg lp->hs_state = LDC_HS_OPEN; 1428a88b5ba8SSam Ravnborg lp->flags |= LDC_FLAG_RESET; 1429a88b5ba8SSam Ravnborg 1430a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1431a88b5ba8SSam Ravnborg 1432a88b5ba8SSam Ravnborg return 0; 1433a88b5ba8SSam Ravnborg 1434a88b5ba8SSam Ravnborg out_err: 1435a88b5ba8SSam Ravnborg sun4v_ldc_tx_qconf(lp->id, 0, 0); 1436a88b5ba8SSam Ravnborg sun4v_ldc_rx_qconf(lp->id, 0, 0); 1437a88b5ba8SSam Ravnborg free_irq(lp->cfg.tx_irq, lp); 1438a88b5ba8SSam Ravnborg free_irq(lp->cfg.rx_irq, lp); 1439a88b5ba8SSam Ravnborg lp->flags &= ~(LDC_FLAG_REGISTERED_IRQS | 1440a88b5ba8SSam Ravnborg LDC_FLAG_REGISTERED_QUEUES); 1441a88b5ba8SSam Ravnborg ldc_set_state(lp, LDC_STATE_INIT); 1442a88b5ba8SSam Ravnborg 1443a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1444a88b5ba8SSam Ravnborg 1445a88b5ba8SSam Ravnborg return err; 1446a88b5ba8SSam Ravnborg } 1447a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_disconnect); 1448a88b5ba8SSam Ravnborg 1449a88b5ba8SSam Ravnborg int ldc_state(struct ldc_channel *lp) 1450a88b5ba8SSam Ravnborg { 1451a88b5ba8SSam Ravnborg return lp->state; 1452a88b5ba8SSam Ravnborg } 1453a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_state); 1454a88b5ba8SSam Ravnborg 145585762a65SJag Raman void ldc_set_state(struct ldc_channel *lp, u8 state) 145685762a65SJag Raman { 145785762a65SJag Raman ldcdbg(STATE, "STATE (%s) --> (%s)\n", 145885762a65SJag Raman state_to_str(lp->state), 145985762a65SJag Raman state_to_str(state)); 146085762a65SJag Raman 146185762a65SJag Raman lp->state = state; 146285762a65SJag Raman } 146385762a65SJag Raman 146485762a65SJag Raman int ldc_mode(struct ldc_channel *lp) 146585762a65SJag Raman { 146685762a65SJag Raman return lp->cfg.mode; 146785762a65SJag Raman } 146885762a65SJag Raman 146985762a65SJag Raman int ldc_rx_reset(struct ldc_channel *lp) 147085762a65SJag Raman { 147185762a65SJag Raman return __set_rx_head(lp, lp->rx_tail); 147285762a65SJag Raman } 147385762a65SJag Raman 147485762a65SJag Raman void __ldc_print(struct ldc_channel *lp, const char *caller) 147585762a65SJag Raman { 147685762a65SJag Raman pr_info("%s: id=0x%lx flags=0x%x state=%s cstate=0x%lx hsstate=0x%x\n" 147785762a65SJag Raman "\trx_h=0x%lx rx_t=0x%lx rx_n=%ld\n" 147885762a65SJag Raman "\ttx_h=0x%lx tx_t=0x%lx tx_n=%ld\n" 147985762a65SJag Raman "\trcv_nxt=%u snd_nxt=%u\n", 148085762a65SJag Raman caller, lp->id, lp->flags, state_to_str(lp->state), 148185762a65SJag Raman lp->chan_state, lp->hs_state, 148285762a65SJag Raman lp->rx_head, lp->rx_tail, lp->rx_num_entries, 148385762a65SJag Raman lp->tx_head, lp->tx_tail, lp->tx_num_entries, 148485762a65SJag Raman lp->rcv_nxt, lp->snd_nxt); 148585762a65SJag Raman } 148685762a65SJag Raman 1487a88b5ba8SSam Ravnborg static int write_raw(struct ldc_channel *lp, const void *buf, unsigned int size) 1488a88b5ba8SSam Ravnborg { 1489a88b5ba8SSam Ravnborg struct ldc_packet *p; 1490fc43b978SJag Raman unsigned long new_tail, hv_err; 1491a88b5ba8SSam Ravnborg int err; 1492a88b5ba8SSam Ravnborg 1493fc43b978SJag Raman hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail, 1494fc43b978SJag Raman &lp->chan_state); 1495fc43b978SJag Raman if (unlikely(hv_err)) 1496fc43b978SJag Raman return -EBUSY; 1497fc43b978SJag Raman 1498fc43b978SJag Raman if (unlikely(lp->chan_state != LDC_CHANNEL_UP)) 1499fc43b978SJag Raman return LDC_ABORT(lp); 1500fc43b978SJag Raman 1501a88b5ba8SSam Ravnborg if (size > LDC_PACKET_SIZE) 1502a88b5ba8SSam Ravnborg return -EMSGSIZE; 1503a88b5ba8SSam Ravnborg 1504a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail); 1505a88b5ba8SSam Ravnborg if (!p) 1506a88b5ba8SSam Ravnborg return -EAGAIN; 1507a88b5ba8SSam Ravnborg 1508a88b5ba8SSam Ravnborg memcpy(p, buf, size); 1509a88b5ba8SSam Ravnborg 1510a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail); 1511a88b5ba8SSam Ravnborg if (!err) 1512a88b5ba8SSam Ravnborg err = size; 1513a88b5ba8SSam Ravnborg 1514a88b5ba8SSam Ravnborg return err; 1515a88b5ba8SSam Ravnborg } 1516a88b5ba8SSam Ravnborg 1517a88b5ba8SSam Ravnborg static int read_raw(struct ldc_channel *lp, void *buf, unsigned int size) 1518a88b5ba8SSam Ravnborg { 1519a88b5ba8SSam Ravnborg struct ldc_packet *p; 1520a88b5ba8SSam Ravnborg unsigned long hv_err, new; 1521a88b5ba8SSam Ravnborg int err; 1522a88b5ba8SSam Ravnborg 1523a88b5ba8SSam Ravnborg if (size < LDC_PACKET_SIZE) 1524a88b5ba8SSam Ravnborg return -EINVAL; 1525a88b5ba8SSam Ravnborg 1526a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 1527a88b5ba8SSam Ravnborg &lp->rx_head, 1528a88b5ba8SSam Ravnborg &lp->rx_tail, 1529a88b5ba8SSam Ravnborg &lp->chan_state); 1530a88b5ba8SSam Ravnborg if (hv_err) 153129693e75SJag Raman return LDC_ABORT(lp); 1532a88b5ba8SSam Ravnborg 1533a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN || 1534a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING) 1535a88b5ba8SSam Ravnborg return -ECONNRESET; 1536a88b5ba8SSam Ravnborg 1537a88b5ba8SSam Ravnborg if (lp->rx_head == lp->rx_tail) 1538a88b5ba8SSam Ravnborg return 0; 1539a88b5ba8SSam Ravnborg 1540a88b5ba8SSam Ravnborg p = lp->rx_base + (lp->rx_head / LDC_PACKET_SIZE); 1541a88b5ba8SSam Ravnborg memcpy(buf, p, LDC_PACKET_SIZE); 1542a88b5ba8SSam Ravnborg 1543a88b5ba8SSam Ravnborg new = rx_advance(lp, lp->rx_head); 1544a88b5ba8SSam Ravnborg lp->rx_head = new; 1545a88b5ba8SSam Ravnborg 1546a88b5ba8SSam Ravnborg err = __set_rx_head(lp, new); 1547a88b5ba8SSam Ravnborg if (err < 0) 1548a88b5ba8SSam Ravnborg err = -ECONNRESET; 1549a88b5ba8SSam Ravnborg else 1550a88b5ba8SSam Ravnborg err = LDC_PACKET_SIZE; 1551a88b5ba8SSam Ravnborg 1552a88b5ba8SSam Ravnborg return err; 1553a88b5ba8SSam Ravnborg } 1554a88b5ba8SSam Ravnborg 1555a88b5ba8SSam Ravnborg static const struct ldc_mode_ops raw_ops = { 1556a88b5ba8SSam Ravnborg .write = write_raw, 1557a88b5ba8SSam Ravnborg .read = read_raw, 1558a88b5ba8SSam Ravnborg }; 1559a88b5ba8SSam Ravnborg 1560a88b5ba8SSam Ravnborg static int write_nonraw(struct ldc_channel *lp, const void *buf, 1561a88b5ba8SSam Ravnborg unsigned int size) 1562a88b5ba8SSam Ravnborg { 1563a88b5ba8SSam Ravnborg unsigned long hv_err, tail; 1564a88b5ba8SSam Ravnborg unsigned int copied; 1565a88b5ba8SSam Ravnborg u32 seq; 1566a88b5ba8SSam Ravnborg int err; 1567a88b5ba8SSam Ravnborg 1568a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_tx_get_state(lp->id, &lp->tx_head, &lp->tx_tail, 1569a88b5ba8SSam Ravnborg &lp->chan_state); 1570a88b5ba8SSam Ravnborg if (unlikely(hv_err)) 1571a88b5ba8SSam Ravnborg return -EBUSY; 1572a88b5ba8SSam Ravnborg 1573a88b5ba8SSam Ravnborg if (unlikely(lp->chan_state != LDC_CHANNEL_UP)) 157429693e75SJag Raman return LDC_ABORT(lp); 1575a88b5ba8SSam Ravnborg 1576a88b5ba8SSam Ravnborg if (!tx_has_space_for(lp, size)) 1577a88b5ba8SSam Ravnborg return -EAGAIN; 1578a88b5ba8SSam Ravnborg 1579a88b5ba8SSam Ravnborg seq = lp->snd_nxt; 1580a88b5ba8SSam Ravnborg copied = 0; 1581a88b5ba8SSam Ravnborg tail = lp->tx_tail; 1582a88b5ba8SSam Ravnborg while (copied < size) { 1583a88b5ba8SSam Ravnborg struct ldc_packet *p = lp->tx_base + (tail / LDC_PACKET_SIZE); 1584a88b5ba8SSam Ravnborg u8 *data = ((lp->cfg.mode == LDC_MODE_UNRELIABLE) ? 1585a88b5ba8SSam Ravnborg p->u.u_data : 1586a88b5ba8SSam Ravnborg p->u.r.r_data); 1587a88b5ba8SSam Ravnborg int data_len; 1588a88b5ba8SSam Ravnborg 1589a88b5ba8SSam Ravnborg p->type = LDC_DATA; 1590a88b5ba8SSam Ravnborg p->stype = LDC_INFO; 1591a88b5ba8SSam Ravnborg p->ctrl = 0; 1592a88b5ba8SSam Ravnborg 1593a88b5ba8SSam Ravnborg data_len = size - copied; 1594a88b5ba8SSam Ravnborg if (data_len > lp->mss) 1595a88b5ba8SSam Ravnborg data_len = lp->mss; 1596a88b5ba8SSam Ravnborg 1597a88b5ba8SSam Ravnborg BUG_ON(data_len > LDC_LEN); 1598a88b5ba8SSam Ravnborg 1599a88b5ba8SSam Ravnborg p->env = (data_len | 1600a88b5ba8SSam Ravnborg (copied == 0 ? LDC_START : 0) | 1601a88b5ba8SSam Ravnborg (data_len == size - copied ? LDC_STOP : 0)); 1602a88b5ba8SSam Ravnborg 1603a88b5ba8SSam Ravnborg p->seqid = ++seq; 1604a88b5ba8SSam Ravnborg 1605a88b5ba8SSam Ravnborg ldcdbg(DATA, "SENT DATA [%02x:%02x:%02x:%02x:%08x]\n", 1606a88b5ba8SSam Ravnborg p->type, 1607a88b5ba8SSam Ravnborg p->stype, 1608a88b5ba8SSam Ravnborg p->ctrl, 1609a88b5ba8SSam Ravnborg p->env, 1610a88b5ba8SSam Ravnborg p->seqid); 1611a88b5ba8SSam Ravnborg 1612a88b5ba8SSam Ravnborg memcpy(data, buf, data_len); 1613a88b5ba8SSam Ravnborg buf += data_len; 1614a88b5ba8SSam Ravnborg copied += data_len; 1615a88b5ba8SSam Ravnborg 1616a88b5ba8SSam Ravnborg tail = tx_advance(lp, tail); 1617a88b5ba8SSam Ravnborg } 1618a88b5ba8SSam Ravnborg 1619a88b5ba8SSam Ravnborg err = set_tx_tail(lp, tail); 1620a88b5ba8SSam Ravnborg if (!err) { 1621a88b5ba8SSam Ravnborg lp->snd_nxt = seq; 1622a88b5ba8SSam Ravnborg err = size; 1623a88b5ba8SSam Ravnborg } 1624a88b5ba8SSam Ravnborg 1625a88b5ba8SSam Ravnborg return err; 1626a88b5ba8SSam Ravnborg } 1627a88b5ba8SSam Ravnborg 1628a88b5ba8SSam Ravnborg static int rx_bad_seq(struct ldc_channel *lp, struct ldc_packet *p, 1629a88b5ba8SSam Ravnborg struct ldc_packet *first_frag) 1630a88b5ba8SSam Ravnborg { 1631a88b5ba8SSam Ravnborg int err; 1632a88b5ba8SSam Ravnborg 1633a88b5ba8SSam Ravnborg if (first_frag) 1634a88b5ba8SSam Ravnborg lp->rcv_nxt = first_frag->seqid - 1; 1635a88b5ba8SSam Ravnborg 1636a88b5ba8SSam Ravnborg err = send_data_nack(lp, p); 1637a88b5ba8SSam Ravnborg if (err) 1638a88b5ba8SSam Ravnborg return err; 1639a88b5ba8SSam Ravnborg 164085762a65SJag Raman err = ldc_rx_reset(lp); 1641a88b5ba8SSam Ravnborg if (err < 0) 164229693e75SJag Raman return LDC_ABORT(lp); 1643a88b5ba8SSam Ravnborg 1644a88b5ba8SSam Ravnborg return 0; 1645a88b5ba8SSam Ravnborg } 1646a88b5ba8SSam Ravnborg 1647a88b5ba8SSam Ravnborg static int data_ack_nack(struct ldc_channel *lp, struct ldc_packet *p) 1648a88b5ba8SSam Ravnborg { 1649a88b5ba8SSam Ravnborg if (p->stype & LDC_ACK) { 1650a88b5ba8SSam Ravnborg int err = process_data_ack(lp, p); 1651a88b5ba8SSam Ravnborg if (err) 1652a88b5ba8SSam Ravnborg return err; 1653a88b5ba8SSam Ravnborg } 1654a88b5ba8SSam Ravnborg if (p->stype & LDC_NACK) 165529693e75SJag Raman return LDC_ABORT(lp); 1656a88b5ba8SSam Ravnborg 1657a88b5ba8SSam Ravnborg return 0; 1658a88b5ba8SSam Ravnborg } 1659a88b5ba8SSam Ravnborg 1660a88b5ba8SSam Ravnborg static int rx_data_wait(struct ldc_channel *lp, unsigned long cur_head) 1661a88b5ba8SSam Ravnborg { 1662a88b5ba8SSam Ravnborg unsigned long dummy; 1663a88b5ba8SSam Ravnborg int limit = 1000; 1664a88b5ba8SSam Ravnborg 1665a88b5ba8SSam Ravnborg ldcdbg(DATA, "DATA WAIT cur_head[%lx] rx_head[%lx] rx_tail[%lx]\n", 1666a88b5ba8SSam Ravnborg cur_head, lp->rx_head, lp->rx_tail); 1667a88b5ba8SSam Ravnborg while (limit-- > 0) { 1668a88b5ba8SSam Ravnborg unsigned long hv_err; 1669a88b5ba8SSam Ravnborg 1670a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 1671a88b5ba8SSam Ravnborg &dummy, 1672a88b5ba8SSam Ravnborg &lp->rx_tail, 1673a88b5ba8SSam Ravnborg &lp->chan_state); 1674a88b5ba8SSam Ravnborg if (hv_err) 167529693e75SJag Raman return LDC_ABORT(lp); 1676a88b5ba8SSam Ravnborg 1677a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN || 1678a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING) 1679a88b5ba8SSam Ravnborg return -ECONNRESET; 1680a88b5ba8SSam Ravnborg 1681a88b5ba8SSam Ravnborg if (cur_head != lp->rx_tail) { 1682a88b5ba8SSam Ravnborg ldcdbg(DATA, "DATA WAIT DONE " 1683a88b5ba8SSam Ravnborg "head[%lx] tail[%lx] chan_state[%lx]\n", 1684a88b5ba8SSam Ravnborg dummy, lp->rx_tail, lp->chan_state); 1685a88b5ba8SSam Ravnborg return 0; 1686a88b5ba8SSam Ravnborg } 1687a88b5ba8SSam Ravnborg 1688a88b5ba8SSam Ravnborg udelay(1); 1689a88b5ba8SSam Ravnborg } 1690a88b5ba8SSam Ravnborg return -EAGAIN; 1691a88b5ba8SSam Ravnborg } 1692a88b5ba8SSam Ravnborg 1693a88b5ba8SSam Ravnborg static int rx_set_head(struct ldc_channel *lp, unsigned long head) 1694a88b5ba8SSam Ravnborg { 1695a88b5ba8SSam Ravnborg int err = __set_rx_head(lp, head); 1696a88b5ba8SSam Ravnborg 1697a88b5ba8SSam Ravnborg if (err < 0) 169829693e75SJag Raman return LDC_ABORT(lp); 1699a88b5ba8SSam Ravnborg 1700a88b5ba8SSam Ravnborg lp->rx_head = head; 1701a88b5ba8SSam Ravnborg return 0; 1702a88b5ba8SSam Ravnborg } 1703a88b5ba8SSam Ravnborg 1704a88b5ba8SSam Ravnborg static void send_data_ack(struct ldc_channel *lp) 1705a88b5ba8SSam Ravnborg { 1706a88b5ba8SSam Ravnborg unsigned long new_tail; 1707a88b5ba8SSam Ravnborg struct ldc_packet *p; 1708a88b5ba8SSam Ravnborg 1709a88b5ba8SSam Ravnborg p = data_get_tx_packet(lp, &new_tail); 1710a88b5ba8SSam Ravnborg if (likely(p)) { 1711a88b5ba8SSam Ravnborg int err; 1712a88b5ba8SSam Ravnborg 1713a88b5ba8SSam Ravnborg memset(p, 0, sizeof(*p)); 1714a88b5ba8SSam Ravnborg p->type = LDC_DATA; 1715a88b5ba8SSam Ravnborg p->stype = LDC_ACK; 1716a88b5ba8SSam Ravnborg p->ctrl = 0; 1717a88b5ba8SSam Ravnborg p->seqid = lp->snd_nxt + 1; 1718a88b5ba8SSam Ravnborg p->u.r.ackid = lp->rcv_nxt; 1719a88b5ba8SSam Ravnborg 1720a88b5ba8SSam Ravnborg err = send_tx_packet(lp, p, new_tail); 1721a88b5ba8SSam Ravnborg if (!err) 1722a88b5ba8SSam Ravnborg lp->snd_nxt++; 1723a88b5ba8SSam Ravnborg } 1724a88b5ba8SSam Ravnborg } 1725a88b5ba8SSam Ravnborg 1726a88b5ba8SSam Ravnborg static int read_nonraw(struct ldc_channel *lp, void *buf, unsigned int size) 1727a88b5ba8SSam Ravnborg { 1728a88b5ba8SSam Ravnborg struct ldc_packet *first_frag; 1729a88b5ba8SSam Ravnborg unsigned long hv_err, new; 1730a88b5ba8SSam Ravnborg int err, copied; 1731a88b5ba8SSam Ravnborg 1732a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_rx_get_state(lp->id, 1733a88b5ba8SSam Ravnborg &lp->rx_head, 1734a88b5ba8SSam Ravnborg &lp->rx_tail, 1735a88b5ba8SSam Ravnborg &lp->chan_state); 1736a88b5ba8SSam Ravnborg if (hv_err) 173729693e75SJag Raman return LDC_ABORT(lp); 1738a88b5ba8SSam Ravnborg 1739a88b5ba8SSam Ravnborg if (lp->chan_state == LDC_CHANNEL_DOWN || 1740a88b5ba8SSam Ravnborg lp->chan_state == LDC_CHANNEL_RESETTING) 1741a88b5ba8SSam Ravnborg return -ECONNRESET; 1742a88b5ba8SSam Ravnborg 1743a88b5ba8SSam Ravnborg if (lp->rx_head == lp->rx_tail) 1744a88b5ba8SSam Ravnborg return 0; 1745a88b5ba8SSam Ravnborg 1746a88b5ba8SSam Ravnborg first_frag = NULL; 1747a88b5ba8SSam Ravnborg copied = err = 0; 1748a88b5ba8SSam Ravnborg new = lp->rx_head; 1749a88b5ba8SSam Ravnborg while (1) { 1750a88b5ba8SSam Ravnborg struct ldc_packet *p; 1751a88b5ba8SSam Ravnborg int pkt_len; 1752a88b5ba8SSam Ravnborg 1753a88b5ba8SSam Ravnborg BUG_ON(new == lp->rx_tail); 1754a88b5ba8SSam Ravnborg p = lp->rx_base + (new / LDC_PACKET_SIZE); 1755a88b5ba8SSam Ravnborg 1756a88b5ba8SSam Ravnborg ldcdbg(RX, "RX read pkt[%02x:%02x:%02x:%02x:%08x:%08x] " 1757a88b5ba8SSam Ravnborg "rcv_nxt[%08x]\n", 1758a88b5ba8SSam Ravnborg p->type, 1759a88b5ba8SSam Ravnborg p->stype, 1760a88b5ba8SSam Ravnborg p->ctrl, 1761a88b5ba8SSam Ravnborg p->env, 1762a88b5ba8SSam Ravnborg p->seqid, 1763a88b5ba8SSam Ravnborg p->u.r.ackid, 1764a88b5ba8SSam Ravnborg lp->rcv_nxt); 1765a88b5ba8SSam Ravnborg 1766a88b5ba8SSam Ravnborg if (unlikely(!rx_seq_ok(lp, p->seqid))) { 1767a88b5ba8SSam Ravnborg err = rx_bad_seq(lp, p, first_frag); 1768a88b5ba8SSam Ravnborg copied = 0; 1769a88b5ba8SSam Ravnborg break; 1770a88b5ba8SSam Ravnborg } 1771a88b5ba8SSam Ravnborg 1772a88b5ba8SSam Ravnborg if (p->type & LDC_CTRL) { 1773a88b5ba8SSam Ravnborg err = process_control_frame(lp, p); 1774a88b5ba8SSam Ravnborg if (err < 0) 1775a88b5ba8SSam Ravnborg break; 1776a88b5ba8SSam Ravnborg err = 0; 1777a88b5ba8SSam Ravnborg } 1778a88b5ba8SSam Ravnborg 1779a88b5ba8SSam Ravnborg lp->rcv_nxt = p->seqid; 1780a88b5ba8SSam Ravnborg 17816c95483bSJag Raman /* 17826c95483bSJag Raman * If this is a control-only packet, there is nothing 17836c95483bSJag Raman * else to do but advance the rx queue since the packet 17846c95483bSJag Raman * was already processed above. 17856c95483bSJag Raman */ 1786a88b5ba8SSam Ravnborg if (!(p->type & LDC_DATA)) { 1787a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 17886c95483bSJag Raman break; 1789a88b5ba8SSam Ravnborg } 1790a88b5ba8SSam Ravnborg if (p->stype & (LDC_ACK | LDC_NACK)) { 1791a88b5ba8SSam Ravnborg err = data_ack_nack(lp, p); 1792a88b5ba8SSam Ravnborg if (err) 1793a88b5ba8SSam Ravnborg break; 1794a88b5ba8SSam Ravnborg } 1795a88b5ba8SSam Ravnborg if (!(p->stype & LDC_INFO)) { 1796a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1797a88b5ba8SSam Ravnborg err = rx_set_head(lp, new); 1798a88b5ba8SSam Ravnborg if (err) 1799a88b5ba8SSam Ravnborg break; 1800a88b5ba8SSam Ravnborg goto no_data; 1801a88b5ba8SSam Ravnborg } 1802a88b5ba8SSam Ravnborg 1803a88b5ba8SSam Ravnborg pkt_len = p->env & LDC_LEN; 1804a88b5ba8SSam Ravnborg 1805a88b5ba8SSam Ravnborg /* Every initial packet starts with the START bit set. 1806a88b5ba8SSam Ravnborg * 1807a88b5ba8SSam Ravnborg * Singleton packets will have both START+STOP set. 1808a88b5ba8SSam Ravnborg * 1809a88b5ba8SSam Ravnborg * Fragments will have START set in the first frame, STOP 1810a88b5ba8SSam Ravnborg * set in the last frame, and neither bit set in middle 1811a88b5ba8SSam Ravnborg * frames of the packet. 1812a88b5ba8SSam Ravnborg * 1813a88b5ba8SSam Ravnborg * Therefore if we are at the beginning of a packet and 1814a88b5ba8SSam Ravnborg * we don't see START, or we are in the middle of a fragmented 1815a88b5ba8SSam Ravnborg * packet and do see START, we are unsynchronized and should 1816a88b5ba8SSam Ravnborg * flush the RX queue. 1817a88b5ba8SSam Ravnborg */ 1818a88b5ba8SSam Ravnborg if ((first_frag == NULL && !(p->env & LDC_START)) || 1819a88b5ba8SSam Ravnborg (first_frag != NULL && (p->env & LDC_START))) { 1820a88b5ba8SSam Ravnborg if (!first_frag) 1821a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1822a88b5ba8SSam Ravnborg 1823a88b5ba8SSam Ravnborg err = rx_set_head(lp, new); 1824a88b5ba8SSam Ravnborg if (err) 1825a88b5ba8SSam Ravnborg break; 1826a88b5ba8SSam Ravnborg 1827a88b5ba8SSam Ravnborg if (!first_frag) 1828a88b5ba8SSam Ravnborg goto no_data; 1829a88b5ba8SSam Ravnborg } 1830a88b5ba8SSam Ravnborg if (!first_frag) 1831a88b5ba8SSam Ravnborg first_frag = p; 1832a88b5ba8SSam Ravnborg 1833a88b5ba8SSam Ravnborg if (pkt_len > size - copied) { 1834a88b5ba8SSam Ravnborg /* User didn't give us a big enough buffer, 1835a88b5ba8SSam Ravnborg * what to do? This is a pretty serious error. 1836a88b5ba8SSam Ravnborg * 1837a88b5ba8SSam Ravnborg * Since we haven't updated the RX ring head to 1838a88b5ba8SSam Ravnborg * consume any of the packets, signal the error 1839a88b5ba8SSam Ravnborg * to the user and just leave the RX ring alone. 1840a88b5ba8SSam Ravnborg * 1841a88b5ba8SSam Ravnborg * This seems the best behavior because this allows 1842a88b5ba8SSam Ravnborg * a user of the LDC layer to start with a small 1843a88b5ba8SSam Ravnborg * RX buffer for ldc_read() calls and use -EMSGSIZE 1844a88b5ba8SSam Ravnborg * as a cue to enlarge it's read buffer. 1845a88b5ba8SSam Ravnborg */ 1846a88b5ba8SSam Ravnborg err = -EMSGSIZE; 1847a88b5ba8SSam Ravnborg break; 1848a88b5ba8SSam Ravnborg } 1849a88b5ba8SSam Ravnborg 1850a88b5ba8SSam Ravnborg /* Ok, we are gonna eat this one. */ 1851a88b5ba8SSam Ravnborg new = rx_advance(lp, new); 1852a88b5ba8SSam Ravnborg 1853a88b5ba8SSam Ravnborg memcpy(buf, 1854a88b5ba8SSam Ravnborg (lp->cfg.mode == LDC_MODE_UNRELIABLE ? 1855a88b5ba8SSam Ravnborg p->u.u_data : p->u.r.r_data), pkt_len); 1856a88b5ba8SSam Ravnborg buf += pkt_len; 1857a88b5ba8SSam Ravnborg copied += pkt_len; 1858a88b5ba8SSam Ravnborg 1859a88b5ba8SSam Ravnborg if (p->env & LDC_STOP) 1860a88b5ba8SSam Ravnborg break; 1861a88b5ba8SSam Ravnborg 1862a88b5ba8SSam Ravnborg no_data: 1863a88b5ba8SSam Ravnborg if (new == lp->rx_tail) { 1864a88b5ba8SSam Ravnborg err = rx_data_wait(lp, new); 1865a88b5ba8SSam Ravnborg if (err) 1866a88b5ba8SSam Ravnborg break; 1867a88b5ba8SSam Ravnborg } 1868a88b5ba8SSam Ravnborg } 1869a88b5ba8SSam Ravnborg 1870a88b5ba8SSam Ravnborg if (!err) 1871a88b5ba8SSam Ravnborg err = rx_set_head(lp, new); 1872a88b5ba8SSam Ravnborg 1873a88b5ba8SSam Ravnborg if (err && first_frag) 1874a88b5ba8SSam Ravnborg lp->rcv_nxt = first_frag->seqid - 1; 1875a88b5ba8SSam Ravnborg 1876a88b5ba8SSam Ravnborg if (!err) { 1877a88b5ba8SSam Ravnborg err = copied; 1878a88b5ba8SSam Ravnborg if (err > 0 && lp->cfg.mode != LDC_MODE_UNRELIABLE) 1879a88b5ba8SSam Ravnborg send_data_ack(lp); 1880a88b5ba8SSam Ravnborg } 1881a88b5ba8SSam Ravnborg 1882a88b5ba8SSam Ravnborg return err; 1883a88b5ba8SSam Ravnborg } 1884a88b5ba8SSam Ravnborg 1885a88b5ba8SSam Ravnborg static const struct ldc_mode_ops nonraw_ops = { 1886a88b5ba8SSam Ravnborg .write = write_nonraw, 1887a88b5ba8SSam Ravnborg .read = read_nonraw, 1888a88b5ba8SSam Ravnborg }; 1889a88b5ba8SSam Ravnborg 1890a88b5ba8SSam Ravnborg static int write_stream(struct ldc_channel *lp, const void *buf, 1891a88b5ba8SSam Ravnborg unsigned int size) 1892a88b5ba8SSam Ravnborg { 1893a88b5ba8SSam Ravnborg if (size > lp->cfg.mtu) 1894a88b5ba8SSam Ravnborg size = lp->cfg.mtu; 1895a88b5ba8SSam Ravnborg return write_nonraw(lp, buf, size); 1896a88b5ba8SSam Ravnborg } 1897a88b5ba8SSam Ravnborg 1898a88b5ba8SSam Ravnborg static int read_stream(struct ldc_channel *lp, void *buf, unsigned int size) 1899a88b5ba8SSam Ravnborg { 1900a88b5ba8SSam Ravnborg if (!lp->mssbuf_len) { 1901a88b5ba8SSam Ravnborg int err = read_nonraw(lp, lp->mssbuf, lp->cfg.mtu); 1902a88b5ba8SSam Ravnborg if (err < 0) 1903a88b5ba8SSam Ravnborg return err; 1904a88b5ba8SSam Ravnborg 1905a88b5ba8SSam Ravnborg lp->mssbuf_len = err; 1906a88b5ba8SSam Ravnborg lp->mssbuf_off = 0; 1907a88b5ba8SSam Ravnborg } 1908a88b5ba8SSam Ravnborg 1909a88b5ba8SSam Ravnborg if (size > lp->mssbuf_len) 1910a88b5ba8SSam Ravnborg size = lp->mssbuf_len; 1911a88b5ba8SSam Ravnborg memcpy(buf, lp->mssbuf + lp->mssbuf_off, size); 1912a88b5ba8SSam Ravnborg 1913a88b5ba8SSam Ravnborg lp->mssbuf_off += size; 1914a88b5ba8SSam Ravnborg lp->mssbuf_len -= size; 1915a88b5ba8SSam Ravnborg 1916a88b5ba8SSam Ravnborg return size; 1917a88b5ba8SSam Ravnborg } 1918a88b5ba8SSam Ravnborg 1919a88b5ba8SSam Ravnborg static const struct ldc_mode_ops stream_ops = { 1920a88b5ba8SSam Ravnborg .write = write_stream, 1921a88b5ba8SSam Ravnborg .read = read_stream, 1922a88b5ba8SSam Ravnborg }; 1923a88b5ba8SSam Ravnborg 1924a88b5ba8SSam Ravnborg int ldc_write(struct ldc_channel *lp, const void *buf, unsigned int size) 1925a88b5ba8SSam Ravnborg { 1926a88b5ba8SSam Ravnborg unsigned long flags; 1927a88b5ba8SSam Ravnborg int err; 1928a88b5ba8SSam Ravnborg 1929a88b5ba8SSam Ravnborg if (!buf) 1930a88b5ba8SSam Ravnborg return -EINVAL; 1931a88b5ba8SSam Ravnborg 1932a88b5ba8SSam Ravnborg if (!size) 1933a88b5ba8SSam Ravnborg return 0; 1934a88b5ba8SSam Ravnborg 1935a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1936a88b5ba8SSam Ravnborg 1937a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE) 1938a88b5ba8SSam Ravnborg err = -ENOTCONN; 1939a88b5ba8SSam Ravnborg else 1940a88b5ba8SSam Ravnborg err = lp->mops->write(lp, buf, size); 1941a88b5ba8SSam Ravnborg 1942a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1943a88b5ba8SSam Ravnborg 1944a88b5ba8SSam Ravnborg return err; 1945a88b5ba8SSam Ravnborg } 1946a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_write); 1947a88b5ba8SSam Ravnborg 1948a88b5ba8SSam Ravnborg int ldc_read(struct ldc_channel *lp, void *buf, unsigned int size) 1949a88b5ba8SSam Ravnborg { 1950a88b5ba8SSam Ravnborg unsigned long flags; 1951a88b5ba8SSam Ravnborg int err; 1952a88b5ba8SSam Ravnborg 1953a88b5ba8SSam Ravnborg if (!buf) 1954a88b5ba8SSam Ravnborg return -EINVAL; 1955a88b5ba8SSam Ravnborg 1956a88b5ba8SSam Ravnborg if (!size) 1957a88b5ba8SSam Ravnborg return 0; 1958a88b5ba8SSam Ravnborg 1959a88b5ba8SSam Ravnborg spin_lock_irqsave(&lp->lock, flags); 1960a88b5ba8SSam Ravnborg 1961a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE) 1962a88b5ba8SSam Ravnborg err = -ENOTCONN; 1963a88b5ba8SSam Ravnborg else 1964a88b5ba8SSam Ravnborg err = lp->mops->read(lp, buf, size); 1965a88b5ba8SSam Ravnborg 1966a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&lp->lock, flags); 1967a88b5ba8SSam Ravnborg 1968a88b5ba8SSam Ravnborg return err; 1969a88b5ba8SSam Ravnborg } 1970a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_read); 1971a88b5ba8SSam Ravnborg 1972a88b5ba8SSam Ravnborg static u64 pagesize_code(void) 1973a88b5ba8SSam Ravnborg { 1974a88b5ba8SSam Ravnborg switch (PAGE_SIZE) { 1975a88b5ba8SSam Ravnborg default: 1976a88b5ba8SSam Ravnborg case (8ULL * 1024ULL): 1977a88b5ba8SSam Ravnborg return 0; 1978a88b5ba8SSam Ravnborg case (64ULL * 1024ULL): 1979a88b5ba8SSam Ravnborg return 1; 1980a88b5ba8SSam Ravnborg case (512ULL * 1024ULL): 1981a88b5ba8SSam Ravnborg return 2; 1982a88b5ba8SSam Ravnborg case (4ULL * 1024ULL * 1024ULL): 1983a88b5ba8SSam Ravnborg return 3; 1984a88b5ba8SSam Ravnborg case (32ULL * 1024ULL * 1024ULL): 1985a88b5ba8SSam Ravnborg return 4; 1986a88b5ba8SSam Ravnborg case (256ULL * 1024ULL * 1024ULL): 1987a88b5ba8SSam Ravnborg return 5; 1988a88b5ba8SSam Ravnborg } 1989a88b5ba8SSam Ravnborg } 1990a88b5ba8SSam Ravnborg 1991a88b5ba8SSam Ravnborg static u64 make_cookie(u64 index, u64 pgsz_code, u64 page_offset) 1992a88b5ba8SSam Ravnborg { 1993a88b5ba8SSam Ravnborg return ((pgsz_code << COOKIE_PGSZ_CODE_SHIFT) | 1994a88b5ba8SSam Ravnborg (index << PAGE_SHIFT) | 1995a88b5ba8SSam Ravnborg page_offset); 1996a88b5ba8SSam Ravnborg } 1997a88b5ba8SSam Ravnborg 1998a88b5ba8SSam Ravnborg 1999a88b5ba8SSam Ravnborg static struct ldc_mtable_entry *alloc_npages(struct ldc_iommu *iommu, 2000a88b5ba8SSam Ravnborg unsigned long npages) 2001a88b5ba8SSam Ravnborg { 2002a88b5ba8SSam Ravnborg long entry; 2003a88b5ba8SSam Ravnborg 20040ae53ed1SSowmini Varadhan entry = iommu_tbl_range_alloc(NULL, &iommu->iommu_map_table, 20050ae53ed1SSowmini Varadhan npages, NULL, (unsigned long)-1, 0); 2006d618382bSDavid S. Miller if (unlikely(entry == IOMMU_ERROR_CODE)) 2007a88b5ba8SSam Ravnborg return NULL; 2008a88b5ba8SSam Ravnborg 2009a88b5ba8SSam Ravnborg return iommu->page_table + entry; 2010a88b5ba8SSam Ravnborg } 2011a88b5ba8SSam Ravnborg 2012a88b5ba8SSam Ravnborg static u64 perm_to_mte(unsigned int map_perm) 2013a88b5ba8SSam Ravnborg { 2014a88b5ba8SSam Ravnborg u64 mte_base; 2015a88b5ba8SSam Ravnborg 2016a88b5ba8SSam Ravnborg mte_base = pagesize_code(); 2017a88b5ba8SSam Ravnborg 2018a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_SHADOW) { 2019a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R) 2020a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_COPY_R; 2021a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W) 2022a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_COPY_W; 2023a88b5ba8SSam Ravnborg } 2024a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_DIRECT) { 2025a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R) 2026a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_READ; 2027a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W) 2028a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_WRITE; 2029a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_X) 2030a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_EXEC; 2031a88b5ba8SSam Ravnborg } 2032a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_IO) { 2033a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_R) 2034a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_IOMMU_R; 2035a88b5ba8SSam Ravnborg if (map_perm & LDC_MAP_W) 2036a88b5ba8SSam Ravnborg mte_base |= LDC_MTE_IOMMU_W; 2037a88b5ba8SSam Ravnborg } 2038a88b5ba8SSam Ravnborg 2039a88b5ba8SSam Ravnborg return mte_base; 2040a88b5ba8SSam Ravnborg } 2041a88b5ba8SSam Ravnborg 2042a88b5ba8SSam Ravnborg static int pages_in_region(unsigned long base, long len) 2043a88b5ba8SSam Ravnborg { 2044a88b5ba8SSam Ravnborg int count = 0; 2045a88b5ba8SSam Ravnborg 2046a88b5ba8SSam Ravnborg do { 2047a88b5ba8SSam Ravnborg unsigned long new = (base + PAGE_SIZE) & PAGE_MASK; 2048a88b5ba8SSam Ravnborg 2049a88b5ba8SSam Ravnborg len -= (new - base); 2050a88b5ba8SSam Ravnborg base = new; 2051a88b5ba8SSam Ravnborg count++; 2052a88b5ba8SSam Ravnborg } while (len > 0); 2053a88b5ba8SSam Ravnborg 2054a88b5ba8SSam Ravnborg return count; 2055a88b5ba8SSam Ravnborg } 2056a88b5ba8SSam Ravnborg 2057a88b5ba8SSam Ravnborg struct cookie_state { 2058a88b5ba8SSam Ravnborg struct ldc_mtable_entry *page_table; 2059a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies; 2060a88b5ba8SSam Ravnborg u64 mte_base; 2061a88b5ba8SSam Ravnborg u64 prev_cookie; 2062a88b5ba8SSam Ravnborg u32 pte_idx; 2063a88b5ba8SSam Ravnborg u32 nc; 2064a88b5ba8SSam Ravnborg }; 2065a88b5ba8SSam Ravnborg 2066a88b5ba8SSam Ravnborg static void fill_cookies(struct cookie_state *sp, unsigned long pa, 2067a88b5ba8SSam Ravnborg unsigned long off, unsigned long len) 2068a88b5ba8SSam Ravnborg { 2069a88b5ba8SSam Ravnborg do { 2070a88b5ba8SSam Ravnborg unsigned long tlen, new = pa + PAGE_SIZE; 2071a88b5ba8SSam Ravnborg u64 this_cookie; 2072a88b5ba8SSam Ravnborg 2073a88b5ba8SSam Ravnborg sp->page_table[sp->pte_idx].mte = sp->mte_base | pa; 2074a88b5ba8SSam Ravnborg 2075a88b5ba8SSam Ravnborg tlen = PAGE_SIZE; 2076a88b5ba8SSam Ravnborg if (off) 2077a88b5ba8SSam Ravnborg tlen = PAGE_SIZE - off; 2078a88b5ba8SSam Ravnborg if (tlen > len) 2079a88b5ba8SSam Ravnborg tlen = len; 2080a88b5ba8SSam Ravnborg 2081a88b5ba8SSam Ravnborg this_cookie = make_cookie(sp->pte_idx, 2082a88b5ba8SSam Ravnborg pagesize_code(), off); 2083a88b5ba8SSam Ravnborg 2084a88b5ba8SSam Ravnborg off = 0; 2085a88b5ba8SSam Ravnborg 2086a88b5ba8SSam Ravnborg if (this_cookie == sp->prev_cookie) { 2087a88b5ba8SSam Ravnborg sp->cookies[sp->nc - 1].cookie_size += tlen; 2088a88b5ba8SSam Ravnborg } else { 2089a88b5ba8SSam Ravnborg sp->cookies[sp->nc].cookie_addr = this_cookie; 2090a88b5ba8SSam Ravnborg sp->cookies[sp->nc].cookie_size = tlen; 2091a88b5ba8SSam Ravnborg sp->nc++; 2092a88b5ba8SSam Ravnborg } 2093a88b5ba8SSam Ravnborg sp->prev_cookie = this_cookie + tlen; 2094a88b5ba8SSam Ravnborg 2095a88b5ba8SSam Ravnborg sp->pte_idx++; 2096a88b5ba8SSam Ravnborg 2097a88b5ba8SSam Ravnborg len -= tlen; 2098a88b5ba8SSam Ravnborg pa = new; 2099a88b5ba8SSam Ravnborg } while (len > 0); 2100a88b5ba8SSam Ravnborg } 2101a88b5ba8SSam Ravnborg 2102a88b5ba8SSam Ravnborg static int sg_count_one(struct scatterlist *sg) 2103a88b5ba8SSam Ravnborg { 2104a88b5ba8SSam Ravnborg unsigned long base = page_to_pfn(sg_page(sg)) << PAGE_SHIFT; 2105a88b5ba8SSam Ravnborg long len = sg->length; 2106a88b5ba8SSam Ravnborg 2107a88b5ba8SSam Ravnborg if ((sg->offset | len) & (8UL - 1)) 2108a88b5ba8SSam Ravnborg return -EFAULT; 2109a88b5ba8SSam Ravnborg 2110a88b5ba8SSam Ravnborg return pages_in_region(base + sg->offset, len); 2111a88b5ba8SSam Ravnborg } 2112a88b5ba8SSam Ravnborg 2113a88b5ba8SSam Ravnborg static int sg_count_pages(struct scatterlist *sg, int num_sg) 2114a88b5ba8SSam Ravnborg { 2115a88b5ba8SSam Ravnborg int count; 2116a88b5ba8SSam Ravnborg int i; 2117a88b5ba8SSam Ravnborg 2118a88b5ba8SSam Ravnborg count = 0; 2119a88b5ba8SSam Ravnborg for (i = 0; i < num_sg; i++) { 2120a88b5ba8SSam Ravnborg int err = sg_count_one(sg + i); 2121a88b5ba8SSam Ravnborg if (err < 0) 2122a88b5ba8SSam Ravnborg return err; 2123a88b5ba8SSam Ravnborg count += err; 2124a88b5ba8SSam Ravnborg } 2125a88b5ba8SSam Ravnborg 2126a88b5ba8SSam Ravnborg return count; 2127a88b5ba8SSam Ravnborg } 2128a88b5ba8SSam Ravnborg 2129a88b5ba8SSam Ravnborg int ldc_map_sg(struct ldc_channel *lp, 2130a88b5ba8SSam Ravnborg struct scatterlist *sg, int num_sg, 2131a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies, 2132a88b5ba8SSam Ravnborg unsigned int map_perm) 2133a88b5ba8SSam Ravnborg { 21340ae53ed1SSowmini Varadhan unsigned long i, npages; 2135a88b5ba8SSam Ravnborg struct ldc_mtable_entry *base; 2136a88b5ba8SSam Ravnborg struct cookie_state state; 2137a88b5ba8SSam Ravnborg struct ldc_iommu *iommu; 2138a88b5ba8SSam Ravnborg int err; 21398c07a308SAkinobu Mita struct scatterlist *s; 2140a88b5ba8SSam Ravnborg 2141a88b5ba8SSam Ravnborg if (map_perm & ~LDC_MAP_ALL) 2142a88b5ba8SSam Ravnborg return -EINVAL; 2143a88b5ba8SSam Ravnborg 2144a88b5ba8SSam Ravnborg err = sg_count_pages(sg, num_sg); 2145a88b5ba8SSam Ravnborg if (err < 0) 2146a88b5ba8SSam Ravnborg return err; 2147a88b5ba8SSam Ravnborg 2148a88b5ba8SSam Ravnborg npages = err; 2149a88b5ba8SSam Ravnborg if (err > ncookies) 2150a88b5ba8SSam Ravnborg return -EMSGSIZE; 2151a88b5ba8SSam Ravnborg 2152a88b5ba8SSam Ravnborg iommu = &lp->iommu; 2153a88b5ba8SSam Ravnborg 2154a88b5ba8SSam Ravnborg base = alloc_npages(iommu, npages); 2155a88b5ba8SSam Ravnborg 2156a88b5ba8SSam Ravnborg if (!base) 2157a88b5ba8SSam Ravnborg return -ENOMEM; 2158a88b5ba8SSam Ravnborg 2159a88b5ba8SSam Ravnborg state.page_table = iommu->page_table; 2160a88b5ba8SSam Ravnborg state.cookies = cookies; 2161a88b5ba8SSam Ravnborg state.mte_base = perm_to_mte(map_perm); 2162a88b5ba8SSam Ravnborg state.prev_cookie = ~(u64)0; 2163a88b5ba8SSam Ravnborg state.pte_idx = (base - iommu->page_table); 2164a88b5ba8SSam Ravnborg state.nc = 0; 2165a88b5ba8SSam Ravnborg 21668c07a308SAkinobu Mita for_each_sg(sg, s, num_sg, i) { 21678c07a308SAkinobu Mita fill_cookies(&state, page_to_pfn(sg_page(s)) << PAGE_SHIFT, 21688c07a308SAkinobu Mita s->offset, s->length); 21698c07a308SAkinobu Mita } 2170a88b5ba8SSam Ravnborg 2171a88b5ba8SSam Ravnborg return state.nc; 2172a88b5ba8SSam Ravnborg } 2173a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_map_sg); 2174a88b5ba8SSam Ravnborg 2175a88b5ba8SSam Ravnborg int ldc_map_single(struct ldc_channel *lp, 2176a88b5ba8SSam Ravnborg void *buf, unsigned int len, 2177a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies, 2178a88b5ba8SSam Ravnborg unsigned int map_perm) 2179a88b5ba8SSam Ravnborg { 21800ae53ed1SSowmini Varadhan unsigned long npages, pa; 2181a88b5ba8SSam Ravnborg struct ldc_mtable_entry *base; 2182a88b5ba8SSam Ravnborg struct cookie_state state; 2183a88b5ba8SSam Ravnborg struct ldc_iommu *iommu; 2184a88b5ba8SSam Ravnborg 2185a88b5ba8SSam Ravnborg if ((map_perm & ~LDC_MAP_ALL) || (ncookies < 1)) 2186a88b5ba8SSam Ravnborg return -EINVAL; 2187a88b5ba8SSam Ravnborg 2188a88b5ba8SSam Ravnborg pa = __pa(buf); 2189a88b5ba8SSam Ravnborg if ((pa | len) & (8UL - 1)) 2190a88b5ba8SSam Ravnborg return -EFAULT; 2191a88b5ba8SSam Ravnborg 2192a88b5ba8SSam Ravnborg npages = pages_in_region(pa, len); 2193a88b5ba8SSam Ravnborg 2194a88b5ba8SSam Ravnborg iommu = &lp->iommu; 2195a88b5ba8SSam Ravnborg 2196a88b5ba8SSam Ravnborg base = alloc_npages(iommu, npages); 2197a88b5ba8SSam Ravnborg 2198a88b5ba8SSam Ravnborg if (!base) 2199a88b5ba8SSam Ravnborg return -ENOMEM; 2200a88b5ba8SSam Ravnborg 2201a88b5ba8SSam Ravnborg state.page_table = iommu->page_table; 2202a88b5ba8SSam Ravnborg state.cookies = cookies; 2203a88b5ba8SSam Ravnborg state.mte_base = perm_to_mte(map_perm); 2204a88b5ba8SSam Ravnborg state.prev_cookie = ~(u64)0; 2205a88b5ba8SSam Ravnborg state.pte_idx = (base - iommu->page_table); 2206a88b5ba8SSam Ravnborg state.nc = 0; 2207a88b5ba8SSam Ravnborg fill_cookies(&state, (pa & PAGE_MASK), (pa & ~PAGE_MASK), len); 220842db672dSDavid L Stevens BUG_ON(state.nc > ncookies); 2209a88b5ba8SSam Ravnborg 2210a88b5ba8SSam Ravnborg return state.nc; 2211a88b5ba8SSam Ravnborg } 2212a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_map_single); 2213a88b5ba8SSam Ravnborg 22140ae53ed1SSowmini Varadhan 2215a88b5ba8SSam Ravnborg static void free_npages(unsigned long id, struct ldc_iommu *iommu, 2216a88b5ba8SSam Ravnborg u64 cookie, u64 size) 2217a88b5ba8SSam Ravnborg { 22180ae53ed1SSowmini Varadhan unsigned long npages, entry; 2219a88b5ba8SSam Ravnborg 2220a88b5ba8SSam Ravnborg npages = PAGE_ALIGN(((cookie & ~PAGE_MASK) + size)) >> PAGE_SHIFT; 2221a88b5ba8SSam Ravnborg 22220ae53ed1SSowmini Varadhan entry = ldc_cookie_to_index(cookie, iommu); 22230ae53ed1SSowmini Varadhan ldc_demap(iommu, id, cookie, entry, npages); 22240ae53ed1SSowmini Varadhan iommu_tbl_range_free(&iommu->iommu_map_table, cookie, npages, entry); 2225a88b5ba8SSam Ravnborg } 2226a88b5ba8SSam Ravnborg 2227a88b5ba8SSam Ravnborg void ldc_unmap(struct ldc_channel *lp, struct ldc_trans_cookie *cookies, 2228a88b5ba8SSam Ravnborg int ncookies) 2229a88b5ba8SSam Ravnborg { 2230a88b5ba8SSam Ravnborg struct ldc_iommu *iommu = &lp->iommu; 2231c12f048fSDavid S. Miller int i; 22320ae53ed1SSowmini Varadhan unsigned long flags; 2233a88b5ba8SSam Ravnborg 2234a88b5ba8SSam Ravnborg spin_lock_irqsave(&iommu->lock, flags); 2235a88b5ba8SSam Ravnborg for (i = 0; i < ncookies; i++) { 2236a88b5ba8SSam Ravnborg u64 addr = cookies[i].cookie_addr; 2237a88b5ba8SSam Ravnborg u64 size = cookies[i].cookie_size; 2238a88b5ba8SSam Ravnborg 2239a88b5ba8SSam Ravnborg free_npages(lp->id, iommu, addr, size); 2240a88b5ba8SSam Ravnborg } 2241a88b5ba8SSam Ravnborg spin_unlock_irqrestore(&iommu->lock, flags); 2242a88b5ba8SSam Ravnborg } 2243a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_unmap); 2244a88b5ba8SSam Ravnborg 2245a88b5ba8SSam Ravnborg int ldc_copy(struct ldc_channel *lp, int copy_dir, 2246a88b5ba8SSam Ravnborg void *buf, unsigned int len, unsigned long offset, 2247a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies) 2248a88b5ba8SSam Ravnborg { 2249a88b5ba8SSam Ravnborg unsigned int orig_len; 2250a88b5ba8SSam Ravnborg unsigned long ra; 2251a88b5ba8SSam Ravnborg int i; 2252a88b5ba8SSam Ravnborg 2253a88b5ba8SSam Ravnborg if (copy_dir != LDC_COPY_IN && copy_dir != LDC_COPY_OUT) { 2254a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Bad copy_dir[%d]\n", 2255a88b5ba8SSam Ravnborg lp->id, copy_dir); 2256a88b5ba8SSam Ravnborg return -EINVAL; 2257a88b5ba8SSam Ravnborg } 2258a88b5ba8SSam Ravnborg 2259a88b5ba8SSam Ravnborg ra = __pa(buf); 2260a88b5ba8SSam Ravnborg if ((ra | len | offset) & (8UL - 1)) { 2261a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Unaligned buffer " 2262a88b5ba8SSam Ravnborg "ra[%lx] len[%x] offset[%lx]\n", 2263a88b5ba8SSam Ravnborg lp->id, ra, len, offset); 2264a88b5ba8SSam Ravnborg return -EFAULT; 2265a88b5ba8SSam Ravnborg } 2266a88b5ba8SSam Ravnborg 2267a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE || 2268a88b5ba8SSam Ravnborg (lp->flags & LDC_FLAG_RESET)) { 2269a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] Link down hs_state[%x] " 2270a88b5ba8SSam Ravnborg "flags[%x]\n", lp->id, lp->hs_state, lp->flags); 2271a88b5ba8SSam Ravnborg return -ECONNRESET; 2272a88b5ba8SSam Ravnborg } 2273a88b5ba8SSam Ravnborg 2274a88b5ba8SSam Ravnborg orig_len = len; 2275a88b5ba8SSam Ravnborg for (i = 0; i < ncookies; i++) { 2276a88b5ba8SSam Ravnborg unsigned long cookie_raddr = cookies[i].cookie_addr; 2277a88b5ba8SSam Ravnborg unsigned long this_len = cookies[i].cookie_size; 2278a88b5ba8SSam Ravnborg unsigned long actual_len; 2279a88b5ba8SSam Ravnborg 2280a88b5ba8SSam Ravnborg if (unlikely(offset)) { 2281a88b5ba8SSam Ravnborg unsigned long this_off = offset; 2282a88b5ba8SSam Ravnborg 2283a88b5ba8SSam Ravnborg if (this_off > this_len) 2284a88b5ba8SSam Ravnborg this_off = this_len; 2285a88b5ba8SSam Ravnborg 2286a88b5ba8SSam Ravnborg offset -= this_off; 2287a88b5ba8SSam Ravnborg this_len -= this_off; 2288a88b5ba8SSam Ravnborg if (!this_len) 2289a88b5ba8SSam Ravnborg continue; 2290a88b5ba8SSam Ravnborg cookie_raddr += this_off; 2291a88b5ba8SSam Ravnborg } 2292a88b5ba8SSam Ravnborg 2293a88b5ba8SSam Ravnborg if (this_len > len) 2294a88b5ba8SSam Ravnborg this_len = len; 2295a88b5ba8SSam Ravnborg 2296a88b5ba8SSam Ravnborg while (1) { 2297a88b5ba8SSam Ravnborg unsigned long hv_err; 2298a88b5ba8SSam Ravnborg 2299a88b5ba8SSam Ravnborg hv_err = sun4v_ldc_copy(lp->id, copy_dir, 2300a88b5ba8SSam Ravnborg cookie_raddr, ra, 2301a88b5ba8SSam Ravnborg this_len, &actual_len); 2302a88b5ba8SSam Ravnborg if (unlikely(hv_err)) { 2303a88b5ba8SSam Ravnborg printk(KERN_ERR PFX "ldc_copy: ID[%lu] " 2304a88b5ba8SSam Ravnborg "HV error %lu\n", 2305a88b5ba8SSam Ravnborg lp->id, hv_err); 2306a88b5ba8SSam Ravnborg if (lp->hs_state != LDC_HS_COMPLETE || 2307a88b5ba8SSam Ravnborg (lp->flags & LDC_FLAG_RESET)) 2308a88b5ba8SSam Ravnborg return -ECONNRESET; 2309a88b5ba8SSam Ravnborg else 2310a88b5ba8SSam Ravnborg return -EFAULT; 2311a88b5ba8SSam Ravnborg } 2312a88b5ba8SSam Ravnborg 2313a88b5ba8SSam Ravnborg cookie_raddr += actual_len; 2314a88b5ba8SSam Ravnborg ra += actual_len; 2315a88b5ba8SSam Ravnborg len -= actual_len; 2316a88b5ba8SSam Ravnborg if (actual_len == this_len) 2317a88b5ba8SSam Ravnborg break; 2318a88b5ba8SSam Ravnborg 2319a88b5ba8SSam Ravnborg this_len -= actual_len; 2320a88b5ba8SSam Ravnborg } 2321a88b5ba8SSam Ravnborg 2322a88b5ba8SSam Ravnborg if (!len) 2323a88b5ba8SSam Ravnborg break; 2324a88b5ba8SSam Ravnborg } 2325a88b5ba8SSam Ravnborg 2326a88b5ba8SSam Ravnborg /* It is caller policy what to do about short copies. 2327a88b5ba8SSam Ravnborg * For example, a networking driver can declare the 2328a88b5ba8SSam Ravnborg * packet a runt and drop it. 2329a88b5ba8SSam Ravnborg */ 2330a88b5ba8SSam Ravnborg 2331a88b5ba8SSam Ravnborg return orig_len - len; 2332a88b5ba8SSam Ravnborg } 2333a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_copy); 2334a88b5ba8SSam Ravnborg 2335a88b5ba8SSam Ravnborg void *ldc_alloc_exp_dring(struct ldc_channel *lp, unsigned int len, 2336a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int *ncookies, 2337a88b5ba8SSam Ravnborg unsigned int map_perm) 2338a88b5ba8SSam Ravnborg { 2339a88b5ba8SSam Ravnborg void *buf; 2340a88b5ba8SSam Ravnborg int err; 2341a88b5ba8SSam Ravnborg 2342a88b5ba8SSam Ravnborg if (len & (8UL - 1)) 2343a88b5ba8SSam Ravnborg return ERR_PTR(-EINVAL); 2344a88b5ba8SSam Ravnborg 23450edfad59SSowmini Varadhan buf = kzalloc(len, GFP_ATOMIC); 2346a88b5ba8SSam Ravnborg if (!buf) 2347a88b5ba8SSam Ravnborg return ERR_PTR(-ENOMEM); 2348a88b5ba8SSam Ravnborg 2349a88b5ba8SSam Ravnborg err = ldc_map_single(lp, buf, len, cookies, *ncookies, map_perm); 2350a88b5ba8SSam Ravnborg if (err < 0) { 2351a88b5ba8SSam Ravnborg kfree(buf); 2352a88b5ba8SSam Ravnborg return ERR_PTR(err); 2353a88b5ba8SSam Ravnborg } 2354a88b5ba8SSam Ravnborg *ncookies = err; 2355a88b5ba8SSam Ravnborg 2356a88b5ba8SSam Ravnborg return buf; 2357a88b5ba8SSam Ravnborg } 2358a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_alloc_exp_dring); 2359a88b5ba8SSam Ravnborg 2360a88b5ba8SSam Ravnborg void ldc_free_exp_dring(struct ldc_channel *lp, void *buf, unsigned int len, 2361a88b5ba8SSam Ravnborg struct ldc_trans_cookie *cookies, int ncookies) 2362a88b5ba8SSam Ravnborg { 2363a88b5ba8SSam Ravnborg ldc_unmap(lp, cookies, ncookies); 2364a88b5ba8SSam Ravnborg kfree(buf); 2365a88b5ba8SSam Ravnborg } 2366a88b5ba8SSam Ravnborg EXPORT_SYMBOL(ldc_free_exp_dring); 2367a88b5ba8SSam Ravnborg 2368a88b5ba8SSam Ravnborg static int __init ldc_init(void) 2369a88b5ba8SSam Ravnborg { 2370a88b5ba8SSam Ravnborg unsigned long major, minor; 2371a88b5ba8SSam Ravnborg struct mdesc_handle *hp; 2372a88b5ba8SSam Ravnborg const u64 *v; 2373a88b5ba8SSam Ravnborg int err; 2374a88b5ba8SSam Ravnborg u64 mp; 2375a88b5ba8SSam Ravnborg 2376a88b5ba8SSam Ravnborg hp = mdesc_grab(); 2377a88b5ba8SSam Ravnborg if (!hp) 2378a88b5ba8SSam Ravnborg return -ENODEV; 2379a88b5ba8SSam Ravnborg 2380a88b5ba8SSam Ravnborg mp = mdesc_node_by_name(hp, MDESC_NODE_NULL, "platform"); 2381a88b5ba8SSam Ravnborg err = -ENODEV; 2382a88b5ba8SSam Ravnborg if (mp == MDESC_NODE_NULL) 2383a88b5ba8SSam Ravnborg goto out; 2384a88b5ba8SSam Ravnborg 2385a88b5ba8SSam Ravnborg v = mdesc_get_property(hp, mp, "domaining-enabled", NULL); 2386a88b5ba8SSam Ravnborg if (!v) 2387a88b5ba8SSam Ravnborg goto out; 2388a88b5ba8SSam Ravnborg 2389a88b5ba8SSam Ravnborg major = 1; 2390a88b5ba8SSam Ravnborg minor = 0; 2391a88b5ba8SSam Ravnborg if (sun4v_hvapi_register(HV_GRP_LDOM, major, &minor)) { 2392a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "Could not register LDOM hvapi.\n"); 2393a88b5ba8SSam Ravnborg goto out; 2394a88b5ba8SSam Ravnborg } 2395a88b5ba8SSam Ravnborg 2396a88b5ba8SSam Ravnborg printk(KERN_INFO "%s", version); 2397a88b5ba8SSam Ravnborg 2398a88b5ba8SSam Ravnborg if (!*v) { 2399a88b5ba8SSam Ravnborg printk(KERN_INFO PFX "Domaining disabled.\n"); 2400a88b5ba8SSam Ravnborg goto out; 2401a88b5ba8SSam Ravnborg } 2402a88b5ba8SSam Ravnborg ldom_domaining_enabled = 1; 2403a88b5ba8SSam Ravnborg err = 0; 2404a88b5ba8SSam Ravnborg 2405a88b5ba8SSam Ravnborg out: 2406a88b5ba8SSam Ravnborg mdesc_release(hp); 2407a88b5ba8SSam Ravnborg return err; 2408a88b5ba8SSam Ravnborg } 2409a88b5ba8SSam Ravnborg 2410a88b5ba8SSam Ravnborg core_initcall(ldc_init); 2411