1 /* 2 * Microsemi Switchtec(tm) PCIe Management Driver 3 * Copyright (c) 2017, Microsemi Corporation 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 */ 15 16 #include <linux/switchtec.h> 17 #include <linux/module.h> 18 #include <linux/delay.h> 19 #include <linux/kthread.h> 20 #include <linux/interrupt.h> 21 #include <linux/ntb.h> 22 #include <linux/pci.h> 23 24 MODULE_DESCRIPTION("Microsemi Switchtec(tm) NTB Driver"); 25 MODULE_VERSION("0.1"); 26 MODULE_LICENSE("GPL"); 27 MODULE_AUTHOR("Microsemi Corporation"); 28 29 static ulong max_mw_size = SZ_2M; 30 module_param(max_mw_size, ulong, 0644); 31 MODULE_PARM_DESC(max_mw_size, 32 "Max memory window size reported to the upper layer"); 33 34 static bool use_lut_mws; 35 module_param(use_lut_mws, bool, 0644); 36 MODULE_PARM_DESC(use_lut_mws, 37 "Enable the use of the LUT based memory windows"); 38 39 #ifndef ioread64 40 #ifdef readq 41 #define ioread64 readq 42 #else 43 #define ioread64 _ioread64 44 static inline u64 _ioread64(void __iomem *mmio) 45 { 46 u64 low, high; 47 48 low = ioread32(mmio); 49 high = ioread32(mmio + sizeof(u32)); 50 return low | (high << 32); 51 } 52 #endif 53 #endif 54 55 #ifndef iowrite64 56 #ifdef writeq 57 #define iowrite64 writeq 58 #else 59 #define iowrite64 _iowrite64 60 static inline void _iowrite64(u64 val, void __iomem *mmio) 61 { 62 iowrite32(val, mmio); 63 iowrite32(val >> 32, mmio + sizeof(u32)); 64 } 65 #endif 66 #endif 67 68 #define SWITCHTEC_NTB_MAGIC 0x45CC0001 69 #define MAX_MWS 128 70 71 struct shared_mw { 72 u32 magic; 73 u32 link_sta; 74 u32 partition_id; 75 u64 mw_sizes[MAX_MWS]; 76 u32 spad[128]; 77 }; 78 79 #define MAX_DIRECT_MW ARRAY_SIZE(((struct ntb_ctrl_regs *)(0))->bar_entry) 80 #define LUT_SIZE SZ_64K 81 82 struct switchtec_ntb { 83 struct ntb_dev ntb; 84 struct switchtec_dev *stdev; 85 86 int self_partition; 87 int peer_partition; 88 89 int doorbell_irq; 90 int message_irq; 91 92 struct ntb_info_regs __iomem *mmio_ntb; 93 struct ntb_ctrl_regs __iomem *mmio_ctrl; 94 struct ntb_dbmsg_regs __iomem *mmio_dbmsg; 95 struct ntb_ctrl_regs __iomem *mmio_self_ctrl; 96 struct ntb_ctrl_regs __iomem *mmio_peer_ctrl; 97 struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg; 98 struct ntb_dbmsg_regs __iomem *mmio_peer_dbmsg; 99 100 void __iomem *mmio_xlink_win; 101 102 struct shared_mw *self_shared; 103 struct shared_mw __iomem *peer_shared; 104 dma_addr_t self_shared_dma; 105 106 u64 db_mask; 107 u64 db_valid_mask; 108 int db_shift; 109 int db_peer_shift; 110 111 /* synchronize rmw access of db_mask and hw reg */ 112 spinlock_t db_mask_lock; 113 114 int nr_direct_mw; 115 int nr_lut_mw; 116 int nr_rsvd_luts; 117 int direct_mw_to_bar[MAX_DIRECT_MW]; 118 119 int peer_nr_direct_mw; 120 int peer_nr_lut_mw; 121 int peer_direct_mw_to_bar[MAX_DIRECT_MW]; 122 123 bool link_is_up; 124 enum ntb_speed link_speed; 125 enum ntb_width link_width; 126 struct work_struct link_reinit_work; 127 }; 128 129 static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb) 130 { 131 return container_of(ntb, struct switchtec_ntb, ntb); 132 } 133 134 static int switchtec_ntb_part_op(struct switchtec_ntb *sndev, 135 struct ntb_ctrl_regs __iomem *ctl, 136 u32 op, int wait_status) 137 { 138 static const char * const op_text[] = { 139 [NTB_CTRL_PART_OP_LOCK] = "lock", 140 [NTB_CTRL_PART_OP_CFG] = "configure", 141 [NTB_CTRL_PART_OP_RESET] = "reset", 142 }; 143 144 int i; 145 u32 ps; 146 int status; 147 148 switch (op) { 149 case NTB_CTRL_PART_OP_LOCK: 150 status = NTB_CTRL_PART_STATUS_LOCKING; 151 break; 152 case NTB_CTRL_PART_OP_CFG: 153 status = NTB_CTRL_PART_STATUS_CONFIGURING; 154 break; 155 case NTB_CTRL_PART_OP_RESET: 156 status = NTB_CTRL_PART_STATUS_RESETTING; 157 break; 158 default: 159 return -EINVAL; 160 } 161 162 iowrite32(op, &ctl->partition_op); 163 164 for (i = 0; i < 1000; i++) { 165 if (msleep_interruptible(50) != 0) { 166 iowrite32(NTB_CTRL_PART_OP_RESET, &ctl->partition_op); 167 return -EINTR; 168 } 169 170 ps = ioread32(&ctl->partition_status) & 0xFFFF; 171 172 if (ps != status) 173 break; 174 } 175 176 if (ps == wait_status) 177 return 0; 178 179 if (ps == status) { 180 dev_err(&sndev->stdev->dev, 181 "Timed out while performing %s (%d). (%08x)\n", 182 op_text[op], op, 183 ioread32(&ctl->partition_status)); 184 185 return -ETIMEDOUT; 186 } 187 188 return -EIO; 189 } 190 191 static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx, 192 u32 val) 193 { 194 if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg)) 195 return -EINVAL; 196 197 iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg); 198 199 return 0; 200 } 201 202 static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx) 203 { 204 struct switchtec_ntb *sndev = ntb_sndev(ntb); 205 int nr_direct_mw = sndev->peer_nr_direct_mw; 206 int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts; 207 208 if (pidx != NTB_DEF_PEER_IDX) 209 return -EINVAL; 210 211 if (!use_lut_mws) 212 nr_lut_mw = 0; 213 214 return nr_direct_mw + nr_lut_mw; 215 } 216 217 static int lut_index(struct switchtec_ntb *sndev, int mw_idx) 218 { 219 return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts; 220 } 221 222 static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx) 223 { 224 return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts; 225 } 226 227 static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, 228 int widx, resource_size_t *addr_align, 229 resource_size_t *size_align, 230 resource_size_t *size_max) 231 { 232 struct switchtec_ntb *sndev = ntb_sndev(ntb); 233 int lut; 234 resource_size_t size; 235 236 if (pidx != NTB_DEF_PEER_IDX) 237 return -EINVAL; 238 239 lut = widx >= sndev->peer_nr_direct_mw; 240 size = ioread64(&sndev->peer_shared->mw_sizes[widx]); 241 242 if (size == 0) 243 return -EINVAL; 244 245 if (addr_align) 246 *addr_align = lut ? size : SZ_4K; 247 248 if (size_align) 249 *size_align = lut ? size : SZ_4K; 250 251 if (size_max) 252 *size_max = size; 253 254 return 0; 255 } 256 257 static void switchtec_ntb_mw_clr_direct(struct switchtec_ntb *sndev, int idx) 258 { 259 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 260 int bar = sndev->peer_direct_mw_to_bar[idx]; 261 u32 ctl_val; 262 263 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 264 ctl_val &= ~NTB_CTRL_BAR_DIR_WIN_EN; 265 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); 266 iowrite32(0, &ctl->bar_entry[bar].win_size); 267 iowrite64(sndev->self_partition, &ctl->bar_entry[bar].xlate_addr); 268 } 269 270 static void switchtec_ntb_mw_clr_lut(struct switchtec_ntb *sndev, int idx) 271 { 272 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 273 274 iowrite64(0, &ctl->lut_entry[peer_lut_index(sndev, idx)]); 275 } 276 277 static void switchtec_ntb_mw_set_direct(struct switchtec_ntb *sndev, int idx, 278 dma_addr_t addr, resource_size_t size) 279 { 280 int xlate_pos = ilog2(size); 281 int bar = sndev->peer_direct_mw_to_bar[idx]; 282 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 283 u32 ctl_val; 284 285 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 286 ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN; 287 288 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); 289 iowrite32(xlate_pos | size, &ctl->bar_entry[bar].win_size); 290 iowrite64(sndev->self_partition | addr, 291 &ctl->bar_entry[bar].xlate_addr); 292 } 293 294 static void switchtec_ntb_mw_set_lut(struct switchtec_ntb *sndev, int idx, 295 dma_addr_t addr, resource_size_t size) 296 { 297 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 298 299 iowrite64((NTB_CTRL_LUT_EN | (sndev->self_partition << 1) | addr), 300 &ctl->lut_entry[peer_lut_index(sndev, idx)]); 301 } 302 303 static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx, 304 dma_addr_t addr, resource_size_t size) 305 { 306 struct switchtec_ntb *sndev = ntb_sndev(ntb); 307 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 308 int xlate_pos = ilog2(size); 309 int nr_direct_mw = sndev->peer_nr_direct_mw; 310 int rc; 311 312 if (pidx != NTB_DEF_PEER_IDX) 313 return -EINVAL; 314 315 dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n", 316 widx, pidx, &addr, &size); 317 318 if (widx >= switchtec_ntb_mw_count(ntb, pidx)) 319 return -EINVAL; 320 321 if (xlate_pos < 12) 322 return -EINVAL; 323 324 if (!IS_ALIGNED(addr, BIT_ULL(xlate_pos))) { 325 /* 326 * In certain circumstances we can get a buffer that is 327 * not aligned to its size. (Most of the time 328 * dma_alloc_coherent ensures this). This can happen when 329 * using large buffers allocated by the CMA 330 * (see CMA_CONFIG_ALIGNMENT) 331 */ 332 dev_err(&sndev->stdev->dev, 333 "ERROR: Memory window address is not aligned to it's size!\n"); 334 return -EINVAL; 335 } 336 337 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 338 NTB_CTRL_PART_STATUS_LOCKED); 339 if (rc) 340 return rc; 341 342 if (addr == 0 || size == 0) { 343 if (widx < nr_direct_mw) 344 switchtec_ntb_mw_clr_direct(sndev, widx); 345 else 346 switchtec_ntb_mw_clr_lut(sndev, widx); 347 } else { 348 if (widx < nr_direct_mw) 349 switchtec_ntb_mw_set_direct(sndev, widx, addr, size); 350 else 351 switchtec_ntb_mw_set_lut(sndev, widx, addr, size); 352 } 353 354 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 355 NTB_CTRL_PART_STATUS_NORMAL); 356 357 if (rc == -EIO) { 358 dev_err(&sndev->stdev->dev, 359 "Hardware reported an error configuring mw %d: %08x\n", 360 widx, ioread32(&ctl->bar_error)); 361 362 if (widx < nr_direct_mw) 363 switchtec_ntb_mw_clr_direct(sndev, widx); 364 else 365 switchtec_ntb_mw_clr_lut(sndev, widx); 366 367 switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 368 NTB_CTRL_PART_STATUS_NORMAL); 369 } 370 371 return rc; 372 } 373 374 static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb) 375 { 376 struct switchtec_ntb *sndev = ntb_sndev(ntb); 377 int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts; 378 379 return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0); 380 } 381 382 static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev, 383 int idx, phys_addr_t *base, 384 resource_size_t *size) 385 { 386 int bar = sndev->direct_mw_to_bar[idx]; 387 size_t offset = 0; 388 389 if (bar < 0) 390 return -EINVAL; 391 392 if (idx == 0) { 393 /* 394 * This is the direct BAR shared with the LUTs 395 * which means the actual window will be offset 396 * by the size of all the LUT entries. 397 */ 398 399 offset = LUT_SIZE * sndev->nr_lut_mw; 400 } 401 402 if (base) 403 *base = pci_resource_start(sndev->ntb.pdev, bar) + offset; 404 405 if (size) { 406 *size = pci_resource_len(sndev->ntb.pdev, bar) - offset; 407 if (offset && *size > offset) 408 *size = offset; 409 410 if (*size > max_mw_size) 411 *size = max_mw_size; 412 } 413 414 return 0; 415 } 416 417 static int switchtec_ntb_lut_get_addr(struct switchtec_ntb *sndev, 418 int idx, phys_addr_t *base, 419 resource_size_t *size) 420 { 421 int bar = sndev->direct_mw_to_bar[0]; 422 int offset; 423 424 offset = LUT_SIZE * lut_index(sndev, idx); 425 426 if (base) 427 *base = pci_resource_start(sndev->ntb.pdev, bar) + offset; 428 429 if (size) 430 *size = LUT_SIZE; 431 432 return 0; 433 } 434 435 static int switchtec_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, 436 phys_addr_t *base, 437 resource_size_t *size) 438 { 439 struct switchtec_ntb *sndev = ntb_sndev(ntb); 440 441 if (idx < sndev->nr_direct_mw) 442 return switchtec_ntb_direct_get_addr(sndev, idx, base, size); 443 else if (idx < switchtec_ntb_peer_mw_count(ntb)) 444 return switchtec_ntb_lut_get_addr(sndev, idx, base, size); 445 else 446 return -EINVAL; 447 } 448 449 static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev, 450 int partition, 451 enum ntb_speed *speed, 452 enum ntb_width *width) 453 { 454 struct switchtec_dev *stdev = sndev->stdev; 455 456 u32 pff = ioread32(&stdev->mmio_part_cfg[partition].vep_pff_inst_id); 457 u32 linksta = ioread32(&stdev->mmio_pff_csr[pff].pci_cap_region[13]); 458 459 if (speed) 460 *speed = (linksta >> 16) & 0xF; 461 462 if (width) 463 *width = (linksta >> 20) & 0x3F; 464 } 465 466 static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev) 467 { 468 enum ntb_speed self_speed, peer_speed; 469 enum ntb_width self_width, peer_width; 470 471 if (!sndev->link_is_up) { 472 sndev->link_speed = NTB_SPEED_NONE; 473 sndev->link_width = NTB_WIDTH_NONE; 474 return; 475 } 476 477 switchtec_ntb_part_link_speed(sndev, sndev->self_partition, 478 &self_speed, &self_width); 479 switchtec_ntb_part_link_speed(sndev, sndev->peer_partition, 480 &peer_speed, &peer_width); 481 482 sndev->link_speed = min(self_speed, peer_speed); 483 sndev->link_width = min(self_width, peer_width); 484 } 485 486 static int crosslink_is_enabled(struct switchtec_ntb *sndev) 487 { 488 struct ntb_info_regs __iomem *inf = sndev->mmio_ntb; 489 490 return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled); 491 } 492 493 static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev) 494 { 495 int i; 496 u32 msg_map = 0; 497 498 if (!crosslink_is_enabled(sndev)) 499 return; 500 501 for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) { 502 int m = i | sndev->self_partition << 2; 503 504 msg_map |= m << i * 8; 505 } 506 507 iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map); 508 iowrite64(sndev->db_valid_mask << sndev->db_peer_shift, 509 &sndev->mmio_peer_dbmsg->odb_mask); 510 } 511 512 enum switchtec_msg { 513 LINK_MESSAGE = 0, 514 MSG_LINK_UP = 1, 515 MSG_LINK_DOWN = 2, 516 MSG_CHECK_LINK = 3, 517 MSG_LINK_FORCE_DOWN = 4, 518 }; 519 520 static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev); 521 522 static void link_reinit_work(struct work_struct *work) 523 { 524 struct switchtec_ntb *sndev; 525 526 sndev = container_of(work, struct switchtec_ntb, link_reinit_work); 527 528 switchtec_ntb_reinit_peer(sndev); 529 } 530 531 static void switchtec_ntb_check_link(struct switchtec_ntb *sndev, 532 enum switchtec_msg msg) 533 { 534 int link_sta; 535 int old = sndev->link_is_up; 536 537 if (msg == MSG_LINK_FORCE_DOWN) { 538 schedule_work(&sndev->link_reinit_work); 539 540 if (sndev->link_is_up) { 541 sndev->link_is_up = 0; 542 ntb_link_event(&sndev->ntb); 543 dev_info(&sndev->stdev->dev, "ntb link forced down\n"); 544 } 545 546 return; 547 } 548 549 link_sta = sndev->self_shared->link_sta; 550 if (link_sta) { 551 u64 peer = ioread64(&sndev->peer_shared->magic); 552 553 if ((peer & 0xFFFFFFFF) == SWITCHTEC_NTB_MAGIC) 554 link_sta = peer >> 32; 555 else 556 link_sta = 0; 557 } 558 559 sndev->link_is_up = link_sta; 560 switchtec_ntb_set_link_speed(sndev); 561 562 if (link_sta != old) { 563 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK); 564 ntb_link_event(&sndev->ntb); 565 dev_info(&sndev->stdev->dev, "ntb link %s\n", 566 link_sta ? "up" : "down"); 567 568 if (link_sta) 569 crosslink_init_dbmsgs(sndev); 570 } 571 } 572 573 static void switchtec_ntb_link_notification(struct switchtec_dev *stdev) 574 { 575 struct switchtec_ntb *sndev = stdev->sndev; 576 577 switchtec_ntb_check_link(sndev, MSG_CHECK_LINK); 578 } 579 580 static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb, 581 enum ntb_speed *speed, 582 enum ntb_width *width) 583 { 584 struct switchtec_ntb *sndev = ntb_sndev(ntb); 585 586 if (speed) 587 *speed = sndev->link_speed; 588 if (width) 589 *width = sndev->link_width; 590 591 return sndev->link_is_up; 592 } 593 594 static int switchtec_ntb_link_enable(struct ntb_dev *ntb, 595 enum ntb_speed max_speed, 596 enum ntb_width max_width) 597 { 598 struct switchtec_ntb *sndev = ntb_sndev(ntb); 599 600 dev_dbg(&sndev->stdev->dev, "enabling link\n"); 601 602 sndev->self_shared->link_sta = 1; 603 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP); 604 605 switchtec_ntb_check_link(sndev, MSG_CHECK_LINK); 606 607 return 0; 608 } 609 610 static int switchtec_ntb_link_disable(struct ntb_dev *ntb) 611 { 612 struct switchtec_ntb *sndev = ntb_sndev(ntb); 613 614 dev_dbg(&sndev->stdev->dev, "disabling link\n"); 615 616 sndev->self_shared->link_sta = 0; 617 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN); 618 619 switchtec_ntb_check_link(sndev, MSG_CHECK_LINK); 620 621 return 0; 622 } 623 624 static u64 switchtec_ntb_db_valid_mask(struct ntb_dev *ntb) 625 { 626 struct switchtec_ntb *sndev = ntb_sndev(ntb); 627 628 return sndev->db_valid_mask; 629 } 630 631 static int switchtec_ntb_db_vector_count(struct ntb_dev *ntb) 632 { 633 return 1; 634 } 635 636 static u64 switchtec_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) 637 { 638 struct switchtec_ntb *sndev = ntb_sndev(ntb); 639 640 if (db_vector < 0 || db_vector > 1) 641 return 0; 642 643 return sndev->db_valid_mask; 644 } 645 646 static u64 switchtec_ntb_db_read(struct ntb_dev *ntb) 647 { 648 u64 ret; 649 struct switchtec_ntb *sndev = ntb_sndev(ntb); 650 651 ret = ioread64(&sndev->mmio_self_dbmsg->idb) >> sndev->db_shift; 652 653 return ret & sndev->db_valid_mask; 654 } 655 656 static int switchtec_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) 657 { 658 struct switchtec_ntb *sndev = ntb_sndev(ntb); 659 660 iowrite64(db_bits << sndev->db_shift, &sndev->mmio_self_dbmsg->idb); 661 662 return 0; 663 } 664 665 static int switchtec_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 666 { 667 unsigned long irqflags; 668 struct switchtec_ntb *sndev = ntb_sndev(ntb); 669 670 if (db_bits & ~sndev->db_valid_mask) 671 return -EINVAL; 672 673 spin_lock_irqsave(&sndev->db_mask_lock, irqflags); 674 675 sndev->db_mask |= db_bits << sndev->db_shift; 676 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 677 678 spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags); 679 680 return 0; 681 } 682 683 static int switchtec_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) 684 { 685 unsigned long irqflags; 686 struct switchtec_ntb *sndev = ntb_sndev(ntb); 687 688 if (db_bits & ~sndev->db_valid_mask) 689 return -EINVAL; 690 691 spin_lock_irqsave(&sndev->db_mask_lock, irqflags); 692 693 sndev->db_mask &= ~(db_bits << sndev->db_shift); 694 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 695 696 spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags); 697 698 return 0; 699 } 700 701 static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb) 702 { 703 struct switchtec_ntb *sndev = ntb_sndev(ntb); 704 705 return (sndev->db_mask >> sndev->db_shift) & sndev->db_valid_mask; 706 } 707 708 static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb, 709 phys_addr_t *db_addr, 710 resource_size_t *db_size) 711 { 712 struct switchtec_ntb *sndev = ntb_sndev(ntb); 713 unsigned long offset; 714 715 offset = (unsigned long)sndev->mmio_peer_dbmsg->odb - 716 (unsigned long)sndev->stdev->mmio; 717 718 offset += sndev->db_shift / 8; 719 720 if (db_addr) 721 *db_addr = pci_resource_start(ntb->pdev, 0) + offset; 722 if (db_size) 723 *db_size = sizeof(u32); 724 725 return 0; 726 } 727 728 static int switchtec_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) 729 { 730 struct switchtec_ntb *sndev = ntb_sndev(ntb); 731 732 iowrite64(db_bits << sndev->db_peer_shift, 733 &sndev->mmio_peer_dbmsg->odb); 734 735 return 0; 736 } 737 738 static int switchtec_ntb_spad_count(struct ntb_dev *ntb) 739 { 740 struct switchtec_ntb *sndev = ntb_sndev(ntb); 741 742 return ARRAY_SIZE(sndev->self_shared->spad); 743 } 744 745 static u32 switchtec_ntb_spad_read(struct ntb_dev *ntb, int idx) 746 { 747 struct switchtec_ntb *sndev = ntb_sndev(ntb); 748 749 if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad)) 750 return 0; 751 752 if (!sndev->self_shared) 753 return 0; 754 755 return sndev->self_shared->spad[idx]; 756 } 757 758 static int switchtec_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val) 759 { 760 struct switchtec_ntb *sndev = ntb_sndev(ntb); 761 762 if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad)) 763 return -EINVAL; 764 765 if (!sndev->self_shared) 766 return -EIO; 767 768 sndev->self_shared->spad[idx] = val; 769 770 return 0; 771 } 772 773 static u32 switchtec_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, 774 int sidx) 775 { 776 struct switchtec_ntb *sndev = ntb_sndev(ntb); 777 778 if (pidx != NTB_DEF_PEER_IDX) 779 return -EINVAL; 780 781 if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad)) 782 return 0; 783 784 if (!sndev->peer_shared) 785 return 0; 786 787 return ioread32(&sndev->peer_shared->spad[sidx]); 788 } 789 790 static int switchtec_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, 791 int sidx, u32 val) 792 { 793 struct switchtec_ntb *sndev = ntb_sndev(ntb); 794 795 if (pidx != NTB_DEF_PEER_IDX) 796 return -EINVAL; 797 798 if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad)) 799 return -EINVAL; 800 801 if (!sndev->peer_shared) 802 return -EIO; 803 804 iowrite32(val, &sndev->peer_shared->spad[sidx]); 805 806 return 0; 807 } 808 809 static int switchtec_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, 810 int sidx, phys_addr_t *spad_addr) 811 { 812 struct switchtec_ntb *sndev = ntb_sndev(ntb); 813 unsigned long offset; 814 815 if (pidx != NTB_DEF_PEER_IDX) 816 return -EINVAL; 817 818 offset = (unsigned long)&sndev->peer_shared->spad[sidx] - 819 (unsigned long)sndev->stdev->mmio; 820 821 if (spad_addr) 822 *spad_addr = pci_resource_start(ntb->pdev, 0) + offset; 823 824 return 0; 825 } 826 827 static const struct ntb_dev_ops switchtec_ntb_ops = { 828 .mw_count = switchtec_ntb_mw_count, 829 .mw_get_align = switchtec_ntb_mw_get_align, 830 .mw_set_trans = switchtec_ntb_mw_set_trans, 831 .peer_mw_count = switchtec_ntb_peer_mw_count, 832 .peer_mw_get_addr = switchtec_ntb_peer_mw_get_addr, 833 .link_is_up = switchtec_ntb_link_is_up, 834 .link_enable = switchtec_ntb_link_enable, 835 .link_disable = switchtec_ntb_link_disable, 836 .db_valid_mask = switchtec_ntb_db_valid_mask, 837 .db_vector_count = switchtec_ntb_db_vector_count, 838 .db_vector_mask = switchtec_ntb_db_vector_mask, 839 .db_read = switchtec_ntb_db_read, 840 .db_clear = switchtec_ntb_db_clear, 841 .db_set_mask = switchtec_ntb_db_set_mask, 842 .db_clear_mask = switchtec_ntb_db_clear_mask, 843 .db_read_mask = switchtec_ntb_db_read_mask, 844 .peer_db_addr = switchtec_ntb_peer_db_addr, 845 .peer_db_set = switchtec_ntb_peer_db_set, 846 .spad_count = switchtec_ntb_spad_count, 847 .spad_read = switchtec_ntb_spad_read, 848 .spad_write = switchtec_ntb_spad_write, 849 .peer_spad_read = switchtec_ntb_peer_spad_read, 850 .peer_spad_write = switchtec_ntb_peer_spad_write, 851 .peer_spad_addr = switchtec_ntb_peer_spad_addr, 852 }; 853 854 static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev) 855 { 856 u64 tpart_vec; 857 int self; 858 u64 part_map; 859 int bit; 860 861 sndev->ntb.pdev = sndev->stdev->pdev; 862 sndev->ntb.topo = NTB_TOPO_SWITCH; 863 sndev->ntb.ops = &switchtec_ntb_ops; 864 865 INIT_WORK(&sndev->link_reinit_work, link_reinit_work); 866 867 sndev->self_partition = sndev->stdev->partition; 868 869 sndev->mmio_ntb = sndev->stdev->mmio_ntb; 870 871 self = sndev->self_partition; 872 tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high); 873 tpart_vec <<= 32; 874 tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low); 875 876 part_map = ioread64(&sndev->mmio_ntb->ep_map); 877 part_map &= ~(1 << sndev->self_partition); 878 879 if (!ffs(tpart_vec)) { 880 if (sndev->stdev->partition_count != 2) { 881 dev_err(&sndev->stdev->dev, 882 "ntb target partition not defined\n"); 883 return -ENODEV; 884 } 885 886 bit = ffs(part_map); 887 if (!bit) { 888 dev_err(&sndev->stdev->dev, 889 "peer partition is not NT partition\n"); 890 return -ENODEV; 891 } 892 893 sndev->peer_partition = bit - 1; 894 } else { 895 if (ffs(tpart_vec) != fls(tpart_vec)) { 896 dev_err(&sndev->stdev->dev, 897 "ntb driver only supports 1 pair of 1-1 ntb mapping\n"); 898 return -ENODEV; 899 } 900 901 sndev->peer_partition = ffs(tpart_vec) - 1; 902 if (!(part_map & (1 << sndev->peer_partition))) { 903 dev_err(&sndev->stdev->dev, 904 "ntb target partition is not NT partition\n"); 905 return -ENODEV; 906 } 907 } 908 909 dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n", 910 sndev->self_partition, sndev->stdev->partition_count); 911 912 sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb + 913 SWITCHTEC_NTB_REG_CTRL_OFFSET; 914 sndev->mmio_dbmsg = (void * __iomem)sndev->mmio_ntb + 915 SWITCHTEC_NTB_REG_DBMSG_OFFSET; 916 917 sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition]; 918 sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition]; 919 sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition]; 920 sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg; 921 922 return 0; 923 } 924 925 static int config_rsvd_lut_win(struct switchtec_ntb *sndev, 926 struct ntb_ctrl_regs __iomem *ctl, 927 int lut_idx, int partition, u64 addr) 928 { 929 int peer_bar = sndev->peer_direct_mw_to_bar[0]; 930 u32 ctl_val; 931 int rc; 932 933 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 934 NTB_CTRL_PART_STATUS_LOCKED); 935 if (rc) 936 return rc; 937 938 ctl_val = ioread32(&ctl->bar_entry[peer_bar].ctl); 939 ctl_val &= 0xFF; 940 ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN; 941 ctl_val |= ilog2(LUT_SIZE) << 8; 942 ctl_val |= (sndev->nr_lut_mw - 1) << 14; 943 iowrite32(ctl_val, &ctl->bar_entry[peer_bar].ctl); 944 945 iowrite64((NTB_CTRL_LUT_EN | (partition << 1) | addr), 946 &ctl->lut_entry[lut_idx]); 947 948 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 949 NTB_CTRL_PART_STATUS_NORMAL); 950 if (rc) { 951 u32 bar_error, lut_error; 952 953 bar_error = ioread32(&ctl->bar_error); 954 lut_error = ioread32(&ctl->lut_error); 955 dev_err(&sndev->stdev->dev, 956 "Error setting up reserved lut window: %08x / %08x\n", 957 bar_error, lut_error); 958 return rc; 959 } 960 961 return 0; 962 } 963 964 static int config_req_id_table(struct switchtec_ntb *sndev, 965 struct ntb_ctrl_regs __iomem *mmio_ctrl, 966 int *req_ids, int count) 967 { 968 int i, rc = 0; 969 u32 error; 970 u32 proxy_id; 971 972 if (ioread32(&mmio_ctrl->req_id_table_size) < count) { 973 dev_err(&sndev->stdev->dev, 974 "Not enough requester IDs available.\n"); 975 return -EFAULT; 976 } 977 978 rc = switchtec_ntb_part_op(sndev, mmio_ctrl, 979 NTB_CTRL_PART_OP_LOCK, 980 NTB_CTRL_PART_STATUS_LOCKED); 981 if (rc) 982 return rc; 983 984 iowrite32(NTB_PART_CTRL_ID_PROT_DIS, 985 &mmio_ctrl->partition_ctrl); 986 987 for (i = 0; i < count; i++) { 988 iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN, 989 &mmio_ctrl->req_id_table[i]); 990 991 proxy_id = ioread32(&mmio_ctrl->req_id_table[i]); 992 dev_dbg(&sndev->stdev->dev, 993 "Requester ID %02X:%02X.%X -> BB:%02X.%X\n", 994 req_ids[i] >> 8, (req_ids[i] >> 3) & 0x1F, 995 req_ids[i] & 0x7, (proxy_id >> 4) & 0x1F, 996 (proxy_id >> 1) & 0x7); 997 } 998 999 rc = switchtec_ntb_part_op(sndev, mmio_ctrl, 1000 NTB_CTRL_PART_OP_CFG, 1001 NTB_CTRL_PART_STATUS_NORMAL); 1002 1003 if (rc == -EIO) { 1004 error = ioread32(&mmio_ctrl->req_id_error); 1005 dev_err(&sndev->stdev->dev, 1006 "Error setting up the requester ID table: %08x\n", 1007 error); 1008 } 1009 1010 return 0; 1011 } 1012 1013 static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx, 1014 u64 *mw_addrs, int mw_count) 1015 { 1016 int rc, i; 1017 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl; 1018 u64 addr; 1019 size_t size, offset; 1020 int bar; 1021 int xlate_pos; 1022 u32 ctl_val; 1023 1024 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 1025 NTB_CTRL_PART_STATUS_LOCKED); 1026 if (rc) 1027 return rc; 1028 1029 for (i = 0; i < sndev->nr_lut_mw; i++) { 1030 if (i == ntb_lut_idx) 1031 continue; 1032 1033 addr = mw_addrs[0] + LUT_SIZE * i; 1034 1035 iowrite64((NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) | 1036 addr), 1037 &ctl->lut_entry[i]); 1038 } 1039 1040 sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count); 1041 1042 for (i = 0; i < sndev->nr_direct_mw; i++) { 1043 bar = sndev->direct_mw_to_bar[i]; 1044 offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0; 1045 addr = mw_addrs[i] + offset; 1046 size = pci_resource_len(sndev->ntb.pdev, bar) - offset; 1047 xlate_pos = ilog2(size); 1048 1049 if (offset && size > offset) 1050 size = offset; 1051 1052 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 1053 ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN; 1054 1055 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); 1056 iowrite32(xlate_pos | size, &ctl->bar_entry[bar].win_size); 1057 iowrite64(sndev->peer_partition | addr, 1058 &ctl->bar_entry[bar].xlate_addr); 1059 } 1060 1061 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 1062 NTB_CTRL_PART_STATUS_NORMAL); 1063 if (rc) { 1064 u32 bar_error, lut_error; 1065 1066 bar_error = ioread32(&ctl->bar_error); 1067 lut_error = ioread32(&ctl->lut_error); 1068 dev_err(&sndev->stdev->dev, 1069 "Error setting up cross link windows: %08x / %08x\n", 1070 bar_error, lut_error); 1071 return rc; 1072 } 1073 1074 return 0; 1075 } 1076 1077 static int crosslink_setup_req_ids(struct switchtec_ntb *sndev, 1078 struct ntb_ctrl_regs __iomem *mmio_ctrl) 1079 { 1080 int req_ids[16]; 1081 int i; 1082 u32 proxy_id; 1083 1084 for (i = 0; i < ARRAY_SIZE(req_ids); i++) { 1085 proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]); 1086 1087 if (!(proxy_id & NTB_CTRL_REQ_ID_EN)) 1088 break; 1089 1090 req_ids[i] = ((proxy_id >> 1) & 0xFF); 1091 } 1092 1093 return config_req_id_table(sndev, mmio_ctrl, req_ids, i); 1094 } 1095 1096 /* 1097 * In crosslink configuration there is a virtual partition in the 1098 * middle of the two switches. The BARs in this partition have to be 1099 * enumerated and assigned addresses. 1100 */ 1101 static int crosslink_enum_partition(struct switchtec_ntb *sndev, 1102 u64 *bar_addrs) 1103 { 1104 struct part_cfg_regs __iomem *part_cfg = 1105 &sndev->stdev->mmio_part_cfg_all[sndev->peer_partition]; 1106 u32 pff = ioread32(&part_cfg->vep_pff_inst_id); 1107 struct pff_csr_regs __iomem *mmio_pff = 1108 &sndev->stdev->mmio_pff_csr[pff]; 1109 const u64 bar_space = 0x1000000000LL; 1110 u64 bar_addr; 1111 int bar_cnt = 0; 1112 int i; 1113 1114 iowrite16(0x6, &mmio_pff->pcicmd); 1115 1116 for (i = 0; i < ARRAY_SIZE(mmio_pff->pci_bar64); i++) { 1117 iowrite64(bar_space * i, &mmio_pff->pci_bar64[i]); 1118 bar_addr = ioread64(&mmio_pff->pci_bar64[i]); 1119 bar_addr &= ~0xf; 1120 1121 dev_dbg(&sndev->stdev->dev, 1122 "Crosslink BAR%d addr: %llx\n", 1123 i, bar_addr); 1124 1125 if (bar_addr != bar_space * i) 1126 continue; 1127 1128 bar_addrs[bar_cnt++] = bar_addr; 1129 } 1130 1131 return bar_cnt; 1132 } 1133 1134 static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev) 1135 { 1136 int rc; 1137 int bar = sndev->direct_mw_to_bar[0]; 1138 const int ntb_lut_idx = 1; 1139 u64 bar_addrs[6]; 1140 u64 addr; 1141 int offset; 1142 int bar_cnt; 1143 1144 if (!crosslink_is_enabled(sndev)) 1145 return 0; 1146 1147 dev_info(&sndev->stdev->dev, "Using crosslink configuration\n"); 1148 sndev->ntb.topo = NTB_TOPO_CROSSLINK; 1149 1150 bar_cnt = crosslink_enum_partition(sndev, bar_addrs); 1151 if (bar_cnt < sndev->nr_direct_mw + 1) { 1152 dev_err(&sndev->stdev->dev, 1153 "Error enumerating crosslink partition\n"); 1154 return -EINVAL; 1155 } 1156 1157 addr = (bar_addrs[0] + SWITCHTEC_GAS_NTB_OFFSET + 1158 SWITCHTEC_NTB_REG_DBMSG_OFFSET + 1159 sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition); 1160 1161 offset = addr & (LUT_SIZE - 1); 1162 addr -= offset; 1163 1164 rc = config_rsvd_lut_win(sndev, sndev->mmio_self_ctrl, ntb_lut_idx, 1165 sndev->peer_partition, addr); 1166 if (rc) 1167 return rc; 1168 1169 rc = crosslink_setup_mws(sndev, ntb_lut_idx, &bar_addrs[1], 1170 bar_cnt - 1); 1171 if (rc) 1172 return rc; 1173 1174 rc = crosslink_setup_req_ids(sndev, sndev->mmio_peer_ctrl); 1175 if (rc) 1176 return rc; 1177 1178 sndev->mmio_xlink_win = pci_iomap_range(sndev->stdev->pdev, bar, 1179 LUT_SIZE, LUT_SIZE); 1180 if (!sndev->mmio_xlink_win) { 1181 rc = -ENOMEM; 1182 return rc; 1183 } 1184 1185 sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset; 1186 sndev->nr_rsvd_luts++; 1187 1188 crosslink_init_dbmsgs(sndev); 1189 1190 return 0; 1191 } 1192 1193 static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev) 1194 { 1195 if (sndev->mmio_xlink_win) 1196 pci_iounmap(sndev->stdev->pdev, sndev->mmio_xlink_win); 1197 } 1198 1199 static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl) 1200 { 1201 int i; 1202 int cnt = 0; 1203 1204 for (i = 0; i < ARRAY_SIZE(ctrl->bar_entry); i++) { 1205 u32 r = ioread32(&ctrl->bar_entry[i].ctl); 1206 1207 if (r & NTB_CTRL_BAR_VALID) 1208 map[cnt++] = i; 1209 } 1210 1211 return cnt; 1212 } 1213 1214 static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev) 1215 { 1216 sndev->nr_direct_mw = map_bars(sndev->direct_mw_to_bar, 1217 sndev->mmio_self_ctrl); 1218 1219 sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries); 1220 sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw); 1221 1222 dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n", 1223 sndev->nr_direct_mw, sndev->nr_lut_mw); 1224 1225 sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar, 1226 sndev->mmio_peer_ctrl); 1227 1228 sndev->peer_nr_lut_mw = 1229 ioread16(&sndev->mmio_peer_ctrl->lut_table_entries); 1230 sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw); 1231 1232 dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n", 1233 sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw); 1234 1235 } 1236 1237 /* 1238 * There are 64 doorbells in the switch hardware but this is 1239 * shared among all partitions. So we must split them in half 1240 * (32 for each partition). However, the message interrupts are 1241 * also shared with the top 4 doorbells so we just limit this to 1242 * 28 doorbells per partition. 1243 * 1244 * In crosslink mode, each side has it's own dbmsg register so 1245 * they can each use all 60 of the available doorbells. 1246 */ 1247 static void switchtec_ntb_init_db(struct switchtec_ntb *sndev) 1248 { 1249 sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL; 1250 1251 if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) { 1252 sndev->db_shift = 0; 1253 sndev->db_peer_shift = 0; 1254 sndev->db_valid_mask = sndev->db_mask; 1255 } else if (sndev->self_partition < sndev->peer_partition) { 1256 sndev->db_shift = 0; 1257 sndev->db_peer_shift = 32; 1258 sndev->db_valid_mask = 0x0FFFFFFF; 1259 } else { 1260 sndev->db_shift = 32; 1261 sndev->db_peer_shift = 0; 1262 sndev->db_valid_mask = 0x0FFFFFFF; 1263 } 1264 1265 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 1266 iowrite64(sndev->db_valid_mask << sndev->db_peer_shift, 1267 &sndev->mmio_peer_dbmsg->odb_mask); 1268 1269 dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n", 1270 sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask); 1271 } 1272 1273 static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev) 1274 { 1275 int i; 1276 u32 msg_map = 0; 1277 1278 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) { 1279 int m = i | sndev->peer_partition << 2; 1280 1281 msg_map |= m << i * 8; 1282 } 1283 1284 iowrite32(msg_map, &sndev->mmio_self_dbmsg->msg_map); 1285 1286 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) 1287 iowrite64(NTB_DBMSG_IMSG_STATUS | NTB_DBMSG_IMSG_MASK, 1288 &sndev->mmio_self_dbmsg->imsg[i]); 1289 } 1290 1291 static int 1292 switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev) 1293 { 1294 int req_ids[2]; 1295 1296 /* 1297 * Root Complex Requester ID (which is 0:00.0) 1298 */ 1299 req_ids[0] = 0; 1300 1301 /* 1302 * Host Bridge Requester ID (as read from the mmap address) 1303 */ 1304 req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id); 1305 1306 return config_req_id_table(sndev, sndev->mmio_self_ctrl, req_ids, 1307 ARRAY_SIZE(req_ids)); 1308 } 1309 1310 static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev) 1311 { 1312 int i; 1313 1314 memset(sndev->self_shared, 0, LUT_SIZE); 1315 sndev->self_shared->magic = SWITCHTEC_NTB_MAGIC; 1316 sndev->self_shared->partition_id = sndev->stdev->partition; 1317 1318 for (i = 0; i < sndev->nr_direct_mw; i++) { 1319 int bar = sndev->direct_mw_to_bar[i]; 1320 resource_size_t sz = pci_resource_len(sndev->stdev->pdev, bar); 1321 1322 if (i == 0) 1323 sz = min_t(resource_size_t, sz, 1324 LUT_SIZE * sndev->nr_lut_mw); 1325 1326 sndev->self_shared->mw_sizes[i] = sz; 1327 } 1328 1329 for (i = 0; i < sndev->nr_lut_mw; i++) { 1330 int idx = sndev->nr_direct_mw + i; 1331 1332 sndev->self_shared->mw_sizes[idx] = LUT_SIZE; 1333 } 1334 } 1335 1336 static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev) 1337 { 1338 int self_bar = sndev->direct_mw_to_bar[0]; 1339 int rc; 1340 1341 sndev->nr_rsvd_luts++; 1342 sndev->self_shared = dma_zalloc_coherent(&sndev->stdev->pdev->dev, 1343 LUT_SIZE, 1344 &sndev->self_shared_dma, 1345 GFP_KERNEL); 1346 if (!sndev->self_shared) { 1347 dev_err(&sndev->stdev->dev, 1348 "unable to allocate memory for shared mw\n"); 1349 return -ENOMEM; 1350 } 1351 1352 switchtec_ntb_init_shared(sndev); 1353 1354 rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0, 1355 sndev->self_partition, 1356 sndev->self_shared_dma); 1357 if (rc) 1358 goto unalloc_and_exit; 1359 1360 sndev->peer_shared = pci_iomap(sndev->stdev->pdev, self_bar, LUT_SIZE); 1361 if (!sndev->peer_shared) { 1362 rc = -ENOMEM; 1363 goto unalloc_and_exit; 1364 } 1365 1366 dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n"); 1367 return 0; 1368 1369 unalloc_and_exit: 1370 dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE, 1371 sndev->self_shared, sndev->self_shared_dma); 1372 1373 return rc; 1374 } 1375 1376 static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev) 1377 { 1378 if (sndev->peer_shared) 1379 pci_iounmap(sndev->stdev->pdev, sndev->peer_shared); 1380 1381 if (sndev->self_shared) 1382 dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE, 1383 sndev->self_shared, 1384 sndev->self_shared_dma); 1385 sndev->nr_rsvd_luts--; 1386 } 1387 1388 static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev) 1389 { 1390 struct switchtec_ntb *sndev = dev; 1391 1392 dev_dbg(&sndev->stdev->dev, "doorbell\n"); 1393 1394 ntb_db_event(&sndev->ntb, 0); 1395 1396 return IRQ_HANDLED; 1397 } 1398 1399 static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev) 1400 { 1401 int i; 1402 struct switchtec_ntb *sndev = dev; 1403 1404 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) { 1405 u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]); 1406 1407 if (msg & NTB_DBMSG_IMSG_STATUS) { 1408 dev_dbg(&sndev->stdev->dev, "message: %d %08x\n", 1409 i, (u32)msg); 1410 iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status); 1411 1412 if (i == LINK_MESSAGE) 1413 switchtec_ntb_check_link(sndev, msg); 1414 } 1415 } 1416 1417 return IRQ_HANDLED; 1418 } 1419 1420 static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev) 1421 { 1422 int i; 1423 int rc; 1424 int doorbell_irq = 0; 1425 int message_irq = 0; 1426 int event_irq; 1427 int idb_vecs = sizeof(sndev->mmio_self_dbmsg->idb_vec_map); 1428 1429 event_irq = ioread32(&sndev->stdev->mmio_part_cfg->vep_vector_number); 1430 1431 while (doorbell_irq == event_irq) 1432 doorbell_irq++; 1433 while (message_irq == doorbell_irq || 1434 message_irq == event_irq) 1435 message_irq++; 1436 1437 dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n", 1438 event_irq, doorbell_irq, message_irq); 1439 1440 for (i = 0; i < idb_vecs - 4; i++) 1441 iowrite8(doorbell_irq, 1442 &sndev->mmio_self_dbmsg->idb_vec_map[i]); 1443 1444 for (; i < idb_vecs; i++) 1445 iowrite8(message_irq, 1446 &sndev->mmio_self_dbmsg->idb_vec_map[i]); 1447 1448 sndev->doorbell_irq = pci_irq_vector(sndev->stdev->pdev, doorbell_irq); 1449 sndev->message_irq = pci_irq_vector(sndev->stdev->pdev, message_irq); 1450 1451 rc = request_irq(sndev->doorbell_irq, 1452 switchtec_ntb_doorbell_isr, 0, 1453 "switchtec_ntb_doorbell", sndev); 1454 if (rc) 1455 return rc; 1456 1457 rc = request_irq(sndev->message_irq, 1458 switchtec_ntb_message_isr, 0, 1459 "switchtec_ntb_message", sndev); 1460 if (rc) { 1461 free_irq(sndev->doorbell_irq, sndev); 1462 return rc; 1463 } 1464 1465 return 0; 1466 } 1467 1468 static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev) 1469 { 1470 free_irq(sndev->doorbell_irq, sndev); 1471 free_irq(sndev->message_irq, sndev); 1472 } 1473 1474 static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev) 1475 { 1476 dev_info(&sndev->stdev->dev, "peer reinitialized\n"); 1477 switchtec_ntb_deinit_shared_mw(sndev); 1478 switchtec_ntb_init_mw(sndev); 1479 return switchtec_ntb_init_shared_mw(sndev); 1480 } 1481 1482 static int switchtec_ntb_add(struct device *dev, 1483 struct class_interface *class_intf) 1484 { 1485 struct switchtec_dev *stdev = to_stdev(dev); 1486 struct switchtec_ntb *sndev; 1487 int rc; 1488 1489 stdev->sndev = NULL; 1490 1491 if (stdev->pdev->class != (PCI_CLASS_BRIDGE_OTHER << 8)) 1492 return -ENODEV; 1493 1494 sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev)); 1495 if (!sndev) 1496 return -ENOMEM; 1497 1498 sndev->stdev = stdev; 1499 rc = switchtec_ntb_init_sndev(sndev); 1500 if (rc) 1501 goto free_and_exit; 1502 1503 switchtec_ntb_init_mw(sndev); 1504 1505 rc = switchtec_ntb_init_req_id_table(sndev); 1506 if (rc) 1507 goto free_and_exit; 1508 1509 rc = switchtec_ntb_init_crosslink(sndev); 1510 if (rc) 1511 goto free_and_exit; 1512 1513 switchtec_ntb_init_db(sndev); 1514 switchtec_ntb_init_msgs(sndev); 1515 1516 rc = switchtec_ntb_init_shared_mw(sndev); 1517 if (rc) 1518 goto deinit_crosslink; 1519 1520 rc = switchtec_ntb_init_db_msg_irq(sndev); 1521 if (rc) 1522 goto deinit_shared_and_exit; 1523 1524 /* 1525 * If this host crashed, the other host may think the link is 1526 * still up. Tell them to force it down (it will go back up 1527 * once we register the ntb device). 1528 */ 1529 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_FORCE_DOWN); 1530 1531 rc = ntb_register_device(&sndev->ntb); 1532 if (rc) 1533 goto deinit_and_exit; 1534 1535 stdev->sndev = sndev; 1536 stdev->link_notifier = switchtec_ntb_link_notification; 1537 dev_info(dev, "NTB device registered\n"); 1538 1539 return 0; 1540 1541 deinit_and_exit: 1542 switchtec_ntb_deinit_db_msg_irq(sndev); 1543 deinit_shared_and_exit: 1544 switchtec_ntb_deinit_shared_mw(sndev); 1545 deinit_crosslink: 1546 switchtec_ntb_deinit_crosslink(sndev); 1547 free_and_exit: 1548 kfree(sndev); 1549 dev_err(dev, "failed to register ntb device: %d\n", rc); 1550 return rc; 1551 } 1552 1553 static void switchtec_ntb_remove(struct device *dev, 1554 struct class_interface *class_intf) 1555 { 1556 struct switchtec_dev *stdev = to_stdev(dev); 1557 struct switchtec_ntb *sndev = stdev->sndev; 1558 1559 if (!sndev) 1560 return; 1561 1562 stdev->link_notifier = NULL; 1563 stdev->sndev = NULL; 1564 ntb_unregister_device(&sndev->ntb); 1565 switchtec_ntb_deinit_db_msg_irq(sndev); 1566 switchtec_ntb_deinit_shared_mw(sndev); 1567 switchtec_ntb_deinit_crosslink(sndev); 1568 kfree(sndev); 1569 dev_info(dev, "ntb device unregistered\n"); 1570 } 1571 1572 static struct class_interface switchtec_interface = { 1573 .add_dev = switchtec_ntb_add, 1574 .remove_dev = switchtec_ntb_remove, 1575 }; 1576 1577 static int __init switchtec_ntb_init(void) 1578 { 1579 switchtec_interface.class = switchtec_class; 1580 return class_interface_register(&switchtec_interface); 1581 } 1582 module_init(switchtec_ntb_init); 1583 1584 static void __exit switchtec_ntb_exit(void) 1585 { 1586 class_interface_unregister(&switchtec_interface); 1587 } 1588 module_exit(switchtec_ntb_exit); 1589