1 /* 2 * Microsemi Switchtec(tm) PCIe Management Driver 3 * Copyright (c) 2017, Microsemi Corporation 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms and conditions of the GNU General Public License, 7 * version 2, as published by the Free Software Foundation. 8 * 9 * This program is distributed in the hope it will be useful, but WITHOUT 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for 12 * more details. 13 * 14 */ 15 16 #include <linux/interrupt.h> 17 #include <linux/io-64-nonatomic-lo-hi.h> 18 #include <linux/delay.h> 19 #include <linux/kthread.h> 20 #include <linux/module.h> 21 #include <linux/ntb.h> 22 #include <linux/pci.h> 23 #include <linux/switchtec.h> 24 25 MODULE_DESCRIPTION("Microsemi Switchtec(tm) NTB Driver"); 26 MODULE_VERSION("0.1"); 27 MODULE_LICENSE("GPL"); 28 MODULE_AUTHOR("Microsemi Corporation"); 29 30 static ulong max_mw_size = SZ_2M; 31 module_param(max_mw_size, ulong, 0644); 32 MODULE_PARM_DESC(max_mw_size, 33 "Max memory window size reported to the upper layer"); 34 35 static bool use_lut_mws; 36 module_param(use_lut_mws, bool, 0644); 37 MODULE_PARM_DESC(use_lut_mws, 38 "Enable the use of the LUT based memory windows"); 39 40 #define SWITCHTEC_NTB_MAGIC 0x45CC0001 41 #define MAX_MWS 128 42 43 struct shared_mw { 44 u32 magic; 45 u32 link_sta; 46 u32 partition_id; 47 u64 mw_sizes[MAX_MWS]; 48 u32 spad[128]; 49 }; 50 51 #define MAX_DIRECT_MW ARRAY_SIZE(((struct ntb_ctrl_regs *)(0))->bar_entry) 52 #define LUT_SIZE SZ_64K 53 54 struct switchtec_ntb { 55 struct ntb_dev ntb; 56 struct switchtec_dev *stdev; 57 58 int self_partition; 59 int peer_partition; 60 61 int doorbell_irq; 62 int message_irq; 63 64 struct ntb_info_regs __iomem *mmio_ntb; 65 struct ntb_ctrl_regs __iomem *mmio_ctrl; 66 struct ntb_dbmsg_regs __iomem *mmio_dbmsg; 67 struct ntb_ctrl_regs __iomem *mmio_self_ctrl; 68 struct ntb_ctrl_regs __iomem *mmio_peer_ctrl; 69 struct ntb_dbmsg_regs __iomem *mmio_self_dbmsg; 70 struct ntb_dbmsg_regs __iomem *mmio_peer_dbmsg; 71 72 void __iomem *mmio_xlink_win; 73 74 struct shared_mw *self_shared; 75 struct shared_mw __iomem *peer_shared; 76 dma_addr_t self_shared_dma; 77 78 u64 db_mask; 79 u64 db_valid_mask; 80 int db_shift; 81 int db_peer_shift; 82 83 /* synchronize rmw access of db_mask and hw reg */ 84 spinlock_t db_mask_lock; 85 86 int nr_direct_mw; 87 int nr_lut_mw; 88 int nr_rsvd_luts; 89 int direct_mw_to_bar[MAX_DIRECT_MW]; 90 91 int peer_nr_direct_mw; 92 int peer_nr_lut_mw; 93 int peer_direct_mw_to_bar[MAX_DIRECT_MW]; 94 95 bool link_is_up; 96 enum ntb_speed link_speed; 97 enum ntb_width link_width; 98 struct work_struct link_reinit_work; 99 }; 100 101 static struct switchtec_ntb *ntb_sndev(struct ntb_dev *ntb) 102 { 103 return container_of(ntb, struct switchtec_ntb, ntb); 104 } 105 106 static int switchtec_ntb_part_op(struct switchtec_ntb *sndev, 107 struct ntb_ctrl_regs __iomem *ctl, 108 u32 op, int wait_status) 109 { 110 static const char * const op_text[] = { 111 [NTB_CTRL_PART_OP_LOCK] = "lock", 112 [NTB_CTRL_PART_OP_CFG] = "configure", 113 [NTB_CTRL_PART_OP_RESET] = "reset", 114 }; 115 116 int i; 117 u32 ps; 118 int status; 119 120 switch (op) { 121 case NTB_CTRL_PART_OP_LOCK: 122 status = NTB_CTRL_PART_STATUS_LOCKING; 123 break; 124 case NTB_CTRL_PART_OP_CFG: 125 status = NTB_CTRL_PART_STATUS_CONFIGURING; 126 break; 127 case NTB_CTRL_PART_OP_RESET: 128 status = NTB_CTRL_PART_STATUS_RESETTING; 129 break; 130 default: 131 return -EINVAL; 132 } 133 134 iowrite32(op, &ctl->partition_op); 135 136 for (i = 0; i < 1000; i++) { 137 if (msleep_interruptible(50) != 0) { 138 iowrite32(NTB_CTRL_PART_OP_RESET, &ctl->partition_op); 139 return -EINTR; 140 } 141 142 ps = ioread32(&ctl->partition_status) & 0xFFFF; 143 144 if (ps != status) 145 break; 146 } 147 148 if (ps == wait_status) 149 return 0; 150 151 if (ps == status) { 152 dev_err(&sndev->stdev->dev, 153 "Timed out while performing %s (%d). (%08x)\n", 154 op_text[op], op, 155 ioread32(&ctl->partition_status)); 156 157 return -ETIMEDOUT; 158 } 159 160 return -EIO; 161 } 162 163 static int switchtec_ntb_send_msg(struct switchtec_ntb *sndev, int idx, 164 u32 val) 165 { 166 if (idx < 0 || idx >= ARRAY_SIZE(sndev->mmio_peer_dbmsg->omsg)) 167 return -EINVAL; 168 169 iowrite32(val, &sndev->mmio_peer_dbmsg->omsg[idx].msg); 170 171 return 0; 172 } 173 174 static int switchtec_ntb_mw_count(struct ntb_dev *ntb, int pidx) 175 { 176 struct switchtec_ntb *sndev = ntb_sndev(ntb); 177 int nr_direct_mw = sndev->peer_nr_direct_mw; 178 int nr_lut_mw = sndev->peer_nr_lut_mw - sndev->nr_rsvd_luts; 179 180 if (pidx != NTB_DEF_PEER_IDX) 181 return -EINVAL; 182 183 if (!use_lut_mws) 184 nr_lut_mw = 0; 185 186 return nr_direct_mw + nr_lut_mw; 187 } 188 189 static int lut_index(struct switchtec_ntb *sndev, int mw_idx) 190 { 191 return mw_idx - sndev->nr_direct_mw + sndev->nr_rsvd_luts; 192 } 193 194 static int peer_lut_index(struct switchtec_ntb *sndev, int mw_idx) 195 { 196 return mw_idx - sndev->peer_nr_direct_mw + sndev->nr_rsvd_luts; 197 } 198 199 static int switchtec_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, 200 int widx, resource_size_t *addr_align, 201 resource_size_t *size_align, 202 resource_size_t *size_max) 203 { 204 struct switchtec_ntb *sndev = ntb_sndev(ntb); 205 int lut; 206 resource_size_t size; 207 208 if (pidx != NTB_DEF_PEER_IDX) 209 return -EINVAL; 210 211 lut = widx >= sndev->peer_nr_direct_mw; 212 size = ioread64(&sndev->peer_shared->mw_sizes[widx]); 213 214 if (size == 0) 215 return -EINVAL; 216 217 if (addr_align) 218 *addr_align = lut ? size : SZ_4K; 219 220 if (size_align) 221 *size_align = lut ? size : SZ_4K; 222 223 if (size_max) 224 *size_max = size; 225 226 return 0; 227 } 228 229 static void switchtec_ntb_mw_clr_direct(struct switchtec_ntb *sndev, int idx) 230 { 231 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 232 int bar = sndev->peer_direct_mw_to_bar[idx]; 233 u32 ctl_val; 234 235 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 236 ctl_val &= ~NTB_CTRL_BAR_DIR_WIN_EN; 237 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); 238 iowrite32(0, &ctl->bar_entry[bar].win_size); 239 iowrite32(0, &ctl->bar_ext_entry[bar].win_size); 240 iowrite64(sndev->self_partition, &ctl->bar_entry[bar].xlate_addr); 241 } 242 243 static void switchtec_ntb_mw_clr_lut(struct switchtec_ntb *sndev, int idx) 244 { 245 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 246 247 iowrite64(0, &ctl->lut_entry[peer_lut_index(sndev, idx)]); 248 } 249 250 static void switchtec_ntb_mw_set_direct(struct switchtec_ntb *sndev, int idx, 251 dma_addr_t addr, resource_size_t size) 252 { 253 int xlate_pos = ilog2(size); 254 int bar = sndev->peer_direct_mw_to_bar[idx]; 255 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 256 u32 ctl_val; 257 258 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 259 ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN; 260 261 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); 262 iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000), 263 &ctl->bar_entry[bar].win_size); 264 iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size); 265 iowrite64(sndev->self_partition | addr, 266 &ctl->bar_entry[bar].xlate_addr); 267 } 268 269 static void switchtec_ntb_mw_set_lut(struct switchtec_ntb *sndev, int idx, 270 dma_addr_t addr, resource_size_t size) 271 { 272 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 273 274 iowrite64((NTB_CTRL_LUT_EN | (sndev->self_partition << 1) | addr), 275 &ctl->lut_entry[peer_lut_index(sndev, idx)]); 276 } 277 278 static int switchtec_ntb_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx, 279 dma_addr_t addr, resource_size_t size) 280 { 281 struct switchtec_ntb *sndev = ntb_sndev(ntb); 282 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_peer_ctrl; 283 int xlate_pos = ilog2(size); 284 int nr_direct_mw = sndev->peer_nr_direct_mw; 285 int rc; 286 287 if (pidx != NTB_DEF_PEER_IDX) 288 return -EINVAL; 289 290 dev_dbg(&sndev->stdev->dev, "MW %d: part %d addr %pad size %pap\n", 291 widx, pidx, &addr, &size); 292 293 if (widx >= switchtec_ntb_mw_count(ntb, pidx)) 294 return -EINVAL; 295 296 if (xlate_pos < 12) 297 return -EINVAL; 298 299 if (!IS_ALIGNED(addr, BIT_ULL(xlate_pos))) { 300 /* 301 * In certain circumstances we can get a buffer that is 302 * not aligned to its size. (Most of the time 303 * dma_alloc_coherent ensures this). This can happen when 304 * using large buffers allocated by the CMA 305 * (see CMA_CONFIG_ALIGNMENT) 306 */ 307 dev_err(&sndev->stdev->dev, 308 "ERROR: Memory window address is not aligned to it's size!\n"); 309 return -EINVAL; 310 } 311 312 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 313 NTB_CTRL_PART_STATUS_LOCKED); 314 if (rc) 315 return rc; 316 317 if (addr == 0 || size == 0) { 318 if (widx < nr_direct_mw) 319 switchtec_ntb_mw_clr_direct(sndev, widx); 320 else 321 switchtec_ntb_mw_clr_lut(sndev, widx); 322 } else { 323 if (widx < nr_direct_mw) 324 switchtec_ntb_mw_set_direct(sndev, widx, addr, size); 325 else 326 switchtec_ntb_mw_set_lut(sndev, widx, addr, size); 327 } 328 329 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 330 NTB_CTRL_PART_STATUS_NORMAL); 331 332 if (rc == -EIO) { 333 dev_err(&sndev->stdev->dev, 334 "Hardware reported an error configuring mw %d: %08x\n", 335 widx, ioread32(&ctl->bar_error)); 336 337 if (widx < nr_direct_mw) 338 switchtec_ntb_mw_clr_direct(sndev, widx); 339 else 340 switchtec_ntb_mw_clr_lut(sndev, widx); 341 342 switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 343 NTB_CTRL_PART_STATUS_NORMAL); 344 } 345 346 return rc; 347 } 348 349 static int switchtec_ntb_peer_mw_count(struct ntb_dev *ntb) 350 { 351 struct switchtec_ntb *sndev = ntb_sndev(ntb); 352 int nr_lut_mw = sndev->nr_lut_mw - sndev->nr_rsvd_luts; 353 354 return sndev->nr_direct_mw + (use_lut_mws ? nr_lut_mw : 0); 355 } 356 357 static int switchtec_ntb_direct_get_addr(struct switchtec_ntb *sndev, 358 int idx, phys_addr_t *base, 359 resource_size_t *size) 360 { 361 int bar = sndev->direct_mw_to_bar[idx]; 362 size_t offset = 0; 363 364 if (bar < 0) 365 return -EINVAL; 366 367 if (idx == 0) { 368 /* 369 * This is the direct BAR shared with the LUTs 370 * which means the actual window will be offset 371 * by the size of all the LUT entries. 372 */ 373 374 offset = LUT_SIZE * sndev->nr_lut_mw; 375 } 376 377 if (base) 378 *base = pci_resource_start(sndev->ntb.pdev, bar) + offset; 379 380 if (size) { 381 *size = pci_resource_len(sndev->ntb.pdev, bar) - offset; 382 if (offset && *size > offset) 383 *size = offset; 384 385 if (*size > max_mw_size) 386 *size = max_mw_size; 387 } 388 389 return 0; 390 } 391 392 static int switchtec_ntb_lut_get_addr(struct switchtec_ntb *sndev, 393 int idx, phys_addr_t *base, 394 resource_size_t *size) 395 { 396 int bar = sndev->direct_mw_to_bar[0]; 397 int offset; 398 399 offset = LUT_SIZE * lut_index(sndev, idx); 400 401 if (base) 402 *base = pci_resource_start(sndev->ntb.pdev, bar) + offset; 403 404 if (size) 405 *size = LUT_SIZE; 406 407 return 0; 408 } 409 410 static int switchtec_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int idx, 411 phys_addr_t *base, 412 resource_size_t *size) 413 { 414 struct switchtec_ntb *sndev = ntb_sndev(ntb); 415 416 if (idx < sndev->nr_direct_mw) 417 return switchtec_ntb_direct_get_addr(sndev, idx, base, size); 418 else if (idx < switchtec_ntb_peer_mw_count(ntb)) 419 return switchtec_ntb_lut_get_addr(sndev, idx, base, size); 420 else 421 return -EINVAL; 422 } 423 424 static void switchtec_ntb_part_link_speed(struct switchtec_ntb *sndev, 425 int partition, 426 enum ntb_speed *speed, 427 enum ntb_width *width) 428 { 429 struct switchtec_dev *stdev = sndev->stdev; 430 431 u32 pff = ioread32(&stdev->mmio_part_cfg[partition].vep_pff_inst_id); 432 u32 linksta = ioread32(&stdev->mmio_pff_csr[pff].pci_cap_region[13]); 433 434 if (speed) 435 *speed = (linksta >> 16) & 0xF; 436 437 if (width) 438 *width = (linksta >> 20) & 0x3F; 439 } 440 441 static void switchtec_ntb_set_link_speed(struct switchtec_ntb *sndev) 442 { 443 enum ntb_speed self_speed, peer_speed; 444 enum ntb_width self_width, peer_width; 445 446 if (!sndev->link_is_up) { 447 sndev->link_speed = NTB_SPEED_NONE; 448 sndev->link_width = NTB_WIDTH_NONE; 449 return; 450 } 451 452 switchtec_ntb_part_link_speed(sndev, sndev->self_partition, 453 &self_speed, &self_width); 454 switchtec_ntb_part_link_speed(sndev, sndev->peer_partition, 455 &peer_speed, &peer_width); 456 457 sndev->link_speed = min(self_speed, peer_speed); 458 sndev->link_width = min(self_width, peer_width); 459 } 460 461 static int crosslink_is_enabled(struct switchtec_ntb *sndev) 462 { 463 struct ntb_info_regs __iomem *inf = sndev->mmio_ntb; 464 465 return ioread8(&inf->ntp_info[sndev->peer_partition].xlink_enabled); 466 } 467 468 static void crosslink_init_dbmsgs(struct switchtec_ntb *sndev) 469 { 470 int i; 471 u32 msg_map = 0; 472 473 if (!crosslink_is_enabled(sndev)) 474 return; 475 476 for (i = 0; i < ARRAY_SIZE(sndev->mmio_peer_dbmsg->imsg); i++) { 477 int m = i | sndev->self_partition << 2; 478 479 msg_map |= m << i * 8; 480 } 481 482 iowrite32(msg_map, &sndev->mmio_peer_dbmsg->msg_map); 483 iowrite64(sndev->db_valid_mask << sndev->db_peer_shift, 484 &sndev->mmio_peer_dbmsg->odb_mask); 485 } 486 487 enum switchtec_msg { 488 LINK_MESSAGE = 0, 489 MSG_LINK_UP = 1, 490 MSG_LINK_DOWN = 2, 491 MSG_CHECK_LINK = 3, 492 MSG_LINK_FORCE_DOWN = 4, 493 }; 494 495 static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev); 496 497 static void link_reinit_work(struct work_struct *work) 498 { 499 struct switchtec_ntb *sndev; 500 501 sndev = container_of(work, struct switchtec_ntb, link_reinit_work); 502 503 switchtec_ntb_reinit_peer(sndev); 504 } 505 506 static void switchtec_ntb_check_link(struct switchtec_ntb *sndev, 507 enum switchtec_msg msg) 508 { 509 int link_sta; 510 int old = sndev->link_is_up; 511 512 if (msg == MSG_LINK_FORCE_DOWN) { 513 schedule_work(&sndev->link_reinit_work); 514 515 if (sndev->link_is_up) { 516 sndev->link_is_up = 0; 517 ntb_link_event(&sndev->ntb); 518 dev_info(&sndev->stdev->dev, "ntb link forced down\n"); 519 } 520 521 return; 522 } 523 524 link_sta = sndev->self_shared->link_sta; 525 if (link_sta) { 526 u64 peer = ioread64(&sndev->peer_shared->magic); 527 528 if ((peer & 0xFFFFFFFF) == SWITCHTEC_NTB_MAGIC) 529 link_sta = peer >> 32; 530 else 531 link_sta = 0; 532 } 533 534 sndev->link_is_up = link_sta; 535 switchtec_ntb_set_link_speed(sndev); 536 537 if (link_sta != old) { 538 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_CHECK_LINK); 539 ntb_link_event(&sndev->ntb); 540 dev_info(&sndev->stdev->dev, "ntb link %s\n", 541 link_sta ? "up" : "down"); 542 543 if (link_sta) 544 crosslink_init_dbmsgs(sndev); 545 } 546 } 547 548 static void switchtec_ntb_link_notification(struct switchtec_dev *stdev) 549 { 550 struct switchtec_ntb *sndev = stdev->sndev; 551 552 switchtec_ntb_check_link(sndev, MSG_CHECK_LINK); 553 } 554 555 static u64 switchtec_ntb_link_is_up(struct ntb_dev *ntb, 556 enum ntb_speed *speed, 557 enum ntb_width *width) 558 { 559 struct switchtec_ntb *sndev = ntb_sndev(ntb); 560 561 if (speed) 562 *speed = sndev->link_speed; 563 if (width) 564 *width = sndev->link_width; 565 566 return sndev->link_is_up; 567 } 568 569 static int switchtec_ntb_link_enable(struct ntb_dev *ntb, 570 enum ntb_speed max_speed, 571 enum ntb_width max_width) 572 { 573 struct switchtec_ntb *sndev = ntb_sndev(ntb); 574 575 dev_dbg(&sndev->stdev->dev, "enabling link\n"); 576 577 sndev->self_shared->link_sta = 1; 578 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_UP); 579 580 switchtec_ntb_check_link(sndev, MSG_CHECK_LINK); 581 582 return 0; 583 } 584 585 static int switchtec_ntb_link_disable(struct ntb_dev *ntb) 586 { 587 struct switchtec_ntb *sndev = ntb_sndev(ntb); 588 589 dev_dbg(&sndev->stdev->dev, "disabling link\n"); 590 591 sndev->self_shared->link_sta = 0; 592 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_DOWN); 593 594 switchtec_ntb_check_link(sndev, MSG_CHECK_LINK); 595 596 return 0; 597 } 598 599 static u64 switchtec_ntb_db_valid_mask(struct ntb_dev *ntb) 600 { 601 struct switchtec_ntb *sndev = ntb_sndev(ntb); 602 603 return sndev->db_valid_mask; 604 } 605 606 static int switchtec_ntb_db_vector_count(struct ntb_dev *ntb) 607 { 608 return 1; 609 } 610 611 static u64 switchtec_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector) 612 { 613 struct switchtec_ntb *sndev = ntb_sndev(ntb); 614 615 if (db_vector < 0 || db_vector > 1) 616 return 0; 617 618 return sndev->db_valid_mask; 619 } 620 621 static u64 switchtec_ntb_db_read(struct ntb_dev *ntb) 622 { 623 u64 ret; 624 struct switchtec_ntb *sndev = ntb_sndev(ntb); 625 626 ret = ioread64(&sndev->mmio_self_dbmsg->idb) >> sndev->db_shift; 627 628 return ret & sndev->db_valid_mask; 629 } 630 631 static int switchtec_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits) 632 { 633 struct switchtec_ntb *sndev = ntb_sndev(ntb); 634 635 iowrite64(db_bits << sndev->db_shift, &sndev->mmio_self_dbmsg->idb); 636 637 return 0; 638 } 639 640 static int switchtec_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits) 641 { 642 unsigned long irqflags; 643 struct switchtec_ntb *sndev = ntb_sndev(ntb); 644 645 if (db_bits & ~sndev->db_valid_mask) 646 return -EINVAL; 647 648 spin_lock_irqsave(&sndev->db_mask_lock, irqflags); 649 650 sndev->db_mask |= db_bits << sndev->db_shift; 651 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 652 653 spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags); 654 655 return 0; 656 } 657 658 static int switchtec_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits) 659 { 660 unsigned long irqflags; 661 struct switchtec_ntb *sndev = ntb_sndev(ntb); 662 663 if (db_bits & ~sndev->db_valid_mask) 664 return -EINVAL; 665 666 spin_lock_irqsave(&sndev->db_mask_lock, irqflags); 667 668 sndev->db_mask &= ~(db_bits << sndev->db_shift); 669 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 670 671 spin_unlock_irqrestore(&sndev->db_mask_lock, irqflags); 672 673 return 0; 674 } 675 676 static u64 switchtec_ntb_db_read_mask(struct ntb_dev *ntb) 677 { 678 struct switchtec_ntb *sndev = ntb_sndev(ntb); 679 680 return (sndev->db_mask >> sndev->db_shift) & sndev->db_valid_mask; 681 } 682 683 static int switchtec_ntb_peer_db_addr(struct ntb_dev *ntb, 684 phys_addr_t *db_addr, 685 resource_size_t *db_size, 686 u64 *db_data, 687 int db_bit) 688 { 689 struct switchtec_ntb *sndev = ntb_sndev(ntb); 690 unsigned long offset; 691 692 if (unlikely(db_bit >= BITS_PER_LONG_LONG)) 693 return -EINVAL; 694 695 offset = (unsigned long)sndev->mmio_peer_dbmsg->odb - 696 (unsigned long)sndev->stdev->mmio; 697 698 offset += sndev->db_shift / 8; 699 700 if (db_addr) 701 *db_addr = pci_resource_start(ntb->pdev, 0) + offset; 702 if (db_size) 703 *db_size = sizeof(u32); 704 if (db_data) 705 *db_data = BIT_ULL(db_bit) << sndev->db_peer_shift; 706 707 return 0; 708 } 709 710 static int switchtec_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits) 711 { 712 struct switchtec_ntb *sndev = ntb_sndev(ntb); 713 714 iowrite64(db_bits << sndev->db_peer_shift, 715 &sndev->mmio_peer_dbmsg->odb); 716 717 return 0; 718 } 719 720 static int switchtec_ntb_spad_count(struct ntb_dev *ntb) 721 { 722 struct switchtec_ntb *sndev = ntb_sndev(ntb); 723 724 return ARRAY_SIZE(sndev->self_shared->spad); 725 } 726 727 static u32 switchtec_ntb_spad_read(struct ntb_dev *ntb, int idx) 728 { 729 struct switchtec_ntb *sndev = ntb_sndev(ntb); 730 731 if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad)) 732 return 0; 733 734 if (!sndev->self_shared) 735 return 0; 736 737 return sndev->self_shared->spad[idx]; 738 } 739 740 static int switchtec_ntb_spad_write(struct ntb_dev *ntb, int idx, u32 val) 741 { 742 struct switchtec_ntb *sndev = ntb_sndev(ntb); 743 744 if (idx < 0 || idx >= ARRAY_SIZE(sndev->self_shared->spad)) 745 return -EINVAL; 746 747 if (!sndev->self_shared) 748 return -EIO; 749 750 sndev->self_shared->spad[idx] = val; 751 752 return 0; 753 } 754 755 static u32 switchtec_ntb_peer_spad_read(struct ntb_dev *ntb, int pidx, 756 int sidx) 757 { 758 struct switchtec_ntb *sndev = ntb_sndev(ntb); 759 760 if (pidx != NTB_DEF_PEER_IDX) 761 return -EINVAL; 762 763 if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad)) 764 return 0; 765 766 if (!sndev->peer_shared) 767 return 0; 768 769 return ioread32(&sndev->peer_shared->spad[sidx]); 770 } 771 772 static int switchtec_ntb_peer_spad_write(struct ntb_dev *ntb, int pidx, 773 int sidx, u32 val) 774 { 775 struct switchtec_ntb *sndev = ntb_sndev(ntb); 776 777 if (pidx != NTB_DEF_PEER_IDX) 778 return -EINVAL; 779 780 if (sidx < 0 || sidx >= ARRAY_SIZE(sndev->peer_shared->spad)) 781 return -EINVAL; 782 783 if (!sndev->peer_shared) 784 return -EIO; 785 786 iowrite32(val, &sndev->peer_shared->spad[sidx]); 787 788 return 0; 789 } 790 791 static int switchtec_ntb_peer_spad_addr(struct ntb_dev *ntb, int pidx, 792 int sidx, phys_addr_t *spad_addr) 793 { 794 struct switchtec_ntb *sndev = ntb_sndev(ntb); 795 unsigned long offset; 796 797 if (pidx != NTB_DEF_PEER_IDX) 798 return -EINVAL; 799 800 offset = (unsigned long)&sndev->peer_shared->spad[sidx] - 801 (unsigned long)sndev->stdev->mmio; 802 803 if (spad_addr) 804 *spad_addr = pci_resource_start(ntb->pdev, 0) + offset; 805 806 return 0; 807 } 808 809 static const struct ntb_dev_ops switchtec_ntb_ops = { 810 .mw_count = switchtec_ntb_mw_count, 811 .mw_get_align = switchtec_ntb_mw_get_align, 812 .mw_set_trans = switchtec_ntb_mw_set_trans, 813 .peer_mw_count = switchtec_ntb_peer_mw_count, 814 .peer_mw_get_addr = switchtec_ntb_peer_mw_get_addr, 815 .link_is_up = switchtec_ntb_link_is_up, 816 .link_enable = switchtec_ntb_link_enable, 817 .link_disable = switchtec_ntb_link_disable, 818 .db_valid_mask = switchtec_ntb_db_valid_mask, 819 .db_vector_count = switchtec_ntb_db_vector_count, 820 .db_vector_mask = switchtec_ntb_db_vector_mask, 821 .db_read = switchtec_ntb_db_read, 822 .db_clear = switchtec_ntb_db_clear, 823 .db_set_mask = switchtec_ntb_db_set_mask, 824 .db_clear_mask = switchtec_ntb_db_clear_mask, 825 .db_read_mask = switchtec_ntb_db_read_mask, 826 .peer_db_addr = switchtec_ntb_peer_db_addr, 827 .peer_db_set = switchtec_ntb_peer_db_set, 828 .spad_count = switchtec_ntb_spad_count, 829 .spad_read = switchtec_ntb_spad_read, 830 .spad_write = switchtec_ntb_spad_write, 831 .peer_spad_read = switchtec_ntb_peer_spad_read, 832 .peer_spad_write = switchtec_ntb_peer_spad_write, 833 .peer_spad_addr = switchtec_ntb_peer_spad_addr, 834 }; 835 836 static int switchtec_ntb_init_sndev(struct switchtec_ntb *sndev) 837 { 838 u64 tpart_vec; 839 int self; 840 u64 part_map; 841 int bit; 842 843 sndev->ntb.pdev = sndev->stdev->pdev; 844 sndev->ntb.topo = NTB_TOPO_SWITCH; 845 sndev->ntb.ops = &switchtec_ntb_ops; 846 847 INIT_WORK(&sndev->link_reinit_work, link_reinit_work); 848 849 sndev->self_partition = sndev->stdev->partition; 850 851 sndev->mmio_ntb = sndev->stdev->mmio_ntb; 852 853 self = sndev->self_partition; 854 tpart_vec = ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_high); 855 tpart_vec <<= 32; 856 tpart_vec |= ioread32(&sndev->mmio_ntb->ntp_info[self].target_part_low); 857 858 part_map = ioread64(&sndev->mmio_ntb->ep_map); 859 part_map &= ~(1 << sndev->self_partition); 860 861 if (!ffs(tpart_vec)) { 862 if (sndev->stdev->partition_count != 2) { 863 dev_err(&sndev->stdev->dev, 864 "ntb target partition not defined\n"); 865 return -ENODEV; 866 } 867 868 bit = ffs(part_map); 869 if (!bit) { 870 dev_err(&sndev->stdev->dev, 871 "peer partition is not NT partition\n"); 872 return -ENODEV; 873 } 874 875 sndev->peer_partition = bit - 1; 876 } else { 877 if (ffs(tpart_vec) != fls(tpart_vec)) { 878 dev_err(&sndev->stdev->dev, 879 "ntb driver only supports 1 pair of 1-1 ntb mapping\n"); 880 return -ENODEV; 881 } 882 883 sndev->peer_partition = ffs(tpart_vec) - 1; 884 if (!(part_map & (1 << sndev->peer_partition))) { 885 dev_err(&sndev->stdev->dev, 886 "ntb target partition is not NT partition\n"); 887 return -ENODEV; 888 } 889 } 890 891 dev_dbg(&sndev->stdev->dev, "Partition ID %d of %d\n", 892 sndev->self_partition, sndev->stdev->partition_count); 893 894 sndev->mmio_ctrl = (void * __iomem)sndev->mmio_ntb + 895 SWITCHTEC_NTB_REG_CTRL_OFFSET; 896 sndev->mmio_dbmsg = (void * __iomem)sndev->mmio_ntb + 897 SWITCHTEC_NTB_REG_DBMSG_OFFSET; 898 899 sndev->mmio_self_ctrl = &sndev->mmio_ctrl[sndev->self_partition]; 900 sndev->mmio_peer_ctrl = &sndev->mmio_ctrl[sndev->peer_partition]; 901 sndev->mmio_self_dbmsg = &sndev->mmio_dbmsg[sndev->self_partition]; 902 sndev->mmio_peer_dbmsg = sndev->mmio_self_dbmsg; 903 904 return 0; 905 } 906 907 static int config_rsvd_lut_win(struct switchtec_ntb *sndev, 908 struct ntb_ctrl_regs __iomem *ctl, 909 int lut_idx, int partition, u64 addr) 910 { 911 int peer_bar = sndev->peer_direct_mw_to_bar[0]; 912 u32 ctl_val; 913 int rc; 914 915 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 916 NTB_CTRL_PART_STATUS_LOCKED); 917 if (rc) 918 return rc; 919 920 ctl_val = ioread32(&ctl->bar_entry[peer_bar].ctl); 921 ctl_val &= 0xFF; 922 ctl_val |= NTB_CTRL_BAR_LUT_WIN_EN; 923 ctl_val |= ilog2(LUT_SIZE) << 8; 924 ctl_val |= (sndev->nr_lut_mw - 1) << 14; 925 iowrite32(ctl_val, &ctl->bar_entry[peer_bar].ctl); 926 927 iowrite64((NTB_CTRL_LUT_EN | (partition << 1) | addr), 928 &ctl->lut_entry[lut_idx]); 929 930 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 931 NTB_CTRL_PART_STATUS_NORMAL); 932 if (rc) { 933 u32 bar_error, lut_error; 934 935 bar_error = ioread32(&ctl->bar_error); 936 lut_error = ioread32(&ctl->lut_error); 937 dev_err(&sndev->stdev->dev, 938 "Error setting up reserved lut window: %08x / %08x\n", 939 bar_error, lut_error); 940 return rc; 941 } 942 943 return 0; 944 } 945 946 static int config_req_id_table(struct switchtec_ntb *sndev, 947 struct ntb_ctrl_regs __iomem *mmio_ctrl, 948 int *req_ids, int count) 949 { 950 int i, rc = 0; 951 u32 error; 952 u32 proxy_id; 953 954 if (ioread32(&mmio_ctrl->req_id_table_size) < count) { 955 dev_err(&sndev->stdev->dev, 956 "Not enough requester IDs available.\n"); 957 return -EFAULT; 958 } 959 960 rc = switchtec_ntb_part_op(sndev, mmio_ctrl, 961 NTB_CTRL_PART_OP_LOCK, 962 NTB_CTRL_PART_STATUS_LOCKED); 963 if (rc) 964 return rc; 965 966 iowrite32(NTB_PART_CTRL_ID_PROT_DIS, 967 &mmio_ctrl->partition_ctrl); 968 969 for (i = 0; i < count; i++) { 970 iowrite32(req_ids[i] << 16 | NTB_CTRL_REQ_ID_EN, 971 &mmio_ctrl->req_id_table[i]); 972 973 proxy_id = ioread32(&mmio_ctrl->req_id_table[i]); 974 dev_dbg(&sndev->stdev->dev, 975 "Requester ID %02X:%02X.%X -> BB:%02X.%X\n", 976 req_ids[i] >> 8, (req_ids[i] >> 3) & 0x1F, 977 req_ids[i] & 0x7, (proxy_id >> 4) & 0x1F, 978 (proxy_id >> 1) & 0x7); 979 } 980 981 rc = switchtec_ntb_part_op(sndev, mmio_ctrl, 982 NTB_CTRL_PART_OP_CFG, 983 NTB_CTRL_PART_STATUS_NORMAL); 984 985 if (rc == -EIO) { 986 error = ioread32(&mmio_ctrl->req_id_error); 987 dev_err(&sndev->stdev->dev, 988 "Error setting up the requester ID table: %08x\n", 989 error); 990 } 991 992 return 0; 993 } 994 995 static int crosslink_setup_mws(struct switchtec_ntb *sndev, int ntb_lut_idx, 996 u64 *mw_addrs, int mw_count) 997 { 998 int rc, i; 999 struct ntb_ctrl_regs __iomem *ctl = sndev->mmio_self_ctrl; 1000 u64 addr; 1001 size_t size, offset; 1002 int bar; 1003 int xlate_pos; 1004 u32 ctl_val; 1005 1006 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_LOCK, 1007 NTB_CTRL_PART_STATUS_LOCKED); 1008 if (rc) 1009 return rc; 1010 1011 for (i = 0; i < sndev->nr_lut_mw; i++) { 1012 if (i == ntb_lut_idx) 1013 continue; 1014 1015 addr = mw_addrs[0] + LUT_SIZE * i; 1016 1017 iowrite64((NTB_CTRL_LUT_EN | (sndev->peer_partition << 1) | 1018 addr), 1019 &ctl->lut_entry[i]); 1020 } 1021 1022 sndev->nr_direct_mw = min_t(int, sndev->nr_direct_mw, mw_count); 1023 1024 for (i = 0; i < sndev->nr_direct_mw; i++) { 1025 bar = sndev->direct_mw_to_bar[i]; 1026 offset = (i == 0) ? LUT_SIZE * sndev->nr_lut_mw : 0; 1027 addr = mw_addrs[i] + offset; 1028 size = pci_resource_len(sndev->ntb.pdev, bar) - offset; 1029 xlate_pos = ilog2(size); 1030 1031 if (offset && size > offset) 1032 size = offset; 1033 1034 ctl_val = ioread32(&ctl->bar_entry[bar].ctl); 1035 ctl_val |= NTB_CTRL_BAR_DIR_WIN_EN; 1036 1037 iowrite32(ctl_val, &ctl->bar_entry[bar].ctl); 1038 iowrite32(xlate_pos | (lower_32_bits(size) & 0xFFFFF000), 1039 &ctl->bar_entry[bar].win_size); 1040 iowrite32(upper_32_bits(size), &ctl->bar_ext_entry[bar].win_size); 1041 iowrite64(sndev->peer_partition | addr, 1042 &ctl->bar_entry[bar].xlate_addr); 1043 } 1044 1045 rc = switchtec_ntb_part_op(sndev, ctl, NTB_CTRL_PART_OP_CFG, 1046 NTB_CTRL_PART_STATUS_NORMAL); 1047 if (rc) { 1048 u32 bar_error, lut_error; 1049 1050 bar_error = ioread32(&ctl->bar_error); 1051 lut_error = ioread32(&ctl->lut_error); 1052 dev_err(&sndev->stdev->dev, 1053 "Error setting up cross link windows: %08x / %08x\n", 1054 bar_error, lut_error); 1055 return rc; 1056 } 1057 1058 return 0; 1059 } 1060 1061 static int crosslink_setup_req_ids(struct switchtec_ntb *sndev, 1062 struct ntb_ctrl_regs __iomem *mmio_ctrl) 1063 { 1064 int req_ids[16]; 1065 int i; 1066 u32 proxy_id; 1067 1068 for (i = 0; i < ARRAY_SIZE(req_ids); i++) { 1069 proxy_id = ioread32(&sndev->mmio_self_ctrl->req_id_table[i]); 1070 1071 if (!(proxy_id & NTB_CTRL_REQ_ID_EN)) 1072 break; 1073 1074 req_ids[i] = ((proxy_id >> 1) & 0xFF); 1075 } 1076 1077 return config_req_id_table(sndev, mmio_ctrl, req_ids, i); 1078 } 1079 1080 /* 1081 * In crosslink configuration there is a virtual partition in the 1082 * middle of the two switches. The BARs in this partition have to be 1083 * enumerated and assigned addresses. 1084 */ 1085 static int crosslink_enum_partition(struct switchtec_ntb *sndev, 1086 u64 *bar_addrs) 1087 { 1088 struct part_cfg_regs __iomem *part_cfg = 1089 &sndev->stdev->mmio_part_cfg_all[sndev->peer_partition]; 1090 u32 pff = ioread32(&part_cfg->vep_pff_inst_id); 1091 struct pff_csr_regs __iomem *mmio_pff = 1092 &sndev->stdev->mmio_pff_csr[pff]; 1093 const u64 bar_space = 0x1000000000LL; 1094 u64 bar_addr; 1095 int bar_cnt = 0; 1096 int i; 1097 1098 iowrite16(0x6, &mmio_pff->pcicmd); 1099 1100 for (i = 0; i < ARRAY_SIZE(mmio_pff->pci_bar64); i++) { 1101 iowrite64(bar_space * i, &mmio_pff->pci_bar64[i]); 1102 bar_addr = ioread64(&mmio_pff->pci_bar64[i]); 1103 bar_addr &= ~0xf; 1104 1105 dev_dbg(&sndev->stdev->dev, 1106 "Crosslink BAR%d addr: %llx\n", 1107 i*2, bar_addr); 1108 1109 if (bar_addr != bar_space * i) 1110 continue; 1111 1112 bar_addrs[bar_cnt++] = bar_addr; 1113 } 1114 1115 return bar_cnt; 1116 } 1117 1118 static int switchtec_ntb_init_crosslink(struct switchtec_ntb *sndev) 1119 { 1120 int rc; 1121 int bar = sndev->direct_mw_to_bar[0]; 1122 const int ntb_lut_idx = 1; 1123 u64 bar_addrs[6]; 1124 u64 addr; 1125 int offset; 1126 int bar_cnt; 1127 1128 if (!crosslink_is_enabled(sndev)) 1129 return 0; 1130 1131 dev_info(&sndev->stdev->dev, "Using crosslink configuration\n"); 1132 sndev->ntb.topo = NTB_TOPO_CROSSLINK; 1133 1134 bar_cnt = crosslink_enum_partition(sndev, bar_addrs); 1135 if (bar_cnt < sndev->nr_direct_mw + 1) { 1136 dev_err(&sndev->stdev->dev, 1137 "Error enumerating crosslink partition\n"); 1138 return -EINVAL; 1139 } 1140 1141 addr = (bar_addrs[0] + SWITCHTEC_GAS_NTB_OFFSET + 1142 SWITCHTEC_NTB_REG_DBMSG_OFFSET + 1143 sizeof(struct ntb_dbmsg_regs) * sndev->peer_partition); 1144 1145 offset = addr & (LUT_SIZE - 1); 1146 addr -= offset; 1147 1148 rc = config_rsvd_lut_win(sndev, sndev->mmio_self_ctrl, ntb_lut_idx, 1149 sndev->peer_partition, addr); 1150 if (rc) 1151 return rc; 1152 1153 rc = crosslink_setup_mws(sndev, ntb_lut_idx, &bar_addrs[1], 1154 bar_cnt - 1); 1155 if (rc) 1156 return rc; 1157 1158 rc = crosslink_setup_req_ids(sndev, sndev->mmio_peer_ctrl); 1159 if (rc) 1160 return rc; 1161 1162 sndev->mmio_xlink_win = pci_iomap_range(sndev->stdev->pdev, bar, 1163 LUT_SIZE, LUT_SIZE); 1164 if (!sndev->mmio_xlink_win) { 1165 rc = -ENOMEM; 1166 return rc; 1167 } 1168 1169 sndev->mmio_peer_dbmsg = sndev->mmio_xlink_win + offset; 1170 sndev->nr_rsvd_luts++; 1171 1172 crosslink_init_dbmsgs(sndev); 1173 1174 return 0; 1175 } 1176 1177 static void switchtec_ntb_deinit_crosslink(struct switchtec_ntb *sndev) 1178 { 1179 if (sndev->mmio_xlink_win) 1180 pci_iounmap(sndev->stdev->pdev, sndev->mmio_xlink_win); 1181 } 1182 1183 static int map_bars(int *map, struct ntb_ctrl_regs __iomem *ctrl) 1184 { 1185 int i; 1186 int cnt = 0; 1187 1188 for (i = 0; i < ARRAY_SIZE(ctrl->bar_entry); i++) { 1189 u32 r = ioread32(&ctrl->bar_entry[i].ctl); 1190 1191 if (r & NTB_CTRL_BAR_VALID) 1192 map[cnt++] = i; 1193 } 1194 1195 return cnt; 1196 } 1197 1198 static void switchtec_ntb_init_mw(struct switchtec_ntb *sndev) 1199 { 1200 sndev->nr_direct_mw = map_bars(sndev->direct_mw_to_bar, 1201 sndev->mmio_self_ctrl); 1202 1203 sndev->nr_lut_mw = ioread16(&sndev->mmio_self_ctrl->lut_table_entries); 1204 sndev->nr_lut_mw = rounddown_pow_of_two(sndev->nr_lut_mw); 1205 1206 dev_dbg(&sndev->stdev->dev, "MWs: %d direct, %d lut\n", 1207 sndev->nr_direct_mw, sndev->nr_lut_mw); 1208 1209 sndev->peer_nr_direct_mw = map_bars(sndev->peer_direct_mw_to_bar, 1210 sndev->mmio_peer_ctrl); 1211 1212 sndev->peer_nr_lut_mw = 1213 ioread16(&sndev->mmio_peer_ctrl->lut_table_entries); 1214 sndev->peer_nr_lut_mw = rounddown_pow_of_two(sndev->peer_nr_lut_mw); 1215 1216 dev_dbg(&sndev->stdev->dev, "Peer MWs: %d direct, %d lut\n", 1217 sndev->peer_nr_direct_mw, sndev->peer_nr_lut_mw); 1218 1219 } 1220 1221 /* 1222 * There are 64 doorbells in the switch hardware but this is 1223 * shared among all partitions. So we must split them in half 1224 * (32 for each partition). However, the message interrupts are 1225 * also shared with the top 4 doorbells so we just limit this to 1226 * 28 doorbells per partition. 1227 * 1228 * In crosslink mode, each side has it's own dbmsg register so 1229 * they can each use all 60 of the available doorbells. 1230 */ 1231 static void switchtec_ntb_init_db(struct switchtec_ntb *sndev) 1232 { 1233 sndev->db_mask = 0x0FFFFFFFFFFFFFFFULL; 1234 1235 if (sndev->mmio_peer_dbmsg != sndev->mmio_self_dbmsg) { 1236 sndev->db_shift = 0; 1237 sndev->db_peer_shift = 0; 1238 sndev->db_valid_mask = sndev->db_mask; 1239 } else if (sndev->self_partition < sndev->peer_partition) { 1240 sndev->db_shift = 0; 1241 sndev->db_peer_shift = 32; 1242 sndev->db_valid_mask = 0x0FFFFFFF; 1243 } else { 1244 sndev->db_shift = 32; 1245 sndev->db_peer_shift = 0; 1246 sndev->db_valid_mask = 0x0FFFFFFF; 1247 } 1248 1249 iowrite64(~sndev->db_mask, &sndev->mmio_self_dbmsg->idb_mask); 1250 iowrite64(sndev->db_valid_mask << sndev->db_peer_shift, 1251 &sndev->mmio_peer_dbmsg->odb_mask); 1252 1253 dev_dbg(&sndev->stdev->dev, "dbs: shift %d/%d, mask %016llx\n", 1254 sndev->db_shift, sndev->db_peer_shift, sndev->db_valid_mask); 1255 } 1256 1257 static void switchtec_ntb_init_msgs(struct switchtec_ntb *sndev) 1258 { 1259 int i; 1260 u32 msg_map = 0; 1261 1262 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) { 1263 int m = i | sndev->peer_partition << 2; 1264 1265 msg_map |= m << i * 8; 1266 } 1267 1268 iowrite32(msg_map, &sndev->mmio_self_dbmsg->msg_map); 1269 1270 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) 1271 iowrite64(NTB_DBMSG_IMSG_STATUS | NTB_DBMSG_IMSG_MASK, 1272 &sndev->mmio_self_dbmsg->imsg[i]); 1273 } 1274 1275 static int 1276 switchtec_ntb_init_req_id_table(struct switchtec_ntb *sndev) 1277 { 1278 int req_ids[2]; 1279 1280 /* 1281 * Root Complex Requester ID (which is 0:00.0) 1282 */ 1283 req_ids[0] = 0; 1284 1285 /* 1286 * Host Bridge Requester ID (as read from the mmap address) 1287 */ 1288 req_ids[1] = ioread16(&sndev->mmio_ntb->requester_id); 1289 1290 return config_req_id_table(sndev, sndev->mmio_self_ctrl, req_ids, 1291 ARRAY_SIZE(req_ids)); 1292 } 1293 1294 static void switchtec_ntb_init_shared(struct switchtec_ntb *sndev) 1295 { 1296 int i; 1297 1298 memset(sndev->self_shared, 0, LUT_SIZE); 1299 sndev->self_shared->magic = SWITCHTEC_NTB_MAGIC; 1300 sndev->self_shared->partition_id = sndev->stdev->partition; 1301 1302 for (i = 0; i < sndev->nr_direct_mw; i++) { 1303 int bar = sndev->direct_mw_to_bar[i]; 1304 resource_size_t sz = pci_resource_len(sndev->stdev->pdev, bar); 1305 1306 if (i == 0) 1307 sz = min_t(resource_size_t, sz, 1308 LUT_SIZE * sndev->nr_lut_mw); 1309 1310 sndev->self_shared->mw_sizes[i] = sz; 1311 } 1312 1313 for (i = 0; i < sndev->nr_lut_mw; i++) { 1314 int idx = sndev->nr_direct_mw + i; 1315 1316 sndev->self_shared->mw_sizes[idx] = LUT_SIZE; 1317 } 1318 } 1319 1320 static int switchtec_ntb_init_shared_mw(struct switchtec_ntb *sndev) 1321 { 1322 int self_bar = sndev->direct_mw_to_bar[0]; 1323 int rc; 1324 1325 sndev->nr_rsvd_luts++; 1326 sndev->self_shared = dma_alloc_coherent(&sndev->stdev->pdev->dev, 1327 LUT_SIZE, 1328 &sndev->self_shared_dma, 1329 GFP_KERNEL); 1330 if (!sndev->self_shared) { 1331 dev_err(&sndev->stdev->dev, 1332 "unable to allocate memory for shared mw\n"); 1333 return -ENOMEM; 1334 } 1335 1336 switchtec_ntb_init_shared(sndev); 1337 1338 rc = config_rsvd_lut_win(sndev, sndev->mmio_peer_ctrl, 0, 1339 sndev->self_partition, 1340 sndev->self_shared_dma); 1341 if (rc) 1342 goto unalloc_and_exit; 1343 1344 sndev->peer_shared = pci_iomap(sndev->stdev->pdev, self_bar, LUT_SIZE); 1345 if (!sndev->peer_shared) { 1346 rc = -ENOMEM; 1347 goto unalloc_and_exit; 1348 } 1349 1350 dev_dbg(&sndev->stdev->dev, "Shared MW Ready\n"); 1351 return 0; 1352 1353 unalloc_and_exit: 1354 dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE, 1355 sndev->self_shared, sndev->self_shared_dma); 1356 1357 return rc; 1358 } 1359 1360 static void switchtec_ntb_deinit_shared_mw(struct switchtec_ntb *sndev) 1361 { 1362 if (sndev->peer_shared) 1363 pci_iounmap(sndev->stdev->pdev, sndev->peer_shared); 1364 1365 if (sndev->self_shared) 1366 dma_free_coherent(&sndev->stdev->pdev->dev, LUT_SIZE, 1367 sndev->self_shared, 1368 sndev->self_shared_dma); 1369 sndev->nr_rsvd_luts--; 1370 } 1371 1372 static irqreturn_t switchtec_ntb_doorbell_isr(int irq, void *dev) 1373 { 1374 struct switchtec_ntb *sndev = dev; 1375 1376 dev_dbg(&sndev->stdev->dev, "doorbell\n"); 1377 1378 ntb_db_event(&sndev->ntb, 0); 1379 1380 return IRQ_HANDLED; 1381 } 1382 1383 static irqreturn_t switchtec_ntb_message_isr(int irq, void *dev) 1384 { 1385 int i; 1386 struct switchtec_ntb *sndev = dev; 1387 1388 for (i = 0; i < ARRAY_SIZE(sndev->mmio_self_dbmsg->imsg); i++) { 1389 u64 msg = ioread64(&sndev->mmio_self_dbmsg->imsg[i]); 1390 1391 if (msg & NTB_DBMSG_IMSG_STATUS) { 1392 dev_dbg(&sndev->stdev->dev, "message: %d %08x\n", 1393 i, (u32)msg); 1394 iowrite8(1, &sndev->mmio_self_dbmsg->imsg[i].status); 1395 1396 if (i == LINK_MESSAGE) 1397 switchtec_ntb_check_link(sndev, msg); 1398 } 1399 } 1400 1401 return IRQ_HANDLED; 1402 } 1403 1404 static int switchtec_ntb_init_db_msg_irq(struct switchtec_ntb *sndev) 1405 { 1406 int i; 1407 int rc; 1408 int doorbell_irq = 0; 1409 int message_irq = 0; 1410 int event_irq; 1411 int idb_vecs = sizeof(sndev->mmio_self_dbmsg->idb_vec_map); 1412 1413 event_irq = ioread32(&sndev->stdev->mmio_part_cfg->vep_vector_number); 1414 1415 while (doorbell_irq == event_irq) 1416 doorbell_irq++; 1417 while (message_irq == doorbell_irq || 1418 message_irq == event_irq) 1419 message_irq++; 1420 1421 dev_dbg(&sndev->stdev->dev, "irqs - event: %d, db: %d, msgs: %d\n", 1422 event_irq, doorbell_irq, message_irq); 1423 1424 for (i = 0; i < idb_vecs - 4; i++) 1425 iowrite8(doorbell_irq, 1426 &sndev->mmio_self_dbmsg->idb_vec_map[i]); 1427 1428 for (; i < idb_vecs; i++) 1429 iowrite8(message_irq, 1430 &sndev->mmio_self_dbmsg->idb_vec_map[i]); 1431 1432 sndev->doorbell_irq = pci_irq_vector(sndev->stdev->pdev, doorbell_irq); 1433 sndev->message_irq = pci_irq_vector(sndev->stdev->pdev, message_irq); 1434 1435 rc = request_irq(sndev->doorbell_irq, 1436 switchtec_ntb_doorbell_isr, 0, 1437 "switchtec_ntb_doorbell", sndev); 1438 if (rc) 1439 return rc; 1440 1441 rc = request_irq(sndev->message_irq, 1442 switchtec_ntb_message_isr, 0, 1443 "switchtec_ntb_message", sndev); 1444 if (rc) { 1445 free_irq(sndev->doorbell_irq, sndev); 1446 return rc; 1447 } 1448 1449 return 0; 1450 } 1451 1452 static void switchtec_ntb_deinit_db_msg_irq(struct switchtec_ntb *sndev) 1453 { 1454 free_irq(sndev->doorbell_irq, sndev); 1455 free_irq(sndev->message_irq, sndev); 1456 } 1457 1458 static int switchtec_ntb_reinit_peer(struct switchtec_ntb *sndev) 1459 { 1460 dev_info(&sndev->stdev->dev, "peer reinitialized\n"); 1461 switchtec_ntb_deinit_shared_mw(sndev); 1462 switchtec_ntb_init_mw(sndev); 1463 return switchtec_ntb_init_shared_mw(sndev); 1464 } 1465 1466 static int switchtec_ntb_add(struct device *dev, 1467 struct class_interface *class_intf) 1468 { 1469 struct switchtec_dev *stdev = to_stdev(dev); 1470 struct switchtec_ntb *sndev; 1471 int rc; 1472 1473 stdev->sndev = NULL; 1474 1475 if (stdev->pdev->class != (PCI_CLASS_BRIDGE_OTHER << 8)) 1476 return -ENODEV; 1477 1478 sndev = kzalloc_node(sizeof(*sndev), GFP_KERNEL, dev_to_node(dev)); 1479 if (!sndev) 1480 return -ENOMEM; 1481 1482 sndev->stdev = stdev; 1483 rc = switchtec_ntb_init_sndev(sndev); 1484 if (rc) 1485 goto free_and_exit; 1486 1487 switchtec_ntb_init_mw(sndev); 1488 1489 rc = switchtec_ntb_init_req_id_table(sndev); 1490 if (rc) 1491 goto free_and_exit; 1492 1493 rc = switchtec_ntb_init_crosslink(sndev); 1494 if (rc) 1495 goto free_and_exit; 1496 1497 switchtec_ntb_init_db(sndev); 1498 switchtec_ntb_init_msgs(sndev); 1499 1500 rc = switchtec_ntb_init_shared_mw(sndev); 1501 if (rc) 1502 goto deinit_crosslink; 1503 1504 rc = switchtec_ntb_init_db_msg_irq(sndev); 1505 if (rc) 1506 goto deinit_shared_and_exit; 1507 1508 /* 1509 * If this host crashed, the other host may think the link is 1510 * still up. Tell them to force it down (it will go back up 1511 * once we register the ntb device). 1512 */ 1513 switchtec_ntb_send_msg(sndev, LINK_MESSAGE, MSG_LINK_FORCE_DOWN); 1514 1515 rc = ntb_register_device(&sndev->ntb); 1516 if (rc) 1517 goto deinit_and_exit; 1518 1519 stdev->sndev = sndev; 1520 stdev->link_notifier = switchtec_ntb_link_notification; 1521 dev_info(dev, "NTB device registered\n"); 1522 1523 return 0; 1524 1525 deinit_and_exit: 1526 switchtec_ntb_deinit_db_msg_irq(sndev); 1527 deinit_shared_and_exit: 1528 switchtec_ntb_deinit_shared_mw(sndev); 1529 deinit_crosslink: 1530 switchtec_ntb_deinit_crosslink(sndev); 1531 free_and_exit: 1532 kfree(sndev); 1533 dev_err(dev, "failed to register ntb device: %d\n", rc); 1534 return rc; 1535 } 1536 1537 static void switchtec_ntb_remove(struct device *dev, 1538 struct class_interface *class_intf) 1539 { 1540 struct switchtec_dev *stdev = to_stdev(dev); 1541 struct switchtec_ntb *sndev = stdev->sndev; 1542 1543 if (!sndev) 1544 return; 1545 1546 stdev->link_notifier = NULL; 1547 stdev->sndev = NULL; 1548 ntb_unregister_device(&sndev->ntb); 1549 switchtec_ntb_deinit_db_msg_irq(sndev); 1550 switchtec_ntb_deinit_shared_mw(sndev); 1551 switchtec_ntb_deinit_crosslink(sndev); 1552 kfree(sndev); 1553 dev_info(dev, "ntb device unregistered\n"); 1554 } 1555 1556 static struct class_interface switchtec_interface = { 1557 .add_dev = switchtec_ntb_add, 1558 .remove_dev = switchtec_ntb_remove, 1559 }; 1560 1561 static int __init switchtec_ntb_init(void) 1562 { 1563 switchtec_interface.class = switchtec_class; 1564 return class_interface_register(&switchtec_interface); 1565 } 1566 module_init(switchtec_ntb_init); 1567 1568 static void __exit switchtec_ntb_exit(void) 1569 { 1570 class_interface_unregister(&switchtec_interface); 1571 } 1572 module_exit(switchtec_ntb_exit); 1573