1 // SPDX-License-Identifier: GPL-2.0 2 /** 3 * Endpoint Function Driver to implement Non-Transparent Bridge functionality 4 * 5 * Copyright (C) 2020 Texas Instruments 6 * Author: Kishon Vijay Abraham I <kishon@ti.com> 7 */ 8 9 /* 10 * The PCI NTB function driver configures the SoC with multiple PCIe Endpoint 11 * (EP) controller instances (see diagram below) in such a way that 12 * transactions from one EP controller are routed to the other EP controller. 13 * Once PCI NTB function driver configures the SoC with multiple EP instances, 14 * HOST1 and HOST2 can communicate with each other using SoC as a bridge. 15 * 16 * +-------------+ +-------------+ 17 * | | | | 18 * | HOST1 | | HOST2 | 19 * | | | | 20 * +------^------+ +------^------+ 21 * | | 22 * | | 23 * +---------|-------------------------------------------------|---------+ 24 * | +------v------+ +------v------+ | 25 * | | | | | | 26 * | | EP | | EP | | 27 * | | CONTROLLER1 | | CONTROLLER2 | | 28 * | | <-----------------------------------> | | 29 * | | | | | | 30 * | | | | | | 31 * | | | SoC With Multiple EP Instances | | | 32 * | | | (Configured using NTB Function) | | | 33 * | +-------------+ +-------------+ | 34 * +---------------------------------------------------------------------+ 35 */ 36 37 #include <linux/delay.h> 38 #include <linux/io.h> 39 #include <linux/module.h> 40 #include <linux/slab.h> 41 42 #include <linux/pci-epc.h> 43 #include <linux/pci-epf.h> 44 45 static struct workqueue_struct *kpcintb_workqueue; 46 47 #define COMMAND_CONFIGURE_DOORBELL 1 48 #define COMMAND_TEARDOWN_DOORBELL 2 49 #define COMMAND_CONFIGURE_MW 3 50 #define COMMAND_TEARDOWN_MW 4 51 #define COMMAND_LINK_UP 5 52 #define COMMAND_LINK_DOWN 6 53 54 #define COMMAND_STATUS_OK 1 55 #define COMMAND_STATUS_ERROR 2 56 57 #define LINK_STATUS_UP BIT(0) 58 59 #define SPAD_COUNT 64 60 #define DB_COUNT 4 61 #define NTB_MW_OFFSET 2 62 #define DB_COUNT_MASK GENMASK(15, 0) 63 #define MSIX_ENABLE BIT(16) 64 #define MAX_DB_COUNT 32 65 #define MAX_MW 4 66 67 enum epf_ntb_bar { 68 BAR_CONFIG, 69 BAR_PEER_SPAD, 70 BAR_DB_MW1, 71 BAR_MW2, 72 BAR_MW3, 73 BAR_MW4, 74 }; 75 76 struct epf_ntb { 77 u32 num_mws; 78 u32 db_count; 79 u32 spad_count; 80 struct pci_epf *epf; 81 u64 mws_size[MAX_MW]; 82 struct config_group group; 83 struct epf_ntb_epc *epc[2]; 84 }; 85 86 #define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group) 87 88 struct epf_ntb_epc { 89 u8 func_no; 90 bool linkup; 91 bool is_msix; 92 int msix_bar; 93 u32 spad_size; 94 struct pci_epc *epc; 95 struct epf_ntb *epf_ntb; 96 void __iomem *mw_addr[6]; 97 size_t msix_table_offset; 98 struct epf_ntb_ctrl *reg; 99 struct pci_epf_bar *epf_bar; 100 enum pci_barno epf_ntb_bar[6]; 101 struct delayed_work cmd_handler; 102 enum pci_epc_interface_type type; 103 const struct pci_epc_features *epc_features; 104 }; 105 106 struct epf_ntb_ctrl { 107 u32 command; 108 u32 argument; 109 u16 command_status; 110 u16 link_status; 111 u32 topology; 112 u64 addr; 113 u64 size; 114 u32 num_mws; 115 u32 mw1_offset; 116 u32 spad_offset; 117 u32 spad_count; 118 u32 db_entry_size; 119 u32 db_data[MAX_DB_COUNT]; 120 u32 db_offset[MAX_DB_COUNT]; 121 } __packed; 122 123 static struct pci_epf_header epf_ntb_header = { 124 .vendorid = PCI_ANY_ID, 125 .deviceid = PCI_ANY_ID, 126 .baseclass_code = PCI_BASE_CLASS_MEMORY, 127 .interrupt_pin = PCI_INTERRUPT_INTA, 128 }; 129 130 /** 131 * epf_ntb_link_up() - Raise link_up interrupt to both the hosts 132 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 133 * @link_up: true or false indicating Link is UP or Down 134 * 135 * Once NTB function in HOST1 and the NTB function in HOST2 invoke 136 * ntb_link_enable(), this NTB function driver will trigger a link event to 137 * the NTB client in both the hosts. 138 */ 139 static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up) 140 { 141 enum pci_epc_interface_type type; 142 enum pci_epc_irq_type irq_type; 143 struct epf_ntb_epc *ntb_epc; 144 struct epf_ntb_ctrl *ctrl; 145 struct pci_epc *epc; 146 bool is_msix; 147 u8 func_no; 148 int ret; 149 150 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { 151 ntb_epc = ntb->epc[type]; 152 epc = ntb_epc->epc; 153 func_no = ntb_epc->func_no; 154 is_msix = ntb_epc->is_msix; 155 ctrl = ntb_epc->reg; 156 if (link_up) 157 ctrl->link_status |= LINK_STATUS_UP; 158 else 159 ctrl->link_status &= ~LINK_STATUS_UP; 160 irq_type = is_msix ? PCI_EPC_IRQ_MSIX : PCI_EPC_IRQ_MSI; 161 ret = pci_epc_raise_irq(epc, func_no, irq_type, 1); 162 if (ret) { 163 dev_err(&epc->dev, 164 "%s intf: Failed to raise Link Up IRQ\n", 165 pci_epc_interface_string(type)); 166 return ret; 167 } 168 } 169 170 return 0; 171 } 172 173 /** 174 * epf_ntb_configure_mw() - Configure the Outbound Address Space for one host 175 * to access the memory window of other host 176 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 177 * @type: PRIMARY interface or SECONDARY interface 178 * @mw: Index of the memory window (either 0, 1, 2 or 3) 179 * 180 * +-----------------+ +---->+----------------+-----------+-----------------+ 181 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | 182 * +-----------------+ | +----------------+ +-----------------+ 183 * | BAR1 | | | Doorbell 2 +---------+ | | 184 * +-----------------+----+ +----------------+ | | | 185 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ 186 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | 187 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ 188 * +-----------------+ | |----------------+ | | | | 189 * | BAR4 | | | | | | +-----------------+ 190 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| 191 * | BAR5 | | | | | | +-----------------+ 192 * +-----------------+ +---->-----------------+ | | | | 193 * EP CONTROLLER 1 | | | | +-----------------+ 194 * | | | +---->+ MSI|X ADDRESS 4 | 195 * +----------------+ | +-----------------+ 196 * (A) EP CONTROLLER 2 | | | 197 * (OB SPACE) | | | 198 * +-------> MW1 | 199 * | | 200 * | | 201 * (B) +-----------------+ 202 * | | 203 * | | 204 * | | 205 * | | 206 * | | 207 * +-----------------+ 208 * PCI Address Space 209 * (Managed by HOST2) 210 * 211 * This function performs stage (B) in the above diagram (see MW1) i.e., map OB 212 * address space of memory window to PCI address space. 213 * 214 * This operation requires 3 parameters 215 * 1) Address in the outbound address space 216 * 2) Address in the PCI Address space 217 * 3) Size of the address region to be mapped 218 * 219 * The address in the outbound address space (for MW1, MW2, MW3 and MW4) is 220 * stored in epf_bar corresponding to BAR_DB_MW1 for MW1 and BAR_MW2, BAR_MW3 221 * BAR_MW4 for rest of the BARs of epf_ntb_epc that is connected to HOST1. This 222 * is populated in epf_ntb_alloc_peer_mem() in this driver. 223 * 224 * The address and size of the PCI address region that has to be mapped would 225 * be provided by HOST2 in ctrl->addr and ctrl->size of epf_ntb_epc that is 226 * connected to HOST2. 227 * 228 * Please note Memory window1 (MW1) and Doorbell registers together will be 229 * mapped to a single BAR (BAR2) above for 32-bit BARs. The exact BAR that's 230 * used for Memory window (MW) can be obtained from epf_ntb_bar[BAR_DB_MW1], 231 * epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2]. 232 */ 233 static int epf_ntb_configure_mw(struct epf_ntb *ntb, 234 enum pci_epc_interface_type type, u32 mw) 235 { 236 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 237 struct pci_epf_bar *peer_epf_bar; 238 enum pci_barno peer_barno; 239 struct epf_ntb_ctrl *ctrl; 240 phys_addr_t phys_addr; 241 struct pci_epc *epc; 242 u64 addr, size; 243 int ret = 0; 244 u8 func_no; 245 246 ntb_epc = ntb->epc[type]; 247 epc = ntb_epc->epc; 248 249 peer_ntb_epc = ntb->epc[!type]; 250 peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET]; 251 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 252 253 phys_addr = peer_epf_bar->phys_addr; 254 ctrl = ntb_epc->reg; 255 addr = ctrl->addr; 256 size = ctrl->size; 257 if (mw + NTB_MW_OFFSET == BAR_DB_MW1) 258 phys_addr += ctrl->mw1_offset; 259 260 if (size > ntb->mws_size[mw]) { 261 dev_err(&epc->dev, 262 "%s intf: MW: %d Req Sz:%llxx > Supported Sz:%llx\n", 263 pci_epc_interface_string(type), mw, size, 264 ntb->mws_size[mw]); 265 ret = -EINVAL; 266 goto err_invalid_size; 267 } 268 269 func_no = ntb_epc->func_no; 270 271 ret = pci_epc_map_addr(epc, func_no, phys_addr, addr, size); 272 if (ret) 273 dev_err(&epc->dev, 274 "%s intf: Failed to map memory window %d address\n", 275 pci_epc_interface_string(type), mw); 276 277 err_invalid_size: 278 279 return ret; 280 } 281 282 /** 283 * epf_ntb_teardown_mw() - Teardown the configured OB ATU 284 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 285 * @type: PRIMARY interface or SECONDARY interface 286 * @mw: Index of the memory window (either 0, 1, 2 or 3) 287 * 288 * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using 289 * pci_epc_unmap_addr() 290 */ 291 static void epf_ntb_teardown_mw(struct epf_ntb *ntb, 292 enum pci_epc_interface_type type, u32 mw) 293 { 294 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 295 struct pci_epf_bar *peer_epf_bar; 296 enum pci_barno peer_barno; 297 struct epf_ntb_ctrl *ctrl; 298 phys_addr_t phys_addr; 299 struct pci_epc *epc; 300 u8 func_no; 301 302 ntb_epc = ntb->epc[type]; 303 epc = ntb_epc->epc; 304 305 peer_ntb_epc = ntb->epc[!type]; 306 peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET]; 307 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 308 309 phys_addr = peer_epf_bar->phys_addr; 310 ctrl = ntb_epc->reg; 311 if (mw + NTB_MW_OFFSET == BAR_DB_MW1) 312 phys_addr += ctrl->mw1_offset; 313 func_no = ntb_epc->func_no; 314 315 pci_epc_unmap_addr(epc, func_no, phys_addr); 316 } 317 318 /** 319 * epf_ntb_configure_msi() - Map OB address space to MSI address 320 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 321 * @type: PRIMARY interface or SECONDARY interface 322 * @db_count: Number of doorbell interrupts to map 323 * 324 *+-----------------+ +----->+----------------+-----------+-----------------+ 325 *| BAR0 | | | Doorbell 1 +---+-------> MSI ADDRESS | 326 *+-----------------+ | +----------------+ | +-----------------+ 327 *| BAR1 | | | Doorbell 2 +---+ | | 328 *+-----------------+----+ +----------------+ | | | 329 *| BAR2 | | Doorbell 3 +---+ | | 330 *+-----------------+----+ +----------------+ | | | 331 *| BAR3 | | | Doorbell 4 +---+ | | 332 *+-----------------+ | |----------------+ | | 333 *| BAR4 | | | | | | 334 *+-----------------+ | | MW1 | | | 335 *| BAR5 | | | | | | 336 *+-----------------+ +----->-----------------+ | | 337 * EP CONTROLLER 1 | | | | 338 * | | | | 339 * +----------------+ +-----------------+ 340 * (A) EP CONTROLLER 2 | | 341 * (OB SPACE) | | 342 * | MW1 | 343 * | | 344 * | | 345 * (B) +-----------------+ 346 * | | 347 * | | 348 * | | 349 * | | 350 * | | 351 * +-----------------+ 352 * PCI Address Space 353 * (Managed by HOST2) 354 * 355 * 356 * This function performs stage (B) in the above diagram (see Doorbell 1, 357 * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to 358 * doorbell to MSI address in PCI address space. 359 * 360 * This operation requires 3 parameters 361 * 1) Address reserved for doorbell in the outbound address space 362 * 2) MSI-X address in the PCIe Address space 363 * 3) Number of MSI-X interrupts that has to be configured 364 * 365 * The address in the outbound address space (for the Doorbell) is stored in 366 * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to 367 * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along 368 * with address for MW1. 369 * 370 * pci_epc_map_msi_irq() takes the MSI address from MSI capability register 371 * and maps the OB address (obtained in epf_ntb_alloc_peer_mem()) to the MSI 372 * address. 373 * 374 * epf_ntb_configure_msi() also stores the MSI data to raise each interrupt 375 * in db_data of the peer's control region. This helps the peer to raise 376 * doorbell of the other host by writing db_data to the BAR corresponding to 377 * BAR_DB_MW1. 378 */ 379 static int epf_ntb_configure_msi(struct epf_ntb *ntb, 380 enum pci_epc_interface_type type, u16 db_count) 381 { 382 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 383 u32 db_entry_size, db_data, db_offset; 384 struct pci_epf_bar *peer_epf_bar; 385 struct epf_ntb_ctrl *peer_ctrl; 386 enum pci_barno peer_barno; 387 phys_addr_t phys_addr; 388 struct pci_epc *epc; 389 u8 func_no; 390 int ret, i; 391 392 ntb_epc = ntb->epc[type]; 393 epc = ntb_epc->epc; 394 395 peer_ntb_epc = ntb->epc[!type]; 396 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; 397 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 398 peer_ctrl = peer_ntb_epc->reg; 399 db_entry_size = peer_ctrl->db_entry_size; 400 401 phys_addr = peer_epf_bar->phys_addr; 402 func_no = ntb_epc->func_no; 403 404 ret = pci_epc_map_msi_irq(epc, func_no, phys_addr, db_count, 405 db_entry_size, &db_data, &db_offset); 406 if (ret) { 407 dev_err(&epc->dev, "%s intf: Failed to map MSI IRQ\n", 408 pci_epc_interface_string(type)); 409 return ret; 410 } 411 412 for (i = 0; i < db_count; i++) { 413 peer_ctrl->db_data[i] = db_data | i; 414 peer_ctrl->db_offset[i] = db_offset; 415 } 416 417 return 0; 418 } 419 420 /** 421 * epf_ntb_configure_msix() - Map OB address space to MSI-X address 422 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 423 * @type: PRIMARY interface or SECONDARY interface 424 * @db_count: Number of doorbell interrupts to map 425 * 426 *+-----------------+ +----->+----------------+-----------+-----------------+ 427 *| BAR0 | | | Doorbell 1 +-----------> MSI-X ADDRESS 1 | 428 *+-----------------+ | +----------------+ +-----------------+ 429 *| BAR1 | | | Doorbell 2 +---------+ | | 430 *+-----------------+----+ +----------------+ | | | 431 *| BAR2 | | Doorbell 3 +-------+ | +-----------------+ 432 *+-----------------+----+ +----------------+ | +-> MSI-X ADDRESS 2 | 433 *| BAR3 | | | Doorbell 4 +-----+ | +-----------------+ 434 *+-----------------+ | |----------------+ | | | | 435 *| BAR4 | | | | | | +-----------------+ 436 *+-----------------+ | | MW1 + | +-->+ MSI-X ADDRESS 3|| 437 *| BAR5 | | | | | +-----------------+ 438 *+-----------------+ +----->-----------------+ | | | 439 * EP CONTROLLER 1 | | | +-----------------+ 440 * | | +---->+ MSI-X ADDRESS 4 | 441 * +----------------+ +-----------------+ 442 * (A) EP CONTROLLER 2 | | 443 * (OB SPACE) | | 444 * | MW1 | 445 * | | 446 * | | 447 * (B) +-----------------+ 448 * | | 449 * | | 450 * | | 451 * | | 452 * | | 453 * +-----------------+ 454 * PCI Address Space 455 * (Managed by HOST2) 456 * 457 * This function performs stage (B) in the above diagram (see Doorbell 1, 458 * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to 459 * doorbell to MSI-X address in PCI address space. 460 * 461 * This operation requires 3 parameters 462 * 1) Address reserved for doorbell in the outbound address space 463 * 2) MSI-X address in the PCIe Address space 464 * 3) Number of MSI-X interrupts that has to be configured 465 * 466 * The address in the outbound address space (for the Doorbell) is stored in 467 * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to 468 * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along 469 * with address for MW1. 470 * 471 * The MSI-X address is in the MSI-X table of EP CONTROLLER 2 and 472 * the count of doorbell is in ctrl->argument of epf_ntb_epc that is connected 473 * to HOST2. MSI-X table is stored memory mapped to ntb_epc->msix_bar and the 474 * offset is in ntb_epc->msix_table_offset. From this epf_ntb_configure_msix() 475 * gets the MSI-X address and data. 476 * 477 * epf_ntb_configure_msix() also stores the MSI-X data to raise each interrupt 478 * in db_data of the peer's control region. This helps the peer to raise 479 * doorbell of the other host by writing db_data to the BAR corresponding to 480 * BAR_DB_MW1. 481 */ 482 static int epf_ntb_configure_msix(struct epf_ntb *ntb, 483 enum pci_epc_interface_type type, 484 u16 db_count) 485 { 486 const struct pci_epc_features *epc_features; 487 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 488 struct pci_epf_bar *peer_epf_bar, *epf_bar; 489 struct pci_epf_msix_tbl *msix_tbl; 490 struct epf_ntb_ctrl *peer_ctrl; 491 u32 db_entry_size, msg_data; 492 enum pci_barno peer_barno; 493 phys_addr_t phys_addr; 494 struct pci_epc *epc; 495 size_t align; 496 u64 msg_addr; 497 u8 func_no; 498 int ret, i; 499 500 ntb_epc = ntb->epc[type]; 501 epc = ntb_epc->epc; 502 503 epf_bar = &ntb_epc->epf_bar[ntb_epc->msix_bar]; 504 msix_tbl = epf_bar->addr + ntb_epc->msix_table_offset; 505 506 peer_ntb_epc = ntb->epc[!type]; 507 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; 508 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 509 phys_addr = peer_epf_bar->phys_addr; 510 peer_ctrl = peer_ntb_epc->reg; 511 epc_features = ntb_epc->epc_features; 512 align = epc_features->align; 513 514 func_no = ntb_epc->func_no; 515 db_entry_size = peer_ctrl->db_entry_size; 516 517 for (i = 0; i < db_count; i++) { 518 msg_addr = ALIGN_DOWN(msix_tbl[i].msg_addr, align); 519 msg_data = msix_tbl[i].msg_data; 520 ret = pci_epc_map_addr(epc, func_no, phys_addr, msg_addr, 521 db_entry_size); 522 if (ret) { 523 dev_err(&epc->dev, 524 "%s intf: Failed to configure MSI-X IRQ\n", 525 pci_epc_interface_string(type)); 526 return ret; 527 } 528 phys_addr = phys_addr + db_entry_size; 529 peer_ctrl->db_data[i] = msg_data; 530 peer_ctrl->db_offset[i] = msix_tbl[i].msg_addr & (align - 1); 531 } 532 ntb_epc->is_msix = true; 533 534 return 0; 535 } 536 537 /** 538 * epf_ntb_configure_db() - Configure the Outbound Address Space for one host 539 * to ring the doorbell of other host 540 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 541 * @type: PRIMARY interface or SECONDARY interface 542 * @db_count: Count of the number of doorbells that has to be configured 543 * @msix: Indicates whether MSI-X or MSI should be used 544 * 545 * Invokes epf_ntb_configure_msix() or epf_ntb_configure_msi() required for 546 * one HOST to ring the doorbell of other HOST. 547 */ 548 static int epf_ntb_configure_db(struct epf_ntb *ntb, 549 enum pci_epc_interface_type type, 550 u16 db_count, bool msix) 551 { 552 struct epf_ntb_epc *ntb_epc; 553 struct pci_epc *epc; 554 int ret; 555 556 if (db_count > MAX_DB_COUNT) 557 return -EINVAL; 558 559 ntb_epc = ntb->epc[type]; 560 epc = ntb_epc->epc; 561 562 if (msix) 563 ret = epf_ntb_configure_msix(ntb, type, db_count); 564 else 565 ret = epf_ntb_configure_msi(ntb, type, db_count); 566 567 if (ret) 568 dev_err(&epc->dev, "%s intf: Failed to configure DB\n", 569 pci_epc_interface_string(type)); 570 571 return ret; 572 } 573 574 /** 575 * epf_ntb_teardown_db() - Unmap address in OB address space to MSI/MSI-X 576 * address 577 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 578 * @type: PRIMARY interface or SECONDARY interface 579 * 580 * Invoke pci_epc_unmap_addr() to unmap OB address to MSI/MSI-X address. 581 */ 582 static void 583 epf_ntb_teardown_db(struct epf_ntb *ntb, enum pci_epc_interface_type type) 584 { 585 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 586 struct pci_epf_bar *peer_epf_bar; 587 enum pci_barno peer_barno; 588 phys_addr_t phys_addr; 589 struct pci_epc *epc; 590 u8 func_no; 591 592 ntb_epc = ntb->epc[type]; 593 epc = ntb_epc->epc; 594 595 peer_ntb_epc = ntb->epc[!type]; 596 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; 597 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 598 phys_addr = peer_epf_bar->phys_addr; 599 func_no = ntb_epc->func_no; 600 601 pci_epc_unmap_addr(epc, func_no, phys_addr); 602 } 603 604 /** 605 * epf_ntb_cmd_handler() - Handle commands provided by the NTB Host 606 * @work: work_struct for the two epf_ntb_epc (PRIMARY and SECONDARY) 607 * 608 * Workqueue function that gets invoked for the two epf_ntb_epc 609 * periodically (once every 5ms) to see if it has received any commands 610 * from NTB host. The host can send commands to configure doorbell or 611 * configure memory window or to update link status. 612 */ 613 static void epf_ntb_cmd_handler(struct work_struct *work) 614 { 615 enum pci_epc_interface_type type; 616 struct epf_ntb_epc *ntb_epc; 617 struct epf_ntb_ctrl *ctrl; 618 u32 command, argument; 619 struct epf_ntb *ntb; 620 struct device *dev; 621 u16 db_count; 622 bool is_msix; 623 int ret; 624 625 ntb_epc = container_of(work, struct epf_ntb_epc, cmd_handler.work); 626 ctrl = ntb_epc->reg; 627 command = ctrl->command; 628 if (!command) 629 goto reset_handler; 630 argument = ctrl->argument; 631 632 ctrl->command = 0; 633 ctrl->argument = 0; 634 635 ctrl = ntb_epc->reg; 636 type = ntb_epc->type; 637 ntb = ntb_epc->epf_ntb; 638 dev = &ntb->epf->dev; 639 640 switch (command) { 641 case COMMAND_CONFIGURE_DOORBELL: 642 db_count = argument & DB_COUNT_MASK; 643 is_msix = argument & MSIX_ENABLE; 644 ret = epf_ntb_configure_db(ntb, type, db_count, is_msix); 645 if (ret < 0) 646 ctrl->command_status = COMMAND_STATUS_ERROR; 647 else 648 ctrl->command_status = COMMAND_STATUS_OK; 649 break; 650 case COMMAND_TEARDOWN_DOORBELL: 651 epf_ntb_teardown_db(ntb, type); 652 ctrl->command_status = COMMAND_STATUS_OK; 653 break; 654 case COMMAND_CONFIGURE_MW: 655 ret = epf_ntb_configure_mw(ntb, type, argument); 656 if (ret < 0) 657 ctrl->command_status = COMMAND_STATUS_ERROR; 658 else 659 ctrl->command_status = COMMAND_STATUS_OK; 660 break; 661 case COMMAND_TEARDOWN_MW: 662 epf_ntb_teardown_mw(ntb, type, argument); 663 ctrl->command_status = COMMAND_STATUS_OK; 664 break; 665 case COMMAND_LINK_UP: 666 ntb_epc->linkup = true; 667 if (ntb->epc[PRIMARY_INTERFACE]->linkup && 668 ntb->epc[SECONDARY_INTERFACE]->linkup) { 669 ret = epf_ntb_link_up(ntb, true); 670 if (ret < 0) 671 ctrl->command_status = COMMAND_STATUS_ERROR; 672 else 673 ctrl->command_status = COMMAND_STATUS_OK; 674 goto reset_handler; 675 } 676 ctrl->command_status = COMMAND_STATUS_OK; 677 break; 678 case COMMAND_LINK_DOWN: 679 ntb_epc->linkup = false; 680 ret = epf_ntb_link_up(ntb, false); 681 if (ret < 0) 682 ctrl->command_status = COMMAND_STATUS_ERROR; 683 else 684 ctrl->command_status = COMMAND_STATUS_OK; 685 break; 686 default: 687 dev_err(dev, "%s intf UNKNOWN command: %d\n", 688 pci_epc_interface_string(type), command); 689 break; 690 } 691 692 reset_handler: 693 queue_delayed_work(kpcintb_workqueue, &ntb_epc->cmd_handler, 694 msecs_to_jiffies(5)); 695 } 696 697 /** 698 * epf_ntb_peer_spad_bar_clear() - Clear Peer Scratchpad BAR 699 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 700 * 701 *+-----------------+------->+------------------+ +-----------------+ 702 *| BAR0 | | CONFIG REGION | | BAR0 | 703 *+-----------------+----+ +------------------+<-------+-----------------+ 704 *| BAR1 | | |SCRATCHPAD REGION | | BAR1 | 705 *+-----------------+ +-->+------------------+<-------+-----------------+ 706 *| BAR2 | Local Memory | BAR2 | 707 *+-----------------+ +-----------------+ 708 *| BAR3 | | BAR3 | 709 *+-----------------+ +-----------------+ 710 *| BAR4 | | BAR4 | 711 *+-----------------+ +-----------------+ 712 *| BAR5 | | BAR5 | 713 *+-----------------+ +-----------------+ 714 * EP CONTROLLER 1 EP CONTROLLER 2 715 * 716 * Clear BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad 717 * region. While BAR1 is the default peer scratchpad BAR, an NTB could have 718 * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs). 719 * This function can get the exact BAR used for peer scratchpad from 720 * epf_ntb_bar[BAR_PEER_SPAD]. 721 * 722 * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function 723 * gets the address of peer scratchpad from 724 * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG]. 725 */ 726 static void epf_ntb_peer_spad_bar_clear(struct epf_ntb_epc *ntb_epc) 727 { 728 struct pci_epf_bar *epf_bar; 729 enum pci_barno barno; 730 struct pci_epc *epc; 731 u8 func_no; 732 733 epc = ntb_epc->epc; 734 func_no = ntb_epc->func_no; 735 barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; 736 epf_bar = &ntb_epc->epf_bar[barno]; 737 pci_epc_clear_bar(epc, func_no, epf_bar); 738 } 739 740 /** 741 * epf_ntb_peer_spad_bar_set() - Set peer scratchpad BAR 742 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 743 * 744 *+-----------------+------->+------------------+ +-----------------+ 745 *| BAR0 | | CONFIG REGION | | BAR0 | 746 *+-----------------+----+ +------------------+<-------+-----------------+ 747 *| BAR1 | | |SCRATCHPAD REGION | | BAR1 | 748 *+-----------------+ +-->+------------------+<-------+-----------------+ 749 *| BAR2 | Local Memory | BAR2 | 750 *+-----------------+ +-----------------+ 751 *| BAR3 | | BAR3 | 752 *+-----------------+ +-----------------+ 753 *| BAR4 | | BAR4 | 754 *+-----------------+ +-----------------+ 755 *| BAR5 | | BAR5 | 756 *+-----------------+ +-----------------+ 757 * EP CONTROLLER 1 EP CONTROLLER 2 758 * 759 * Set BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad 760 * region. While BAR1 is the default peer scratchpad BAR, an NTB could have 761 * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs). 762 * This function can get the exact BAR used for peer scratchpad from 763 * epf_ntb_bar[BAR_PEER_SPAD]. 764 * 765 * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function 766 * gets the address of peer scratchpad from 767 * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG]. 768 */ 769 static int epf_ntb_peer_spad_bar_set(struct epf_ntb *ntb, 770 enum pci_epc_interface_type type) 771 { 772 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 773 struct pci_epf_bar *peer_epf_bar, *epf_bar; 774 enum pci_barno peer_barno, barno; 775 u32 peer_spad_offset; 776 struct pci_epc *epc; 777 struct device *dev; 778 u8 func_no; 779 int ret; 780 781 dev = &ntb->epf->dev; 782 783 peer_ntb_epc = ntb->epc[!type]; 784 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_CONFIG]; 785 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 786 787 ntb_epc = ntb->epc[type]; 788 barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; 789 epf_bar = &ntb_epc->epf_bar[barno]; 790 func_no = ntb_epc->func_no; 791 epc = ntb_epc->epc; 792 793 peer_spad_offset = peer_ntb_epc->reg->spad_offset; 794 epf_bar->phys_addr = peer_epf_bar->phys_addr + peer_spad_offset; 795 epf_bar->size = peer_ntb_epc->spad_size; 796 epf_bar->barno = barno; 797 epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32; 798 799 ret = pci_epc_set_bar(epc, func_no, epf_bar); 800 if (ret) { 801 dev_err(dev, "%s intf: peer SPAD BAR set failed\n", 802 pci_epc_interface_string(type)); 803 return ret; 804 } 805 806 return 0; 807 } 808 809 /** 810 * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR 811 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 812 * 813 * +-----------------+------->+------------------+ +-----------------+ 814 * | BAR0 | | CONFIG REGION | | BAR0 | 815 * +-----------------+----+ +------------------+<-------+-----------------+ 816 * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | 817 * +-----------------+ +-->+------------------+<-------+-----------------+ 818 * | BAR2 | Local Memory | BAR2 | 819 * +-----------------+ +-----------------+ 820 * | BAR3 | | BAR3 | 821 * +-----------------+ +-----------------+ 822 * | BAR4 | | BAR4 | 823 * +-----------------+ +-----------------+ 824 * | BAR5 | | BAR5 | 825 * +-----------------+ +-----------------+ 826 * EP CONTROLLER 1 EP CONTROLLER 2 827 * 828 * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and 829 * self scratchpad region (removes inbound ATU configuration). While BAR0 is 830 * the default self scratchpad BAR, an NTB could have other BARs for self 831 * scratchpad (because of reserved BARs). This function can get the exact BAR 832 * used for self scratchpad from epf_ntb_bar[BAR_CONFIG]. 833 * 834 * Please note the self scratchpad region and config region is combined to 835 * a single region and mapped using the same BAR. Also note HOST2's peer 836 * scratchpad is HOST1's self scratchpad. 837 */ 838 static void epf_ntb_config_sspad_bar_clear(struct epf_ntb_epc *ntb_epc) 839 { 840 struct pci_epf_bar *epf_bar; 841 enum pci_barno barno; 842 struct pci_epc *epc; 843 u8 func_no; 844 845 epc = ntb_epc->epc; 846 func_no = ntb_epc->func_no; 847 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; 848 epf_bar = &ntb_epc->epf_bar[barno]; 849 pci_epc_clear_bar(epc, func_no, epf_bar); 850 } 851 852 /** 853 * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR 854 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 855 * 856 * +-----------------+------->+------------------+ +-----------------+ 857 * | BAR0 | | CONFIG REGION | | BAR0 | 858 * +-----------------+----+ +------------------+<-------+-----------------+ 859 * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | 860 * +-----------------+ +-->+------------------+<-------+-----------------+ 861 * | BAR2 | Local Memory | BAR2 | 862 * +-----------------+ +-----------------+ 863 * | BAR3 | | BAR3 | 864 * +-----------------+ +-----------------+ 865 * | BAR4 | | BAR4 | 866 * +-----------------+ +-----------------+ 867 * | BAR5 | | BAR5 | 868 * +-----------------+ +-----------------+ 869 * EP CONTROLLER 1 EP CONTROLLER 2 870 * 871 * Map BAR0 of EP CONTROLLER 1 which contains the HOST1's config and 872 * self scratchpad region. While BAR0 is the default self scratchpad BAR, an 873 * NTB could have other BARs for self scratchpad (because of reserved BARs). 874 * This function can get the exact BAR used for self scratchpad from 875 * epf_ntb_bar[BAR_CONFIG]. 876 * 877 * Please note the self scratchpad region and config region is combined to 878 * a single region and mapped using the same BAR. Also note HOST2's peer 879 * scratchpad is HOST1's self scratchpad. 880 */ 881 static int epf_ntb_config_sspad_bar_set(struct epf_ntb_epc *ntb_epc) 882 { 883 struct pci_epf_bar *epf_bar; 884 enum pci_barno barno; 885 struct epf_ntb *ntb; 886 struct pci_epc *epc; 887 struct device *dev; 888 u8 func_no; 889 int ret; 890 891 ntb = ntb_epc->epf_ntb; 892 dev = &ntb->epf->dev; 893 894 epc = ntb_epc->epc; 895 func_no = ntb_epc->func_no; 896 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; 897 epf_bar = &ntb_epc->epf_bar[barno]; 898 899 ret = pci_epc_set_bar(epc, func_no, epf_bar); 900 if (ret) { 901 dev_err(dev, "%s inft: Config/Status/SPAD BAR set failed\n", 902 pci_epc_interface_string(ntb_epc->type)); 903 return ret; 904 } 905 906 return 0; 907 } 908 909 /** 910 * epf_ntb_config_spad_bar_free() - Free the physical memory associated with 911 * config + scratchpad region 912 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 913 * 914 * +-----------------+------->+------------------+ +-----------------+ 915 * | BAR0 | | CONFIG REGION | | BAR0 | 916 * +-----------------+----+ +------------------+<-------+-----------------+ 917 * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | 918 * +-----------------+ +-->+------------------+<-------+-----------------+ 919 * | BAR2 | Local Memory | BAR2 | 920 * +-----------------+ +-----------------+ 921 * | BAR3 | | BAR3 | 922 * +-----------------+ +-----------------+ 923 * | BAR4 | | BAR4 | 924 * +-----------------+ +-----------------+ 925 * | BAR5 | | BAR5 | 926 * +-----------------+ +-----------------+ 927 * EP CONTROLLER 1 EP CONTROLLER 2 928 * 929 * Free the Local Memory mentioned in the above diagram. After invoking this 930 * function, any of config + self scratchpad region of HOST1 or peer scratchpad 931 * region of HOST2 should not be accessed. 932 */ 933 static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb) 934 { 935 enum pci_epc_interface_type type; 936 struct epf_ntb_epc *ntb_epc; 937 enum pci_barno barno; 938 struct pci_epf *epf; 939 940 epf = ntb->epf; 941 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { 942 ntb_epc = ntb->epc[type]; 943 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; 944 if (ntb_epc->reg) 945 pci_epf_free_space(epf, ntb_epc->reg, barno, type); 946 } 947 } 948 949 /** 950 * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad 951 * region 952 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 953 * @type: PRIMARY interface or SECONDARY interface 954 * 955 * +-----------------+------->+------------------+ +-----------------+ 956 * | BAR0 | | CONFIG REGION | | BAR0 | 957 * +-----------------+----+ +------------------+<-------+-----------------+ 958 * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | 959 * +-----------------+ +-->+------------------+<-------+-----------------+ 960 * | BAR2 | Local Memory | BAR2 | 961 * +-----------------+ +-----------------+ 962 * | BAR3 | | BAR3 | 963 * +-----------------+ +-----------------+ 964 * | BAR4 | | BAR4 | 965 * +-----------------+ +-----------------+ 966 * | BAR5 | | BAR5 | 967 * +-----------------+ +-----------------+ 968 * EP CONTROLLER 1 EP CONTROLLER 2 969 * 970 * Allocate the Local Memory mentioned in the above diagram. The size of 971 * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION 972 * is obtained from "spad-count" configfs entry. 973 * 974 * The size of both config region and scratchpad region has to be aligned, 975 * since the scratchpad region will also be mapped as PEER SCRATCHPAD of 976 * other host using a separate BAR. 977 */ 978 static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb, 979 enum pci_epc_interface_type type) 980 { 981 const struct pci_epc_features *peer_epc_features, *epc_features; 982 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 983 size_t msix_table_size, pba_size, align; 984 enum pci_barno peer_barno, barno; 985 struct epf_ntb_ctrl *ctrl; 986 u32 spad_size, ctrl_size; 987 u64 size, peer_size; 988 struct pci_epf *epf; 989 struct device *dev; 990 bool msix_capable; 991 u32 spad_count; 992 void *base; 993 994 epf = ntb->epf; 995 dev = &epf->dev; 996 ntb_epc = ntb->epc[type]; 997 998 epc_features = ntb_epc->epc_features; 999 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; 1000 size = epc_features->bar_fixed_size[barno]; 1001 align = epc_features->align; 1002 1003 peer_ntb_epc = ntb->epc[!type]; 1004 peer_epc_features = peer_ntb_epc->epc_features; 1005 peer_barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; 1006 peer_size = peer_epc_features->bar_fixed_size[peer_barno]; 1007 1008 /* Check if epc_features is populated incorrectly */ 1009 if ((!IS_ALIGNED(size, align))) 1010 return -EINVAL; 1011 1012 spad_count = ntb->spad_count; 1013 1014 ctrl_size = sizeof(struct epf_ntb_ctrl); 1015 spad_size = spad_count * 4; 1016 1017 msix_capable = epc_features->msix_capable; 1018 if (msix_capable) { 1019 msix_table_size = PCI_MSIX_ENTRY_SIZE * ntb->db_count; 1020 ctrl_size = ALIGN(ctrl_size, 8); 1021 ntb_epc->msix_table_offset = ctrl_size; 1022 ntb_epc->msix_bar = barno; 1023 /* Align to QWORD or 8 Bytes */ 1024 pba_size = ALIGN(DIV_ROUND_UP(ntb->db_count, 8), 8); 1025 ctrl_size = ctrl_size + msix_table_size + pba_size; 1026 } 1027 1028 if (!align) { 1029 ctrl_size = roundup_pow_of_two(ctrl_size); 1030 spad_size = roundup_pow_of_two(spad_size); 1031 } else { 1032 ctrl_size = ALIGN(ctrl_size, align); 1033 spad_size = ALIGN(spad_size, align); 1034 } 1035 1036 if (peer_size) { 1037 if (peer_size < spad_size) 1038 spad_count = peer_size / 4; 1039 spad_size = peer_size; 1040 } 1041 1042 /* 1043 * In order to make sure SPAD offset is aligned to its size, 1044 * expand control region size to the size of SPAD if SPAD size 1045 * is greater than control region size. 1046 */ 1047 if (spad_size > ctrl_size) 1048 ctrl_size = spad_size; 1049 1050 if (!size) 1051 size = ctrl_size + spad_size; 1052 else if (size < ctrl_size + spad_size) 1053 return -EINVAL; 1054 1055 base = pci_epf_alloc_space(epf, size, barno, align, type); 1056 if (!base) { 1057 dev_err(dev, "%s intf: Config/Status/SPAD alloc region fail\n", 1058 pci_epc_interface_string(type)); 1059 return -ENOMEM; 1060 } 1061 1062 ntb_epc->reg = base; 1063 1064 ctrl = ntb_epc->reg; 1065 ctrl->spad_offset = ctrl_size; 1066 ctrl->spad_count = spad_count; 1067 ctrl->num_mws = ntb->num_mws; 1068 ctrl->db_entry_size = align ? align : 4; 1069 ntb_epc->spad_size = spad_size; 1070 1071 return 0; 1072 } 1073 1074 /** 1075 * epf_ntb_config_spad_bar_alloc_interface() - Allocate memory for config + 1076 * scratchpad region for each of PRIMARY and SECONDARY interface 1077 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1078 * 1079 * Wrapper for epf_ntb_config_spad_bar_alloc() which allocates memory for 1080 * config + scratchpad region for a specific interface 1081 */ 1082 static int epf_ntb_config_spad_bar_alloc_interface(struct epf_ntb *ntb) 1083 { 1084 enum pci_epc_interface_type type; 1085 struct device *dev; 1086 int ret; 1087 1088 dev = &ntb->epf->dev; 1089 1090 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { 1091 ret = epf_ntb_config_spad_bar_alloc(ntb, type); 1092 if (ret) { 1093 dev_err(dev, "%s intf: Config/SPAD BAR alloc failed\n", 1094 pci_epc_interface_string(type)); 1095 return ret; 1096 } 1097 } 1098 1099 return 0; 1100 } 1101 1102 /** 1103 * epf_ntb_free_peer_mem() - Free memory allocated in peers outbound address 1104 * space 1105 * @ntb_epc: EPC associated with one of the HOST which holds peers outbound 1106 * address regions 1107 * 1108 * +-----------------+ +---->+----------------+-----------+-----------------+ 1109 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | 1110 * +-----------------+ | +----------------+ +-----------------+ 1111 * | BAR1 | | | Doorbell 2 +---------+ | | 1112 * +-----------------+----+ +----------------+ | | | 1113 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ 1114 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | 1115 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ 1116 * +-----------------+ | |----------------+ | | | | 1117 * | BAR4 | | | | | | +-----------------+ 1118 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| 1119 * | BAR5 | | | | | | +-----------------+ 1120 * +-----------------+ +---->-----------------+ | | | | 1121 * EP CONTROLLER 1 | | | | +-----------------+ 1122 * | | | +---->+ MSI|X ADDRESS 4 | 1123 * +----------------+ | +-----------------+ 1124 * (A) EP CONTROLLER 2 | | | 1125 * (OB SPACE) | | | 1126 * +-------> MW1 | 1127 * | | 1128 * | | 1129 * (B) +-----------------+ 1130 * | | 1131 * | | 1132 * | | 1133 * | | 1134 * | | 1135 * +-----------------+ 1136 * PCI Address Space 1137 * (Managed by HOST2) 1138 * 1139 * Free memory allocated in EP CONTROLLER 2 (OB SPACE) in the above diagram. 1140 * It'll free Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3, 1141 * MW4). 1142 */ 1143 static void epf_ntb_free_peer_mem(struct epf_ntb_epc *ntb_epc) 1144 { 1145 struct pci_epf_bar *epf_bar; 1146 void __iomem *mw_addr; 1147 phys_addr_t phys_addr; 1148 enum epf_ntb_bar bar; 1149 enum pci_barno barno; 1150 struct pci_epc *epc; 1151 size_t size; 1152 1153 epc = ntb_epc->epc; 1154 1155 for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) { 1156 barno = ntb_epc->epf_ntb_bar[bar]; 1157 mw_addr = ntb_epc->mw_addr[barno]; 1158 epf_bar = &ntb_epc->epf_bar[barno]; 1159 phys_addr = epf_bar->phys_addr; 1160 size = epf_bar->size; 1161 if (mw_addr) { 1162 pci_epc_mem_free_addr(epc, phys_addr, mw_addr, size); 1163 ntb_epc->mw_addr[barno] = NULL; 1164 } 1165 } 1166 } 1167 1168 /** 1169 * epf_ntb_db_mw_bar_clear() - Clear doorbell and memory BAR 1170 * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound 1171 * address 1172 * 1173 * +-----------------+ +---->+----------------+-----------+-----------------+ 1174 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | 1175 * +-----------------+ | +----------------+ +-----------------+ 1176 * | BAR1 | | | Doorbell 2 +---------+ | | 1177 * +-----------------+----+ +----------------+ | | | 1178 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ 1179 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | 1180 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ 1181 * +-----------------+ | |----------------+ | | | | 1182 * | BAR4 | | | | | | +-----------------+ 1183 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| 1184 * | BAR5 | | | | | | +-----------------+ 1185 * +-----------------+ +---->-----------------+ | | | | 1186 * EP CONTROLLER 1 | | | | +-----------------+ 1187 * | | | +---->+ MSI|X ADDRESS 4 | 1188 * +----------------+ | +-----------------+ 1189 * (A) EP CONTROLLER 2 | | | 1190 * (OB SPACE) | | | 1191 * +-------> MW1 | 1192 * | | 1193 * | | 1194 * (B) +-----------------+ 1195 * | | 1196 * | | 1197 * | | 1198 * | | 1199 * | | 1200 * +-----------------+ 1201 * PCI Address Space 1202 * (Managed by HOST2) 1203 * 1204 * Clear doorbell and memory BARs (remove inbound ATU configuration). In the above 1205 * diagram it clears BAR2 TO BAR5 of EP CONTROLLER 1 (Doorbell BAR, MW1 BAR, MW2 1206 * BAR, MW3 BAR and MW4 BAR). 1207 */ 1208 static void epf_ntb_db_mw_bar_clear(struct epf_ntb_epc *ntb_epc) 1209 { 1210 struct pci_epf_bar *epf_bar; 1211 enum epf_ntb_bar bar; 1212 enum pci_barno barno; 1213 struct pci_epc *epc; 1214 u8 func_no; 1215 1216 epc = ntb_epc->epc; 1217 1218 func_no = ntb_epc->func_no; 1219 1220 for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) { 1221 barno = ntb_epc->epf_ntb_bar[bar]; 1222 epf_bar = &ntb_epc->epf_bar[barno]; 1223 pci_epc_clear_bar(epc, func_no, epf_bar); 1224 } 1225 } 1226 1227 /** 1228 * epf_ntb_db_mw_bar_cleanup() - Clear doorbell/memory BAR and free memory 1229 * allocated in peers outbound address space 1230 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1231 * @type: PRIMARY interface or SECONDARY interface 1232 * 1233 * Wrapper for epf_ntb_db_mw_bar_clear() to clear HOST1's BAR and 1234 * epf_ntb_free_peer_mem() which frees up HOST2 outbound memory. 1235 */ 1236 static void epf_ntb_db_mw_bar_cleanup(struct epf_ntb *ntb, 1237 enum pci_epc_interface_type type) 1238 { 1239 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 1240 1241 ntb_epc = ntb->epc[type]; 1242 peer_ntb_epc = ntb->epc[!type]; 1243 1244 epf_ntb_db_mw_bar_clear(ntb_epc); 1245 epf_ntb_free_peer_mem(peer_ntb_epc); 1246 } 1247 1248 /** 1249 * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capaiblity 1250 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1251 * @type: PRIMARY interface or SECONDARY interface 1252 * 1253 * Configure MSI/MSI-X capability for each interface with number of 1254 * interrupts equal to "db_count" configfs entry. 1255 */ 1256 static int epf_ntb_configure_interrupt(struct epf_ntb *ntb, 1257 enum pci_epc_interface_type type) 1258 { 1259 const struct pci_epc_features *epc_features; 1260 bool msix_capable, msi_capable; 1261 struct epf_ntb_epc *ntb_epc; 1262 struct pci_epc *epc; 1263 struct device *dev; 1264 u32 db_count; 1265 u8 func_no; 1266 int ret; 1267 1268 ntb_epc = ntb->epc[type]; 1269 dev = &ntb->epf->dev; 1270 1271 epc_features = ntb_epc->epc_features; 1272 msix_capable = epc_features->msix_capable; 1273 msi_capable = epc_features->msi_capable; 1274 1275 if (!(msix_capable || msi_capable)) { 1276 dev_err(dev, "MSI or MSI-X is required for doorbell\n"); 1277 return -EINVAL; 1278 } 1279 1280 func_no = ntb_epc->func_no; 1281 1282 db_count = ntb->db_count; 1283 if (db_count > MAX_DB_COUNT) { 1284 dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT); 1285 return -EINVAL; 1286 } 1287 1288 ntb->db_count = db_count; 1289 epc = ntb_epc->epc; 1290 1291 if (msi_capable) { 1292 ret = pci_epc_set_msi(epc, func_no, db_count); 1293 if (ret) { 1294 dev_err(dev, "%s intf: MSI configuration failed\n", 1295 pci_epc_interface_string(type)); 1296 return ret; 1297 } 1298 } 1299 1300 if (msix_capable) { 1301 ret = pci_epc_set_msix(epc, func_no, db_count, 1302 ntb_epc->msix_bar, 1303 ntb_epc->msix_table_offset); 1304 if (ret) { 1305 dev_err(dev, "MSI configuration failed\n"); 1306 return ret; 1307 } 1308 } 1309 1310 return 0; 1311 } 1312 1313 /** 1314 * epf_ntb_alloc_peer_mem() - Allocate memory in peer's outbound address space 1315 * @ntb_epc: EPC associated with one of the HOST whose BAR holds peer's outbound 1316 * address 1317 * @bar: BAR of @ntb_epc in for which memory has to be allocated (could be 1318 * BAR_DB_MW1, BAR_MW2, BAR_MW3, BAR_MW4) 1319 * @peer_ntb_epc: EPC associated with HOST whose outbound address space is 1320 * used by @ntb_epc 1321 * @size: Size of the address region that has to be allocated in peers OB SPACE 1322 * 1323 * 1324 * +-----------------+ +---->+----------------+-----------+-----------------+ 1325 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | 1326 * +-----------------+ | +----------------+ +-----------------+ 1327 * | BAR1 | | | Doorbell 2 +---------+ | | 1328 * +-----------------+----+ +----------------+ | | | 1329 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ 1330 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | 1331 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ 1332 * +-----------------+ | |----------------+ | | | | 1333 * | BAR4 | | | | | | +-----------------+ 1334 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| 1335 * | BAR5 | | | | | | +-----------------+ 1336 * +-----------------+ +---->-----------------+ | | | | 1337 * EP CONTROLLER 1 | | | | +-----------------+ 1338 * | | | +---->+ MSI|X ADDRESS 4 | 1339 * +----------------+ | +-----------------+ 1340 * (A) EP CONTROLLER 2 | | | 1341 * (OB SPACE) | | | 1342 * +-------> MW1 | 1343 * | | 1344 * | | 1345 * (B) +-----------------+ 1346 * | | 1347 * | | 1348 * | | 1349 * | | 1350 * | | 1351 * +-----------------+ 1352 * PCI Address Space 1353 * (Managed by HOST2) 1354 * 1355 * Allocate memory in OB space of EP CONTROLLER 2 in the above diagram. Allocate 1356 * for Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3, MW4). 1357 */ 1358 static int epf_ntb_alloc_peer_mem(struct device *dev, 1359 struct epf_ntb_epc *ntb_epc, 1360 enum epf_ntb_bar bar, 1361 struct epf_ntb_epc *peer_ntb_epc, 1362 size_t size) 1363 { 1364 const struct pci_epc_features *epc_features; 1365 struct pci_epf_bar *epf_bar; 1366 struct pci_epc *peer_epc; 1367 phys_addr_t phys_addr; 1368 void __iomem *mw_addr; 1369 enum pci_barno barno; 1370 size_t align; 1371 1372 epc_features = ntb_epc->epc_features; 1373 align = epc_features->align; 1374 1375 if (size < 128) 1376 size = 128; 1377 1378 if (align) 1379 size = ALIGN(size, align); 1380 else 1381 size = roundup_pow_of_two(size); 1382 1383 peer_epc = peer_ntb_epc->epc; 1384 mw_addr = pci_epc_mem_alloc_addr(peer_epc, &phys_addr, size); 1385 if (!mw_addr) { 1386 dev_err(dev, "%s intf: Failed to allocate OB address\n", 1387 pci_epc_interface_string(peer_ntb_epc->type)); 1388 return -ENOMEM; 1389 } 1390 1391 barno = ntb_epc->epf_ntb_bar[bar]; 1392 epf_bar = &ntb_epc->epf_bar[barno]; 1393 ntb_epc->mw_addr[barno] = mw_addr; 1394 1395 epf_bar->phys_addr = phys_addr; 1396 epf_bar->size = size; 1397 epf_bar->barno = barno; 1398 epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32; 1399 1400 return 0; 1401 } 1402 1403 /** 1404 * epf_ntb_db_mw_bar_init() - Configure Doorbell and Memory window BARs 1405 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1406 * @type: PRIMARY interface or SECONDARY interface 1407 * 1408 * Wrapper for epf_ntb_alloc_peer_mem() and pci_epc_set_bar() that allocates 1409 * memory in OB address space of HOST2 and configures BAR of HOST1 1410 */ 1411 static int epf_ntb_db_mw_bar_init(struct epf_ntb *ntb, 1412 enum pci_epc_interface_type type) 1413 { 1414 const struct pci_epc_features *epc_features; 1415 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 1416 struct pci_epf_bar *epf_bar; 1417 struct epf_ntb_ctrl *ctrl; 1418 u32 num_mws, db_count; 1419 enum epf_ntb_bar bar; 1420 enum pci_barno barno; 1421 struct pci_epc *epc; 1422 struct device *dev; 1423 size_t align; 1424 int ret, i; 1425 u8 func_no; 1426 u64 size; 1427 1428 ntb_epc = ntb->epc[type]; 1429 peer_ntb_epc = ntb->epc[!type]; 1430 1431 dev = &ntb->epf->dev; 1432 epc_features = ntb_epc->epc_features; 1433 align = epc_features->align; 1434 func_no = ntb_epc->func_no; 1435 epc = ntb_epc->epc; 1436 num_mws = ntb->num_mws; 1437 db_count = ntb->db_count; 1438 1439 for (bar = BAR_DB_MW1, i = 0; i < num_mws; bar++, i++) { 1440 if (bar == BAR_DB_MW1) { 1441 align = align ? align : 4; 1442 size = db_count * align; 1443 size = ALIGN(size, ntb->mws_size[i]); 1444 ctrl = ntb_epc->reg; 1445 ctrl->mw1_offset = size; 1446 size += ntb->mws_size[i]; 1447 } else { 1448 size = ntb->mws_size[i]; 1449 } 1450 1451 ret = epf_ntb_alloc_peer_mem(dev, ntb_epc, bar, 1452 peer_ntb_epc, size); 1453 if (ret) { 1454 dev_err(dev, "%s intf: DoorBell mem alloc failed\n", 1455 pci_epc_interface_string(type)); 1456 goto err_alloc_peer_mem; 1457 } 1458 1459 barno = ntb_epc->epf_ntb_bar[bar]; 1460 epf_bar = &ntb_epc->epf_bar[barno]; 1461 1462 ret = pci_epc_set_bar(epc, func_no, epf_bar); 1463 if (ret) { 1464 dev_err(dev, "%s intf: DoorBell BAR set failed\n", 1465 pci_epc_interface_string(type)); 1466 goto err_alloc_peer_mem; 1467 } 1468 } 1469 1470 return 0; 1471 1472 err_alloc_peer_mem: 1473 epf_ntb_db_mw_bar_cleanup(ntb, type); 1474 1475 return ret; 1476 } 1477 1478 /** 1479 * epf_ntb_epc_destroy_interface() - Cleanup NTB EPC interface 1480 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1481 * @type: PRIMARY interface or SECONDARY interface 1482 * 1483 * Unbind NTB function device from EPC and relinquish reference to pci_epc 1484 * for each of the interface. 1485 */ 1486 static void epf_ntb_epc_destroy_interface(struct epf_ntb *ntb, 1487 enum pci_epc_interface_type type) 1488 { 1489 struct epf_ntb_epc *ntb_epc; 1490 struct pci_epc *epc; 1491 struct pci_epf *epf; 1492 1493 if (type < 0) 1494 return; 1495 1496 epf = ntb->epf; 1497 ntb_epc = ntb->epc[type]; 1498 if (!ntb_epc) 1499 return; 1500 epc = ntb_epc->epc; 1501 pci_epc_remove_epf(epc, epf, type); 1502 pci_epc_put(epc); 1503 } 1504 1505 /** 1506 * epf_ntb_epc_destroy() - Cleanup NTB EPC interface 1507 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1508 * 1509 * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces 1510 */ 1511 static void epf_ntb_epc_destroy(struct epf_ntb *ntb) 1512 { 1513 enum pci_epc_interface_type type; 1514 1515 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) 1516 epf_ntb_epc_destroy_interface(ntb, type); 1517 } 1518 1519 /** 1520 * epf_ntb_epc_create_interface() - Create and initialize NTB EPC interface 1521 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1522 * @epc: struct pci_epc to which a particular NTB interface should be associated 1523 * @type: PRIMARY interface or SECONDARY interface 1524 * 1525 * Allocate memory for NTB EPC interface and initialize it. 1526 */ 1527 static int epf_ntb_epc_create_interface(struct epf_ntb *ntb, 1528 struct pci_epc *epc, 1529 enum pci_epc_interface_type type) 1530 { 1531 const struct pci_epc_features *epc_features; 1532 struct pci_epf_bar *epf_bar; 1533 struct epf_ntb_epc *ntb_epc; 1534 struct pci_epf *epf; 1535 struct device *dev; 1536 u8 func_no; 1537 1538 dev = &ntb->epf->dev; 1539 1540 ntb_epc = devm_kzalloc(dev, sizeof(*ntb_epc), GFP_KERNEL); 1541 if (!ntb_epc) 1542 return -ENOMEM; 1543 1544 epf = ntb->epf; 1545 if (type == PRIMARY_INTERFACE) { 1546 func_no = epf->func_no; 1547 epf_bar = epf->bar; 1548 } else { 1549 func_no = epf->sec_epc_func_no; 1550 epf_bar = epf->sec_epc_bar; 1551 } 1552 1553 ntb_epc->linkup = false; 1554 ntb_epc->epc = epc; 1555 ntb_epc->func_no = func_no; 1556 ntb_epc->type = type; 1557 ntb_epc->epf_bar = epf_bar; 1558 ntb_epc->epf_ntb = ntb; 1559 1560 epc_features = pci_epc_get_features(epc, func_no); 1561 if (!epc_features) 1562 return -EINVAL; 1563 ntb_epc->epc_features = epc_features; 1564 1565 ntb->epc[type] = ntb_epc; 1566 1567 return 0; 1568 } 1569 1570 /** 1571 * epf_ntb_epc_create() - Create and initialize NTB EPC interface 1572 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1573 * 1574 * Get a reference to EPC device and bind NTB function device to that EPC 1575 * for each of the interface. It is also a wrapper to 1576 * epf_ntb_epc_create_interface() to allocate memory for NTB EPC interface 1577 * and initialize it 1578 */ 1579 static int epf_ntb_epc_create(struct epf_ntb *ntb) 1580 { 1581 struct pci_epf *epf; 1582 struct device *dev; 1583 int ret; 1584 1585 epf = ntb->epf; 1586 dev = &epf->dev; 1587 1588 ret = epf_ntb_epc_create_interface(ntb, epf->epc, PRIMARY_INTERFACE); 1589 if (ret) { 1590 dev_err(dev, "PRIMARY intf: Fail to create NTB EPC\n"); 1591 return ret; 1592 } 1593 1594 ret = epf_ntb_epc_create_interface(ntb, epf->sec_epc, 1595 SECONDARY_INTERFACE); 1596 if (ret) { 1597 dev_err(dev, "SECONDARY intf: Fail to create NTB EPC\n"); 1598 goto err_epc_create; 1599 } 1600 1601 return 0; 1602 1603 err_epc_create: 1604 epf_ntb_epc_destroy_interface(ntb, PRIMARY_INTERFACE); 1605 1606 return ret; 1607 } 1608 1609 /** 1610 * epf_ntb_init_epc_bar_interface() - Identify BARs to be used for each of 1611 * the NTB constructs (scratchpad region, doorbell, memorywindow) 1612 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1613 * @type: PRIMARY interface or SECONDARY interface 1614 * 1615 * Identify the free BARs to be used for each of BAR_CONFIG, BAR_PEER_SPAD, 1616 * BAR_DB_MW1, BAR_MW2, BAR_MW3 and BAR_MW4. 1617 */ 1618 static int epf_ntb_init_epc_bar_interface(struct epf_ntb *ntb, 1619 enum pci_epc_interface_type type) 1620 { 1621 const struct pci_epc_features *epc_features; 1622 struct epf_ntb_epc *ntb_epc; 1623 enum pci_barno barno; 1624 enum epf_ntb_bar bar; 1625 struct device *dev; 1626 u32 num_mws; 1627 int i; 1628 1629 barno = BAR_0; 1630 ntb_epc = ntb->epc[type]; 1631 num_mws = ntb->num_mws; 1632 dev = &ntb->epf->dev; 1633 epc_features = ntb_epc->epc_features; 1634 1635 /* These are required BARs which are mandatory for NTB functionality */ 1636 for (bar = BAR_CONFIG; bar <= BAR_DB_MW1; bar++, barno++) { 1637 barno = pci_epc_get_next_free_bar(epc_features, barno); 1638 if (barno < 0) { 1639 dev_err(dev, "%s intf: Fail to get NTB function BAR\n", 1640 pci_epc_interface_string(type)); 1641 return barno; 1642 } 1643 ntb_epc->epf_ntb_bar[bar] = barno; 1644 } 1645 1646 /* These are optional BARs which don't impact NTB functionality */ 1647 for (bar = BAR_MW2, i = 1; i < num_mws; bar++, barno++, i++) { 1648 barno = pci_epc_get_next_free_bar(epc_features, barno); 1649 if (barno < 0) { 1650 ntb->num_mws = i; 1651 dev_dbg(dev, "BAR not available for > MW%d\n", i + 1); 1652 } 1653 ntb_epc->epf_ntb_bar[bar] = barno; 1654 } 1655 1656 return 0; 1657 } 1658 1659 /** 1660 * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB 1661 * constructs (scratchpad region, doorbell, memorywindow) 1662 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1663 * @type: PRIMARY interface or SECONDARY interface 1664 * 1665 * Wrapper to epf_ntb_init_epc_bar_interface() to identify the free BARs 1666 * to be used for each of BAR_CONFIG, BAR_PEER_SPAD, BAR_DB_MW1, BAR_MW2, 1667 * BAR_MW3 and BAR_MW4 for all the interfaces. 1668 */ 1669 static int epf_ntb_init_epc_bar(struct epf_ntb *ntb) 1670 { 1671 enum pci_epc_interface_type type; 1672 struct device *dev; 1673 int ret; 1674 1675 dev = &ntb->epf->dev; 1676 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { 1677 ret = epf_ntb_init_epc_bar_interface(ntb, type); 1678 if (ret) { 1679 dev_err(dev, "Fail to init EPC bar for %s interface\n", 1680 pci_epc_interface_string(type)); 1681 return ret; 1682 } 1683 } 1684 1685 return 0; 1686 } 1687 1688 /** 1689 * epf_ntb_epc_init_interface() - Initialize NTB interface 1690 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1691 * @type: PRIMARY interface or SECONDARY interface 1692 * 1693 * Wrapper to initialize a particular EPC interface and start the workqueue 1694 * to check for commands from host. This function will write to the 1695 * EP controller HW for configuring it. 1696 */ 1697 static int epf_ntb_epc_init_interface(struct epf_ntb *ntb, 1698 enum pci_epc_interface_type type) 1699 { 1700 struct epf_ntb_epc *ntb_epc; 1701 struct pci_epc *epc; 1702 struct pci_epf *epf; 1703 struct device *dev; 1704 u8 func_no; 1705 int ret; 1706 1707 ntb_epc = ntb->epc[type]; 1708 epf = ntb->epf; 1709 dev = &epf->dev; 1710 epc = ntb_epc->epc; 1711 func_no = ntb_epc->func_no; 1712 1713 ret = epf_ntb_config_sspad_bar_set(ntb->epc[type]); 1714 if (ret) { 1715 dev_err(dev, "%s intf: Config/self SPAD BAR init failed\n", 1716 pci_epc_interface_string(type)); 1717 return ret; 1718 } 1719 1720 ret = epf_ntb_peer_spad_bar_set(ntb, type); 1721 if (ret) { 1722 dev_err(dev, "%s intf: Peer SPAD BAR init failed\n", 1723 pci_epc_interface_string(type)); 1724 goto err_peer_spad_bar_init; 1725 } 1726 1727 ret = epf_ntb_configure_interrupt(ntb, type); 1728 if (ret) { 1729 dev_err(dev, "%s intf: Interrupt configuration failed\n", 1730 pci_epc_interface_string(type)); 1731 goto err_peer_spad_bar_init; 1732 } 1733 1734 ret = epf_ntb_db_mw_bar_init(ntb, type); 1735 if (ret) { 1736 dev_err(dev, "%s intf: DB/MW BAR init failed\n", 1737 pci_epc_interface_string(type)); 1738 goto err_db_mw_bar_init; 1739 } 1740 1741 ret = pci_epc_write_header(epc, func_no, epf->header); 1742 if (ret) { 1743 dev_err(dev, "%s intf: Configuration header write failed\n", 1744 pci_epc_interface_string(type)); 1745 goto err_write_header; 1746 } 1747 1748 INIT_DELAYED_WORK(&ntb->epc[type]->cmd_handler, epf_ntb_cmd_handler); 1749 queue_work(kpcintb_workqueue, &ntb->epc[type]->cmd_handler.work); 1750 1751 return 0; 1752 1753 err_write_header: 1754 epf_ntb_db_mw_bar_cleanup(ntb, type); 1755 1756 err_db_mw_bar_init: 1757 epf_ntb_peer_spad_bar_clear(ntb->epc[type]); 1758 1759 err_peer_spad_bar_init: 1760 epf_ntb_config_sspad_bar_clear(ntb->epc[type]); 1761 1762 return ret; 1763 } 1764 1765 /** 1766 * epf_ntb_epc_cleanup_interface() - Cleanup NTB interface 1767 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1768 * @type: PRIMARY interface or SECONDARY interface 1769 * 1770 * Wrapper to cleanup a particular NTB interface. 1771 */ 1772 static void epf_ntb_epc_cleanup_interface(struct epf_ntb *ntb, 1773 enum pci_epc_interface_type type) 1774 { 1775 struct epf_ntb_epc *ntb_epc; 1776 1777 if (type < 0) 1778 return; 1779 1780 ntb_epc = ntb->epc[type]; 1781 cancel_delayed_work(&ntb_epc->cmd_handler); 1782 epf_ntb_db_mw_bar_cleanup(ntb, type); 1783 epf_ntb_peer_spad_bar_clear(ntb_epc); 1784 epf_ntb_config_sspad_bar_clear(ntb_epc); 1785 } 1786 1787 /** 1788 * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces 1789 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1790 * 1791 * Wrapper to cleanup all NTB interfaces. 1792 */ 1793 static void epf_ntb_epc_cleanup(struct epf_ntb *ntb) 1794 { 1795 enum pci_epc_interface_type type; 1796 1797 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) 1798 epf_ntb_epc_cleanup_interface(ntb, type); 1799 } 1800 1801 /** 1802 * epf_ntb_epc_init() - Initialize all NTB interfaces 1803 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1804 * 1805 * Wrapper to initialize all NTB interface and start the workqueue 1806 * to check for commands from host. 1807 */ 1808 static int epf_ntb_epc_init(struct epf_ntb *ntb) 1809 { 1810 enum pci_epc_interface_type type; 1811 struct device *dev; 1812 int ret; 1813 1814 dev = &ntb->epf->dev; 1815 1816 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { 1817 ret = epf_ntb_epc_init_interface(ntb, type); 1818 if (ret) { 1819 dev_err(dev, "%s intf: Failed to initialize\n", 1820 pci_epc_interface_string(type)); 1821 goto err_init_type; 1822 } 1823 } 1824 1825 return 0; 1826 1827 err_init_type: 1828 epf_ntb_epc_cleanup_interface(ntb, type - 1); 1829 1830 return ret; 1831 } 1832 1833 /** 1834 * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality 1835 * @epf: NTB endpoint function device 1836 * 1837 * Initialize both the endpoint controllers associated with NTB function device. 1838 * Invoked when a primary interface or secondary interface is bound to EPC 1839 * device. This function will succeed only when EPC is bound to both the 1840 * interfaces. 1841 */ 1842 static int epf_ntb_bind(struct pci_epf *epf) 1843 { 1844 struct epf_ntb *ntb = epf_get_drvdata(epf); 1845 struct device *dev = &epf->dev; 1846 int ret; 1847 1848 if (!epf->epc) { 1849 dev_dbg(dev, "PRIMARY EPC interface not yet bound\n"); 1850 return 0; 1851 } 1852 1853 if (!epf->sec_epc) { 1854 dev_dbg(dev, "SECONDARY EPC interface not yet bound\n"); 1855 return 0; 1856 } 1857 1858 ret = epf_ntb_epc_create(ntb); 1859 if (ret) { 1860 dev_err(dev, "Failed to create NTB EPC\n"); 1861 return ret; 1862 } 1863 1864 ret = epf_ntb_init_epc_bar(ntb); 1865 if (ret) { 1866 dev_err(dev, "Failed to create NTB EPC\n"); 1867 goto err_bar_init; 1868 } 1869 1870 ret = epf_ntb_config_spad_bar_alloc_interface(ntb); 1871 if (ret) { 1872 dev_err(dev, "Failed to allocate BAR memory\n"); 1873 goto err_bar_alloc; 1874 } 1875 1876 ret = epf_ntb_epc_init(ntb); 1877 if (ret) { 1878 dev_err(dev, "Failed to initialize EPC\n"); 1879 goto err_bar_alloc; 1880 } 1881 1882 epf_set_drvdata(epf, ntb); 1883 1884 return 0; 1885 1886 err_bar_alloc: 1887 epf_ntb_config_spad_bar_free(ntb); 1888 1889 err_bar_init: 1890 epf_ntb_epc_destroy(ntb); 1891 1892 return ret; 1893 } 1894 1895 /** 1896 * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind() 1897 * @epf: NTB endpoint function device 1898 * 1899 * Cleanup the initialization from epf_ntb_bind() 1900 */ 1901 static void epf_ntb_unbind(struct pci_epf *epf) 1902 { 1903 struct epf_ntb *ntb = epf_get_drvdata(epf); 1904 1905 epf_ntb_epc_cleanup(ntb); 1906 epf_ntb_config_spad_bar_free(ntb); 1907 epf_ntb_epc_destroy(ntb); 1908 } 1909 1910 #define EPF_NTB_R(_name) \ 1911 static ssize_t epf_ntb_##_name##_show(struct config_item *item, \ 1912 char *page) \ 1913 { \ 1914 struct config_group *group = to_config_group(item); \ 1915 struct epf_ntb *ntb = to_epf_ntb(group); \ 1916 \ 1917 return sprintf(page, "%d\n", ntb->_name); \ 1918 } 1919 1920 #define EPF_NTB_W(_name) \ 1921 static ssize_t epf_ntb_##_name##_store(struct config_item *item, \ 1922 const char *page, size_t len) \ 1923 { \ 1924 struct config_group *group = to_config_group(item); \ 1925 struct epf_ntb *ntb = to_epf_ntb(group); \ 1926 u32 val; \ 1927 int ret; \ 1928 \ 1929 ret = kstrtou32(page, 0, &val); \ 1930 if (ret) \ 1931 return ret; \ 1932 \ 1933 ntb->_name = val; \ 1934 \ 1935 return len; \ 1936 } 1937 1938 #define EPF_NTB_MW_R(_name) \ 1939 static ssize_t epf_ntb_##_name##_show(struct config_item *item, \ 1940 char *page) \ 1941 { \ 1942 struct config_group *group = to_config_group(item); \ 1943 struct epf_ntb *ntb = to_epf_ntb(group); \ 1944 int win_no; \ 1945 \ 1946 sscanf(#_name, "mw%d", &win_no); \ 1947 \ 1948 return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]); \ 1949 } 1950 1951 #define EPF_NTB_MW_W(_name) \ 1952 static ssize_t epf_ntb_##_name##_store(struct config_item *item, \ 1953 const char *page, size_t len) \ 1954 { \ 1955 struct config_group *group = to_config_group(item); \ 1956 struct epf_ntb *ntb = to_epf_ntb(group); \ 1957 struct device *dev = &ntb->epf->dev; \ 1958 int win_no; \ 1959 u64 val; \ 1960 int ret; \ 1961 \ 1962 ret = kstrtou64(page, 0, &val); \ 1963 if (ret) \ 1964 return ret; \ 1965 \ 1966 if (sscanf(#_name, "mw%d", &win_no) != 1) \ 1967 return -EINVAL; \ 1968 \ 1969 if (ntb->num_mws < win_no) { \ 1970 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \ 1971 return -EINVAL; \ 1972 } \ 1973 \ 1974 ntb->mws_size[win_no - 1] = val; \ 1975 \ 1976 return len; \ 1977 } 1978 1979 static ssize_t epf_ntb_num_mws_store(struct config_item *item, 1980 const char *page, size_t len) 1981 { 1982 struct config_group *group = to_config_group(item); 1983 struct epf_ntb *ntb = to_epf_ntb(group); 1984 u32 val; 1985 int ret; 1986 1987 ret = kstrtou32(page, 0, &val); 1988 if (ret) 1989 return ret; 1990 1991 if (val > MAX_MW) 1992 return -EINVAL; 1993 1994 ntb->num_mws = val; 1995 1996 return len; 1997 } 1998 1999 EPF_NTB_R(spad_count) 2000 EPF_NTB_W(spad_count) 2001 EPF_NTB_R(db_count) 2002 EPF_NTB_W(db_count) 2003 EPF_NTB_R(num_mws) 2004 EPF_NTB_MW_R(mw1) 2005 EPF_NTB_MW_W(mw1) 2006 EPF_NTB_MW_R(mw2) 2007 EPF_NTB_MW_W(mw2) 2008 EPF_NTB_MW_R(mw3) 2009 EPF_NTB_MW_W(mw3) 2010 EPF_NTB_MW_R(mw4) 2011 EPF_NTB_MW_W(mw4) 2012 2013 CONFIGFS_ATTR(epf_ntb_, spad_count); 2014 CONFIGFS_ATTR(epf_ntb_, db_count); 2015 CONFIGFS_ATTR(epf_ntb_, num_mws); 2016 CONFIGFS_ATTR(epf_ntb_, mw1); 2017 CONFIGFS_ATTR(epf_ntb_, mw2); 2018 CONFIGFS_ATTR(epf_ntb_, mw3); 2019 CONFIGFS_ATTR(epf_ntb_, mw4); 2020 2021 static struct configfs_attribute *epf_ntb_attrs[] = { 2022 &epf_ntb_attr_spad_count, 2023 &epf_ntb_attr_db_count, 2024 &epf_ntb_attr_num_mws, 2025 &epf_ntb_attr_mw1, 2026 &epf_ntb_attr_mw2, 2027 &epf_ntb_attr_mw3, 2028 &epf_ntb_attr_mw4, 2029 NULL, 2030 }; 2031 2032 static const struct config_item_type ntb_group_type = { 2033 .ct_attrs = epf_ntb_attrs, 2034 .ct_owner = THIS_MODULE, 2035 }; 2036 2037 /** 2038 * epf_ntb_add_cfs() - Add configfs directory specific to NTB 2039 * @epf: NTB endpoint function device 2040 * 2041 * Add configfs directory specific to NTB. This directory will hold 2042 * NTB specific properties like db_count, spad_count, num_mws etc., 2043 */ 2044 static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf, 2045 struct config_group *group) 2046 { 2047 struct epf_ntb *ntb = epf_get_drvdata(epf); 2048 struct config_group *ntb_group = &ntb->group; 2049 struct device *dev = &epf->dev; 2050 2051 config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type); 2052 2053 return ntb_group; 2054 } 2055 2056 /** 2057 * epf_ntb_probe() - Probe NTB function driver 2058 * @epf: NTB endpoint function device 2059 * 2060 * Probe NTB function driver when endpoint function bus detects a NTB 2061 * endpoint function. 2062 */ 2063 static int epf_ntb_probe(struct pci_epf *epf) 2064 { 2065 struct epf_ntb *ntb; 2066 struct device *dev; 2067 2068 dev = &epf->dev; 2069 2070 ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL); 2071 if (!ntb) 2072 return -ENOMEM; 2073 2074 epf->header = &epf_ntb_header; 2075 ntb->epf = epf; 2076 epf_set_drvdata(epf, ntb); 2077 2078 return 0; 2079 } 2080 2081 static struct pci_epf_ops epf_ntb_ops = { 2082 .bind = epf_ntb_bind, 2083 .unbind = epf_ntb_unbind, 2084 .add_cfs = epf_ntb_add_cfs, 2085 }; 2086 2087 static const struct pci_epf_device_id epf_ntb_ids[] = { 2088 { 2089 .name = "pci_epf_ntb", 2090 }, 2091 {}, 2092 }; 2093 2094 static struct pci_epf_driver epf_ntb_driver = { 2095 .driver.name = "pci_epf_ntb", 2096 .probe = epf_ntb_probe, 2097 .id_table = epf_ntb_ids, 2098 .ops = &epf_ntb_ops, 2099 .owner = THIS_MODULE, 2100 }; 2101 2102 static int __init epf_ntb_init(void) 2103 { 2104 int ret; 2105 2106 kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM | 2107 WQ_HIGHPRI, 0); 2108 ret = pci_epf_register_driver(&epf_ntb_driver); 2109 if (ret) { 2110 destroy_workqueue(kpcintb_workqueue); 2111 pr_err("Failed to register pci epf ntb driver --> %d\n", ret); 2112 return ret; 2113 } 2114 2115 return 0; 2116 } 2117 module_init(epf_ntb_init); 2118 2119 static void __exit epf_ntb_exit(void) 2120 { 2121 pci_epf_unregister_driver(&epf_ntb_driver); 2122 destroy_workqueue(kpcintb_workqueue); 2123 } 2124 module_exit(epf_ntb_exit); 2125 2126 MODULE_DESCRIPTION("PCI EPF NTB DRIVER"); 2127 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>"); 2128 MODULE_LICENSE("GPL v2"); 2129