1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Endpoint Function Driver to implement Non-Transparent Bridge functionality 4 * 5 * Copyright (C) 2020 Texas Instruments 6 * Author: Kishon Vijay Abraham I <kishon@ti.com> 7 */ 8 9 /* 10 * The PCI NTB function driver configures the SoC with multiple PCIe Endpoint 11 * (EP) controller instances (see diagram below) in such a way that 12 * transactions from one EP controller are routed to the other EP controller. 13 * Once PCI NTB function driver configures the SoC with multiple EP instances, 14 * HOST1 and HOST2 can communicate with each other using SoC as a bridge. 15 * 16 * +-------------+ +-------------+ 17 * | | | | 18 * | HOST1 | | HOST2 | 19 * | | | | 20 * +------^------+ +------^------+ 21 * | | 22 * | | 23 * +---------|-------------------------------------------------|---------+ 24 * | +------v------+ +------v------+ | 25 * | | | | | | 26 * | | EP | | EP | | 27 * | | CONTROLLER1 | | CONTROLLER2 | | 28 * | | <-----------------------------------> | | 29 * | | | | | | 30 * | | | | | | 31 * | | | SoC With Multiple EP Instances | | | 32 * | | | (Configured using NTB Function) | | | 33 * | +-------------+ +-------------+ | 34 * +---------------------------------------------------------------------+ 35 */ 36 37 #include <linux/delay.h> 38 #include <linux/io.h> 39 #include <linux/module.h> 40 #include <linux/slab.h> 41 42 #include <linux/pci-epc.h> 43 #include <linux/pci-epf.h> 44 45 static struct workqueue_struct *kpcintb_workqueue; 46 47 #define COMMAND_CONFIGURE_DOORBELL 1 48 #define COMMAND_TEARDOWN_DOORBELL 2 49 #define COMMAND_CONFIGURE_MW 3 50 #define COMMAND_TEARDOWN_MW 4 51 #define COMMAND_LINK_UP 5 52 #define COMMAND_LINK_DOWN 6 53 54 #define COMMAND_STATUS_OK 1 55 #define COMMAND_STATUS_ERROR 2 56 57 #define LINK_STATUS_UP BIT(0) 58 59 #define SPAD_COUNT 64 60 #define DB_COUNT 4 61 #define NTB_MW_OFFSET 2 62 #define DB_COUNT_MASK GENMASK(15, 0) 63 #define MSIX_ENABLE BIT(16) 64 #define MAX_DB_COUNT 32 65 #define MAX_MW 4 66 67 enum epf_ntb_bar { 68 BAR_CONFIG, 69 BAR_PEER_SPAD, 70 BAR_DB_MW1, 71 BAR_MW2, 72 BAR_MW3, 73 BAR_MW4, 74 }; 75 76 struct epf_ntb { 77 u32 num_mws; 78 u32 db_count; 79 u32 spad_count; 80 struct pci_epf *epf; 81 u64 mws_size[MAX_MW]; 82 struct config_group group; 83 struct epf_ntb_epc *epc[2]; 84 }; 85 86 #define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group) 87 88 struct epf_ntb_epc { 89 u8 func_no; 90 bool linkup; 91 bool is_msix; 92 int msix_bar; 93 u32 spad_size; 94 struct pci_epc *epc; 95 struct epf_ntb *epf_ntb; 96 void __iomem *mw_addr[6]; 97 size_t msix_table_offset; 98 struct epf_ntb_ctrl *reg; 99 struct pci_epf_bar *epf_bar; 100 enum pci_barno epf_ntb_bar[6]; 101 struct delayed_work cmd_handler; 102 enum pci_epc_interface_type type; 103 const struct pci_epc_features *epc_features; 104 }; 105 106 struct epf_ntb_ctrl { 107 u32 command; 108 u32 argument; 109 u16 command_status; 110 u16 link_status; 111 u32 topology; 112 u64 addr; 113 u64 size; 114 u32 num_mws; 115 u32 mw1_offset; 116 u32 spad_offset; 117 u32 spad_count; 118 u32 db_entry_size; 119 u32 db_data[MAX_DB_COUNT]; 120 u32 db_offset[MAX_DB_COUNT]; 121 } __packed; 122 123 static struct pci_epf_header epf_ntb_header = { 124 .vendorid = PCI_ANY_ID, 125 .deviceid = PCI_ANY_ID, 126 .baseclass_code = PCI_BASE_CLASS_MEMORY, 127 .interrupt_pin = PCI_INTERRUPT_INTA, 128 }; 129 130 /** 131 * epf_ntb_link_up() - Raise link_up interrupt to both the hosts 132 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 133 * @link_up: true or false indicating Link is UP or Down 134 * 135 * Once NTB function in HOST1 and the NTB function in HOST2 invoke 136 * ntb_link_enable(), this NTB function driver will trigger a link event to 137 * the NTB client in both the hosts. 138 */ 139 static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up) 140 { 141 enum pci_epc_interface_type type; 142 enum pci_epc_irq_type irq_type; 143 struct epf_ntb_epc *ntb_epc; 144 struct epf_ntb_ctrl *ctrl; 145 struct pci_epc *epc; 146 bool is_msix; 147 u8 func_no; 148 int ret; 149 150 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { 151 ntb_epc = ntb->epc[type]; 152 epc = ntb_epc->epc; 153 func_no = ntb_epc->func_no; 154 is_msix = ntb_epc->is_msix; 155 ctrl = ntb_epc->reg; 156 if (link_up) 157 ctrl->link_status |= LINK_STATUS_UP; 158 else 159 ctrl->link_status &= ~LINK_STATUS_UP; 160 irq_type = is_msix ? PCI_EPC_IRQ_MSIX : PCI_EPC_IRQ_MSI; 161 ret = pci_epc_raise_irq(epc, func_no, irq_type, 1); 162 if (ret) { 163 dev_err(&epc->dev, 164 "%s intf: Failed to raise Link Up IRQ\n", 165 pci_epc_interface_string(type)); 166 return ret; 167 } 168 } 169 170 return 0; 171 } 172 173 /** 174 * epf_ntb_configure_mw() - Configure the Outbound Address Space for one host 175 * to access the memory window of other host 176 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 177 * @type: PRIMARY interface or SECONDARY interface 178 * @mw: Index of the memory window (either 0, 1, 2 or 3) 179 * 180 * +-----------------+ +---->+----------------+-----------+-----------------+ 181 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | 182 * +-----------------+ | +----------------+ +-----------------+ 183 * | BAR1 | | | Doorbell 2 +---------+ | | 184 * +-----------------+----+ +----------------+ | | | 185 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ 186 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | 187 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ 188 * +-----------------+ | |----------------+ | | | | 189 * | BAR4 | | | | | | +-----------------+ 190 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| 191 * | BAR5 | | | | | | +-----------------+ 192 * +-----------------+ +---->-----------------+ | | | | 193 * EP CONTROLLER 1 | | | | +-----------------+ 194 * | | | +---->+ MSI|X ADDRESS 4 | 195 * +----------------+ | +-----------------+ 196 * (A) EP CONTROLLER 2 | | | 197 * (OB SPACE) | | | 198 * +-------> MW1 | 199 * | | 200 * | | 201 * (B) +-----------------+ 202 * | | 203 * | | 204 * | | 205 * | | 206 * | | 207 * +-----------------+ 208 * PCI Address Space 209 * (Managed by HOST2) 210 * 211 * This function performs stage (B) in the above diagram (see MW1) i.e., map OB 212 * address space of memory window to PCI address space. 213 * 214 * This operation requires 3 parameters 215 * 1) Address in the outbound address space 216 * 2) Address in the PCI Address space 217 * 3) Size of the address region to be mapped 218 * 219 * The address in the outbound address space (for MW1, MW2, MW3 and MW4) is 220 * stored in epf_bar corresponding to BAR_DB_MW1 for MW1 and BAR_MW2, BAR_MW3 221 * BAR_MW4 for rest of the BARs of epf_ntb_epc that is connected to HOST1. This 222 * is populated in epf_ntb_alloc_peer_mem() in this driver. 223 * 224 * The address and size of the PCI address region that has to be mapped would 225 * be provided by HOST2 in ctrl->addr and ctrl->size of epf_ntb_epc that is 226 * connected to HOST2. 227 * 228 * Please note Memory window1 (MW1) and Doorbell registers together will be 229 * mapped to a single BAR (BAR2) above for 32-bit BARs. The exact BAR that's 230 * used for Memory window (MW) can be obtained from epf_ntb_bar[BAR_DB_MW1], 231 * epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2], epf_ntb_bar[BAR_MW2]. 232 */ 233 static int epf_ntb_configure_mw(struct epf_ntb *ntb, 234 enum pci_epc_interface_type type, u32 mw) 235 { 236 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 237 struct pci_epf_bar *peer_epf_bar; 238 enum pci_barno peer_barno; 239 struct epf_ntb_ctrl *ctrl; 240 phys_addr_t phys_addr; 241 struct pci_epc *epc; 242 u64 addr, size; 243 int ret = 0; 244 u8 func_no; 245 246 ntb_epc = ntb->epc[type]; 247 epc = ntb_epc->epc; 248 249 peer_ntb_epc = ntb->epc[!type]; 250 peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET]; 251 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 252 253 phys_addr = peer_epf_bar->phys_addr; 254 ctrl = ntb_epc->reg; 255 addr = ctrl->addr; 256 size = ctrl->size; 257 if (mw + NTB_MW_OFFSET == BAR_DB_MW1) 258 phys_addr += ctrl->mw1_offset; 259 260 if (size > ntb->mws_size[mw]) { 261 dev_err(&epc->dev, 262 "%s intf: MW: %d Req Sz:%llxx > Supported Sz:%llx\n", 263 pci_epc_interface_string(type), mw, size, 264 ntb->mws_size[mw]); 265 ret = -EINVAL; 266 goto err_invalid_size; 267 } 268 269 func_no = ntb_epc->func_no; 270 271 ret = pci_epc_map_addr(epc, func_no, phys_addr, addr, size); 272 if (ret) 273 dev_err(&epc->dev, 274 "%s intf: Failed to map memory window %d address\n", 275 pci_epc_interface_string(type), mw); 276 277 err_invalid_size: 278 279 return ret; 280 } 281 282 /** 283 * epf_ntb_teardown_mw() - Teardown the configured OB ATU 284 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 285 * @type: PRIMARY interface or SECONDARY interface 286 * @mw: Index of the memory window (either 0, 1, 2 or 3) 287 * 288 * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using 289 * pci_epc_unmap_addr() 290 */ 291 static void epf_ntb_teardown_mw(struct epf_ntb *ntb, 292 enum pci_epc_interface_type type, u32 mw) 293 { 294 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 295 struct pci_epf_bar *peer_epf_bar; 296 enum pci_barno peer_barno; 297 struct epf_ntb_ctrl *ctrl; 298 phys_addr_t phys_addr; 299 struct pci_epc *epc; 300 u8 func_no; 301 302 ntb_epc = ntb->epc[type]; 303 epc = ntb_epc->epc; 304 305 peer_ntb_epc = ntb->epc[!type]; 306 peer_barno = peer_ntb_epc->epf_ntb_bar[mw + NTB_MW_OFFSET]; 307 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 308 309 phys_addr = peer_epf_bar->phys_addr; 310 ctrl = ntb_epc->reg; 311 if (mw + NTB_MW_OFFSET == BAR_DB_MW1) 312 phys_addr += ctrl->mw1_offset; 313 func_no = ntb_epc->func_no; 314 315 pci_epc_unmap_addr(epc, func_no, phys_addr); 316 } 317 318 /** 319 * epf_ntb_configure_msi() - Map OB address space to MSI address 320 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 321 * @type: PRIMARY interface or SECONDARY interface 322 * @db_count: Number of doorbell interrupts to map 323 * 324 *+-----------------+ +----->+----------------+-----------+-----------------+ 325 *| BAR0 | | | Doorbell 1 +---+-------> MSI ADDRESS | 326 *+-----------------+ | +----------------+ | +-----------------+ 327 *| BAR1 | | | Doorbell 2 +---+ | | 328 *+-----------------+----+ +----------------+ | | | 329 *| BAR2 | | Doorbell 3 +---+ | | 330 *+-----------------+----+ +----------------+ | | | 331 *| BAR3 | | | Doorbell 4 +---+ | | 332 *+-----------------+ | |----------------+ | | 333 *| BAR4 | | | | | | 334 *+-----------------+ | | MW1 | | | 335 *| BAR5 | | | | | | 336 *+-----------------+ +----->-----------------+ | | 337 * EP CONTROLLER 1 | | | | 338 * | | | | 339 * +----------------+ +-----------------+ 340 * (A) EP CONTROLLER 2 | | 341 * (OB SPACE) | | 342 * | MW1 | 343 * | | 344 * | | 345 * (B) +-----------------+ 346 * | | 347 * | | 348 * | | 349 * | | 350 * | | 351 * +-----------------+ 352 * PCI Address Space 353 * (Managed by HOST2) 354 * 355 * 356 * This function performs stage (B) in the above diagram (see Doorbell 1, 357 * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to 358 * doorbell to MSI address in PCI address space. 359 * 360 * This operation requires 3 parameters 361 * 1) Address reserved for doorbell in the outbound address space 362 * 2) MSI-X address in the PCIe Address space 363 * 3) Number of MSI-X interrupts that has to be configured 364 * 365 * The address in the outbound address space (for the Doorbell) is stored in 366 * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to 367 * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along 368 * with address for MW1. 369 * 370 * pci_epc_map_msi_irq() takes the MSI address from MSI capability register 371 * and maps the OB address (obtained in epf_ntb_alloc_peer_mem()) to the MSI 372 * address. 373 * 374 * epf_ntb_configure_msi() also stores the MSI data to raise each interrupt 375 * in db_data of the peer's control region. This helps the peer to raise 376 * doorbell of the other host by writing db_data to the BAR corresponding to 377 * BAR_DB_MW1. 378 */ 379 static int epf_ntb_configure_msi(struct epf_ntb *ntb, 380 enum pci_epc_interface_type type, u16 db_count) 381 { 382 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 383 u32 db_entry_size, db_data, db_offset; 384 struct pci_epf_bar *peer_epf_bar; 385 struct epf_ntb_ctrl *peer_ctrl; 386 enum pci_barno peer_barno; 387 phys_addr_t phys_addr; 388 struct pci_epc *epc; 389 u8 func_no; 390 int ret, i; 391 392 ntb_epc = ntb->epc[type]; 393 epc = ntb_epc->epc; 394 395 peer_ntb_epc = ntb->epc[!type]; 396 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; 397 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 398 peer_ctrl = peer_ntb_epc->reg; 399 db_entry_size = peer_ctrl->db_entry_size; 400 401 phys_addr = peer_epf_bar->phys_addr; 402 func_no = ntb_epc->func_no; 403 404 ret = pci_epc_map_msi_irq(epc, func_no, phys_addr, db_count, 405 db_entry_size, &db_data, &db_offset); 406 if (ret) { 407 dev_err(&epc->dev, "%s intf: Failed to map MSI IRQ\n", 408 pci_epc_interface_string(type)); 409 return ret; 410 } 411 412 for (i = 0; i < db_count; i++) { 413 peer_ctrl->db_data[i] = db_data | i; 414 peer_ctrl->db_offset[i] = db_offset; 415 } 416 417 return 0; 418 } 419 420 /** 421 * epf_ntb_configure_msix() - Map OB address space to MSI-X address 422 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 423 * @type: PRIMARY interface or SECONDARY interface 424 * @db_count: Number of doorbell interrupts to map 425 * 426 *+-----------------+ +----->+----------------+-----------+-----------------+ 427 *| BAR0 | | | Doorbell 1 +-----------> MSI-X ADDRESS 1 | 428 *+-----------------+ | +----------------+ +-----------------+ 429 *| BAR1 | | | Doorbell 2 +---------+ | | 430 *+-----------------+----+ +----------------+ | | | 431 *| BAR2 | | Doorbell 3 +-------+ | +-----------------+ 432 *+-----------------+----+ +----------------+ | +-> MSI-X ADDRESS 2 | 433 *| BAR3 | | | Doorbell 4 +-----+ | +-----------------+ 434 *+-----------------+ | |----------------+ | | | | 435 *| BAR4 | | | | | | +-----------------+ 436 *+-----------------+ | | MW1 + | +-->+ MSI-X ADDRESS 3|| 437 *| BAR5 | | | | | +-----------------+ 438 *+-----------------+ +----->-----------------+ | | | 439 * EP CONTROLLER 1 | | | +-----------------+ 440 * | | +---->+ MSI-X ADDRESS 4 | 441 * +----------------+ +-----------------+ 442 * (A) EP CONTROLLER 2 | | 443 * (OB SPACE) | | 444 * | MW1 | 445 * | | 446 * | | 447 * (B) +-----------------+ 448 * | | 449 * | | 450 * | | 451 * | | 452 * | | 453 * +-----------------+ 454 * PCI Address Space 455 * (Managed by HOST2) 456 * 457 * This function performs stage (B) in the above diagram (see Doorbell 1, 458 * Doorbell 2, Doorbell 3, Doorbell 4) i.e map OB address space corresponding to 459 * doorbell to MSI-X address in PCI address space. 460 * 461 * This operation requires 3 parameters 462 * 1) Address reserved for doorbell in the outbound address space 463 * 2) MSI-X address in the PCIe Address space 464 * 3) Number of MSI-X interrupts that has to be configured 465 * 466 * The address in the outbound address space (for the Doorbell) is stored in 467 * epf_bar corresponding to BAR_DB_MW1 of epf_ntb_epc that is connected to 468 * HOST1. This is populated in epf_ntb_alloc_peer_mem() in this driver along 469 * with address for MW1. 470 * 471 * The MSI-X address is in the MSI-X table of EP CONTROLLER 2 and 472 * the count of doorbell is in ctrl->argument of epf_ntb_epc that is connected 473 * to HOST2. MSI-X table is stored memory mapped to ntb_epc->msix_bar and the 474 * offset is in ntb_epc->msix_table_offset. From this epf_ntb_configure_msix() 475 * gets the MSI-X address and data. 476 * 477 * epf_ntb_configure_msix() also stores the MSI-X data to raise each interrupt 478 * in db_data of the peer's control region. This helps the peer to raise 479 * doorbell of the other host by writing db_data to the BAR corresponding to 480 * BAR_DB_MW1. 481 */ 482 static int epf_ntb_configure_msix(struct epf_ntb *ntb, 483 enum pci_epc_interface_type type, 484 u16 db_count) 485 { 486 const struct pci_epc_features *epc_features; 487 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 488 struct pci_epf_bar *peer_epf_bar, *epf_bar; 489 struct pci_epf_msix_tbl *msix_tbl; 490 struct epf_ntb_ctrl *peer_ctrl; 491 u32 db_entry_size, msg_data; 492 enum pci_barno peer_barno; 493 phys_addr_t phys_addr; 494 struct pci_epc *epc; 495 size_t align; 496 u64 msg_addr; 497 u8 func_no; 498 int ret, i; 499 500 ntb_epc = ntb->epc[type]; 501 epc = ntb_epc->epc; 502 503 epf_bar = &ntb_epc->epf_bar[ntb_epc->msix_bar]; 504 msix_tbl = epf_bar->addr + ntb_epc->msix_table_offset; 505 506 peer_ntb_epc = ntb->epc[!type]; 507 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; 508 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 509 phys_addr = peer_epf_bar->phys_addr; 510 peer_ctrl = peer_ntb_epc->reg; 511 epc_features = ntb_epc->epc_features; 512 align = epc_features->align; 513 514 func_no = ntb_epc->func_no; 515 db_entry_size = peer_ctrl->db_entry_size; 516 517 for (i = 0; i < db_count; i++) { 518 msg_addr = ALIGN_DOWN(msix_tbl[i].msg_addr, align); 519 msg_data = msix_tbl[i].msg_data; 520 ret = pci_epc_map_addr(epc, func_no, phys_addr, msg_addr, 521 db_entry_size); 522 if (ret) { 523 dev_err(&epc->dev, 524 "%s intf: Failed to configure MSI-X IRQ\n", 525 pci_epc_interface_string(type)); 526 return ret; 527 } 528 phys_addr = phys_addr + db_entry_size; 529 peer_ctrl->db_data[i] = msg_data; 530 peer_ctrl->db_offset[i] = msix_tbl[i].msg_addr & (align - 1); 531 } 532 ntb_epc->is_msix = true; 533 534 return 0; 535 } 536 537 /** 538 * epf_ntb_configure_db() - Configure the Outbound Address Space for one host 539 * to ring the doorbell of other host 540 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 541 * @type: PRIMARY interface or SECONDARY interface 542 * @db_count: Count of the number of doorbells that has to be configured 543 * @msix: Indicates whether MSI-X or MSI should be used 544 * 545 * Invokes epf_ntb_configure_msix() or epf_ntb_configure_msi() required for 546 * one HOST to ring the doorbell of other HOST. 547 */ 548 static int epf_ntb_configure_db(struct epf_ntb *ntb, 549 enum pci_epc_interface_type type, 550 u16 db_count, bool msix) 551 { 552 struct epf_ntb_epc *ntb_epc; 553 struct pci_epc *epc; 554 int ret; 555 556 if (db_count > MAX_DB_COUNT) 557 return -EINVAL; 558 559 ntb_epc = ntb->epc[type]; 560 epc = ntb_epc->epc; 561 562 if (msix) 563 ret = epf_ntb_configure_msix(ntb, type, db_count); 564 else 565 ret = epf_ntb_configure_msi(ntb, type, db_count); 566 567 if (ret) 568 dev_err(&epc->dev, "%s intf: Failed to configure DB\n", 569 pci_epc_interface_string(type)); 570 571 return ret; 572 } 573 574 /** 575 * epf_ntb_teardown_db() - Unmap address in OB address space to MSI/MSI-X 576 * address 577 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 578 * @type: PRIMARY interface or SECONDARY interface 579 * 580 * Invoke pci_epc_unmap_addr() to unmap OB address to MSI/MSI-X address. 581 */ 582 static void 583 epf_ntb_teardown_db(struct epf_ntb *ntb, enum pci_epc_interface_type type) 584 { 585 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 586 struct pci_epf_bar *peer_epf_bar; 587 enum pci_barno peer_barno; 588 phys_addr_t phys_addr; 589 struct pci_epc *epc; 590 u8 func_no; 591 592 ntb_epc = ntb->epc[type]; 593 epc = ntb_epc->epc; 594 595 peer_ntb_epc = ntb->epc[!type]; 596 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_DB_MW1]; 597 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 598 phys_addr = peer_epf_bar->phys_addr; 599 func_no = ntb_epc->func_no; 600 601 pci_epc_unmap_addr(epc, func_no, phys_addr); 602 } 603 604 /** 605 * epf_ntb_cmd_handler() - Handle commands provided by the NTB Host 606 * @work: work_struct for the two epf_ntb_epc (PRIMARY and SECONDARY) 607 * 608 * Workqueue function that gets invoked for the two epf_ntb_epc 609 * periodically (once every 5ms) to see if it has received any commands 610 * from NTB host. The host can send commands to configure doorbell or 611 * configure memory window or to update link status. 612 */ 613 static void epf_ntb_cmd_handler(struct work_struct *work) 614 { 615 enum pci_epc_interface_type type; 616 struct epf_ntb_epc *ntb_epc; 617 struct epf_ntb_ctrl *ctrl; 618 u32 command, argument; 619 struct epf_ntb *ntb; 620 struct device *dev; 621 u16 db_count; 622 bool is_msix; 623 int ret; 624 625 ntb_epc = container_of(work, struct epf_ntb_epc, cmd_handler.work); 626 ctrl = ntb_epc->reg; 627 command = ctrl->command; 628 if (!command) 629 goto reset_handler; 630 argument = ctrl->argument; 631 632 ctrl->command = 0; 633 ctrl->argument = 0; 634 635 ctrl = ntb_epc->reg; 636 type = ntb_epc->type; 637 ntb = ntb_epc->epf_ntb; 638 dev = &ntb->epf->dev; 639 640 switch (command) { 641 case COMMAND_CONFIGURE_DOORBELL: 642 db_count = argument & DB_COUNT_MASK; 643 is_msix = argument & MSIX_ENABLE; 644 ret = epf_ntb_configure_db(ntb, type, db_count, is_msix); 645 if (ret < 0) 646 ctrl->command_status = COMMAND_STATUS_ERROR; 647 else 648 ctrl->command_status = COMMAND_STATUS_OK; 649 break; 650 case COMMAND_TEARDOWN_DOORBELL: 651 epf_ntb_teardown_db(ntb, type); 652 ctrl->command_status = COMMAND_STATUS_OK; 653 break; 654 case COMMAND_CONFIGURE_MW: 655 ret = epf_ntb_configure_mw(ntb, type, argument); 656 if (ret < 0) 657 ctrl->command_status = COMMAND_STATUS_ERROR; 658 else 659 ctrl->command_status = COMMAND_STATUS_OK; 660 break; 661 case COMMAND_TEARDOWN_MW: 662 epf_ntb_teardown_mw(ntb, type, argument); 663 ctrl->command_status = COMMAND_STATUS_OK; 664 break; 665 case COMMAND_LINK_UP: 666 ntb_epc->linkup = true; 667 if (ntb->epc[PRIMARY_INTERFACE]->linkup && 668 ntb->epc[SECONDARY_INTERFACE]->linkup) { 669 ret = epf_ntb_link_up(ntb, true); 670 if (ret < 0) 671 ctrl->command_status = COMMAND_STATUS_ERROR; 672 else 673 ctrl->command_status = COMMAND_STATUS_OK; 674 goto reset_handler; 675 } 676 ctrl->command_status = COMMAND_STATUS_OK; 677 break; 678 case COMMAND_LINK_DOWN: 679 ntb_epc->linkup = false; 680 ret = epf_ntb_link_up(ntb, false); 681 if (ret < 0) 682 ctrl->command_status = COMMAND_STATUS_ERROR; 683 else 684 ctrl->command_status = COMMAND_STATUS_OK; 685 break; 686 default: 687 dev_err(dev, "%s intf UNKNOWN command: %d\n", 688 pci_epc_interface_string(type), command); 689 break; 690 } 691 692 reset_handler: 693 queue_delayed_work(kpcintb_workqueue, &ntb_epc->cmd_handler, 694 msecs_to_jiffies(5)); 695 } 696 697 /** 698 * epf_ntb_peer_spad_bar_clear() - Clear Peer Scratchpad BAR 699 * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound 700 * address. 701 * 702 *+-----------------+------->+------------------+ +-----------------+ 703 *| BAR0 | | CONFIG REGION | | BAR0 | 704 *+-----------------+----+ +------------------+<-------+-----------------+ 705 *| BAR1 | | |SCRATCHPAD REGION | | BAR1 | 706 *+-----------------+ +-->+------------------+<-------+-----------------+ 707 *| BAR2 | Local Memory | BAR2 | 708 *+-----------------+ +-----------------+ 709 *| BAR3 | | BAR3 | 710 *+-----------------+ +-----------------+ 711 *| BAR4 | | BAR4 | 712 *+-----------------+ +-----------------+ 713 *| BAR5 | | BAR5 | 714 *+-----------------+ +-----------------+ 715 * EP CONTROLLER 1 EP CONTROLLER 2 716 * 717 * Clear BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad 718 * region. While BAR1 is the default peer scratchpad BAR, an NTB could have 719 * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs). 720 * This function can get the exact BAR used for peer scratchpad from 721 * epf_ntb_bar[BAR_PEER_SPAD]. 722 * 723 * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function 724 * gets the address of peer scratchpad from 725 * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG]. 726 */ 727 static void epf_ntb_peer_spad_bar_clear(struct epf_ntb_epc *ntb_epc) 728 { 729 struct pci_epf_bar *epf_bar; 730 enum pci_barno barno; 731 struct pci_epc *epc; 732 u8 func_no; 733 734 epc = ntb_epc->epc; 735 func_no = ntb_epc->func_no; 736 barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; 737 epf_bar = &ntb_epc->epf_bar[barno]; 738 pci_epc_clear_bar(epc, func_no, epf_bar); 739 } 740 741 /** 742 * epf_ntb_peer_spad_bar_set() - Set peer scratchpad BAR 743 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 744 * @type: PRIMARY interface or SECONDARY interface 745 * 746 *+-----------------+------->+------------------+ +-----------------+ 747 *| BAR0 | | CONFIG REGION | | BAR0 | 748 *+-----------------+----+ +------------------+<-------+-----------------+ 749 *| BAR1 | | |SCRATCHPAD REGION | | BAR1 | 750 *+-----------------+ +-->+------------------+<-------+-----------------+ 751 *| BAR2 | Local Memory | BAR2 | 752 *+-----------------+ +-----------------+ 753 *| BAR3 | | BAR3 | 754 *+-----------------+ +-----------------+ 755 *| BAR4 | | BAR4 | 756 *+-----------------+ +-----------------+ 757 *| BAR5 | | BAR5 | 758 *+-----------------+ +-----------------+ 759 * EP CONTROLLER 1 EP CONTROLLER 2 760 * 761 * Set BAR1 of EP CONTROLLER 2 which contains the HOST2's peer scratchpad 762 * region. While BAR1 is the default peer scratchpad BAR, an NTB could have 763 * other BARs for peer scratchpad (because of 64-bit BARs or reserved BARs). 764 * This function can get the exact BAR used for peer scratchpad from 765 * epf_ntb_bar[BAR_PEER_SPAD]. 766 * 767 * Since HOST2's peer scratchpad is also HOST1's self scratchpad, this function 768 * gets the address of peer scratchpad from 769 * peer_ntb_epc->epf_ntb_bar[BAR_CONFIG]. 770 */ 771 static int epf_ntb_peer_spad_bar_set(struct epf_ntb *ntb, 772 enum pci_epc_interface_type type) 773 { 774 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 775 struct pci_epf_bar *peer_epf_bar, *epf_bar; 776 enum pci_barno peer_barno, barno; 777 u32 peer_spad_offset; 778 struct pci_epc *epc; 779 struct device *dev; 780 u8 func_no; 781 int ret; 782 783 dev = &ntb->epf->dev; 784 785 peer_ntb_epc = ntb->epc[!type]; 786 peer_barno = peer_ntb_epc->epf_ntb_bar[BAR_CONFIG]; 787 peer_epf_bar = &peer_ntb_epc->epf_bar[peer_barno]; 788 789 ntb_epc = ntb->epc[type]; 790 barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; 791 epf_bar = &ntb_epc->epf_bar[barno]; 792 func_no = ntb_epc->func_no; 793 epc = ntb_epc->epc; 794 795 peer_spad_offset = peer_ntb_epc->reg->spad_offset; 796 epf_bar->phys_addr = peer_epf_bar->phys_addr + peer_spad_offset; 797 epf_bar->size = peer_ntb_epc->spad_size; 798 epf_bar->barno = barno; 799 epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32; 800 801 ret = pci_epc_set_bar(epc, func_no, epf_bar); 802 if (ret) { 803 dev_err(dev, "%s intf: peer SPAD BAR set failed\n", 804 pci_epc_interface_string(type)); 805 return ret; 806 } 807 808 return 0; 809 } 810 811 /** 812 * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR 813 * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound 814 * address. 815 * 816 * +-----------------+------->+------------------+ +-----------------+ 817 * | BAR0 | | CONFIG REGION | | BAR0 | 818 * +-----------------+----+ +------------------+<-------+-----------------+ 819 * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | 820 * +-----------------+ +-->+------------------+<-------+-----------------+ 821 * | BAR2 | Local Memory | BAR2 | 822 * +-----------------+ +-----------------+ 823 * | BAR3 | | BAR3 | 824 * +-----------------+ +-----------------+ 825 * | BAR4 | | BAR4 | 826 * +-----------------+ +-----------------+ 827 * | BAR5 | | BAR5 | 828 * +-----------------+ +-----------------+ 829 * EP CONTROLLER 1 EP CONTROLLER 2 830 * 831 * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and 832 * self scratchpad region (removes inbound ATU configuration). While BAR0 is 833 * the default self scratchpad BAR, an NTB could have other BARs for self 834 * scratchpad (because of reserved BARs). This function can get the exact BAR 835 * used for self scratchpad from epf_ntb_bar[BAR_CONFIG]. 836 * 837 * Please note the self scratchpad region and config region is combined to 838 * a single region and mapped using the same BAR. Also note HOST2's peer 839 * scratchpad is HOST1's self scratchpad. 840 */ 841 static void epf_ntb_config_sspad_bar_clear(struct epf_ntb_epc *ntb_epc) 842 { 843 struct pci_epf_bar *epf_bar; 844 enum pci_barno barno; 845 struct pci_epc *epc; 846 u8 func_no; 847 848 epc = ntb_epc->epc; 849 func_no = ntb_epc->func_no; 850 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; 851 epf_bar = &ntb_epc->epf_bar[barno]; 852 pci_epc_clear_bar(epc, func_no, epf_bar); 853 } 854 855 /** 856 * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR 857 * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound 858 * address. 859 * 860 * +-----------------+------->+------------------+ +-----------------+ 861 * | BAR0 | | CONFIG REGION | | BAR0 | 862 * +-----------------+----+ +------------------+<-------+-----------------+ 863 * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | 864 * +-----------------+ +-->+------------------+<-------+-----------------+ 865 * | BAR2 | Local Memory | BAR2 | 866 * +-----------------+ +-----------------+ 867 * | BAR3 | | BAR3 | 868 * +-----------------+ +-----------------+ 869 * | BAR4 | | BAR4 | 870 * +-----------------+ +-----------------+ 871 * | BAR5 | | BAR5 | 872 * +-----------------+ +-----------------+ 873 * EP CONTROLLER 1 EP CONTROLLER 2 874 * 875 * Map BAR0 of EP CONTROLLER 1 which contains the HOST1's config and 876 * self scratchpad region. While BAR0 is the default self scratchpad BAR, an 877 * NTB could have other BARs for self scratchpad (because of reserved BARs). 878 * This function can get the exact BAR used for self scratchpad from 879 * epf_ntb_bar[BAR_CONFIG]. 880 * 881 * Please note the self scratchpad region and config region is combined to 882 * a single region and mapped using the same BAR. Also note HOST2's peer 883 * scratchpad is HOST1's self scratchpad. 884 */ 885 static int epf_ntb_config_sspad_bar_set(struct epf_ntb_epc *ntb_epc) 886 { 887 struct pci_epf_bar *epf_bar; 888 enum pci_barno barno; 889 struct epf_ntb *ntb; 890 struct pci_epc *epc; 891 struct device *dev; 892 u8 func_no; 893 int ret; 894 895 ntb = ntb_epc->epf_ntb; 896 dev = &ntb->epf->dev; 897 898 epc = ntb_epc->epc; 899 func_no = ntb_epc->func_no; 900 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; 901 epf_bar = &ntb_epc->epf_bar[barno]; 902 903 ret = pci_epc_set_bar(epc, func_no, epf_bar); 904 if (ret) { 905 dev_err(dev, "%s inft: Config/Status/SPAD BAR set failed\n", 906 pci_epc_interface_string(ntb_epc->type)); 907 return ret; 908 } 909 910 return 0; 911 } 912 913 /** 914 * epf_ntb_config_spad_bar_free() - Free the physical memory associated with 915 * config + scratchpad region 916 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 917 * 918 * +-----------------+------->+------------------+ +-----------------+ 919 * | BAR0 | | CONFIG REGION | | BAR0 | 920 * +-----------------+----+ +------------------+<-------+-----------------+ 921 * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | 922 * +-----------------+ +-->+------------------+<-------+-----------------+ 923 * | BAR2 | Local Memory | BAR2 | 924 * +-----------------+ +-----------------+ 925 * | BAR3 | | BAR3 | 926 * +-----------------+ +-----------------+ 927 * | BAR4 | | BAR4 | 928 * +-----------------+ +-----------------+ 929 * | BAR5 | | BAR5 | 930 * +-----------------+ +-----------------+ 931 * EP CONTROLLER 1 EP CONTROLLER 2 932 * 933 * Free the Local Memory mentioned in the above diagram. After invoking this 934 * function, any of config + self scratchpad region of HOST1 or peer scratchpad 935 * region of HOST2 should not be accessed. 936 */ 937 static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb) 938 { 939 enum pci_epc_interface_type type; 940 struct epf_ntb_epc *ntb_epc; 941 enum pci_barno barno; 942 struct pci_epf *epf; 943 944 epf = ntb->epf; 945 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { 946 ntb_epc = ntb->epc[type]; 947 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; 948 if (ntb_epc->reg) 949 pci_epf_free_space(epf, ntb_epc->reg, barno, type); 950 } 951 } 952 953 /** 954 * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad 955 * region 956 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 957 * @type: PRIMARY interface or SECONDARY interface 958 * 959 * +-----------------+------->+------------------+ +-----------------+ 960 * | BAR0 | | CONFIG REGION | | BAR0 | 961 * +-----------------+----+ +------------------+<-------+-----------------+ 962 * | BAR1 | | |SCRATCHPAD REGION | | BAR1 | 963 * +-----------------+ +-->+------------------+<-------+-----------------+ 964 * | BAR2 | Local Memory | BAR2 | 965 * +-----------------+ +-----------------+ 966 * | BAR3 | | BAR3 | 967 * +-----------------+ +-----------------+ 968 * | BAR4 | | BAR4 | 969 * +-----------------+ +-----------------+ 970 * | BAR5 | | BAR5 | 971 * +-----------------+ +-----------------+ 972 * EP CONTROLLER 1 EP CONTROLLER 2 973 * 974 * Allocate the Local Memory mentioned in the above diagram. The size of 975 * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION 976 * is obtained from "spad-count" configfs entry. 977 * 978 * The size of both config region and scratchpad region has to be aligned, 979 * since the scratchpad region will also be mapped as PEER SCRATCHPAD of 980 * other host using a separate BAR. 981 */ 982 static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb, 983 enum pci_epc_interface_type type) 984 { 985 const struct pci_epc_features *peer_epc_features, *epc_features; 986 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 987 size_t msix_table_size, pba_size, align; 988 enum pci_barno peer_barno, barno; 989 struct epf_ntb_ctrl *ctrl; 990 u32 spad_size, ctrl_size; 991 u64 size, peer_size; 992 struct pci_epf *epf; 993 struct device *dev; 994 bool msix_capable; 995 u32 spad_count; 996 void *base; 997 998 epf = ntb->epf; 999 dev = &epf->dev; 1000 ntb_epc = ntb->epc[type]; 1001 1002 epc_features = ntb_epc->epc_features; 1003 barno = ntb_epc->epf_ntb_bar[BAR_CONFIG]; 1004 size = epc_features->bar_fixed_size[barno]; 1005 align = epc_features->align; 1006 1007 peer_ntb_epc = ntb->epc[!type]; 1008 peer_epc_features = peer_ntb_epc->epc_features; 1009 peer_barno = ntb_epc->epf_ntb_bar[BAR_PEER_SPAD]; 1010 peer_size = peer_epc_features->bar_fixed_size[peer_barno]; 1011 1012 /* Check if epc_features is populated incorrectly */ 1013 if ((!IS_ALIGNED(size, align))) 1014 return -EINVAL; 1015 1016 spad_count = ntb->spad_count; 1017 1018 ctrl_size = sizeof(struct epf_ntb_ctrl); 1019 spad_size = spad_count * 4; 1020 1021 msix_capable = epc_features->msix_capable; 1022 if (msix_capable) { 1023 msix_table_size = PCI_MSIX_ENTRY_SIZE * ntb->db_count; 1024 ctrl_size = ALIGN(ctrl_size, 8); 1025 ntb_epc->msix_table_offset = ctrl_size; 1026 ntb_epc->msix_bar = barno; 1027 /* Align to QWORD or 8 Bytes */ 1028 pba_size = ALIGN(DIV_ROUND_UP(ntb->db_count, 8), 8); 1029 ctrl_size = ctrl_size + msix_table_size + pba_size; 1030 } 1031 1032 if (!align) { 1033 ctrl_size = roundup_pow_of_two(ctrl_size); 1034 spad_size = roundup_pow_of_two(spad_size); 1035 } else { 1036 ctrl_size = ALIGN(ctrl_size, align); 1037 spad_size = ALIGN(spad_size, align); 1038 } 1039 1040 if (peer_size) { 1041 if (peer_size < spad_size) 1042 spad_count = peer_size / 4; 1043 spad_size = peer_size; 1044 } 1045 1046 /* 1047 * In order to make sure SPAD offset is aligned to its size, 1048 * expand control region size to the size of SPAD if SPAD size 1049 * is greater than control region size. 1050 */ 1051 if (spad_size > ctrl_size) 1052 ctrl_size = spad_size; 1053 1054 if (!size) 1055 size = ctrl_size + spad_size; 1056 else if (size < ctrl_size + spad_size) 1057 return -EINVAL; 1058 1059 base = pci_epf_alloc_space(epf, size, barno, align, type); 1060 if (!base) { 1061 dev_err(dev, "%s intf: Config/Status/SPAD alloc region fail\n", 1062 pci_epc_interface_string(type)); 1063 return -ENOMEM; 1064 } 1065 1066 ntb_epc->reg = base; 1067 1068 ctrl = ntb_epc->reg; 1069 ctrl->spad_offset = ctrl_size; 1070 ctrl->spad_count = spad_count; 1071 ctrl->num_mws = ntb->num_mws; 1072 ctrl->db_entry_size = align ? align : 4; 1073 ntb_epc->spad_size = spad_size; 1074 1075 return 0; 1076 } 1077 1078 /** 1079 * epf_ntb_config_spad_bar_alloc_interface() - Allocate memory for config + 1080 * scratchpad region for each of PRIMARY and SECONDARY interface 1081 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1082 * 1083 * Wrapper for epf_ntb_config_spad_bar_alloc() which allocates memory for 1084 * config + scratchpad region for a specific interface 1085 */ 1086 static int epf_ntb_config_spad_bar_alloc_interface(struct epf_ntb *ntb) 1087 { 1088 enum pci_epc_interface_type type; 1089 struct device *dev; 1090 int ret; 1091 1092 dev = &ntb->epf->dev; 1093 1094 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { 1095 ret = epf_ntb_config_spad_bar_alloc(ntb, type); 1096 if (ret) { 1097 dev_err(dev, "%s intf: Config/SPAD BAR alloc failed\n", 1098 pci_epc_interface_string(type)); 1099 return ret; 1100 } 1101 } 1102 1103 return 0; 1104 } 1105 1106 /** 1107 * epf_ntb_free_peer_mem() - Free memory allocated in peers outbound address 1108 * space 1109 * @ntb_epc: EPC associated with one of the HOST which holds peers outbound 1110 * address regions 1111 * 1112 * +-----------------+ +---->+----------------+-----------+-----------------+ 1113 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | 1114 * +-----------------+ | +----------------+ +-----------------+ 1115 * | BAR1 | | | Doorbell 2 +---------+ | | 1116 * +-----------------+----+ +----------------+ | | | 1117 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ 1118 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | 1119 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ 1120 * +-----------------+ | |----------------+ | | | | 1121 * | BAR4 | | | | | | +-----------------+ 1122 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| 1123 * | BAR5 | | | | | | +-----------------+ 1124 * +-----------------+ +---->-----------------+ | | | | 1125 * EP CONTROLLER 1 | | | | +-----------------+ 1126 * | | | +---->+ MSI|X ADDRESS 4 | 1127 * +----------------+ | +-----------------+ 1128 * (A) EP CONTROLLER 2 | | | 1129 * (OB SPACE) | | | 1130 * +-------> MW1 | 1131 * | | 1132 * | | 1133 * (B) +-----------------+ 1134 * | | 1135 * | | 1136 * | | 1137 * | | 1138 * | | 1139 * +-----------------+ 1140 * PCI Address Space 1141 * (Managed by HOST2) 1142 * 1143 * Free memory allocated in EP CONTROLLER 2 (OB SPACE) in the above diagram. 1144 * It'll free Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3, 1145 * MW4). 1146 */ 1147 static void epf_ntb_free_peer_mem(struct epf_ntb_epc *ntb_epc) 1148 { 1149 struct pci_epf_bar *epf_bar; 1150 void __iomem *mw_addr; 1151 phys_addr_t phys_addr; 1152 enum epf_ntb_bar bar; 1153 enum pci_barno barno; 1154 struct pci_epc *epc; 1155 size_t size; 1156 1157 epc = ntb_epc->epc; 1158 1159 for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) { 1160 barno = ntb_epc->epf_ntb_bar[bar]; 1161 mw_addr = ntb_epc->mw_addr[barno]; 1162 epf_bar = &ntb_epc->epf_bar[barno]; 1163 phys_addr = epf_bar->phys_addr; 1164 size = epf_bar->size; 1165 if (mw_addr) { 1166 pci_epc_mem_free_addr(epc, phys_addr, mw_addr, size); 1167 ntb_epc->mw_addr[barno] = NULL; 1168 } 1169 } 1170 } 1171 1172 /** 1173 * epf_ntb_db_mw_bar_clear() - Clear doorbell and memory BAR 1174 * @ntb_epc: EPC associated with one of the HOST which holds peer's outbound 1175 * address 1176 * 1177 * +-----------------+ +---->+----------------+-----------+-----------------+ 1178 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | 1179 * +-----------------+ | +----------------+ +-----------------+ 1180 * | BAR1 | | | Doorbell 2 +---------+ | | 1181 * +-----------------+----+ +----------------+ | | | 1182 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ 1183 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | 1184 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ 1185 * +-----------------+ | |----------------+ | | | | 1186 * | BAR4 | | | | | | +-----------------+ 1187 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| 1188 * | BAR5 | | | | | | +-----------------+ 1189 * +-----------------+ +---->-----------------+ | | | | 1190 * EP CONTROLLER 1 | | | | +-----------------+ 1191 * | | | +---->+ MSI|X ADDRESS 4 | 1192 * +----------------+ | +-----------------+ 1193 * (A) EP CONTROLLER 2 | | | 1194 * (OB SPACE) | | | 1195 * +-------> MW1 | 1196 * | | 1197 * | | 1198 * (B) +-----------------+ 1199 * | | 1200 * | | 1201 * | | 1202 * | | 1203 * | | 1204 * +-----------------+ 1205 * PCI Address Space 1206 * (Managed by HOST2) 1207 * 1208 * Clear doorbell and memory BARs (remove inbound ATU configuration). In the above 1209 * diagram it clears BAR2 TO BAR5 of EP CONTROLLER 1 (Doorbell BAR, MW1 BAR, MW2 1210 * BAR, MW3 BAR and MW4 BAR). 1211 */ 1212 static void epf_ntb_db_mw_bar_clear(struct epf_ntb_epc *ntb_epc) 1213 { 1214 struct pci_epf_bar *epf_bar; 1215 enum epf_ntb_bar bar; 1216 enum pci_barno barno; 1217 struct pci_epc *epc; 1218 u8 func_no; 1219 1220 epc = ntb_epc->epc; 1221 1222 func_no = ntb_epc->func_no; 1223 1224 for (bar = BAR_DB_MW1; bar < BAR_MW4; bar++) { 1225 barno = ntb_epc->epf_ntb_bar[bar]; 1226 epf_bar = &ntb_epc->epf_bar[barno]; 1227 pci_epc_clear_bar(epc, func_no, epf_bar); 1228 } 1229 } 1230 1231 /** 1232 * epf_ntb_db_mw_bar_cleanup() - Clear doorbell/memory BAR and free memory 1233 * allocated in peers outbound address space 1234 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1235 * @type: PRIMARY interface or SECONDARY interface 1236 * 1237 * Wrapper for epf_ntb_db_mw_bar_clear() to clear HOST1's BAR and 1238 * epf_ntb_free_peer_mem() which frees up HOST2 outbound memory. 1239 */ 1240 static void epf_ntb_db_mw_bar_cleanup(struct epf_ntb *ntb, 1241 enum pci_epc_interface_type type) 1242 { 1243 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 1244 1245 ntb_epc = ntb->epc[type]; 1246 peer_ntb_epc = ntb->epc[!type]; 1247 1248 epf_ntb_db_mw_bar_clear(ntb_epc); 1249 epf_ntb_free_peer_mem(peer_ntb_epc); 1250 } 1251 1252 /** 1253 * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capaiblity 1254 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1255 * @type: PRIMARY interface or SECONDARY interface 1256 * 1257 * Configure MSI/MSI-X capability for each interface with number of 1258 * interrupts equal to "db_count" configfs entry. 1259 */ 1260 static int epf_ntb_configure_interrupt(struct epf_ntb *ntb, 1261 enum pci_epc_interface_type type) 1262 { 1263 const struct pci_epc_features *epc_features; 1264 bool msix_capable, msi_capable; 1265 struct epf_ntb_epc *ntb_epc; 1266 struct pci_epc *epc; 1267 struct device *dev; 1268 u32 db_count; 1269 u8 func_no; 1270 int ret; 1271 1272 ntb_epc = ntb->epc[type]; 1273 dev = &ntb->epf->dev; 1274 1275 epc_features = ntb_epc->epc_features; 1276 msix_capable = epc_features->msix_capable; 1277 msi_capable = epc_features->msi_capable; 1278 1279 if (!(msix_capable || msi_capable)) { 1280 dev_err(dev, "MSI or MSI-X is required for doorbell\n"); 1281 return -EINVAL; 1282 } 1283 1284 func_no = ntb_epc->func_no; 1285 1286 db_count = ntb->db_count; 1287 if (db_count > MAX_DB_COUNT) { 1288 dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT); 1289 return -EINVAL; 1290 } 1291 1292 ntb->db_count = db_count; 1293 epc = ntb_epc->epc; 1294 1295 if (msi_capable) { 1296 ret = pci_epc_set_msi(epc, func_no, db_count); 1297 if (ret) { 1298 dev_err(dev, "%s intf: MSI configuration failed\n", 1299 pci_epc_interface_string(type)); 1300 return ret; 1301 } 1302 } 1303 1304 if (msix_capable) { 1305 ret = pci_epc_set_msix(epc, func_no, db_count, 1306 ntb_epc->msix_bar, 1307 ntb_epc->msix_table_offset); 1308 if (ret) { 1309 dev_err(dev, "MSI configuration failed\n"); 1310 return ret; 1311 } 1312 } 1313 1314 return 0; 1315 } 1316 1317 /** 1318 * epf_ntb_alloc_peer_mem() - Allocate memory in peer's outbound address space 1319 * @dev: The PCI device. 1320 * @ntb_epc: EPC associated with one of the HOST whose BAR holds peer's outbound 1321 * address 1322 * @bar: BAR of @ntb_epc in for which memory has to be allocated (could be 1323 * BAR_DB_MW1, BAR_MW2, BAR_MW3, BAR_MW4) 1324 * @peer_ntb_epc: EPC associated with HOST whose outbound address space is 1325 * used by @ntb_epc 1326 * @size: Size of the address region that has to be allocated in peers OB SPACE 1327 * 1328 * 1329 * +-----------------+ +---->+----------------+-----------+-----------------+ 1330 * | BAR0 | | | Doorbell 1 +-----------> MSI|X ADDRESS 1 | 1331 * +-----------------+ | +----------------+ +-----------------+ 1332 * | BAR1 | | | Doorbell 2 +---------+ | | 1333 * +-----------------+----+ +----------------+ | | | 1334 * | BAR2 | | Doorbell 3 +-------+ | +-----------------+ 1335 * +-----------------+----+ +----------------+ | +-> MSI|X ADDRESS 2 | 1336 * | BAR3 | | | Doorbell 4 +-----+ | +-----------------+ 1337 * +-----------------+ | |----------------+ | | | | 1338 * | BAR4 | | | | | | +-----------------+ 1339 * +-----------------+ | | MW1 +---+ | +-->+ MSI|X ADDRESS 3|| 1340 * | BAR5 | | | | | | +-----------------+ 1341 * +-----------------+ +---->-----------------+ | | | | 1342 * EP CONTROLLER 1 | | | | +-----------------+ 1343 * | | | +---->+ MSI|X ADDRESS 4 | 1344 * +----------------+ | +-----------------+ 1345 * (A) EP CONTROLLER 2 | | | 1346 * (OB SPACE) | | | 1347 * +-------> MW1 | 1348 * | | 1349 * | | 1350 * (B) +-----------------+ 1351 * | | 1352 * | | 1353 * | | 1354 * | | 1355 * | | 1356 * +-----------------+ 1357 * PCI Address Space 1358 * (Managed by HOST2) 1359 * 1360 * Allocate memory in OB space of EP CONTROLLER 2 in the above diagram. Allocate 1361 * for Doorbell 1, Doorbell 2, Doorbell 3, Doorbell 4, MW1 (and MW2, MW3, MW4). 1362 */ 1363 static int epf_ntb_alloc_peer_mem(struct device *dev, 1364 struct epf_ntb_epc *ntb_epc, 1365 enum epf_ntb_bar bar, 1366 struct epf_ntb_epc *peer_ntb_epc, 1367 size_t size) 1368 { 1369 const struct pci_epc_features *epc_features; 1370 struct pci_epf_bar *epf_bar; 1371 struct pci_epc *peer_epc; 1372 phys_addr_t phys_addr; 1373 void __iomem *mw_addr; 1374 enum pci_barno barno; 1375 size_t align; 1376 1377 epc_features = ntb_epc->epc_features; 1378 align = epc_features->align; 1379 1380 if (size < 128) 1381 size = 128; 1382 1383 if (align) 1384 size = ALIGN(size, align); 1385 else 1386 size = roundup_pow_of_two(size); 1387 1388 peer_epc = peer_ntb_epc->epc; 1389 mw_addr = pci_epc_mem_alloc_addr(peer_epc, &phys_addr, size); 1390 if (!mw_addr) { 1391 dev_err(dev, "%s intf: Failed to allocate OB address\n", 1392 pci_epc_interface_string(peer_ntb_epc->type)); 1393 return -ENOMEM; 1394 } 1395 1396 barno = ntb_epc->epf_ntb_bar[bar]; 1397 epf_bar = &ntb_epc->epf_bar[barno]; 1398 ntb_epc->mw_addr[barno] = mw_addr; 1399 1400 epf_bar->phys_addr = phys_addr; 1401 epf_bar->size = size; 1402 epf_bar->barno = barno; 1403 epf_bar->flags = PCI_BASE_ADDRESS_MEM_TYPE_32; 1404 1405 return 0; 1406 } 1407 1408 /** 1409 * epf_ntb_db_mw_bar_init() - Configure Doorbell and Memory window BARs 1410 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1411 * @type: PRIMARY interface or SECONDARY interface 1412 * 1413 * Wrapper for epf_ntb_alloc_peer_mem() and pci_epc_set_bar() that allocates 1414 * memory in OB address space of HOST2 and configures BAR of HOST1 1415 */ 1416 static int epf_ntb_db_mw_bar_init(struct epf_ntb *ntb, 1417 enum pci_epc_interface_type type) 1418 { 1419 const struct pci_epc_features *epc_features; 1420 struct epf_ntb_epc *peer_ntb_epc, *ntb_epc; 1421 struct pci_epf_bar *epf_bar; 1422 struct epf_ntb_ctrl *ctrl; 1423 u32 num_mws, db_count; 1424 enum epf_ntb_bar bar; 1425 enum pci_barno barno; 1426 struct pci_epc *epc; 1427 struct device *dev; 1428 size_t align; 1429 int ret, i; 1430 u8 func_no; 1431 u64 size; 1432 1433 ntb_epc = ntb->epc[type]; 1434 peer_ntb_epc = ntb->epc[!type]; 1435 1436 dev = &ntb->epf->dev; 1437 epc_features = ntb_epc->epc_features; 1438 align = epc_features->align; 1439 func_no = ntb_epc->func_no; 1440 epc = ntb_epc->epc; 1441 num_mws = ntb->num_mws; 1442 db_count = ntb->db_count; 1443 1444 for (bar = BAR_DB_MW1, i = 0; i < num_mws; bar++, i++) { 1445 if (bar == BAR_DB_MW1) { 1446 align = align ? align : 4; 1447 size = db_count * align; 1448 size = ALIGN(size, ntb->mws_size[i]); 1449 ctrl = ntb_epc->reg; 1450 ctrl->mw1_offset = size; 1451 size += ntb->mws_size[i]; 1452 } else { 1453 size = ntb->mws_size[i]; 1454 } 1455 1456 ret = epf_ntb_alloc_peer_mem(dev, ntb_epc, bar, 1457 peer_ntb_epc, size); 1458 if (ret) { 1459 dev_err(dev, "%s intf: DoorBell mem alloc failed\n", 1460 pci_epc_interface_string(type)); 1461 goto err_alloc_peer_mem; 1462 } 1463 1464 barno = ntb_epc->epf_ntb_bar[bar]; 1465 epf_bar = &ntb_epc->epf_bar[barno]; 1466 1467 ret = pci_epc_set_bar(epc, func_no, epf_bar); 1468 if (ret) { 1469 dev_err(dev, "%s intf: DoorBell BAR set failed\n", 1470 pci_epc_interface_string(type)); 1471 goto err_alloc_peer_mem; 1472 } 1473 } 1474 1475 return 0; 1476 1477 err_alloc_peer_mem: 1478 epf_ntb_db_mw_bar_cleanup(ntb, type); 1479 1480 return ret; 1481 } 1482 1483 /** 1484 * epf_ntb_epc_destroy_interface() - Cleanup NTB EPC interface 1485 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1486 * @type: PRIMARY interface or SECONDARY interface 1487 * 1488 * Unbind NTB function device from EPC and relinquish reference to pci_epc 1489 * for each of the interface. 1490 */ 1491 static void epf_ntb_epc_destroy_interface(struct epf_ntb *ntb, 1492 enum pci_epc_interface_type type) 1493 { 1494 struct epf_ntb_epc *ntb_epc; 1495 struct pci_epc *epc; 1496 struct pci_epf *epf; 1497 1498 if (type < 0) 1499 return; 1500 1501 epf = ntb->epf; 1502 ntb_epc = ntb->epc[type]; 1503 if (!ntb_epc) 1504 return; 1505 epc = ntb_epc->epc; 1506 pci_epc_remove_epf(epc, epf, type); 1507 pci_epc_put(epc); 1508 } 1509 1510 /** 1511 * epf_ntb_epc_destroy() - Cleanup NTB EPC interface 1512 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1513 * 1514 * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces 1515 */ 1516 static void epf_ntb_epc_destroy(struct epf_ntb *ntb) 1517 { 1518 enum pci_epc_interface_type type; 1519 1520 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) 1521 epf_ntb_epc_destroy_interface(ntb, type); 1522 } 1523 1524 /** 1525 * epf_ntb_epc_create_interface() - Create and initialize NTB EPC interface 1526 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1527 * @epc: struct pci_epc to which a particular NTB interface should be associated 1528 * @type: PRIMARY interface or SECONDARY interface 1529 * 1530 * Allocate memory for NTB EPC interface and initialize it. 1531 */ 1532 static int epf_ntb_epc_create_interface(struct epf_ntb *ntb, 1533 struct pci_epc *epc, 1534 enum pci_epc_interface_type type) 1535 { 1536 const struct pci_epc_features *epc_features; 1537 struct pci_epf_bar *epf_bar; 1538 struct epf_ntb_epc *ntb_epc; 1539 struct pci_epf *epf; 1540 struct device *dev; 1541 u8 func_no; 1542 1543 dev = &ntb->epf->dev; 1544 1545 ntb_epc = devm_kzalloc(dev, sizeof(*ntb_epc), GFP_KERNEL); 1546 if (!ntb_epc) 1547 return -ENOMEM; 1548 1549 epf = ntb->epf; 1550 if (type == PRIMARY_INTERFACE) { 1551 func_no = epf->func_no; 1552 epf_bar = epf->bar; 1553 } else { 1554 func_no = epf->sec_epc_func_no; 1555 epf_bar = epf->sec_epc_bar; 1556 } 1557 1558 ntb_epc->linkup = false; 1559 ntb_epc->epc = epc; 1560 ntb_epc->func_no = func_no; 1561 ntb_epc->type = type; 1562 ntb_epc->epf_bar = epf_bar; 1563 ntb_epc->epf_ntb = ntb; 1564 1565 epc_features = pci_epc_get_features(epc, func_no); 1566 if (!epc_features) 1567 return -EINVAL; 1568 ntb_epc->epc_features = epc_features; 1569 1570 ntb->epc[type] = ntb_epc; 1571 1572 return 0; 1573 } 1574 1575 /** 1576 * epf_ntb_epc_create() - Create and initialize NTB EPC interface 1577 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1578 * 1579 * Get a reference to EPC device and bind NTB function device to that EPC 1580 * for each of the interface. It is also a wrapper to 1581 * epf_ntb_epc_create_interface() to allocate memory for NTB EPC interface 1582 * and initialize it 1583 */ 1584 static int epf_ntb_epc_create(struct epf_ntb *ntb) 1585 { 1586 struct pci_epf *epf; 1587 struct device *dev; 1588 int ret; 1589 1590 epf = ntb->epf; 1591 dev = &epf->dev; 1592 1593 ret = epf_ntb_epc_create_interface(ntb, epf->epc, PRIMARY_INTERFACE); 1594 if (ret) { 1595 dev_err(dev, "PRIMARY intf: Fail to create NTB EPC\n"); 1596 return ret; 1597 } 1598 1599 ret = epf_ntb_epc_create_interface(ntb, epf->sec_epc, 1600 SECONDARY_INTERFACE); 1601 if (ret) { 1602 dev_err(dev, "SECONDARY intf: Fail to create NTB EPC\n"); 1603 goto err_epc_create; 1604 } 1605 1606 return 0; 1607 1608 err_epc_create: 1609 epf_ntb_epc_destroy_interface(ntb, PRIMARY_INTERFACE); 1610 1611 return ret; 1612 } 1613 1614 /** 1615 * epf_ntb_init_epc_bar_interface() - Identify BARs to be used for each of 1616 * the NTB constructs (scratchpad region, doorbell, memorywindow) 1617 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1618 * @type: PRIMARY interface or SECONDARY interface 1619 * 1620 * Identify the free BARs to be used for each of BAR_CONFIG, BAR_PEER_SPAD, 1621 * BAR_DB_MW1, BAR_MW2, BAR_MW3 and BAR_MW4. 1622 */ 1623 static int epf_ntb_init_epc_bar_interface(struct epf_ntb *ntb, 1624 enum pci_epc_interface_type type) 1625 { 1626 const struct pci_epc_features *epc_features; 1627 struct epf_ntb_epc *ntb_epc; 1628 enum pci_barno barno; 1629 enum epf_ntb_bar bar; 1630 struct device *dev; 1631 u32 num_mws; 1632 int i; 1633 1634 barno = BAR_0; 1635 ntb_epc = ntb->epc[type]; 1636 num_mws = ntb->num_mws; 1637 dev = &ntb->epf->dev; 1638 epc_features = ntb_epc->epc_features; 1639 1640 /* These are required BARs which are mandatory for NTB functionality */ 1641 for (bar = BAR_CONFIG; bar <= BAR_DB_MW1; bar++, barno++) { 1642 barno = pci_epc_get_next_free_bar(epc_features, barno); 1643 if (barno < 0) { 1644 dev_err(dev, "%s intf: Fail to get NTB function BAR\n", 1645 pci_epc_interface_string(type)); 1646 return barno; 1647 } 1648 ntb_epc->epf_ntb_bar[bar] = barno; 1649 } 1650 1651 /* These are optional BARs which don't impact NTB functionality */ 1652 for (bar = BAR_MW2, i = 1; i < num_mws; bar++, barno++, i++) { 1653 barno = pci_epc_get_next_free_bar(epc_features, barno); 1654 if (barno < 0) { 1655 ntb->num_mws = i; 1656 dev_dbg(dev, "BAR not available for > MW%d\n", i + 1); 1657 } 1658 ntb_epc->epf_ntb_bar[bar] = barno; 1659 } 1660 1661 return 0; 1662 } 1663 1664 /** 1665 * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB 1666 * constructs (scratchpad region, doorbell, memorywindow) 1667 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1668 * 1669 * Wrapper to epf_ntb_init_epc_bar_interface() to identify the free BARs 1670 * to be used for each of BAR_CONFIG, BAR_PEER_SPAD, BAR_DB_MW1, BAR_MW2, 1671 * BAR_MW3 and BAR_MW4 for all the interfaces. 1672 */ 1673 static int epf_ntb_init_epc_bar(struct epf_ntb *ntb) 1674 { 1675 enum pci_epc_interface_type type; 1676 struct device *dev; 1677 int ret; 1678 1679 dev = &ntb->epf->dev; 1680 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { 1681 ret = epf_ntb_init_epc_bar_interface(ntb, type); 1682 if (ret) { 1683 dev_err(dev, "Fail to init EPC bar for %s interface\n", 1684 pci_epc_interface_string(type)); 1685 return ret; 1686 } 1687 } 1688 1689 return 0; 1690 } 1691 1692 /** 1693 * epf_ntb_epc_init_interface() - Initialize NTB interface 1694 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1695 * @type: PRIMARY interface or SECONDARY interface 1696 * 1697 * Wrapper to initialize a particular EPC interface and start the workqueue 1698 * to check for commands from host. This function will write to the 1699 * EP controller HW for configuring it. 1700 */ 1701 static int epf_ntb_epc_init_interface(struct epf_ntb *ntb, 1702 enum pci_epc_interface_type type) 1703 { 1704 struct epf_ntb_epc *ntb_epc; 1705 struct pci_epc *epc; 1706 struct pci_epf *epf; 1707 struct device *dev; 1708 u8 func_no; 1709 int ret; 1710 1711 ntb_epc = ntb->epc[type]; 1712 epf = ntb->epf; 1713 dev = &epf->dev; 1714 epc = ntb_epc->epc; 1715 func_no = ntb_epc->func_no; 1716 1717 ret = epf_ntb_config_sspad_bar_set(ntb->epc[type]); 1718 if (ret) { 1719 dev_err(dev, "%s intf: Config/self SPAD BAR init failed\n", 1720 pci_epc_interface_string(type)); 1721 return ret; 1722 } 1723 1724 ret = epf_ntb_peer_spad_bar_set(ntb, type); 1725 if (ret) { 1726 dev_err(dev, "%s intf: Peer SPAD BAR init failed\n", 1727 pci_epc_interface_string(type)); 1728 goto err_peer_spad_bar_init; 1729 } 1730 1731 ret = epf_ntb_configure_interrupt(ntb, type); 1732 if (ret) { 1733 dev_err(dev, "%s intf: Interrupt configuration failed\n", 1734 pci_epc_interface_string(type)); 1735 goto err_peer_spad_bar_init; 1736 } 1737 1738 ret = epf_ntb_db_mw_bar_init(ntb, type); 1739 if (ret) { 1740 dev_err(dev, "%s intf: DB/MW BAR init failed\n", 1741 pci_epc_interface_string(type)); 1742 goto err_db_mw_bar_init; 1743 } 1744 1745 ret = pci_epc_write_header(epc, func_no, epf->header); 1746 if (ret) { 1747 dev_err(dev, "%s intf: Configuration header write failed\n", 1748 pci_epc_interface_string(type)); 1749 goto err_write_header; 1750 } 1751 1752 INIT_DELAYED_WORK(&ntb->epc[type]->cmd_handler, epf_ntb_cmd_handler); 1753 queue_work(kpcintb_workqueue, &ntb->epc[type]->cmd_handler.work); 1754 1755 return 0; 1756 1757 err_write_header: 1758 epf_ntb_db_mw_bar_cleanup(ntb, type); 1759 1760 err_db_mw_bar_init: 1761 epf_ntb_peer_spad_bar_clear(ntb->epc[type]); 1762 1763 err_peer_spad_bar_init: 1764 epf_ntb_config_sspad_bar_clear(ntb->epc[type]); 1765 1766 return ret; 1767 } 1768 1769 /** 1770 * epf_ntb_epc_cleanup_interface() - Cleanup NTB interface 1771 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1772 * @type: PRIMARY interface or SECONDARY interface 1773 * 1774 * Wrapper to cleanup a particular NTB interface. 1775 */ 1776 static void epf_ntb_epc_cleanup_interface(struct epf_ntb *ntb, 1777 enum pci_epc_interface_type type) 1778 { 1779 struct epf_ntb_epc *ntb_epc; 1780 1781 if (type < 0) 1782 return; 1783 1784 ntb_epc = ntb->epc[type]; 1785 cancel_delayed_work(&ntb_epc->cmd_handler); 1786 epf_ntb_db_mw_bar_cleanup(ntb, type); 1787 epf_ntb_peer_spad_bar_clear(ntb_epc); 1788 epf_ntb_config_sspad_bar_clear(ntb_epc); 1789 } 1790 1791 /** 1792 * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces 1793 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1794 * 1795 * Wrapper to cleanup all NTB interfaces. 1796 */ 1797 static void epf_ntb_epc_cleanup(struct epf_ntb *ntb) 1798 { 1799 enum pci_epc_interface_type type; 1800 1801 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) 1802 epf_ntb_epc_cleanup_interface(ntb, type); 1803 } 1804 1805 /** 1806 * epf_ntb_epc_init() - Initialize all NTB interfaces 1807 * @ntb: NTB device that facilitates communication between HOST1 and HOST2 1808 * 1809 * Wrapper to initialize all NTB interface and start the workqueue 1810 * to check for commands from host. 1811 */ 1812 static int epf_ntb_epc_init(struct epf_ntb *ntb) 1813 { 1814 enum pci_epc_interface_type type; 1815 struct device *dev; 1816 int ret; 1817 1818 dev = &ntb->epf->dev; 1819 1820 for (type = PRIMARY_INTERFACE; type <= SECONDARY_INTERFACE; type++) { 1821 ret = epf_ntb_epc_init_interface(ntb, type); 1822 if (ret) { 1823 dev_err(dev, "%s intf: Failed to initialize\n", 1824 pci_epc_interface_string(type)); 1825 goto err_init_type; 1826 } 1827 } 1828 1829 return 0; 1830 1831 err_init_type: 1832 epf_ntb_epc_cleanup_interface(ntb, type - 1); 1833 1834 return ret; 1835 } 1836 1837 /** 1838 * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality 1839 * @epf: NTB endpoint function device 1840 * 1841 * Initialize both the endpoint controllers associated with NTB function device. 1842 * Invoked when a primary interface or secondary interface is bound to EPC 1843 * device. This function will succeed only when EPC is bound to both the 1844 * interfaces. 1845 */ 1846 static int epf_ntb_bind(struct pci_epf *epf) 1847 { 1848 struct epf_ntb *ntb = epf_get_drvdata(epf); 1849 struct device *dev = &epf->dev; 1850 int ret; 1851 1852 if (!epf->epc) { 1853 dev_dbg(dev, "PRIMARY EPC interface not yet bound\n"); 1854 return 0; 1855 } 1856 1857 if (!epf->sec_epc) { 1858 dev_dbg(dev, "SECONDARY EPC interface not yet bound\n"); 1859 return 0; 1860 } 1861 1862 ret = epf_ntb_epc_create(ntb); 1863 if (ret) { 1864 dev_err(dev, "Failed to create NTB EPC\n"); 1865 return ret; 1866 } 1867 1868 ret = epf_ntb_init_epc_bar(ntb); 1869 if (ret) { 1870 dev_err(dev, "Failed to create NTB EPC\n"); 1871 goto err_bar_init; 1872 } 1873 1874 ret = epf_ntb_config_spad_bar_alloc_interface(ntb); 1875 if (ret) { 1876 dev_err(dev, "Failed to allocate BAR memory\n"); 1877 goto err_bar_alloc; 1878 } 1879 1880 ret = epf_ntb_epc_init(ntb); 1881 if (ret) { 1882 dev_err(dev, "Failed to initialize EPC\n"); 1883 goto err_bar_alloc; 1884 } 1885 1886 epf_set_drvdata(epf, ntb); 1887 1888 return 0; 1889 1890 err_bar_alloc: 1891 epf_ntb_config_spad_bar_free(ntb); 1892 1893 err_bar_init: 1894 epf_ntb_epc_destroy(ntb); 1895 1896 return ret; 1897 } 1898 1899 /** 1900 * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind() 1901 * @epf: NTB endpoint function device 1902 * 1903 * Cleanup the initialization from epf_ntb_bind() 1904 */ 1905 static void epf_ntb_unbind(struct pci_epf *epf) 1906 { 1907 struct epf_ntb *ntb = epf_get_drvdata(epf); 1908 1909 epf_ntb_epc_cleanup(ntb); 1910 epf_ntb_config_spad_bar_free(ntb); 1911 epf_ntb_epc_destroy(ntb); 1912 } 1913 1914 #define EPF_NTB_R(_name) \ 1915 static ssize_t epf_ntb_##_name##_show(struct config_item *item, \ 1916 char *page) \ 1917 { \ 1918 struct config_group *group = to_config_group(item); \ 1919 struct epf_ntb *ntb = to_epf_ntb(group); \ 1920 \ 1921 return sprintf(page, "%d\n", ntb->_name); \ 1922 } 1923 1924 #define EPF_NTB_W(_name) \ 1925 static ssize_t epf_ntb_##_name##_store(struct config_item *item, \ 1926 const char *page, size_t len) \ 1927 { \ 1928 struct config_group *group = to_config_group(item); \ 1929 struct epf_ntb *ntb = to_epf_ntb(group); \ 1930 u32 val; \ 1931 int ret; \ 1932 \ 1933 ret = kstrtou32(page, 0, &val); \ 1934 if (ret) \ 1935 return ret; \ 1936 \ 1937 ntb->_name = val; \ 1938 \ 1939 return len; \ 1940 } 1941 1942 #define EPF_NTB_MW_R(_name) \ 1943 static ssize_t epf_ntb_##_name##_show(struct config_item *item, \ 1944 char *page) \ 1945 { \ 1946 struct config_group *group = to_config_group(item); \ 1947 struct epf_ntb *ntb = to_epf_ntb(group); \ 1948 int win_no; \ 1949 \ 1950 sscanf(#_name, "mw%d", &win_no); \ 1951 \ 1952 return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]); \ 1953 } 1954 1955 #define EPF_NTB_MW_W(_name) \ 1956 static ssize_t epf_ntb_##_name##_store(struct config_item *item, \ 1957 const char *page, size_t len) \ 1958 { \ 1959 struct config_group *group = to_config_group(item); \ 1960 struct epf_ntb *ntb = to_epf_ntb(group); \ 1961 struct device *dev = &ntb->epf->dev; \ 1962 int win_no; \ 1963 u64 val; \ 1964 int ret; \ 1965 \ 1966 ret = kstrtou64(page, 0, &val); \ 1967 if (ret) \ 1968 return ret; \ 1969 \ 1970 if (sscanf(#_name, "mw%d", &win_no) != 1) \ 1971 return -EINVAL; \ 1972 \ 1973 if (ntb->num_mws < win_no) { \ 1974 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \ 1975 return -EINVAL; \ 1976 } \ 1977 \ 1978 ntb->mws_size[win_no - 1] = val; \ 1979 \ 1980 return len; \ 1981 } 1982 1983 static ssize_t epf_ntb_num_mws_store(struct config_item *item, 1984 const char *page, size_t len) 1985 { 1986 struct config_group *group = to_config_group(item); 1987 struct epf_ntb *ntb = to_epf_ntb(group); 1988 u32 val; 1989 int ret; 1990 1991 ret = kstrtou32(page, 0, &val); 1992 if (ret) 1993 return ret; 1994 1995 if (val > MAX_MW) 1996 return -EINVAL; 1997 1998 ntb->num_mws = val; 1999 2000 return len; 2001 } 2002 2003 EPF_NTB_R(spad_count) 2004 EPF_NTB_W(spad_count) 2005 EPF_NTB_R(db_count) 2006 EPF_NTB_W(db_count) 2007 EPF_NTB_R(num_mws) 2008 EPF_NTB_MW_R(mw1) 2009 EPF_NTB_MW_W(mw1) 2010 EPF_NTB_MW_R(mw2) 2011 EPF_NTB_MW_W(mw2) 2012 EPF_NTB_MW_R(mw3) 2013 EPF_NTB_MW_W(mw3) 2014 EPF_NTB_MW_R(mw4) 2015 EPF_NTB_MW_W(mw4) 2016 2017 CONFIGFS_ATTR(epf_ntb_, spad_count); 2018 CONFIGFS_ATTR(epf_ntb_, db_count); 2019 CONFIGFS_ATTR(epf_ntb_, num_mws); 2020 CONFIGFS_ATTR(epf_ntb_, mw1); 2021 CONFIGFS_ATTR(epf_ntb_, mw2); 2022 CONFIGFS_ATTR(epf_ntb_, mw3); 2023 CONFIGFS_ATTR(epf_ntb_, mw4); 2024 2025 static struct configfs_attribute *epf_ntb_attrs[] = { 2026 &epf_ntb_attr_spad_count, 2027 &epf_ntb_attr_db_count, 2028 &epf_ntb_attr_num_mws, 2029 &epf_ntb_attr_mw1, 2030 &epf_ntb_attr_mw2, 2031 &epf_ntb_attr_mw3, 2032 &epf_ntb_attr_mw4, 2033 NULL, 2034 }; 2035 2036 static const struct config_item_type ntb_group_type = { 2037 .ct_attrs = epf_ntb_attrs, 2038 .ct_owner = THIS_MODULE, 2039 }; 2040 2041 /** 2042 * epf_ntb_add_cfs() - Add configfs directory specific to NTB 2043 * @epf: NTB endpoint function device 2044 * @group: A pointer to the config_group structure referencing a group of 2045 * config_items of a specific type that belong to a specific sub-system. 2046 * 2047 * Add configfs directory specific to NTB. This directory will hold 2048 * NTB specific properties like db_count, spad_count, num_mws etc., 2049 */ 2050 static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf, 2051 struct config_group *group) 2052 { 2053 struct epf_ntb *ntb = epf_get_drvdata(epf); 2054 struct config_group *ntb_group = &ntb->group; 2055 struct device *dev = &epf->dev; 2056 2057 config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type); 2058 2059 return ntb_group; 2060 } 2061 2062 /** 2063 * epf_ntb_probe() - Probe NTB function driver 2064 * @epf: NTB endpoint function device 2065 * 2066 * Probe NTB function driver when endpoint function bus detects a NTB 2067 * endpoint function. 2068 */ 2069 static int epf_ntb_probe(struct pci_epf *epf) 2070 { 2071 struct epf_ntb *ntb; 2072 struct device *dev; 2073 2074 dev = &epf->dev; 2075 2076 ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL); 2077 if (!ntb) 2078 return -ENOMEM; 2079 2080 epf->header = &epf_ntb_header; 2081 ntb->epf = epf; 2082 epf_set_drvdata(epf, ntb); 2083 2084 return 0; 2085 } 2086 2087 static struct pci_epf_ops epf_ntb_ops = { 2088 .bind = epf_ntb_bind, 2089 .unbind = epf_ntb_unbind, 2090 .add_cfs = epf_ntb_add_cfs, 2091 }; 2092 2093 static const struct pci_epf_device_id epf_ntb_ids[] = { 2094 { 2095 .name = "pci_epf_ntb", 2096 }, 2097 {}, 2098 }; 2099 2100 static struct pci_epf_driver epf_ntb_driver = { 2101 .driver.name = "pci_epf_ntb", 2102 .probe = epf_ntb_probe, 2103 .id_table = epf_ntb_ids, 2104 .ops = &epf_ntb_ops, 2105 .owner = THIS_MODULE, 2106 }; 2107 2108 static int __init epf_ntb_init(void) 2109 { 2110 int ret; 2111 2112 kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM | 2113 WQ_HIGHPRI, 0); 2114 ret = pci_epf_register_driver(&epf_ntb_driver); 2115 if (ret) { 2116 destroy_workqueue(kpcintb_workqueue); 2117 pr_err("Failed to register pci epf ntb driver --> %d\n", ret); 2118 return ret; 2119 } 2120 2121 return 0; 2122 } 2123 module_init(epf_ntb_init); 2124 2125 static void __exit epf_ntb_exit(void) 2126 { 2127 pci_epf_unregister_driver(&epf_ntb_driver); 2128 destroy_workqueue(kpcintb_workqueue); 2129 } 2130 module_exit(epf_ntb_exit); 2131 2132 MODULE_DESCRIPTION("PCI EPF NTB DRIVER"); 2133 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>"); 2134 MODULE_LICENSE("GPL v2"); 2135