1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Endpoint Function Driver to implement Non-Transparent Bridge functionality
4 * Between PCI RC and EP
5 *
6 * Copyright (C) 2020 Texas Instruments
7 * Copyright (C) 2022 NXP
8 *
9 * Based on pci-epf-ntb.c
10 * Author: Frank Li <Frank.Li@nxp.com>
11 * Author: Kishon Vijay Abraham I <kishon@ti.com>
12 */
13
14 /*
15 * +------------+ +---------------------------------------+
16 * | | | |
17 * +------------+ | +--------------+
18 * | NTB | | | NTB |
19 * | NetDev | | | NetDev |
20 * +------------+ | +--------------+
21 * | NTB | | | NTB |
22 * | Transfer | | | Transfer |
23 * +------------+ | +--------------+
24 * | | | | |
25 * | PCI NTB | | | |
26 * | EPF | | | |
27 * | Driver | | | PCI Virtual |
28 * | | +---------------+ | NTB Driver |
29 * | | | PCI EP NTB |<------>| |
30 * | | | FN Driver | | |
31 * +------------+ +---------------+ +--------------+
32 * | | | | | |
33 * | PCI Bus | <-----> | PCI EP Bus | | Virtual PCI |
34 * | | PCI | | | Bus |
35 * +------------+ +---------------+--------+--------------+
36 * PCIe Root Port PCI EP
37 */
38
39 #include <linux/delay.h>
40 #include <linux/io.h>
41 #include <linux/module.h>
42 #include <linux/slab.h>
43
44 #include <linux/pci-epc.h>
45 #include <linux/pci-epf.h>
46 #include <linux/ntb.h>
47
48 static struct workqueue_struct *kpcintb_workqueue;
49
50 #define COMMAND_CONFIGURE_DOORBELL 1
51 #define COMMAND_TEARDOWN_DOORBELL 2
52 #define COMMAND_CONFIGURE_MW 3
53 #define COMMAND_TEARDOWN_MW 4
54 #define COMMAND_LINK_UP 5
55 #define COMMAND_LINK_DOWN 6
56
57 #define COMMAND_STATUS_OK 1
58 #define COMMAND_STATUS_ERROR 2
59
60 #define LINK_STATUS_UP BIT(0)
61
62 #define SPAD_COUNT 64
63 #define DB_COUNT 4
64 #define NTB_MW_OFFSET 2
65 #define DB_COUNT_MASK GENMASK(15, 0)
66 #define MSIX_ENABLE BIT(16)
67 #define MAX_DB_COUNT 32
68 #define MAX_MW 4
69
70 enum epf_ntb_bar {
71 BAR_CONFIG,
72 BAR_DB,
73 BAR_MW0,
74 BAR_MW1,
75 BAR_MW2,
76 };
77
78 /*
79 * +--------------------------------------------------+ Base
80 * | |
81 * | |
82 * | |
83 * | Common Control Register |
84 * | |
85 * | |
86 * | |
87 * +-----------------------+--------------------------+ Base+spad_offset
88 * | | |
89 * | Peer Spad Space | Spad Space |
90 * | | |
91 * | | |
92 * +-----------------------+--------------------------+ Base+spad_offset
93 * | | | +spad_count * 4
94 * | | |
95 * | Spad Space | Peer Spad Space |
96 * | | |
97 * +-----------------------+--------------------------+
98 * Virtual PCI PCIe Endpoint
99 * NTB Driver NTB Driver
100 */
101 struct epf_ntb_ctrl {
102 u32 command;
103 u32 argument;
104 u16 command_status;
105 u16 link_status;
106 u32 topology;
107 u64 addr;
108 u64 size;
109 u32 num_mws;
110 u32 reserved;
111 u32 spad_offset;
112 u32 spad_count;
113 u32 db_entry_size;
114 u32 db_data[MAX_DB_COUNT];
115 u32 db_offset[MAX_DB_COUNT];
116 } __packed;
117
118 struct epf_ntb {
119 struct ntb_dev ntb;
120 struct pci_epf *epf;
121 struct config_group group;
122
123 u32 num_mws;
124 u32 db_count;
125 u32 spad_count;
126 u64 mws_size[MAX_MW];
127 u64 db;
128 u32 vbus_number;
129 u16 vntb_pid;
130 u16 vntb_vid;
131
132 bool linkup;
133 u32 spad_size;
134
135 enum pci_barno epf_ntb_bar[6];
136
137 struct epf_ntb_ctrl *reg;
138
139 u32 *epf_db;
140
141 phys_addr_t vpci_mw_phy[MAX_MW];
142 void __iomem *vpci_mw_addr[MAX_MW];
143
144 struct delayed_work cmd_handler;
145 };
146
147 #define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group)
148 #define ntb_ndev(__ntb) container_of(__ntb, struct epf_ntb, ntb)
149
150 static struct pci_epf_header epf_ntb_header = {
151 .vendorid = PCI_ANY_ID,
152 .deviceid = PCI_ANY_ID,
153 .baseclass_code = PCI_BASE_CLASS_MEMORY,
154 .interrupt_pin = PCI_INTERRUPT_INTA,
155 };
156
157 /**
158 * epf_ntb_link_up() - Raise link_up interrupt to Virtual Host (VHOST)
159 * @ntb: NTB device that facilitates communication between HOST and VHOST
160 * @link_up: true or false indicating Link is UP or Down
161 *
162 * Once NTB function in HOST invoke ntb_link_enable(),
163 * this NTB function driver will trigger a link event to VHOST.
164 *
165 * Returns: Zero for success, or an error code in case of failure
166 */
epf_ntb_link_up(struct epf_ntb * ntb,bool link_up)167 static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up)
168 {
169 if (link_up)
170 ntb->reg->link_status |= LINK_STATUS_UP;
171 else
172 ntb->reg->link_status &= ~LINK_STATUS_UP;
173
174 ntb_link_event(&ntb->ntb);
175 return 0;
176 }
177
178 /**
179 * epf_ntb_configure_mw() - Configure the Outbound Address Space for VHOST
180 * to access the memory window of HOST
181 * @ntb: NTB device that facilitates communication between HOST and VHOST
182 * @mw: Index of the memory window (either 0, 1, 2 or 3)
183 *
184 * EP Outbound Window
185 * +--------+ +-----------+
186 * | | | |
187 * | | | |
188 * | | | |
189 * | | | |
190 * | | +-----------+
191 * | Virtual| | Memory Win|
192 * | NTB | -----------> | |
193 * | Driver | | |
194 * | | +-----------+
195 * | | | |
196 * | | | |
197 * +--------+ +-----------+
198 * VHOST PCI EP
199 *
200 * Returns: Zero for success, or an error code in case of failure
201 */
epf_ntb_configure_mw(struct epf_ntb * ntb,u32 mw)202 static int epf_ntb_configure_mw(struct epf_ntb *ntb, u32 mw)
203 {
204 phys_addr_t phys_addr;
205 u8 func_no, vfunc_no;
206 u64 addr, size;
207 int ret = 0;
208
209 phys_addr = ntb->vpci_mw_phy[mw];
210 addr = ntb->reg->addr;
211 size = ntb->reg->size;
212
213 func_no = ntb->epf->func_no;
214 vfunc_no = ntb->epf->vfunc_no;
215
216 ret = pci_epc_map_addr(ntb->epf->epc, func_no, vfunc_no, phys_addr, addr, size);
217 if (ret)
218 dev_err(&ntb->epf->epc->dev,
219 "Failed to map memory window %d address\n", mw);
220 return ret;
221 }
222
223 /**
224 * epf_ntb_teardown_mw() - Teardown the configured OB ATU
225 * @ntb: NTB device that facilitates communication between HOST and VHOST
226 * @mw: Index of the memory window (either 0, 1, 2 or 3)
227 *
228 * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using
229 * pci_epc_unmap_addr()
230 */
epf_ntb_teardown_mw(struct epf_ntb * ntb,u32 mw)231 static void epf_ntb_teardown_mw(struct epf_ntb *ntb, u32 mw)
232 {
233 pci_epc_unmap_addr(ntb->epf->epc,
234 ntb->epf->func_no,
235 ntb->epf->vfunc_no,
236 ntb->vpci_mw_phy[mw]);
237 }
238
239 /**
240 * epf_ntb_cmd_handler() - Handle commands provided by the NTB HOST
241 * @work: work_struct for the epf_ntb_epc
242 *
243 * Workqueue function that gets invoked for the two epf_ntb_epc
244 * periodically (once every 5ms) to see if it has received any commands
245 * from NTB HOST. The HOST can send commands to configure doorbell or
246 * configure memory window or to update link status.
247 */
epf_ntb_cmd_handler(struct work_struct * work)248 static void epf_ntb_cmd_handler(struct work_struct *work)
249 {
250 struct epf_ntb_ctrl *ctrl;
251 u32 command, argument;
252 struct epf_ntb *ntb;
253 struct device *dev;
254 int ret;
255 int i;
256
257 ntb = container_of(work, struct epf_ntb, cmd_handler.work);
258
259 for (i = 1; i < ntb->db_count; i++) {
260 if (ntb->epf_db[i]) {
261 ntb->db |= 1 << (i - 1);
262 ntb_db_event(&ntb->ntb, i);
263 ntb->epf_db[i] = 0;
264 }
265 }
266
267 ctrl = ntb->reg;
268 command = ctrl->command;
269 if (!command)
270 goto reset_handler;
271 argument = ctrl->argument;
272
273 ctrl->command = 0;
274 ctrl->argument = 0;
275
276 ctrl = ntb->reg;
277 dev = &ntb->epf->dev;
278
279 switch (command) {
280 case COMMAND_CONFIGURE_DOORBELL:
281 ctrl->command_status = COMMAND_STATUS_OK;
282 break;
283 case COMMAND_TEARDOWN_DOORBELL:
284 ctrl->command_status = COMMAND_STATUS_OK;
285 break;
286 case COMMAND_CONFIGURE_MW:
287 ret = epf_ntb_configure_mw(ntb, argument);
288 if (ret < 0)
289 ctrl->command_status = COMMAND_STATUS_ERROR;
290 else
291 ctrl->command_status = COMMAND_STATUS_OK;
292 break;
293 case COMMAND_TEARDOWN_MW:
294 epf_ntb_teardown_mw(ntb, argument);
295 ctrl->command_status = COMMAND_STATUS_OK;
296 break;
297 case COMMAND_LINK_UP:
298 ntb->linkup = true;
299 ret = epf_ntb_link_up(ntb, true);
300 if (ret < 0)
301 ctrl->command_status = COMMAND_STATUS_ERROR;
302 else
303 ctrl->command_status = COMMAND_STATUS_OK;
304 goto reset_handler;
305 case COMMAND_LINK_DOWN:
306 ntb->linkup = false;
307 ret = epf_ntb_link_up(ntb, false);
308 if (ret < 0)
309 ctrl->command_status = COMMAND_STATUS_ERROR;
310 else
311 ctrl->command_status = COMMAND_STATUS_OK;
312 break;
313 default:
314 dev_err(dev, "UNKNOWN command: %d\n", command);
315 break;
316 }
317
318 reset_handler:
319 queue_delayed_work(kpcintb_workqueue, &ntb->cmd_handler,
320 msecs_to_jiffies(5));
321 }
322
323 /**
324 * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR
325 * @ntb: EPC associated with one of the HOST which holds peer's outbound
326 * address.
327 *
328 * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and
329 * self scratchpad region (removes inbound ATU configuration). While BAR0 is
330 * the default self scratchpad BAR, an NTB could have other BARs for self
331 * scratchpad (because of reserved BARs). This function can get the exact BAR
332 * used for self scratchpad from epf_ntb_bar[BAR_CONFIG].
333 *
334 * Please note the self scratchpad region and config region is combined to
335 * a single region and mapped using the same BAR. Also note VHOST's peer
336 * scratchpad is HOST's self scratchpad.
337 *
338 * Returns: void
339 */
epf_ntb_config_sspad_bar_clear(struct epf_ntb * ntb)340 static void epf_ntb_config_sspad_bar_clear(struct epf_ntb *ntb)
341 {
342 struct pci_epf_bar *epf_bar;
343 enum pci_barno barno;
344
345 barno = ntb->epf_ntb_bar[BAR_CONFIG];
346 epf_bar = &ntb->epf->bar[barno];
347
348 pci_epc_clear_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar);
349 }
350
351 /**
352 * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR
353 * @ntb: NTB device that facilitates communication between HOST and VHOST
354 *
355 * Map BAR0 of EP CONTROLLER which contains the VHOST's config and
356 * self scratchpad region.
357 *
358 * Please note the self scratchpad region and config region is combined to
359 * a single region and mapped using the same BAR.
360 *
361 * Returns: Zero for success, or an error code in case of failure
362 */
epf_ntb_config_sspad_bar_set(struct epf_ntb * ntb)363 static int epf_ntb_config_sspad_bar_set(struct epf_ntb *ntb)
364 {
365 struct pci_epf_bar *epf_bar;
366 enum pci_barno barno;
367 u8 func_no, vfunc_no;
368 struct device *dev;
369 int ret;
370
371 dev = &ntb->epf->dev;
372 func_no = ntb->epf->func_no;
373 vfunc_no = ntb->epf->vfunc_no;
374 barno = ntb->epf_ntb_bar[BAR_CONFIG];
375 epf_bar = &ntb->epf->bar[barno];
376
377 ret = pci_epc_set_bar(ntb->epf->epc, func_no, vfunc_no, epf_bar);
378 if (ret) {
379 dev_err(dev, "inft: Config/Status/SPAD BAR set failed\n");
380 return ret;
381 }
382 return 0;
383 }
384
385 /**
386 * epf_ntb_config_spad_bar_free() - Free the physical memory associated with
387 * config + scratchpad region
388 * @ntb: NTB device that facilitates communication between HOST and VHOST
389 */
epf_ntb_config_spad_bar_free(struct epf_ntb * ntb)390 static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb)
391 {
392 enum pci_barno barno;
393
394 barno = ntb->epf_ntb_bar[BAR_CONFIG];
395 pci_epf_free_space(ntb->epf, ntb->reg, barno, 0);
396 }
397
398 /**
399 * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad
400 * region
401 * @ntb: NTB device that facilitates communication between HOST and VHOST
402 *
403 * Allocate the Local Memory mentioned in the above diagram. The size of
404 * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION
405 * is obtained from "spad-count" configfs entry.
406 *
407 * Returns: Zero for success, or an error code in case of failure
408 */
epf_ntb_config_spad_bar_alloc(struct epf_ntb * ntb)409 static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb)
410 {
411 size_t align;
412 enum pci_barno barno;
413 struct epf_ntb_ctrl *ctrl;
414 u32 spad_size, ctrl_size;
415 u64 size;
416 struct pci_epf *epf = ntb->epf;
417 struct device *dev = &epf->dev;
418 u32 spad_count;
419 void *base;
420 int i;
421 const struct pci_epc_features *epc_features = pci_epc_get_features(epf->epc,
422 epf->func_no,
423 epf->vfunc_no);
424 barno = ntb->epf_ntb_bar[BAR_CONFIG];
425 size = epc_features->bar_fixed_size[barno];
426 align = epc_features->align;
427
428 if ((!IS_ALIGNED(size, align)))
429 return -EINVAL;
430
431 spad_count = ntb->spad_count;
432
433 ctrl_size = sizeof(struct epf_ntb_ctrl);
434 spad_size = 2 * spad_count * sizeof(u32);
435
436 if (!align) {
437 ctrl_size = roundup_pow_of_two(ctrl_size);
438 spad_size = roundup_pow_of_two(spad_size);
439 } else {
440 ctrl_size = ALIGN(ctrl_size, align);
441 spad_size = ALIGN(spad_size, align);
442 }
443
444 if (!size)
445 size = ctrl_size + spad_size;
446 else if (size < ctrl_size + spad_size)
447 return -EINVAL;
448
449 base = pci_epf_alloc_space(epf, size, barno, align, 0);
450 if (!base) {
451 dev_err(dev, "Config/Status/SPAD alloc region fail\n");
452 return -ENOMEM;
453 }
454
455 ntb->reg = base;
456
457 ctrl = ntb->reg;
458 ctrl->spad_offset = ctrl_size;
459
460 ctrl->spad_count = spad_count;
461 ctrl->num_mws = ntb->num_mws;
462 ntb->spad_size = spad_size;
463
464 ctrl->db_entry_size = sizeof(u32);
465
466 for (i = 0; i < ntb->db_count; i++) {
467 ntb->reg->db_data[i] = 1 + i;
468 ntb->reg->db_offset[i] = 0;
469 }
470
471 return 0;
472 }
473
474 /**
475 * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capability
476 * @ntb: NTB device that facilitates communication between HOST and VHOST
477 *
478 * Configure MSI/MSI-X capability for each interface with number of
479 * interrupts equal to "db_count" configfs entry.
480 *
481 * Returns: Zero for success, or an error code in case of failure
482 */
epf_ntb_configure_interrupt(struct epf_ntb * ntb)483 static int epf_ntb_configure_interrupt(struct epf_ntb *ntb)
484 {
485 const struct pci_epc_features *epc_features;
486 struct device *dev;
487 u32 db_count;
488 int ret;
489
490 dev = &ntb->epf->dev;
491
492 epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no);
493
494 if (!(epc_features->msix_capable || epc_features->msi_capable)) {
495 dev_err(dev, "MSI or MSI-X is required for doorbell\n");
496 return -EINVAL;
497 }
498
499 db_count = ntb->db_count;
500 if (db_count > MAX_DB_COUNT) {
501 dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT);
502 return -EINVAL;
503 }
504
505 ntb->db_count = db_count;
506
507 if (epc_features->msi_capable) {
508 ret = pci_epc_set_msi(ntb->epf->epc,
509 ntb->epf->func_no,
510 ntb->epf->vfunc_no,
511 16);
512 if (ret) {
513 dev_err(dev, "MSI configuration failed\n");
514 return ret;
515 }
516 }
517
518 return 0;
519 }
520
521 /**
522 * epf_ntb_db_bar_init() - Configure Doorbell window BARs
523 * @ntb: NTB device that facilitates communication between HOST and VHOST
524 *
525 * Returns: Zero for success, or an error code in case of failure
526 */
epf_ntb_db_bar_init(struct epf_ntb * ntb)527 static int epf_ntb_db_bar_init(struct epf_ntb *ntb)
528 {
529 const struct pci_epc_features *epc_features;
530 u32 align;
531 struct device *dev = &ntb->epf->dev;
532 int ret;
533 struct pci_epf_bar *epf_bar;
534 void __iomem *mw_addr;
535 enum pci_barno barno;
536 size_t size = sizeof(u32) * ntb->db_count;
537
538 epc_features = pci_epc_get_features(ntb->epf->epc,
539 ntb->epf->func_no,
540 ntb->epf->vfunc_no);
541 align = epc_features->align;
542
543 if (size < 128)
544 size = 128;
545
546 if (align)
547 size = ALIGN(size, align);
548 else
549 size = roundup_pow_of_two(size);
550
551 barno = ntb->epf_ntb_bar[BAR_DB];
552
553 mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, align, 0);
554 if (!mw_addr) {
555 dev_err(dev, "Failed to allocate OB address\n");
556 return -ENOMEM;
557 }
558
559 ntb->epf_db = mw_addr;
560
561 epf_bar = &ntb->epf->bar[barno];
562
563 ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar);
564 if (ret) {
565 dev_err(dev, "Doorbell BAR set failed\n");
566 goto err_alloc_peer_mem;
567 }
568 return ret;
569
570 err_alloc_peer_mem:
571 pci_epf_free_space(ntb->epf, mw_addr, barno, 0);
572 return -1;
573 }
574
575 static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws);
576
577 /**
578 * epf_ntb_db_bar_clear() - Clear doorbell BAR and free memory
579 * allocated in peer's outbound address space
580 * @ntb: NTB device that facilitates communication between HOST and VHOST
581 */
epf_ntb_db_bar_clear(struct epf_ntb * ntb)582 static void epf_ntb_db_bar_clear(struct epf_ntb *ntb)
583 {
584 enum pci_barno barno;
585
586 barno = ntb->epf_ntb_bar[BAR_DB];
587 pci_epf_free_space(ntb->epf, ntb->epf_db, barno, 0);
588 pci_epc_clear_bar(ntb->epf->epc,
589 ntb->epf->func_no,
590 ntb->epf->vfunc_no,
591 &ntb->epf->bar[barno]);
592 }
593
594 /**
595 * epf_ntb_mw_bar_init() - Configure Memory window BARs
596 * @ntb: NTB device that facilitates communication between HOST and VHOST
597 *
598 * Returns: Zero for success, or an error code in case of failure
599 */
epf_ntb_mw_bar_init(struct epf_ntb * ntb)600 static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
601 {
602 int ret = 0;
603 int i;
604 u64 size;
605 enum pci_barno barno;
606 struct device *dev = &ntb->epf->dev;
607
608 for (i = 0; i < ntb->num_mws; i++) {
609 size = ntb->mws_size[i];
610 barno = ntb->epf_ntb_bar[BAR_MW0 + i];
611
612 ntb->epf->bar[barno].barno = barno;
613 ntb->epf->bar[barno].size = size;
614 ntb->epf->bar[barno].addr = NULL;
615 ntb->epf->bar[barno].phys_addr = 0;
616 ntb->epf->bar[barno].flags |= upper_32_bits(size) ?
617 PCI_BASE_ADDRESS_MEM_TYPE_64 :
618 PCI_BASE_ADDRESS_MEM_TYPE_32;
619
620 ret = pci_epc_set_bar(ntb->epf->epc,
621 ntb->epf->func_no,
622 ntb->epf->vfunc_no,
623 &ntb->epf->bar[barno]);
624 if (ret) {
625 dev_err(dev, "MW set failed\n");
626 goto err_alloc_mem;
627 }
628
629 /* Allocate EPC outbound memory windows to vpci vntb device */
630 ntb->vpci_mw_addr[i] = pci_epc_mem_alloc_addr(ntb->epf->epc,
631 &ntb->vpci_mw_phy[i],
632 size);
633 if (!ntb->vpci_mw_addr[i]) {
634 ret = -ENOMEM;
635 dev_err(dev, "Failed to allocate source address\n");
636 goto err_set_bar;
637 }
638 }
639
640 return ret;
641
642 err_set_bar:
643 pci_epc_clear_bar(ntb->epf->epc,
644 ntb->epf->func_no,
645 ntb->epf->vfunc_no,
646 &ntb->epf->bar[barno]);
647 err_alloc_mem:
648 epf_ntb_mw_bar_clear(ntb, i);
649 return ret;
650 }
651
652 /**
653 * epf_ntb_mw_bar_clear() - Clear Memory window BARs
654 * @ntb: NTB device that facilitates communication between HOST and VHOST
655 * @num_mws: the number of Memory window BARs that to be cleared
656 */
epf_ntb_mw_bar_clear(struct epf_ntb * ntb,int num_mws)657 static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws)
658 {
659 enum pci_barno barno;
660 int i;
661
662 for (i = 0; i < num_mws; i++) {
663 barno = ntb->epf_ntb_bar[BAR_MW0 + i];
664 pci_epc_clear_bar(ntb->epf->epc,
665 ntb->epf->func_no,
666 ntb->epf->vfunc_no,
667 &ntb->epf->bar[barno]);
668
669 pci_epc_mem_free_addr(ntb->epf->epc,
670 ntb->vpci_mw_phy[i],
671 ntb->vpci_mw_addr[i],
672 ntb->mws_size[i]);
673 }
674 }
675
676 /**
677 * epf_ntb_epc_destroy() - Cleanup NTB EPC interface
678 * @ntb: NTB device that facilitates communication between HOST and VHOST
679 *
680 * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces
681 */
epf_ntb_epc_destroy(struct epf_ntb * ntb)682 static void epf_ntb_epc_destroy(struct epf_ntb *ntb)
683 {
684 pci_epc_remove_epf(ntb->epf->epc, ntb->epf, 0);
685 pci_epc_put(ntb->epf->epc);
686 }
687
688 /**
689 * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB
690 * constructs (scratchpad region, doorbell, memorywindow)
691 * @ntb: NTB device that facilitates communication between HOST and VHOST
692 *
693 * Returns: Zero for success, or an error code in case of failure
694 */
epf_ntb_init_epc_bar(struct epf_ntb * ntb)695 static int epf_ntb_init_epc_bar(struct epf_ntb *ntb)
696 {
697 const struct pci_epc_features *epc_features;
698 enum pci_barno barno;
699 enum epf_ntb_bar bar;
700 struct device *dev;
701 u32 num_mws;
702 int i;
703
704 barno = BAR_0;
705 num_mws = ntb->num_mws;
706 dev = &ntb->epf->dev;
707 epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no);
708
709 /* These are required BARs which are mandatory for NTB functionality */
710 for (bar = BAR_CONFIG; bar <= BAR_MW0; bar++, barno++) {
711 barno = pci_epc_get_next_free_bar(epc_features, barno);
712 if (barno < 0) {
713 dev_err(dev, "Fail to get NTB function BAR\n");
714 return barno;
715 }
716 ntb->epf_ntb_bar[bar] = barno;
717 }
718
719 /* These are optional BARs which don't impact NTB functionality */
720 for (bar = BAR_MW1, i = 1; i < num_mws; bar++, barno++, i++) {
721 barno = pci_epc_get_next_free_bar(epc_features, barno);
722 if (barno < 0) {
723 ntb->num_mws = i;
724 dev_dbg(dev, "BAR not available for > MW%d\n", i + 1);
725 }
726 ntb->epf_ntb_bar[bar] = barno;
727 }
728
729 return 0;
730 }
731
732 /**
733 * epf_ntb_epc_init() - Initialize NTB interface
734 * @ntb: NTB device that facilitates communication between HOST and VHOST
735 *
736 * Wrapper to initialize a particular EPC interface and start the workqueue
737 * to check for commands from HOST. This function will write to the
738 * EP controller HW for configuring it.
739 *
740 * Returns: Zero for success, or an error code in case of failure
741 */
epf_ntb_epc_init(struct epf_ntb * ntb)742 static int epf_ntb_epc_init(struct epf_ntb *ntb)
743 {
744 u8 func_no, vfunc_no;
745 struct pci_epc *epc;
746 struct pci_epf *epf;
747 struct device *dev;
748 int ret;
749
750 epf = ntb->epf;
751 dev = &epf->dev;
752 epc = epf->epc;
753 func_no = ntb->epf->func_no;
754 vfunc_no = ntb->epf->vfunc_no;
755
756 ret = epf_ntb_config_sspad_bar_set(ntb);
757 if (ret) {
758 dev_err(dev, "Config/self SPAD BAR init failed");
759 return ret;
760 }
761
762 ret = epf_ntb_configure_interrupt(ntb);
763 if (ret) {
764 dev_err(dev, "Interrupt configuration failed\n");
765 goto err_config_interrupt;
766 }
767
768 ret = epf_ntb_db_bar_init(ntb);
769 if (ret) {
770 dev_err(dev, "DB BAR init failed\n");
771 goto err_db_bar_init;
772 }
773
774 ret = epf_ntb_mw_bar_init(ntb);
775 if (ret) {
776 dev_err(dev, "MW BAR init failed\n");
777 goto err_mw_bar_init;
778 }
779
780 if (vfunc_no <= 1) {
781 ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header);
782 if (ret) {
783 dev_err(dev, "Configuration header write failed\n");
784 goto err_write_header;
785 }
786 }
787
788 INIT_DELAYED_WORK(&ntb->cmd_handler, epf_ntb_cmd_handler);
789 queue_work(kpcintb_workqueue, &ntb->cmd_handler.work);
790
791 return 0;
792
793 err_write_header:
794 epf_ntb_mw_bar_clear(ntb, ntb->num_mws);
795 err_mw_bar_init:
796 epf_ntb_db_bar_clear(ntb);
797 err_db_bar_init:
798 err_config_interrupt:
799 epf_ntb_config_sspad_bar_clear(ntb);
800
801 return ret;
802 }
803
804
805 /**
806 * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces
807 * @ntb: NTB device that facilitates communication between HOST and VHOST
808 *
809 * Wrapper to cleanup all NTB interfaces.
810 */
epf_ntb_epc_cleanup(struct epf_ntb * ntb)811 static void epf_ntb_epc_cleanup(struct epf_ntb *ntb)
812 {
813 epf_ntb_mw_bar_clear(ntb, ntb->num_mws);
814 epf_ntb_db_bar_clear(ntb);
815 epf_ntb_config_sspad_bar_clear(ntb);
816 }
817
818 #define EPF_NTB_R(_name) \
819 static ssize_t epf_ntb_##_name##_show(struct config_item *item, \
820 char *page) \
821 { \
822 struct config_group *group = to_config_group(item); \
823 struct epf_ntb *ntb = to_epf_ntb(group); \
824 \
825 return sprintf(page, "%d\n", ntb->_name); \
826 }
827
828 #define EPF_NTB_W(_name) \
829 static ssize_t epf_ntb_##_name##_store(struct config_item *item, \
830 const char *page, size_t len) \
831 { \
832 struct config_group *group = to_config_group(item); \
833 struct epf_ntb *ntb = to_epf_ntb(group); \
834 u32 val; \
835 int ret; \
836 \
837 ret = kstrtou32(page, 0, &val); \
838 if (ret) \
839 return ret; \
840 \
841 ntb->_name = val; \
842 \
843 return len; \
844 }
845
846 #define EPF_NTB_MW_R(_name) \
847 static ssize_t epf_ntb_##_name##_show(struct config_item *item, \
848 char *page) \
849 { \
850 struct config_group *group = to_config_group(item); \
851 struct epf_ntb *ntb = to_epf_ntb(group); \
852 struct device *dev = &ntb->epf->dev; \
853 int win_no; \
854 \
855 if (sscanf(#_name, "mw%d", &win_no) != 1) \
856 return -EINVAL; \
857 \
858 if (win_no <= 0 || win_no > ntb->num_mws) { \
859 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
860 return -EINVAL; \
861 } \
862 \
863 return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]); \
864 }
865
866 #define EPF_NTB_MW_W(_name) \
867 static ssize_t epf_ntb_##_name##_store(struct config_item *item, \
868 const char *page, size_t len) \
869 { \
870 struct config_group *group = to_config_group(item); \
871 struct epf_ntb *ntb = to_epf_ntb(group); \
872 struct device *dev = &ntb->epf->dev; \
873 int win_no; \
874 u64 val; \
875 int ret; \
876 \
877 ret = kstrtou64(page, 0, &val); \
878 if (ret) \
879 return ret; \
880 \
881 if (sscanf(#_name, "mw%d", &win_no) != 1) \
882 return -EINVAL; \
883 \
884 if (win_no <= 0 || win_no > ntb->num_mws) { \
885 dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
886 return -EINVAL; \
887 } \
888 \
889 ntb->mws_size[win_no - 1] = val; \
890 \
891 return len; \
892 }
893
epf_ntb_num_mws_store(struct config_item * item,const char * page,size_t len)894 static ssize_t epf_ntb_num_mws_store(struct config_item *item,
895 const char *page, size_t len)
896 {
897 struct config_group *group = to_config_group(item);
898 struct epf_ntb *ntb = to_epf_ntb(group);
899 u32 val;
900 int ret;
901
902 ret = kstrtou32(page, 0, &val);
903 if (ret)
904 return ret;
905
906 if (val > MAX_MW)
907 return -EINVAL;
908
909 ntb->num_mws = val;
910
911 return len;
912 }
913
914 EPF_NTB_R(spad_count)
915 EPF_NTB_W(spad_count)
916 EPF_NTB_R(db_count)
917 EPF_NTB_W(db_count)
918 EPF_NTB_R(num_mws)
919 EPF_NTB_R(vbus_number)
920 EPF_NTB_W(vbus_number)
921 EPF_NTB_R(vntb_pid)
922 EPF_NTB_W(vntb_pid)
923 EPF_NTB_R(vntb_vid)
924 EPF_NTB_W(vntb_vid)
925 EPF_NTB_MW_R(mw1)
926 EPF_NTB_MW_W(mw1)
927 EPF_NTB_MW_R(mw2)
928 EPF_NTB_MW_W(mw2)
929 EPF_NTB_MW_R(mw3)
930 EPF_NTB_MW_W(mw3)
931 EPF_NTB_MW_R(mw4)
932 EPF_NTB_MW_W(mw4)
933
934 CONFIGFS_ATTR(epf_ntb_, spad_count);
935 CONFIGFS_ATTR(epf_ntb_, db_count);
936 CONFIGFS_ATTR(epf_ntb_, num_mws);
937 CONFIGFS_ATTR(epf_ntb_, mw1);
938 CONFIGFS_ATTR(epf_ntb_, mw2);
939 CONFIGFS_ATTR(epf_ntb_, mw3);
940 CONFIGFS_ATTR(epf_ntb_, mw4);
941 CONFIGFS_ATTR(epf_ntb_, vbus_number);
942 CONFIGFS_ATTR(epf_ntb_, vntb_pid);
943 CONFIGFS_ATTR(epf_ntb_, vntb_vid);
944
945 static struct configfs_attribute *epf_ntb_attrs[] = {
946 &epf_ntb_attr_spad_count,
947 &epf_ntb_attr_db_count,
948 &epf_ntb_attr_num_mws,
949 &epf_ntb_attr_mw1,
950 &epf_ntb_attr_mw2,
951 &epf_ntb_attr_mw3,
952 &epf_ntb_attr_mw4,
953 &epf_ntb_attr_vbus_number,
954 &epf_ntb_attr_vntb_pid,
955 &epf_ntb_attr_vntb_vid,
956 NULL,
957 };
958
959 static const struct config_item_type ntb_group_type = {
960 .ct_attrs = epf_ntb_attrs,
961 .ct_owner = THIS_MODULE,
962 };
963
964 /**
965 * epf_ntb_add_cfs() - Add configfs directory specific to NTB
966 * @epf: NTB endpoint function device
967 * @group: A pointer to the config_group structure referencing a group of
968 * config_items of a specific type that belong to a specific sub-system.
969 *
970 * Add configfs directory specific to NTB. This directory will hold
971 * NTB specific properties like db_count, spad_count, num_mws etc.,
972 *
973 * Returns: Pointer to config_group
974 */
epf_ntb_add_cfs(struct pci_epf * epf,struct config_group * group)975 static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf,
976 struct config_group *group)
977 {
978 struct epf_ntb *ntb = epf_get_drvdata(epf);
979 struct config_group *ntb_group = &ntb->group;
980 struct device *dev = &epf->dev;
981
982 config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type);
983
984 return ntb_group;
985 }
986
987 /*==== virtual PCI bus driver, which only load virtual NTB PCI driver ====*/
988
989 static u32 pci_space[] = {
990 0xffffffff, /* Device ID, Vendor ID */
991 0, /* Status, Command */
992 0xffffffff, /* Base Class, Subclass, Prog Intf, Revision ID */
993 0x40, /* BIST, Header Type, Latency Timer, Cache Line Size */
994 0, /* BAR 0 */
995 0, /* BAR 1 */
996 0, /* BAR 2 */
997 0, /* BAR 3 */
998 0, /* BAR 4 */
999 0, /* BAR 5 */
1000 0, /* Cardbus CIS Pointer */
1001 0, /* Subsystem ID, Subsystem Vendor ID */
1002 0, /* ROM Base Address */
1003 0, /* Reserved, Capabilities Pointer */
1004 0, /* Reserved */
1005 0, /* Max_Lat, Min_Gnt, Interrupt Pin, Interrupt Line */
1006 };
1007
pci_read(struct pci_bus * bus,unsigned int devfn,int where,int size,u32 * val)1008 static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val)
1009 {
1010 if (devfn == 0) {
1011 memcpy(val, ((u8 *)pci_space) + where, size);
1012 return PCIBIOS_SUCCESSFUL;
1013 }
1014 return PCIBIOS_DEVICE_NOT_FOUND;
1015 }
1016
pci_write(struct pci_bus * bus,unsigned int devfn,int where,int size,u32 val)1017 static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val)
1018 {
1019 return 0;
1020 }
1021
1022 static struct pci_ops vpci_ops = {
1023 .read = pci_read,
1024 .write = pci_write,
1025 };
1026
vpci_scan_bus(void * sysdata)1027 static int vpci_scan_bus(void *sysdata)
1028 {
1029 struct pci_bus *vpci_bus;
1030 struct epf_ntb *ndev = sysdata;
1031
1032 vpci_bus = pci_scan_bus(ndev->vbus_number, &vpci_ops, sysdata);
1033 if (!vpci_bus) {
1034 pr_err("create pci bus failed\n");
1035 return -EINVAL;
1036 }
1037
1038 pci_bus_add_devices(vpci_bus);
1039
1040 return 0;
1041 }
1042
1043 /*==================== Virtual PCIe NTB driver ==========================*/
1044
vntb_epf_mw_count(struct ntb_dev * ntb,int pidx)1045 static int vntb_epf_mw_count(struct ntb_dev *ntb, int pidx)
1046 {
1047 struct epf_ntb *ndev = ntb_ndev(ntb);
1048
1049 return ndev->num_mws;
1050 }
1051
vntb_epf_spad_count(struct ntb_dev * ntb)1052 static int vntb_epf_spad_count(struct ntb_dev *ntb)
1053 {
1054 return ntb_ndev(ntb)->spad_count;
1055 }
1056
vntb_epf_peer_mw_count(struct ntb_dev * ntb)1057 static int vntb_epf_peer_mw_count(struct ntb_dev *ntb)
1058 {
1059 return ntb_ndev(ntb)->num_mws;
1060 }
1061
vntb_epf_db_valid_mask(struct ntb_dev * ntb)1062 static u64 vntb_epf_db_valid_mask(struct ntb_dev *ntb)
1063 {
1064 return BIT_ULL(ntb_ndev(ntb)->db_count) - 1;
1065 }
1066
vntb_epf_db_set_mask(struct ntb_dev * ntb,u64 db_bits)1067 static int vntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1068 {
1069 return 0;
1070 }
1071
vntb_epf_mw_set_trans(struct ntb_dev * ndev,int pidx,int idx,dma_addr_t addr,resource_size_t size)1072 static int vntb_epf_mw_set_trans(struct ntb_dev *ndev, int pidx, int idx,
1073 dma_addr_t addr, resource_size_t size)
1074 {
1075 struct epf_ntb *ntb = ntb_ndev(ndev);
1076 struct pci_epf_bar *epf_bar;
1077 enum pci_barno barno;
1078 int ret;
1079 struct device *dev;
1080
1081 dev = &ntb->ntb.dev;
1082 barno = ntb->epf_ntb_bar[BAR_MW0 + idx];
1083 epf_bar = &ntb->epf->bar[barno];
1084 epf_bar->phys_addr = addr;
1085 epf_bar->barno = barno;
1086 epf_bar->size = size;
1087
1088 ret = pci_epc_set_bar(ntb->epf->epc, 0, 0, epf_bar);
1089 if (ret) {
1090 dev_err(dev, "failure set mw trans\n");
1091 return ret;
1092 }
1093 return 0;
1094 }
1095
vntb_epf_mw_clear_trans(struct ntb_dev * ntb,int pidx,int idx)1096 static int vntb_epf_mw_clear_trans(struct ntb_dev *ntb, int pidx, int idx)
1097 {
1098 return 0;
1099 }
1100
vntb_epf_peer_mw_get_addr(struct ntb_dev * ndev,int idx,phys_addr_t * base,resource_size_t * size)1101 static int vntb_epf_peer_mw_get_addr(struct ntb_dev *ndev, int idx,
1102 phys_addr_t *base, resource_size_t *size)
1103 {
1104
1105 struct epf_ntb *ntb = ntb_ndev(ndev);
1106
1107 if (base)
1108 *base = ntb->vpci_mw_phy[idx];
1109
1110 if (size)
1111 *size = ntb->mws_size[idx];
1112
1113 return 0;
1114 }
1115
vntb_epf_link_enable(struct ntb_dev * ntb,enum ntb_speed max_speed,enum ntb_width max_width)1116 static int vntb_epf_link_enable(struct ntb_dev *ntb,
1117 enum ntb_speed max_speed,
1118 enum ntb_width max_width)
1119 {
1120 return 0;
1121 }
1122
vntb_epf_spad_read(struct ntb_dev * ndev,int idx)1123 static u32 vntb_epf_spad_read(struct ntb_dev *ndev, int idx)
1124 {
1125 struct epf_ntb *ntb = ntb_ndev(ndev);
1126 int off = ntb->reg->spad_offset, ct = ntb->reg->spad_count * sizeof(u32);
1127 u32 val;
1128 void __iomem *base = (void __iomem *)ntb->reg;
1129
1130 val = readl(base + off + ct + idx * sizeof(u32));
1131 return val;
1132 }
1133
vntb_epf_spad_write(struct ntb_dev * ndev,int idx,u32 val)1134 static int vntb_epf_spad_write(struct ntb_dev *ndev, int idx, u32 val)
1135 {
1136 struct epf_ntb *ntb = ntb_ndev(ndev);
1137 struct epf_ntb_ctrl *ctrl = ntb->reg;
1138 int off = ctrl->spad_offset, ct = ctrl->spad_count * sizeof(u32);
1139 void __iomem *base = (void __iomem *)ntb->reg;
1140
1141 writel(val, base + off + ct + idx * sizeof(u32));
1142 return 0;
1143 }
1144
vntb_epf_peer_spad_read(struct ntb_dev * ndev,int pidx,int idx)1145 static u32 vntb_epf_peer_spad_read(struct ntb_dev *ndev, int pidx, int idx)
1146 {
1147 struct epf_ntb *ntb = ntb_ndev(ndev);
1148 struct epf_ntb_ctrl *ctrl = ntb->reg;
1149 int off = ctrl->spad_offset;
1150 void __iomem *base = (void __iomem *)ntb->reg;
1151 u32 val;
1152
1153 val = readl(base + off + idx * sizeof(u32));
1154 return val;
1155 }
1156
vntb_epf_peer_spad_write(struct ntb_dev * ndev,int pidx,int idx,u32 val)1157 static int vntb_epf_peer_spad_write(struct ntb_dev *ndev, int pidx, int idx, u32 val)
1158 {
1159 struct epf_ntb *ntb = ntb_ndev(ndev);
1160 struct epf_ntb_ctrl *ctrl = ntb->reg;
1161 int off = ctrl->spad_offset;
1162 void __iomem *base = (void __iomem *)ntb->reg;
1163
1164 writel(val, base + off + idx * sizeof(u32));
1165 return 0;
1166 }
1167
vntb_epf_peer_db_set(struct ntb_dev * ndev,u64 db_bits)1168 static int vntb_epf_peer_db_set(struct ntb_dev *ndev, u64 db_bits)
1169 {
1170 u32 interrupt_num = ffs(db_bits) + 1;
1171 struct epf_ntb *ntb = ntb_ndev(ndev);
1172 u8 func_no, vfunc_no;
1173 int ret;
1174
1175 func_no = ntb->epf->func_no;
1176 vfunc_no = ntb->epf->vfunc_no;
1177
1178 ret = pci_epc_raise_irq(ntb->epf->epc,
1179 func_no,
1180 vfunc_no,
1181 PCI_EPC_IRQ_MSI,
1182 interrupt_num + 1);
1183 if (ret)
1184 dev_err(&ntb->ntb.dev, "Failed to raise IRQ\n");
1185
1186 return ret;
1187 }
1188
vntb_epf_db_read(struct ntb_dev * ndev)1189 static u64 vntb_epf_db_read(struct ntb_dev *ndev)
1190 {
1191 struct epf_ntb *ntb = ntb_ndev(ndev);
1192
1193 return ntb->db;
1194 }
1195
vntb_epf_mw_get_align(struct ntb_dev * ndev,int pidx,int idx,resource_size_t * addr_align,resource_size_t * size_align,resource_size_t * size_max)1196 static int vntb_epf_mw_get_align(struct ntb_dev *ndev, int pidx, int idx,
1197 resource_size_t *addr_align,
1198 resource_size_t *size_align,
1199 resource_size_t *size_max)
1200 {
1201 struct epf_ntb *ntb = ntb_ndev(ndev);
1202
1203 if (addr_align)
1204 *addr_align = SZ_4K;
1205
1206 if (size_align)
1207 *size_align = 1;
1208
1209 if (size_max)
1210 *size_max = ntb->mws_size[idx];
1211
1212 return 0;
1213 }
1214
vntb_epf_link_is_up(struct ntb_dev * ndev,enum ntb_speed * speed,enum ntb_width * width)1215 static u64 vntb_epf_link_is_up(struct ntb_dev *ndev,
1216 enum ntb_speed *speed,
1217 enum ntb_width *width)
1218 {
1219 struct epf_ntb *ntb = ntb_ndev(ndev);
1220
1221 return ntb->reg->link_status;
1222 }
1223
vntb_epf_db_clear_mask(struct ntb_dev * ndev,u64 db_bits)1224 static int vntb_epf_db_clear_mask(struct ntb_dev *ndev, u64 db_bits)
1225 {
1226 return 0;
1227 }
1228
vntb_epf_db_clear(struct ntb_dev * ndev,u64 db_bits)1229 static int vntb_epf_db_clear(struct ntb_dev *ndev, u64 db_bits)
1230 {
1231 struct epf_ntb *ntb = ntb_ndev(ndev);
1232
1233 ntb->db &= ~db_bits;
1234 return 0;
1235 }
1236
vntb_epf_link_disable(struct ntb_dev * ntb)1237 static int vntb_epf_link_disable(struct ntb_dev *ntb)
1238 {
1239 return 0;
1240 }
1241
1242 static const struct ntb_dev_ops vntb_epf_ops = {
1243 .mw_count = vntb_epf_mw_count,
1244 .spad_count = vntb_epf_spad_count,
1245 .peer_mw_count = vntb_epf_peer_mw_count,
1246 .db_valid_mask = vntb_epf_db_valid_mask,
1247 .db_set_mask = vntb_epf_db_set_mask,
1248 .mw_set_trans = vntb_epf_mw_set_trans,
1249 .mw_clear_trans = vntb_epf_mw_clear_trans,
1250 .peer_mw_get_addr = vntb_epf_peer_mw_get_addr,
1251 .link_enable = vntb_epf_link_enable,
1252 .spad_read = vntb_epf_spad_read,
1253 .spad_write = vntb_epf_spad_write,
1254 .peer_spad_read = vntb_epf_peer_spad_read,
1255 .peer_spad_write = vntb_epf_peer_spad_write,
1256 .peer_db_set = vntb_epf_peer_db_set,
1257 .db_read = vntb_epf_db_read,
1258 .mw_get_align = vntb_epf_mw_get_align,
1259 .link_is_up = vntb_epf_link_is_up,
1260 .db_clear_mask = vntb_epf_db_clear_mask,
1261 .db_clear = vntb_epf_db_clear,
1262 .link_disable = vntb_epf_link_disable,
1263 };
1264
pci_vntb_probe(struct pci_dev * pdev,const struct pci_device_id * id)1265 static int pci_vntb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1266 {
1267 int ret;
1268 struct epf_ntb *ndev = (struct epf_ntb *)pdev->sysdata;
1269 struct device *dev = &pdev->dev;
1270
1271 ndev->ntb.pdev = pdev;
1272 ndev->ntb.topo = NTB_TOPO_NONE;
1273 ndev->ntb.ops = &vntb_epf_ops;
1274
1275 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
1276 if (ret) {
1277 dev_err(dev, "Cannot set DMA mask\n");
1278 return -EINVAL;
1279 }
1280
1281 ret = ntb_register_device(&ndev->ntb);
1282 if (ret) {
1283 dev_err(dev, "Failed to register NTB device\n");
1284 return ret;
1285 }
1286
1287 dev_dbg(dev, "PCI Virtual NTB driver loaded\n");
1288 return 0;
1289 }
1290
1291 static struct pci_device_id pci_vntb_table[] = {
1292 {
1293 PCI_DEVICE(0xffff, 0xffff),
1294 },
1295 {},
1296 };
1297
1298 static struct pci_driver vntb_pci_driver = {
1299 .name = "pci-vntb",
1300 .id_table = pci_vntb_table,
1301 .probe = pci_vntb_probe,
1302 };
1303
1304 /* ============ PCIe EPF Driver Bind ====================*/
1305
1306 /**
1307 * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality
1308 * @epf: NTB endpoint function device
1309 *
1310 * Initialize both the endpoint controllers associated with NTB function device.
1311 * Invoked when a primary interface or secondary interface is bound to EPC
1312 * device. This function will succeed only when EPC is bound to both the
1313 * interfaces.
1314 *
1315 * Returns: Zero for success, or an error code in case of failure
1316 */
epf_ntb_bind(struct pci_epf * epf)1317 static int epf_ntb_bind(struct pci_epf *epf)
1318 {
1319 struct epf_ntb *ntb = epf_get_drvdata(epf);
1320 struct device *dev = &epf->dev;
1321 int ret;
1322
1323 if (!epf->epc) {
1324 dev_dbg(dev, "PRIMARY EPC interface not yet bound\n");
1325 return 0;
1326 }
1327
1328 ret = epf_ntb_init_epc_bar(ntb);
1329 if (ret) {
1330 dev_err(dev, "Failed to create NTB EPC\n");
1331 goto err_bar_init;
1332 }
1333
1334 ret = epf_ntb_config_spad_bar_alloc(ntb);
1335 if (ret) {
1336 dev_err(dev, "Failed to allocate BAR memory\n");
1337 goto err_bar_alloc;
1338 }
1339
1340 ret = epf_ntb_epc_init(ntb);
1341 if (ret) {
1342 dev_err(dev, "Failed to initialize EPC\n");
1343 goto err_bar_alloc;
1344 }
1345
1346 epf_set_drvdata(epf, ntb);
1347
1348 pci_space[0] = (ntb->vntb_pid << 16) | ntb->vntb_vid;
1349 pci_vntb_table[0].vendor = ntb->vntb_vid;
1350 pci_vntb_table[0].device = ntb->vntb_pid;
1351
1352 ret = pci_register_driver(&vntb_pci_driver);
1353 if (ret) {
1354 dev_err(dev, "failure register vntb pci driver\n");
1355 goto err_epc_cleanup;
1356 }
1357
1358 ret = vpci_scan_bus(ntb);
1359 if (ret)
1360 goto err_unregister;
1361
1362 return 0;
1363
1364 err_unregister:
1365 pci_unregister_driver(&vntb_pci_driver);
1366 err_epc_cleanup:
1367 epf_ntb_epc_cleanup(ntb);
1368 err_bar_alloc:
1369 epf_ntb_config_spad_bar_free(ntb);
1370
1371 err_bar_init:
1372 epf_ntb_epc_destroy(ntb);
1373
1374 return ret;
1375 }
1376
1377 /**
1378 * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind()
1379 * @epf: NTB endpoint function device
1380 *
1381 * Cleanup the initialization from epf_ntb_bind()
1382 */
epf_ntb_unbind(struct pci_epf * epf)1383 static void epf_ntb_unbind(struct pci_epf *epf)
1384 {
1385 struct epf_ntb *ntb = epf_get_drvdata(epf);
1386
1387 epf_ntb_epc_cleanup(ntb);
1388 epf_ntb_config_spad_bar_free(ntb);
1389 epf_ntb_epc_destroy(ntb);
1390
1391 pci_unregister_driver(&vntb_pci_driver);
1392 }
1393
1394 // EPF driver probe
1395 static struct pci_epf_ops epf_ntb_ops = {
1396 .bind = epf_ntb_bind,
1397 .unbind = epf_ntb_unbind,
1398 .add_cfs = epf_ntb_add_cfs,
1399 };
1400
1401 /**
1402 * epf_ntb_probe() - Probe NTB function driver
1403 * @epf: NTB endpoint function device
1404 * @id: NTB endpoint function device ID
1405 *
1406 * Probe NTB function driver when endpoint function bus detects a NTB
1407 * endpoint function.
1408 *
1409 * Returns: Zero for success, or an error code in case of failure
1410 */
epf_ntb_probe(struct pci_epf * epf,const struct pci_epf_device_id * id)1411 static int epf_ntb_probe(struct pci_epf *epf,
1412 const struct pci_epf_device_id *id)
1413 {
1414 struct epf_ntb *ntb;
1415 struct device *dev;
1416
1417 dev = &epf->dev;
1418
1419 ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL);
1420 if (!ntb)
1421 return -ENOMEM;
1422
1423 epf->header = &epf_ntb_header;
1424 ntb->epf = epf;
1425 ntb->vbus_number = 0xff;
1426 epf_set_drvdata(epf, ntb);
1427
1428 dev_info(dev, "pci-ep epf driver loaded\n");
1429 return 0;
1430 }
1431
1432 static const struct pci_epf_device_id epf_ntb_ids[] = {
1433 {
1434 .name = "pci_epf_vntb",
1435 },
1436 {},
1437 };
1438
1439 static struct pci_epf_driver epf_ntb_driver = {
1440 .driver.name = "pci_epf_vntb",
1441 .probe = epf_ntb_probe,
1442 .id_table = epf_ntb_ids,
1443 .ops = &epf_ntb_ops,
1444 .owner = THIS_MODULE,
1445 };
1446
epf_ntb_init(void)1447 static int __init epf_ntb_init(void)
1448 {
1449 int ret;
1450
1451 kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM |
1452 WQ_HIGHPRI, 0);
1453 ret = pci_epf_register_driver(&epf_ntb_driver);
1454 if (ret) {
1455 destroy_workqueue(kpcintb_workqueue);
1456 pr_err("Failed to register pci epf ntb driver --> %d\n", ret);
1457 return ret;
1458 }
1459
1460 return 0;
1461 }
1462 module_init(epf_ntb_init);
1463
epf_ntb_exit(void)1464 static void __exit epf_ntb_exit(void)
1465 {
1466 pci_epf_unregister_driver(&epf_ntb_driver);
1467 destroy_workqueue(kpcintb_workqueue);
1468 }
1469 module_exit(epf_ntb_exit);
1470
1471 MODULE_DESCRIPTION("PCI EPF NTB DRIVER");
1472 MODULE_AUTHOR("Frank Li <Frank.li@nxp.com>");
1473 MODULE_LICENSE("GPL v2");
1474