1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Endpoint Function Driver to implement Non-Transparent Bridge functionality
4  * Between PCI RC and EP
5  *
6  * Copyright (C) 2020 Texas Instruments
7  * Copyright (C) 2022 NXP
8  *
9  * Based on pci-epf-ntb.c
10  * Author: Frank Li <Frank.Li@nxp.com>
11  * Author: Kishon Vijay Abraham I <kishon@ti.com>
12  */
13 
14 /*
15  * +------------+         +---------------------------------------+
16  * |            |         |                                       |
17  * +------------+         |                        +--------------+
18  * | NTB        |         |                        | NTB          |
19  * | NetDev     |         |                        | NetDev       |
20  * +------------+         |                        +--------------+
21  * | NTB        |         |                        | NTB          |
22  * | Transfer   |         |                        | Transfer     |
23  * +------------+         |                        +--------------+
24  * |            |         |                        |              |
25  * |  PCI NTB   |         |                        |              |
26  * |    EPF     |         |                        |              |
27  * |   Driver   |         |                        | PCI Virtual  |
28  * |            |         +---------------+        | NTB Driver   |
29  * |            |         | PCI EP NTB    |<------>|              |
30  * |            |         |  FN Driver    |        |              |
31  * +------------+         +---------------+        +--------------+
32  * |            |         |               |        |              |
33  * |  PCI Bus   | <-----> |  PCI EP Bus   |        |  Virtual PCI |
34  * |            |  PCI    |               |        |     Bus      |
35  * +------------+         +---------------+--------+--------------+
36  * PCIe Root Port                        PCI EP
37  */
38 
39 #include <linux/delay.h>
40 #include <linux/io.h>
41 #include <linux/module.h>
42 #include <linux/slab.h>
43 
44 #include <linux/pci-epc.h>
45 #include <linux/pci-epf.h>
46 #include <linux/ntb.h>
47 
48 static struct workqueue_struct *kpcintb_workqueue;
49 
50 #define COMMAND_CONFIGURE_DOORBELL	1
51 #define COMMAND_TEARDOWN_DOORBELL	2
52 #define COMMAND_CONFIGURE_MW		3
53 #define COMMAND_TEARDOWN_MW		4
54 #define COMMAND_LINK_UP			5
55 #define COMMAND_LINK_DOWN		6
56 
57 #define COMMAND_STATUS_OK		1
58 #define COMMAND_STATUS_ERROR		2
59 
60 #define LINK_STATUS_UP			BIT(0)
61 
62 #define SPAD_COUNT			64
63 #define DB_COUNT			4
64 #define NTB_MW_OFFSET			2
65 #define DB_COUNT_MASK			GENMASK(15, 0)
66 #define MSIX_ENABLE			BIT(16)
67 #define MAX_DB_COUNT			32
68 #define MAX_MW				4
69 
70 enum epf_ntb_bar {
71 	BAR_CONFIG,
72 	BAR_DB,
73 	BAR_MW0,
74 	BAR_MW1,
75 	BAR_MW2,
76 };
77 
78 /*
79  * +--------------------------------------------------+ Base
80  * |                                                  |
81  * |                                                  |
82  * |                                                  |
83  * |          Common Control Register                 |
84  * |                                                  |
85  * |                                                  |
86  * |                                                  |
87  * +-----------------------+--------------------------+ Base+spad_offset
88  * |                       |                          |
89  * |    Peer Spad Space    |    Spad Space            |
90  * |                       |                          |
91  * |                       |                          |
92  * +-----------------------+--------------------------+ Base+spad_offset
93  * |                       |                          |     +spad_count * 4
94  * |                       |                          |
95  * |     Spad Space        |   Peer Spad Space        |
96  * |                       |                          |
97  * +-----------------------+--------------------------+
98  *       Virtual PCI             PCIe Endpoint
99  *       NTB Driver               NTB Driver
100  */
101 struct epf_ntb_ctrl {
102 	u32 command;
103 	u32 argument;
104 	u16 command_status;
105 	u16 link_status;
106 	u32 topology;
107 	u64 addr;
108 	u64 size;
109 	u32 num_mws;
110 	u32 reserved;
111 	u32 spad_offset;
112 	u32 spad_count;
113 	u32 db_entry_size;
114 	u32 db_data[MAX_DB_COUNT];
115 	u32 db_offset[MAX_DB_COUNT];
116 } __packed;
117 
118 struct epf_ntb {
119 	struct ntb_dev ntb;
120 	struct pci_epf *epf;
121 	struct config_group group;
122 
123 	u32 num_mws;
124 	u32 db_count;
125 	u32 spad_count;
126 	u64 mws_size[MAX_MW];
127 	u64 db;
128 	u32 vbus_number;
129 	u16 vntb_pid;
130 	u16 vntb_vid;
131 
132 	bool linkup;
133 	u32 spad_size;
134 
135 	enum pci_barno epf_ntb_bar[6];
136 
137 	struct epf_ntb_ctrl *reg;
138 
139 	u32 *epf_db;
140 
141 	phys_addr_t vpci_mw_phy[MAX_MW];
142 	void __iomem *vpci_mw_addr[MAX_MW];
143 
144 	struct delayed_work cmd_handler;
145 };
146 
147 #define to_epf_ntb(epf_group) container_of((epf_group), struct epf_ntb, group)
148 #define ntb_ndev(__ntb) container_of(__ntb, struct epf_ntb, ntb)
149 
150 static struct pci_epf_header epf_ntb_header = {
151 	.vendorid	= PCI_ANY_ID,
152 	.deviceid	= PCI_ANY_ID,
153 	.baseclass_code	= PCI_BASE_CLASS_MEMORY,
154 	.interrupt_pin	= PCI_INTERRUPT_INTA,
155 };
156 
157 /**
158  * epf_ntb_link_up() - Raise link_up interrupt to Virtual Host (VHOST)
159  * @ntb: NTB device that facilitates communication between HOST and VHOST
160  * @link_up: true or false indicating Link is UP or Down
161  *
162  * Once NTB function in HOST invoke ntb_link_enable(),
163  * this NTB function driver will trigger a link event to VHOST.
164  *
165  * Returns: Zero for success, or an error code in case of failure
166  */
epf_ntb_link_up(struct epf_ntb * ntb,bool link_up)167 static int epf_ntb_link_up(struct epf_ntb *ntb, bool link_up)
168 {
169 	if (link_up)
170 		ntb->reg->link_status |= LINK_STATUS_UP;
171 	else
172 		ntb->reg->link_status &= ~LINK_STATUS_UP;
173 
174 	ntb_link_event(&ntb->ntb);
175 	return 0;
176 }
177 
178 /**
179  * epf_ntb_configure_mw() - Configure the Outbound Address Space for VHOST
180  *   to access the memory window of HOST
181  * @ntb: NTB device that facilitates communication between HOST and VHOST
182  * @mw: Index of the memory window (either 0, 1, 2 or 3)
183  *
184  *                          EP Outbound Window
185  * +--------+              +-----------+
186  * |        |              |           |
187  * |        |              |           |
188  * |        |              |           |
189  * |        |              |           |
190  * |        |              +-----------+
191  * | Virtual|              | Memory Win|
192  * | NTB    | -----------> |           |
193  * | Driver |              |           |
194  * |        |              +-----------+
195  * |        |              |           |
196  * |        |              |           |
197  * +--------+              +-----------+
198  *  VHOST                   PCI EP
199  *
200  * Returns: Zero for success, or an error code in case of failure
201  */
epf_ntb_configure_mw(struct epf_ntb * ntb,u32 mw)202 static int epf_ntb_configure_mw(struct epf_ntb *ntb, u32 mw)
203 {
204 	phys_addr_t phys_addr;
205 	u8 func_no, vfunc_no;
206 	u64 addr, size;
207 	int ret = 0;
208 
209 	phys_addr = ntb->vpci_mw_phy[mw];
210 	addr = ntb->reg->addr;
211 	size = ntb->reg->size;
212 
213 	func_no = ntb->epf->func_no;
214 	vfunc_no = ntb->epf->vfunc_no;
215 
216 	ret = pci_epc_map_addr(ntb->epf->epc, func_no, vfunc_no, phys_addr, addr, size);
217 	if (ret)
218 		dev_err(&ntb->epf->epc->dev,
219 			"Failed to map memory window %d address\n", mw);
220 	return ret;
221 }
222 
223 /**
224  * epf_ntb_teardown_mw() - Teardown the configured OB ATU
225  * @ntb: NTB device that facilitates communication between HOST and VHOST
226  * @mw: Index of the memory window (either 0, 1, 2 or 3)
227  *
228  * Teardown the configured OB ATU configured in epf_ntb_configure_mw() using
229  * pci_epc_unmap_addr()
230  */
epf_ntb_teardown_mw(struct epf_ntb * ntb,u32 mw)231 static void epf_ntb_teardown_mw(struct epf_ntb *ntb, u32 mw)
232 {
233 	pci_epc_unmap_addr(ntb->epf->epc,
234 			   ntb->epf->func_no,
235 			   ntb->epf->vfunc_no,
236 			   ntb->vpci_mw_phy[mw]);
237 }
238 
239 /**
240  * epf_ntb_cmd_handler() - Handle commands provided by the NTB HOST
241  * @work: work_struct for the epf_ntb_epc
242  *
243  * Workqueue function that gets invoked for the two epf_ntb_epc
244  * periodically (once every 5ms) to see if it has received any commands
245  * from NTB HOST. The HOST can send commands to configure doorbell or
246  * configure memory window or to update link status.
247  */
epf_ntb_cmd_handler(struct work_struct * work)248 static void epf_ntb_cmd_handler(struct work_struct *work)
249 {
250 	struct epf_ntb_ctrl *ctrl;
251 	u32 command, argument;
252 	struct epf_ntb *ntb;
253 	struct device *dev;
254 	int ret;
255 	int i;
256 
257 	ntb = container_of(work, struct epf_ntb, cmd_handler.work);
258 
259 	for (i = 1; i < ntb->db_count; i++) {
260 		if (ntb->epf_db[i]) {
261 			ntb->db |= 1 << (i - 1);
262 			ntb_db_event(&ntb->ntb, i);
263 			ntb->epf_db[i] = 0;
264 		}
265 	}
266 
267 	ctrl = ntb->reg;
268 	command = ctrl->command;
269 	if (!command)
270 		goto reset_handler;
271 	argument = ctrl->argument;
272 
273 	ctrl->command = 0;
274 	ctrl->argument = 0;
275 
276 	ctrl = ntb->reg;
277 	dev = &ntb->epf->dev;
278 
279 	switch (command) {
280 	case COMMAND_CONFIGURE_DOORBELL:
281 		ctrl->command_status = COMMAND_STATUS_OK;
282 		break;
283 	case COMMAND_TEARDOWN_DOORBELL:
284 		ctrl->command_status = COMMAND_STATUS_OK;
285 		break;
286 	case COMMAND_CONFIGURE_MW:
287 		ret = epf_ntb_configure_mw(ntb, argument);
288 		if (ret < 0)
289 			ctrl->command_status = COMMAND_STATUS_ERROR;
290 		else
291 			ctrl->command_status = COMMAND_STATUS_OK;
292 		break;
293 	case COMMAND_TEARDOWN_MW:
294 		epf_ntb_teardown_mw(ntb, argument);
295 		ctrl->command_status = COMMAND_STATUS_OK;
296 		break;
297 	case COMMAND_LINK_UP:
298 		ntb->linkup = true;
299 		ret = epf_ntb_link_up(ntb, true);
300 		if (ret < 0)
301 			ctrl->command_status = COMMAND_STATUS_ERROR;
302 		else
303 			ctrl->command_status = COMMAND_STATUS_OK;
304 		goto reset_handler;
305 	case COMMAND_LINK_DOWN:
306 		ntb->linkup = false;
307 		ret = epf_ntb_link_up(ntb, false);
308 		if (ret < 0)
309 			ctrl->command_status = COMMAND_STATUS_ERROR;
310 		else
311 			ctrl->command_status = COMMAND_STATUS_OK;
312 		break;
313 	default:
314 		dev_err(dev, "UNKNOWN command: %d\n", command);
315 		break;
316 	}
317 
318 reset_handler:
319 	queue_delayed_work(kpcintb_workqueue, &ntb->cmd_handler,
320 			   msecs_to_jiffies(5));
321 }
322 
323 /**
324  * epf_ntb_config_sspad_bar_clear() - Clear Config + Self scratchpad BAR
325  * @ntb: EPC associated with one of the HOST which holds peer's outbound
326  *	 address.
327  *
328  * Clear BAR0 of EP CONTROLLER 1 which contains the HOST1's config and
329  * self scratchpad region (removes inbound ATU configuration). While BAR0 is
330  * the default self scratchpad BAR, an NTB could have other BARs for self
331  * scratchpad (because of reserved BARs). This function can get the exact BAR
332  * used for self scratchpad from epf_ntb_bar[BAR_CONFIG].
333  *
334  * Please note the self scratchpad region and config region is combined to
335  * a single region and mapped using the same BAR. Also note VHOST's peer
336  * scratchpad is HOST's self scratchpad.
337  *
338  * Returns: void
339  */
epf_ntb_config_sspad_bar_clear(struct epf_ntb * ntb)340 static void epf_ntb_config_sspad_bar_clear(struct epf_ntb *ntb)
341 {
342 	struct pci_epf_bar *epf_bar;
343 	enum pci_barno barno;
344 
345 	barno = ntb->epf_ntb_bar[BAR_CONFIG];
346 	epf_bar = &ntb->epf->bar[barno];
347 
348 	pci_epc_clear_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar);
349 }
350 
351 /**
352  * epf_ntb_config_sspad_bar_set() - Set Config + Self scratchpad BAR
353  * @ntb: NTB device that facilitates communication between HOST and VHOST
354  *
355  * Map BAR0 of EP CONTROLLER which contains the VHOST's config and
356  * self scratchpad region.
357  *
358  * Please note the self scratchpad region and config region is combined to
359  * a single region and mapped using the same BAR.
360  *
361  * Returns: Zero for success, or an error code in case of failure
362  */
epf_ntb_config_sspad_bar_set(struct epf_ntb * ntb)363 static int epf_ntb_config_sspad_bar_set(struct epf_ntb *ntb)
364 {
365 	struct pci_epf_bar *epf_bar;
366 	enum pci_barno barno;
367 	u8 func_no, vfunc_no;
368 	struct device *dev;
369 	int ret;
370 
371 	dev = &ntb->epf->dev;
372 	func_no = ntb->epf->func_no;
373 	vfunc_no = ntb->epf->vfunc_no;
374 	barno = ntb->epf_ntb_bar[BAR_CONFIG];
375 	epf_bar = &ntb->epf->bar[barno];
376 
377 	ret = pci_epc_set_bar(ntb->epf->epc, func_no, vfunc_no, epf_bar);
378 	if (ret) {
379 		dev_err(dev, "inft: Config/Status/SPAD BAR set failed\n");
380 		return ret;
381 	}
382 	return 0;
383 }
384 
385 /**
386  * epf_ntb_config_spad_bar_free() - Free the physical memory associated with
387  *   config + scratchpad region
388  * @ntb: NTB device that facilitates communication between HOST and VHOST
389  */
epf_ntb_config_spad_bar_free(struct epf_ntb * ntb)390 static void epf_ntb_config_spad_bar_free(struct epf_ntb *ntb)
391 {
392 	enum pci_barno barno;
393 
394 	barno = ntb->epf_ntb_bar[BAR_CONFIG];
395 	pci_epf_free_space(ntb->epf, ntb->reg, barno, 0);
396 }
397 
398 /**
399  * epf_ntb_config_spad_bar_alloc() - Allocate memory for config + scratchpad
400  *   region
401  * @ntb: NTB device that facilitates communication between HOST and VHOST
402  *
403  * Allocate the Local Memory mentioned in the above diagram. The size of
404  * CONFIG REGION is sizeof(struct epf_ntb_ctrl) and size of SCRATCHPAD REGION
405  * is obtained from "spad-count" configfs entry.
406  *
407  * Returns: Zero for success, or an error code in case of failure
408  */
epf_ntb_config_spad_bar_alloc(struct epf_ntb * ntb)409 static int epf_ntb_config_spad_bar_alloc(struct epf_ntb *ntb)
410 {
411 	size_t align;
412 	enum pci_barno barno;
413 	struct epf_ntb_ctrl *ctrl;
414 	u32 spad_size, ctrl_size;
415 	u64 size;
416 	struct pci_epf *epf = ntb->epf;
417 	struct device *dev = &epf->dev;
418 	u32 spad_count;
419 	void *base;
420 	int i;
421 	const struct pci_epc_features *epc_features = pci_epc_get_features(epf->epc,
422 								epf->func_no,
423 								epf->vfunc_no);
424 	barno = ntb->epf_ntb_bar[BAR_CONFIG];
425 	size = epc_features->bar_fixed_size[barno];
426 	align = epc_features->align;
427 
428 	if ((!IS_ALIGNED(size, align)))
429 		return -EINVAL;
430 
431 	spad_count = ntb->spad_count;
432 
433 	ctrl_size = sizeof(struct epf_ntb_ctrl);
434 	spad_size = 2 * spad_count * sizeof(u32);
435 
436 	if (!align) {
437 		ctrl_size = roundup_pow_of_two(ctrl_size);
438 		spad_size = roundup_pow_of_two(spad_size);
439 	} else {
440 		ctrl_size = ALIGN(ctrl_size, align);
441 		spad_size = ALIGN(spad_size, align);
442 	}
443 
444 	if (!size)
445 		size = ctrl_size + spad_size;
446 	else if (size < ctrl_size + spad_size)
447 		return -EINVAL;
448 
449 	base = pci_epf_alloc_space(epf, size, barno, align, 0);
450 	if (!base) {
451 		dev_err(dev, "Config/Status/SPAD alloc region fail\n");
452 		return -ENOMEM;
453 	}
454 
455 	ntb->reg = base;
456 
457 	ctrl = ntb->reg;
458 	ctrl->spad_offset = ctrl_size;
459 
460 	ctrl->spad_count = spad_count;
461 	ctrl->num_mws = ntb->num_mws;
462 	ntb->spad_size = spad_size;
463 
464 	ctrl->db_entry_size = sizeof(u32);
465 
466 	for (i = 0; i < ntb->db_count; i++) {
467 		ntb->reg->db_data[i] = 1 + i;
468 		ntb->reg->db_offset[i] = 0;
469 	}
470 
471 	return 0;
472 }
473 
474 /**
475  * epf_ntb_configure_interrupt() - Configure MSI/MSI-X capability
476  * @ntb: NTB device that facilitates communication between HOST and VHOST
477  *
478  * Configure MSI/MSI-X capability for each interface with number of
479  * interrupts equal to "db_count" configfs entry.
480  *
481  * Returns: Zero for success, or an error code in case of failure
482  */
epf_ntb_configure_interrupt(struct epf_ntb * ntb)483 static int epf_ntb_configure_interrupt(struct epf_ntb *ntb)
484 {
485 	const struct pci_epc_features *epc_features;
486 	struct device *dev;
487 	u32 db_count;
488 	int ret;
489 
490 	dev = &ntb->epf->dev;
491 
492 	epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no);
493 
494 	if (!(epc_features->msix_capable || epc_features->msi_capable)) {
495 		dev_err(dev, "MSI or MSI-X is required for doorbell\n");
496 		return -EINVAL;
497 	}
498 
499 	db_count = ntb->db_count;
500 	if (db_count > MAX_DB_COUNT) {
501 		dev_err(dev, "DB count cannot be more than %d\n", MAX_DB_COUNT);
502 		return -EINVAL;
503 	}
504 
505 	ntb->db_count = db_count;
506 
507 	if (epc_features->msi_capable) {
508 		ret = pci_epc_set_msi(ntb->epf->epc,
509 				      ntb->epf->func_no,
510 				      ntb->epf->vfunc_no,
511 				      16);
512 		if (ret) {
513 			dev_err(dev, "MSI configuration failed\n");
514 			return ret;
515 		}
516 	}
517 
518 	return 0;
519 }
520 
521 /**
522  * epf_ntb_db_bar_init() - Configure Doorbell window BARs
523  * @ntb: NTB device that facilitates communication between HOST and VHOST
524  *
525  * Returns: Zero for success, or an error code in case of failure
526  */
epf_ntb_db_bar_init(struct epf_ntb * ntb)527 static int epf_ntb_db_bar_init(struct epf_ntb *ntb)
528 {
529 	const struct pci_epc_features *epc_features;
530 	u32 align;
531 	struct device *dev = &ntb->epf->dev;
532 	int ret;
533 	struct pci_epf_bar *epf_bar;
534 	void __iomem *mw_addr;
535 	enum pci_barno barno;
536 	size_t size = sizeof(u32) * ntb->db_count;
537 
538 	epc_features = pci_epc_get_features(ntb->epf->epc,
539 					    ntb->epf->func_no,
540 					    ntb->epf->vfunc_no);
541 	align = epc_features->align;
542 
543 	if (size < 128)
544 		size = 128;
545 
546 	if (align)
547 		size = ALIGN(size, align);
548 	else
549 		size = roundup_pow_of_two(size);
550 
551 	barno = ntb->epf_ntb_bar[BAR_DB];
552 
553 	mw_addr = pci_epf_alloc_space(ntb->epf, size, barno, align, 0);
554 	if (!mw_addr) {
555 		dev_err(dev, "Failed to allocate OB address\n");
556 		return -ENOMEM;
557 	}
558 
559 	ntb->epf_db = mw_addr;
560 
561 	epf_bar = &ntb->epf->bar[barno];
562 
563 	ret = pci_epc_set_bar(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no, epf_bar);
564 	if (ret) {
565 		dev_err(dev, "Doorbell BAR set failed\n");
566 			goto err_alloc_peer_mem;
567 	}
568 	return ret;
569 
570 err_alloc_peer_mem:
571 	pci_epf_free_space(ntb->epf, mw_addr, barno, 0);
572 	return -1;
573 }
574 
575 static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws);
576 
577 /**
578  * epf_ntb_db_bar_clear() - Clear doorbell BAR and free memory
579  *   allocated in peer's outbound address space
580  * @ntb: NTB device that facilitates communication between HOST and VHOST
581  */
epf_ntb_db_bar_clear(struct epf_ntb * ntb)582 static void epf_ntb_db_bar_clear(struct epf_ntb *ntb)
583 {
584 	enum pci_barno barno;
585 
586 	barno = ntb->epf_ntb_bar[BAR_DB];
587 	pci_epf_free_space(ntb->epf, ntb->epf_db, barno, 0);
588 	pci_epc_clear_bar(ntb->epf->epc,
589 			  ntb->epf->func_no,
590 			  ntb->epf->vfunc_no,
591 			  &ntb->epf->bar[barno]);
592 }
593 
594 /**
595  * epf_ntb_mw_bar_init() - Configure Memory window BARs
596  * @ntb: NTB device that facilitates communication between HOST and VHOST
597  *
598  * Returns: Zero for success, or an error code in case of failure
599  */
epf_ntb_mw_bar_init(struct epf_ntb * ntb)600 static int epf_ntb_mw_bar_init(struct epf_ntb *ntb)
601 {
602 	int ret = 0;
603 	int i;
604 	u64 size;
605 	enum pci_barno barno;
606 	struct device *dev = &ntb->epf->dev;
607 
608 	for (i = 0; i < ntb->num_mws; i++) {
609 		size = ntb->mws_size[i];
610 		barno = ntb->epf_ntb_bar[BAR_MW0 + i];
611 
612 		ntb->epf->bar[barno].barno = barno;
613 		ntb->epf->bar[barno].size = size;
614 		ntb->epf->bar[barno].addr = NULL;
615 		ntb->epf->bar[barno].phys_addr = 0;
616 		ntb->epf->bar[barno].flags |= upper_32_bits(size) ?
617 				PCI_BASE_ADDRESS_MEM_TYPE_64 :
618 				PCI_BASE_ADDRESS_MEM_TYPE_32;
619 
620 		ret = pci_epc_set_bar(ntb->epf->epc,
621 				      ntb->epf->func_no,
622 				      ntb->epf->vfunc_no,
623 				      &ntb->epf->bar[barno]);
624 		if (ret) {
625 			dev_err(dev, "MW set failed\n");
626 			goto err_alloc_mem;
627 		}
628 
629 		/* Allocate EPC outbound memory windows to vpci vntb device */
630 		ntb->vpci_mw_addr[i] = pci_epc_mem_alloc_addr(ntb->epf->epc,
631 							      &ntb->vpci_mw_phy[i],
632 							      size);
633 		if (!ntb->vpci_mw_addr[i]) {
634 			ret = -ENOMEM;
635 			dev_err(dev, "Failed to allocate source address\n");
636 			goto err_set_bar;
637 		}
638 	}
639 
640 	return ret;
641 
642 err_set_bar:
643 	pci_epc_clear_bar(ntb->epf->epc,
644 			  ntb->epf->func_no,
645 			  ntb->epf->vfunc_no,
646 			  &ntb->epf->bar[barno]);
647 err_alloc_mem:
648 	epf_ntb_mw_bar_clear(ntb, i);
649 	return ret;
650 }
651 
652 /**
653  * epf_ntb_mw_bar_clear() - Clear Memory window BARs
654  * @ntb: NTB device that facilitates communication between HOST and VHOST
655  * @num_mws: the number of Memory window BARs that to be cleared
656  */
epf_ntb_mw_bar_clear(struct epf_ntb * ntb,int num_mws)657 static void epf_ntb_mw_bar_clear(struct epf_ntb *ntb, int num_mws)
658 {
659 	enum pci_barno barno;
660 	int i;
661 
662 	for (i = 0; i < num_mws; i++) {
663 		barno = ntb->epf_ntb_bar[BAR_MW0 + i];
664 		pci_epc_clear_bar(ntb->epf->epc,
665 				  ntb->epf->func_no,
666 				  ntb->epf->vfunc_no,
667 				  &ntb->epf->bar[barno]);
668 
669 		pci_epc_mem_free_addr(ntb->epf->epc,
670 				      ntb->vpci_mw_phy[i],
671 				      ntb->vpci_mw_addr[i],
672 				      ntb->mws_size[i]);
673 	}
674 }
675 
676 /**
677  * epf_ntb_epc_destroy() - Cleanup NTB EPC interface
678  * @ntb: NTB device that facilitates communication between HOST and VHOST
679  *
680  * Wrapper for epf_ntb_epc_destroy_interface() to cleanup all the NTB interfaces
681  */
epf_ntb_epc_destroy(struct epf_ntb * ntb)682 static void epf_ntb_epc_destroy(struct epf_ntb *ntb)
683 {
684 	pci_epc_remove_epf(ntb->epf->epc, ntb->epf, 0);
685 	pci_epc_put(ntb->epf->epc);
686 }
687 
688 /**
689  * epf_ntb_init_epc_bar() - Identify BARs to be used for each of the NTB
690  * constructs (scratchpad region, doorbell, memorywindow)
691  * @ntb: NTB device that facilitates communication between HOST and VHOST
692  *
693  * Returns: Zero for success, or an error code in case of failure
694  */
epf_ntb_init_epc_bar(struct epf_ntb * ntb)695 static int epf_ntb_init_epc_bar(struct epf_ntb *ntb)
696 {
697 	const struct pci_epc_features *epc_features;
698 	enum pci_barno barno;
699 	enum epf_ntb_bar bar;
700 	struct device *dev;
701 	u32 num_mws;
702 	int i;
703 
704 	barno = BAR_0;
705 	num_mws = ntb->num_mws;
706 	dev = &ntb->epf->dev;
707 	epc_features = pci_epc_get_features(ntb->epf->epc, ntb->epf->func_no, ntb->epf->vfunc_no);
708 
709 	/* These are required BARs which are mandatory for NTB functionality */
710 	for (bar = BAR_CONFIG; bar <= BAR_MW0; bar++, barno++) {
711 		barno = pci_epc_get_next_free_bar(epc_features, barno);
712 		if (barno < 0) {
713 			dev_err(dev, "Fail to get NTB function BAR\n");
714 			return barno;
715 		}
716 		ntb->epf_ntb_bar[bar] = barno;
717 	}
718 
719 	/* These are optional BARs which don't impact NTB functionality */
720 	for (bar = BAR_MW1, i = 1; i < num_mws; bar++, barno++, i++) {
721 		barno = pci_epc_get_next_free_bar(epc_features, barno);
722 		if (barno < 0) {
723 			ntb->num_mws = i;
724 			dev_dbg(dev, "BAR not available for > MW%d\n", i + 1);
725 		}
726 		ntb->epf_ntb_bar[bar] = barno;
727 	}
728 
729 	return 0;
730 }
731 
732 /**
733  * epf_ntb_epc_init() - Initialize NTB interface
734  * @ntb: NTB device that facilitates communication between HOST and VHOST
735  *
736  * Wrapper to initialize a particular EPC interface and start the workqueue
737  * to check for commands from HOST. This function will write to the
738  * EP controller HW for configuring it.
739  *
740  * Returns: Zero for success, or an error code in case of failure
741  */
epf_ntb_epc_init(struct epf_ntb * ntb)742 static int epf_ntb_epc_init(struct epf_ntb *ntb)
743 {
744 	u8 func_no, vfunc_no;
745 	struct pci_epc *epc;
746 	struct pci_epf *epf;
747 	struct device *dev;
748 	int ret;
749 
750 	epf = ntb->epf;
751 	dev = &epf->dev;
752 	epc = epf->epc;
753 	func_no = ntb->epf->func_no;
754 	vfunc_no = ntb->epf->vfunc_no;
755 
756 	ret = epf_ntb_config_sspad_bar_set(ntb);
757 	if (ret) {
758 		dev_err(dev, "Config/self SPAD BAR init failed");
759 		return ret;
760 	}
761 
762 	ret = epf_ntb_configure_interrupt(ntb);
763 	if (ret) {
764 		dev_err(dev, "Interrupt configuration failed\n");
765 		goto err_config_interrupt;
766 	}
767 
768 	ret = epf_ntb_db_bar_init(ntb);
769 	if (ret) {
770 		dev_err(dev, "DB BAR init failed\n");
771 		goto err_db_bar_init;
772 	}
773 
774 	ret = epf_ntb_mw_bar_init(ntb);
775 	if (ret) {
776 		dev_err(dev, "MW BAR init failed\n");
777 		goto err_mw_bar_init;
778 	}
779 
780 	if (vfunc_no <= 1) {
781 		ret = pci_epc_write_header(epc, func_no, vfunc_no, epf->header);
782 		if (ret) {
783 			dev_err(dev, "Configuration header write failed\n");
784 			goto err_write_header;
785 		}
786 	}
787 
788 	INIT_DELAYED_WORK(&ntb->cmd_handler, epf_ntb_cmd_handler);
789 	queue_work(kpcintb_workqueue, &ntb->cmd_handler.work);
790 
791 	return 0;
792 
793 err_write_header:
794 	epf_ntb_mw_bar_clear(ntb, ntb->num_mws);
795 err_mw_bar_init:
796 	epf_ntb_db_bar_clear(ntb);
797 err_db_bar_init:
798 err_config_interrupt:
799 	epf_ntb_config_sspad_bar_clear(ntb);
800 
801 	return ret;
802 }
803 
804 
805 /**
806  * epf_ntb_epc_cleanup() - Cleanup all NTB interfaces
807  * @ntb: NTB device that facilitates communication between HOST and VHOST
808  *
809  * Wrapper to cleanup all NTB interfaces.
810  */
epf_ntb_epc_cleanup(struct epf_ntb * ntb)811 static void epf_ntb_epc_cleanup(struct epf_ntb *ntb)
812 {
813 	epf_ntb_mw_bar_clear(ntb, ntb->num_mws);
814 	epf_ntb_db_bar_clear(ntb);
815 	epf_ntb_config_sspad_bar_clear(ntb);
816 }
817 
818 #define EPF_NTB_R(_name)						\
819 static ssize_t epf_ntb_##_name##_show(struct config_item *item,		\
820 				      char *page)			\
821 {									\
822 	struct config_group *group = to_config_group(item);		\
823 	struct epf_ntb *ntb = to_epf_ntb(group);			\
824 									\
825 	return sprintf(page, "%d\n", ntb->_name);			\
826 }
827 
828 #define EPF_NTB_W(_name)						\
829 static ssize_t epf_ntb_##_name##_store(struct config_item *item,	\
830 				       const char *page, size_t len)	\
831 {									\
832 	struct config_group *group = to_config_group(item);		\
833 	struct epf_ntb *ntb = to_epf_ntb(group);			\
834 	u32 val;							\
835 	int ret;							\
836 									\
837 	ret = kstrtou32(page, 0, &val);					\
838 	if (ret)							\
839 		return ret;						\
840 									\
841 	ntb->_name = val;						\
842 									\
843 	return len;							\
844 }
845 
846 #define EPF_NTB_MW_R(_name)						\
847 static ssize_t epf_ntb_##_name##_show(struct config_item *item,		\
848 				      char *page)			\
849 {									\
850 	struct config_group *group = to_config_group(item);		\
851 	struct epf_ntb *ntb = to_epf_ntb(group);			\
852 	struct device *dev = &ntb->epf->dev;				\
853 	int win_no;							\
854 									\
855 	if (sscanf(#_name, "mw%d", &win_no) != 1)			\
856 		return -EINVAL;						\
857 									\
858 	if (win_no <= 0 || win_no > ntb->num_mws) {			\
859 		dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
860 		return -EINVAL;						\
861 	}								\
862 									\
863 	return sprintf(page, "%lld\n", ntb->mws_size[win_no - 1]);	\
864 }
865 
866 #define EPF_NTB_MW_W(_name)						\
867 static ssize_t epf_ntb_##_name##_store(struct config_item *item,	\
868 				       const char *page, size_t len)	\
869 {									\
870 	struct config_group *group = to_config_group(item);		\
871 	struct epf_ntb *ntb = to_epf_ntb(group);			\
872 	struct device *dev = &ntb->epf->dev;				\
873 	int win_no;							\
874 	u64 val;							\
875 	int ret;							\
876 									\
877 	ret = kstrtou64(page, 0, &val);					\
878 	if (ret)							\
879 		return ret;						\
880 									\
881 	if (sscanf(#_name, "mw%d", &win_no) != 1)			\
882 		return -EINVAL;						\
883 									\
884 	if (win_no <= 0 || win_no > ntb->num_mws) {			\
885 		dev_err(dev, "Invalid num_nws: %d value\n", ntb->num_mws); \
886 		return -EINVAL;						\
887 	}								\
888 									\
889 	ntb->mws_size[win_no - 1] = val;				\
890 									\
891 	return len;							\
892 }
893 
epf_ntb_num_mws_store(struct config_item * item,const char * page,size_t len)894 static ssize_t epf_ntb_num_mws_store(struct config_item *item,
895 				     const char *page, size_t len)
896 {
897 	struct config_group *group = to_config_group(item);
898 	struct epf_ntb *ntb = to_epf_ntb(group);
899 	u32 val;
900 	int ret;
901 
902 	ret = kstrtou32(page, 0, &val);
903 	if (ret)
904 		return ret;
905 
906 	if (val > MAX_MW)
907 		return -EINVAL;
908 
909 	ntb->num_mws = val;
910 
911 	return len;
912 }
913 
914 EPF_NTB_R(spad_count)
915 EPF_NTB_W(spad_count)
916 EPF_NTB_R(db_count)
917 EPF_NTB_W(db_count)
918 EPF_NTB_R(num_mws)
919 EPF_NTB_R(vbus_number)
920 EPF_NTB_W(vbus_number)
921 EPF_NTB_R(vntb_pid)
922 EPF_NTB_W(vntb_pid)
923 EPF_NTB_R(vntb_vid)
924 EPF_NTB_W(vntb_vid)
925 EPF_NTB_MW_R(mw1)
926 EPF_NTB_MW_W(mw1)
927 EPF_NTB_MW_R(mw2)
928 EPF_NTB_MW_W(mw2)
929 EPF_NTB_MW_R(mw3)
930 EPF_NTB_MW_W(mw3)
931 EPF_NTB_MW_R(mw4)
932 EPF_NTB_MW_W(mw4)
933 
934 CONFIGFS_ATTR(epf_ntb_, spad_count);
935 CONFIGFS_ATTR(epf_ntb_, db_count);
936 CONFIGFS_ATTR(epf_ntb_, num_mws);
937 CONFIGFS_ATTR(epf_ntb_, mw1);
938 CONFIGFS_ATTR(epf_ntb_, mw2);
939 CONFIGFS_ATTR(epf_ntb_, mw3);
940 CONFIGFS_ATTR(epf_ntb_, mw4);
941 CONFIGFS_ATTR(epf_ntb_, vbus_number);
942 CONFIGFS_ATTR(epf_ntb_, vntb_pid);
943 CONFIGFS_ATTR(epf_ntb_, vntb_vid);
944 
945 static struct configfs_attribute *epf_ntb_attrs[] = {
946 	&epf_ntb_attr_spad_count,
947 	&epf_ntb_attr_db_count,
948 	&epf_ntb_attr_num_mws,
949 	&epf_ntb_attr_mw1,
950 	&epf_ntb_attr_mw2,
951 	&epf_ntb_attr_mw3,
952 	&epf_ntb_attr_mw4,
953 	&epf_ntb_attr_vbus_number,
954 	&epf_ntb_attr_vntb_pid,
955 	&epf_ntb_attr_vntb_vid,
956 	NULL,
957 };
958 
959 static const struct config_item_type ntb_group_type = {
960 	.ct_attrs	= epf_ntb_attrs,
961 	.ct_owner	= THIS_MODULE,
962 };
963 
964 /**
965  * epf_ntb_add_cfs() - Add configfs directory specific to NTB
966  * @epf: NTB endpoint function device
967  * @group: A pointer to the config_group structure referencing a group of
968  *	   config_items of a specific type that belong to a specific sub-system.
969  *
970  * Add configfs directory specific to NTB. This directory will hold
971  * NTB specific properties like db_count, spad_count, num_mws etc.,
972  *
973  * Returns: Pointer to config_group
974  */
epf_ntb_add_cfs(struct pci_epf * epf,struct config_group * group)975 static struct config_group *epf_ntb_add_cfs(struct pci_epf *epf,
976 					    struct config_group *group)
977 {
978 	struct epf_ntb *ntb = epf_get_drvdata(epf);
979 	struct config_group *ntb_group = &ntb->group;
980 	struct device *dev = &epf->dev;
981 
982 	config_group_init_type_name(ntb_group, dev_name(dev), &ntb_group_type);
983 
984 	return ntb_group;
985 }
986 
987 /*==== virtual PCI bus driver, which only load virtual NTB PCI driver ====*/
988 
989 static u32 pci_space[] = {
990 	0xffffffff,	/* Device ID, Vendor ID */
991 	0,		/* Status, Command */
992 	0xffffffff,	/* Base Class, Subclass, Prog Intf, Revision ID */
993 	0x40,		/* BIST, Header Type, Latency Timer, Cache Line Size */
994 	0,		/* BAR 0 */
995 	0,		/* BAR 1 */
996 	0,		/* BAR 2 */
997 	0,		/* BAR 3 */
998 	0,		/* BAR 4 */
999 	0,		/* BAR 5 */
1000 	0,		/* Cardbus CIS Pointer */
1001 	0,		/* Subsystem ID, Subsystem Vendor ID */
1002 	0,		/* ROM Base Address */
1003 	0,		/* Reserved, Capabilities Pointer */
1004 	0,		/* Reserved */
1005 	0,		/* Max_Lat, Min_Gnt, Interrupt Pin, Interrupt Line */
1006 };
1007 
pci_read(struct pci_bus * bus,unsigned int devfn,int where,int size,u32 * val)1008 static int pci_read(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 *val)
1009 {
1010 	if (devfn == 0) {
1011 		memcpy(val, ((u8 *)pci_space) + where, size);
1012 		return PCIBIOS_SUCCESSFUL;
1013 	}
1014 	return PCIBIOS_DEVICE_NOT_FOUND;
1015 }
1016 
pci_write(struct pci_bus * bus,unsigned int devfn,int where,int size,u32 val)1017 static int pci_write(struct pci_bus *bus, unsigned int devfn, int where, int size, u32 val)
1018 {
1019 	return 0;
1020 }
1021 
1022 static struct pci_ops vpci_ops = {
1023 	.read = pci_read,
1024 	.write = pci_write,
1025 };
1026 
vpci_scan_bus(void * sysdata)1027 static int vpci_scan_bus(void *sysdata)
1028 {
1029 	struct pci_bus *vpci_bus;
1030 	struct epf_ntb *ndev = sysdata;
1031 
1032 	vpci_bus = pci_scan_bus(ndev->vbus_number, &vpci_ops, sysdata);
1033 	if (!vpci_bus) {
1034 		pr_err("create pci bus failed\n");
1035 		return -EINVAL;
1036 	}
1037 
1038 	pci_bus_add_devices(vpci_bus);
1039 
1040 	return 0;
1041 }
1042 
1043 /*==================== Virtual PCIe NTB driver ==========================*/
1044 
vntb_epf_mw_count(struct ntb_dev * ntb,int pidx)1045 static int vntb_epf_mw_count(struct ntb_dev *ntb, int pidx)
1046 {
1047 	struct epf_ntb *ndev = ntb_ndev(ntb);
1048 
1049 	return ndev->num_mws;
1050 }
1051 
vntb_epf_spad_count(struct ntb_dev * ntb)1052 static int vntb_epf_spad_count(struct ntb_dev *ntb)
1053 {
1054 	return ntb_ndev(ntb)->spad_count;
1055 }
1056 
vntb_epf_peer_mw_count(struct ntb_dev * ntb)1057 static int vntb_epf_peer_mw_count(struct ntb_dev *ntb)
1058 {
1059 	return ntb_ndev(ntb)->num_mws;
1060 }
1061 
vntb_epf_db_valid_mask(struct ntb_dev * ntb)1062 static u64 vntb_epf_db_valid_mask(struct ntb_dev *ntb)
1063 {
1064 	return BIT_ULL(ntb_ndev(ntb)->db_count) - 1;
1065 }
1066 
vntb_epf_db_set_mask(struct ntb_dev * ntb,u64 db_bits)1067 static int vntb_epf_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1068 {
1069 	return 0;
1070 }
1071 
vntb_epf_mw_set_trans(struct ntb_dev * ndev,int pidx,int idx,dma_addr_t addr,resource_size_t size)1072 static int vntb_epf_mw_set_trans(struct ntb_dev *ndev, int pidx, int idx,
1073 		dma_addr_t addr, resource_size_t size)
1074 {
1075 	struct epf_ntb *ntb = ntb_ndev(ndev);
1076 	struct pci_epf_bar *epf_bar;
1077 	enum pci_barno barno;
1078 	int ret;
1079 	struct device *dev;
1080 
1081 	dev = &ntb->ntb.dev;
1082 	barno = ntb->epf_ntb_bar[BAR_MW0 + idx];
1083 	epf_bar = &ntb->epf->bar[barno];
1084 	epf_bar->phys_addr = addr;
1085 	epf_bar->barno = barno;
1086 	epf_bar->size = size;
1087 
1088 	ret = pci_epc_set_bar(ntb->epf->epc, 0, 0, epf_bar);
1089 	if (ret) {
1090 		dev_err(dev, "failure set mw trans\n");
1091 		return ret;
1092 	}
1093 	return 0;
1094 }
1095 
vntb_epf_mw_clear_trans(struct ntb_dev * ntb,int pidx,int idx)1096 static int vntb_epf_mw_clear_trans(struct ntb_dev *ntb, int pidx, int idx)
1097 {
1098 	return 0;
1099 }
1100 
vntb_epf_peer_mw_get_addr(struct ntb_dev * ndev,int idx,phys_addr_t * base,resource_size_t * size)1101 static int vntb_epf_peer_mw_get_addr(struct ntb_dev *ndev, int idx,
1102 				phys_addr_t *base, resource_size_t *size)
1103 {
1104 
1105 	struct epf_ntb *ntb = ntb_ndev(ndev);
1106 
1107 	if (base)
1108 		*base = ntb->vpci_mw_phy[idx];
1109 
1110 	if (size)
1111 		*size = ntb->mws_size[idx];
1112 
1113 	return 0;
1114 }
1115 
vntb_epf_link_enable(struct ntb_dev * ntb,enum ntb_speed max_speed,enum ntb_width max_width)1116 static int vntb_epf_link_enable(struct ntb_dev *ntb,
1117 			enum ntb_speed max_speed,
1118 			enum ntb_width max_width)
1119 {
1120 	return 0;
1121 }
1122 
vntb_epf_spad_read(struct ntb_dev * ndev,int idx)1123 static u32 vntb_epf_spad_read(struct ntb_dev *ndev, int idx)
1124 {
1125 	struct epf_ntb *ntb = ntb_ndev(ndev);
1126 	int off = ntb->reg->spad_offset, ct = ntb->reg->spad_count * sizeof(u32);
1127 	u32 val;
1128 	void __iomem *base = (void __iomem *)ntb->reg;
1129 
1130 	val = readl(base + off + ct + idx * sizeof(u32));
1131 	return val;
1132 }
1133 
vntb_epf_spad_write(struct ntb_dev * ndev,int idx,u32 val)1134 static int vntb_epf_spad_write(struct ntb_dev *ndev, int idx, u32 val)
1135 {
1136 	struct epf_ntb *ntb = ntb_ndev(ndev);
1137 	struct epf_ntb_ctrl *ctrl = ntb->reg;
1138 	int off = ctrl->spad_offset, ct = ctrl->spad_count * sizeof(u32);
1139 	void __iomem *base = (void __iomem *)ntb->reg;
1140 
1141 	writel(val, base + off + ct + idx * sizeof(u32));
1142 	return 0;
1143 }
1144 
vntb_epf_peer_spad_read(struct ntb_dev * ndev,int pidx,int idx)1145 static u32 vntb_epf_peer_spad_read(struct ntb_dev *ndev, int pidx, int idx)
1146 {
1147 	struct epf_ntb *ntb = ntb_ndev(ndev);
1148 	struct epf_ntb_ctrl *ctrl = ntb->reg;
1149 	int off = ctrl->spad_offset;
1150 	void __iomem *base = (void __iomem *)ntb->reg;
1151 	u32 val;
1152 
1153 	val = readl(base + off + idx * sizeof(u32));
1154 	return val;
1155 }
1156 
vntb_epf_peer_spad_write(struct ntb_dev * ndev,int pidx,int idx,u32 val)1157 static int vntb_epf_peer_spad_write(struct ntb_dev *ndev, int pidx, int idx, u32 val)
1158 {
1159 	struct epf_ntb *ntb = ntb_ndev(ndev);
1160 	struct epf_ntb_ctrl *ctrl = ntb->reg;
1161 	int off = ctrl->spad_offset;
1162 	void __iomem *base = (void __iomem *)ntb->reg;
1163 
1164 	writel(val, base + off + idx * sizeof(u32));
1165 	return 0;
1166 }
1167 
vntb_epf_peer_db_set(struct ntb_dev * ndev,u64 db_bits)1168 static int vntb_epf_peer_db_set(struct ntb_dev *ndev, u64 db_bits)
1169 {
1170 	u32 interrupt_num = ffs(db_bits) + 1;
1171 	struct epf_ntb *ntb = ntb_ndev(ndev);
1172 	u8 func_no, vfunc_no;
1173 	int ret;
1174 
1175 	func_no = ntb->epf->func_no;
1176 	vfunc_no = ntb->epf->vfunc_no;
1177 
1178 	ret = pci_epc_raise_irq(ntb->epf->epc,
1179 				func_no,
1180 				vfunc_no,
1181 				PCI_EPC_IRQ_MSI,
1182 				interrupt_num + 1);
1183 	if (ret)
1184 		dev_err(&ntb->ntb.dev, "Failed to raise IRQ\n");
1185 
1186 	return ret;
1187 }
1188 
vntb_epf_db_read(struct ntb_dev * ndev)1189 static u64 vntb_epf_db_read(struct ntb_dev *ndev)
1190 {
1191 	struct epf_ntb *ntb = ntb_ndev(ndev);
1192 
1193 	return ntb->db;
1194 }
1195 
vntb_epf_mw_get_align(struct ntb_dev * ndev,int pidx,int idx,resource_size_t * addr_align,resource_size_t * size_align,resource_size_t * size_max)1196 static int vntb_epf_mw_get_align(struct ntb_dev *ndev, int pidx, int idx,
1197 			resource_size_t *addr_align,
1198 			resource_size_t *size_align,
1199 			resource_size_t *size_max)
1200 {
1201 	struct epf_ntb *ntb = ntb_ndev(ndev);
1202 
1203 	if (addr_align)
1204 		*addr_align = SZ_4K;
1205 
1206 	if (size_align)
1207 		*size_align = 1;
1208 
1209 	if (size_max)
1210 		*size_max = ntb->mws_size[idx];
1211 
1212 	return 0;
1213 }
1214 
vntb_epf_link_is_up(struct ntb_dev * ndev,enum ntb_speed * speed,enum ntb_width * width)1215 static u64 vntb_epf_link_is_up(struct ntb_dev *ndev,
1216 			enum ntb_speed *speed,
1217 			enum ntb_width *width)
1218 {
1219 	struct epf_ntb *ntb = ntb_ndev(ndev);
1220 
1221 	return ntb->reg->link_status;
1222 }
1223 
vntb_epf_db_clear_mask(struct ntb_dev * ndev,u64 db_bits)1224 static int vntb_epf_db_clear_mask(struct ntb_dev *ndev, u64 db_bits)
1225 {
1226 	return 0;
1227 }
1228 
vntb_epf_db_clear(struct ntb_dev * ndev,u64 db_bits)1229 static int vntb_epf_db_clear(struct ntb_dev *ndev, u64 db_bits)
1230 {
1231 	struct epf_ntb *ntb = ntb_ndev(ndev);
1232 
1233 	ntb->db &= ~db_bits;
1234 	return 0;
1235 }
1236 
vntb_epf_link_disable(struct ntb_dev * ntb)1237 static int vntb_epf_link_disable(struct ntb_dev *ntb)
1238 {
1239 	return 0;
1240 }
1241 
1242 static const struct ntb_dev_ops vntb_epf_ops = {
1243 	.mw_count		= vntb_epf_mw_count,
1244 	.spad_count		= vntb_epf_spad_count,
1245 	.peer_mw_count		= vntb_epf_peer_mw_count,
1246 	.db_valid_mask		= vntb_epf_db_valid_mask,
1247 	.db_set_mask		= vntb_epf_db_set_mask,
1248 	.mw_set_trans		= vntb_epf_mw_set_trans,
1249 	.mw_clear_trans		= vntb_epf_mw_clear_trans,
1250 	.peer_mw_get_addr	= vntb_epf_peer_mw_get_addr,
1251 	.link_enable		= vntb_epf_link_enable,
1252 	.spad_read		= vntb_epf_spad_read,
1253 	.spad_write		= vntb_epf_spad_write,
1254 	.peer_spad_read		= vntb_epf_peer_spad_read,
1255 	.peer_spad_write	= vntb_epf_peer_spad_write,
1256 	.peer_db_set		= vntb_epf_peer_db_set,
1257 	.db_read		= vntb_epf_db_read,
1258 	.mw_get_align		= vntb_epf_mw_get_align,
1259 	.link_is_up		= vntb_epf_link_is_up,
1260 	.db_clear_mask		= vntb_epf_db_clear_mask,
1261 	.db_clear		= vntb_epf_db_clear,
1262 	.link_disable		= vntb_epf_link_disable,
1263 };
1264 
pci_vntb_probe(struct pci_dev * pdev,const struct pci_device_id * id)1265 static int pci_vntb_probe(struct pci_dev *pdev, const struct pci_device_id *id)
1266 {
1267 	int ret;
1268 	struct epf_ntb *ndev = (struct epf_ntb *)pdev->sysdata;
1269 	struct device *dev = &pdev->dev;
1270 
1271 	ndev->ntb.pdev = pdev;
1272 	ndev->ntb.topo = NTB_TOPO_NONE;
1273 	ndev->ntb.ops =  &vntb_epf_ops;
1274 
1275 	ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(32));
1276 	if (ret) {
1277 		dev_err(dev, "Cannot set DMA mask\n");
1278 		return -EINVAL;
1279 	}
1280 
1281 	ret = ntb_register_device(&ndev->ntb);
1282 	if (ret) {
1283 		dev_err(dev, "Failed to register NTB device\n");
1284 		return ret;
1285 	}
1286 
1287 	dev_dbg(dev, "PCI Virtual NTB driver loaded\n");
1288 	return 0;
1289 }
1290 
1291 static struct pci_device_id pci_vntb_table[] = {
1292 	{
1293 		PCI_DEVICE(0xffff, 0xffff),
1294 	},
1295 	{},
1296 };
1297 
1298 static struct pci_driver vntb_pci_driver = {
1299 	.name           = "pci-vntb",
1300 	.id_table       = pci_vntb_table,
1301 	.probe          = pci_vntb_probe,
1302 };
1303 
1304 /* ============ PCIe EPF Driver Bind ====================*/
1305 
1306 /**
1307  * epf_ntb_bind() - Initialize endpoint controller to provide NTB functionality
1308  * @epf: NTB endpoint function device
1309  *
1310  * Initialize both the endpoint controllers associated with NTB function device.
1311  * Invoked when a primary interface or secondary interface is bound to EPC
1312  * device. This function will succeed only when EPC is bound to both the
1313  * interfaces.
1314  *
1315  * Returns: Zero for success, or an error code in case of failure
1316  */
epf_ntb_bind(struct pci_epf * epf)1317 static int epf_ntb_bind(struct pci_epf *epf)
1318 {
1319 	struct epf_ntb *ntb = epf_get_drvdata(epf);
1320 	struct device *dev = &epf->dev;
1321 	int ret;
1322 
1323 	if (!epf->epc) {
1324 		dev_dbg(dev, "PRIMARY EPC interface not yet bound\n");
1325 		return 0;
1326 	}
1327 
1328 	ret = epf_ntb_init_epc_bar(ntb);
1329 	if (ret) {
1330 		dev_err(dev, "Failed to create NTB EPC\n");
1331 		goto err_bar_init;
1332 	}
1333 
1334 	ret = epf_ntb_config_spad_bar_alloc(ntb);
1335 	if (ret) {
1336 		dev_err(dev, "Failed to allocate BAR memory\n");
1337 		goto err_bar_alloc;
1338 	}
1339 
1340 	ret = epf_ntb_epc_init(ntb);
1341 	if (ret) {
1342 		dev_err(dev, "Failed to initialize EPC\n");
1343 		goto err_bar_alloc;
1344 	}
1345 
1346 	epf_set_drvdata(epf, ntb);
1347 
1348 	pci_space[0] = (ntb->vntb_pid << 16) | ntb->vntb_vid;
1349 	pci_vntb_table[0].vendor = ntb->vntb_vid;
1350 	pci_vntb_table[0].device = ntb->vntb_pid;
1351 
1352 	ret = pci_register_driver(&vntb_pci_driver);
1353 	if (ret) {
1354 		dev_err(dev, "failure register vntb pci driver\n");
1355 		goto err_epc_cleanup;
1356 	}
1357 
1358 	ret = vpci_scan_bus(ntb);
1359 	if (ret)
1360 		goto err_unregister;
1361 
1362 	return 0;
1363 
1364 err_unregister:
1365 	pci_unregister_driver(&vntb_pci_driver);
1366 err_epc_cleanup:
1367 	epf_ntb_epc_cleanup(ntb);
1368 err_bar_alloc:
1369 	epf_ntb_config_spad_bar_free(ntb);
1370 
1371 err_bar_init:
1372 	epf_ntb_epc_destroy(ntb);
1373 
1374 	return ret;
1375 }
1376 
1377 /**
1378  * epf_ntb_unbind() - Cleanup the initialization from epf_ntb_bind()
1379  * @epf: NTB endpoint function device
1380  *
1381  * Cleanup the initialization from epf_ntb_bind()
1382  */
epf_ntb_unbind(struct pci_epf * epf)1383 static void epf_ntb_unbind(struct pci_epf *epf)
1384 {
1385 	struct epf_ntb *ntb = epf_get_drvdata(epf);
1386 
1387 	epf_ntb_epc_cleanup(ntb);
1388 	epf_ntb_config_spad_bar_free(ntb);
1389 	epf_ntb_epc_destroy(ntb);
1390 
1391 	pci_unregister_driver(&vntb_pci_driver);
1392 }
1393 
1394 // EPF driver probe
1395 static struct pci_epf_ops epf_ntb_ops = {
1396 	.bind   = epf_ntb_bind,
1397 	.unbind = epf_ntb_unbind,
1398 	.add_cfs = epf_ntb_add_cfs,
1399 };
1400 
1401 /**
1402  * epf_ntb_probe() - Probe NTB function driver
1403  * @epf: NTB endpoint function device
1404  * @id: NTB endpoint function device ID
1405  *
1406  * Probe NTB function driver when endpoint function bus detects a NTB
1407  * endpoint function.
1408  *
1409  * Returns: Zero for success, or an error code in case of failure
1410  */
epf_ntb_probe(struct pci_epf * epf,const struct pci_epf_device_id * id)1411 static int epf_ntb_probe(struct pci_epf *epf,
1412 			 const struct pci_epf_device_id *id)
1413 {
1414 	struct epf_ntb *ntb;
1415 	struct device *dev;
1416 
1417 	dev = &epf->dev;
1418 
1419 	ntb = devm_kzalloc(dev, sizeof(*ntb), GFP_KERNEL);
1420 	if (!ntb)
1421 		return -ENOMEM;
1422 
1423 	epf->header = &epf_ntb_header;
1424 	ntb->epf = epf;
1425 	ntb->vbus_number = 0xff;
1426 	epf_set_drvdata(epf, ntb);
1427 
1428 	dev_info(dev, "pci-ep epf driver loaded\n");
1429 	return 0;
1430 }
1431 
1432 static const struct pci_epf_device_id epf_ntb_ids[] = {
1433 	{
1434 		.name = "pci_epf_vntb",
1435 	},
1436 	{},
1437 };
1438 
1439 static struct pci_epf_driver epf_ntb_driver = {
1440 	.driver.name    = "pci_epf_vntb",
1441 	.probe          = epf_ntb_probe,
1442 	.id_table       = epf_ntb_ids,
1443 	.ops            = &epf_ntb_ops,
1444 	.owner          = THIS_MODULE,
1445 };
1446 
epf_ntb_init(void)1447 static int __init epf_ntb_init(void)
1448 {
1449 	int ret;
1450 
1451 	kpcintb_workqueue = alloc_workqueue("kpcintb", WQ_MEM_RECLAIM |
1452 					    WQ_HIGHPRI, 0);
1453 	ret = pci_epf_register_driver(&epf_ntb_driver);
1454 	if (ret) {
1455 		destroy_workqueue(kpcintb_workqueue);
1456 		pr_err("Failed to register pci epf ntb driver --> %d\n", ret);
1457 		return ret;
1458 	}
1459 
1460 	return 0;
1461 }
1462 module_init(epf_ntb_init);
1463 
epf_ntb_exit(void)1464 static void __exit epf_ntb_exit(void)
1465 {
1466 	pci_epf_unregister_driver(&epf_ntb_driver);
1467 	destroy_workqueue(kpcintb_workqueue);
1468 }
1469 module_exit(epf_ntb_exit);
1470 
1471 MODULE_DESCRIPTION("PCI EPF NTB DRIVER");
1472 MODULE_AUTHOR("Frank Li <Frank.li@nxp.com>");
1473 MODULE_LICENSE("GPL v2");
1474