1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Test driver to test endpoint functionality
4  *
5  * Copyright (C) 2017 Texas Instruments
6  * Author: Kishon Vijay Abraham I <kishon@ti.com>
7  */
8 
9 #include <linux/crc32.h>
10 #include <linux/delay.h>
11 #include <linux/dmaengine.h>
12 #include <linux/io.h>
13 #include <linux/module.h>
14 #include <linux/slab.h>
15 #include <linux/pci_ids.h>
16 #include <linux/random.h>
17 
18 #include <linux/pci-epc.h>
19 #include <linux/pci-epf.h>
20 #include <linux/pci_regs.h>
21 
22 #define IRQ_TYPE_LEGACY			0
23 #define IRQ_TYPE_MSI			1
24 #define IRQ_TYPE_MSIX			2
25 
26 #define COMMAND_RAISE_LEGACY_IRQ	BIT(0)
27 #define COMMAND_RAISE_MSI_IRQ		BIT(1)
28 #define COMMAND_RAISE_MSIX_IRQ		BIT(2)
29 #define COMMAND_READ			BIT(3)
30 #define COMMAND_WRITE			BIT(4)
31 #define COMMAND_COPY			BIT(5)
32 
33 #define STATUS_READ_SUCCESS		BIT(0)
34 #define STATUS_READ_FAIL		BIT(1)
35 #define STATUS_WRITE_SUCCESS		BIT(2)
36 #define STATUS_WRITE_FAIL		BIT(3)
37 #define STATUS_COPY_SUCCESS		BIT(4)
38 #define STATUS_COPY_FAIL		BIT(5)
39 #define STATUS_IRQ_RAISED		BIT(6)
40 #define STATUS_SRC_ADDR_INVALID		BIT(7)
41 #define STATUS_DST_ADDR_INVALID		BIT(8)
42 
43 #define FLAG_USE_DMA			BIT(0)
44 
45 #define TIMER_RESOLUTION		1
46 
47 static struct workqueue_struct *kpcitest_workqueue;
48 
49 struct pci_epf_test {
50 	void			*reg[PCI_STD_NUM_BARS];
51 	struct pci_epf		*epf;
52 	enum pci_barno		test_reg_bar;
53 	size_t			msix_table_offset;
54 	struct delayed_work	cmd_handler;
55 	struct dma_chan		*dma_chan_tx;
56 	struct dma_chan		*dma_chan_rx;
57 	struct completion	transfer_complete;
58 	bool			dma_supported;
59 	bool			dma_private;
60 	const struct pci_epc_features *epc_features;
61 };
62 
63 struct pci_epf_test_reg {
64 	u32	magic;
65 	u32	command;
66 	u32	status;
67 	u64	src_addr;
68 	u64	dst_addr;
69 	u32	size;
70 	u32	checksum;
71 	u32	irq_type;
72 	u32	irq_number;
73 	u32	flags;
74 } __packed;
75 
76 static struct pci_epf_header test_header = {
77 	.vendorid	= PCI_ANY_ID,
78 	.deviceid	= PCI_ANY_ID,
79 	.baseclass_code = PCI_CLASS_OTHERS,
80 	.interrupt_pin	= PCI_INTERRUPT_INTA,
81 };
82 
83 static size_t bar_size[] = { 512, 512, 1024, 16384, 131072, 1048576 };
84 
85 static void pci_epf_test_dma_callback(void *param)
86 {
87 	struct pci_epf_test *epf_test = param;
88 
89 	complete(&epf_test->transfer_complete);
90 }
91 
92 /**
93  * pci_epf_test_data_transfer() - Function that uses dmaengine API to transfer
94  *				  data between PCIe EP and remote PCIe RC
95  * @epf_test: the EPF test device that performs the data transfer operation
96  * @dma_dst: The destination address of the data transfer. It can be a physical
97  *	     address given by pci_epc_mem_alloc_addr or DMA mapping APIs.
98  * @dma_src: The source address of the data transfer. It can be a physical
99  *	     address given by pci_epc_mem_alloc_addr or DMA mapping APIs.
100  * @len: The size of the data transfer
101  * @dma_remote: remote RC physical address
102  * @dir: DMA transfer direction
103  *
104  * Function that uses dmaengine API to transfer data between PCIe EP and remote
105  * PCIe RC. The source and destination address can be a physical address given
106  * by pci_epc_mem_alloc_addr or the one obtained using DMA mapping APIs.
107  *
108  * The function returns '0' on success and negative value on failure.
109  */
110 static int pci_epf_test_data_transfer(struct pci_epf_test *epf_test,
111 				      dma_addr_t dma_dst, dma_addr_t dma_src,
112 				      size_t len, dma_addr_t dma_remote,
113 				      enum dma_transfer_direction dir)
114 {
115 	struct dma_chan *chan = (dir == DMA_DEV_TO_MEM) ?
116 				 epf_test->dma_chan_tx : epf_test->dma_chan_rx;
117 	dma_addr_t dma_local = (dir == DMA_MEM_TO_DEV) ? dma_src : dma_dst;
118 	enum dma_ctrl_flags flags = DMA_CTRL_ACK | DMA_PREP_INTERRUPT;
119 	struct pci_epf *epf = epf_test->epf;
120 	struct dma_async_tx_descriptor *tx;
121 	struct dma_slave_config sconf = {};
122 	struct device *dev = &epf->dev;
123 	dma_cookie_t cookie;
124 	int ret;
125 
126 	if (IS_ERR_OR_NULL(chan)) {
127 		dev_err(dev, "Invalid DMA memcpy channel\n");
128 		return -EINVAL;
129 	}
130 
131 	if (epf_test->dma_private) {
132 		sconf.direction = dir;
133 		if (dir == DMA_MEM_TO_DEV)
134 			sconf.dst_addr = dma_remote;
135 		else
136 			sconf.src_addr = dma_remote;
137 
138 		if (dmaengine_slave_config(chan, &sconf)) {
139 			dev_err(dev, "DMA slave config fail\n");
140 			return -EIO;
141 		}
142 		tx = dmaengine_prep_slave_single(chan, dma_local, len, dir,
143 						 flags);
144 	} else {
145 		tx = dmaengine_prep_dma_memcpy(chan, dma_dst, dma_src, len,
146 					       flags);
147 	}
148 
149 	if (!tx) {
150 		dev_err(dev, "Failed to prepare DMA memcpy\n");
151 		return -EIO;
152 	}
153 
154 	tx->callback = pci_epf_test_dma_callback;
155 	tx->callback_param = epf_test;
156 	cookie = tx->tx_submit(tx);
157 	reinit_completion(&epf_test->transfer_complete);
158 
159 	ret = dma_submit_error(cookie);
160 	if (ret) {
161 		dev_err(dev, "Failed to do DMA tx_submit %d\n", cookie);
162 		return -EIO;
163 	}
164 
165 	dma_async_issue_pending(chan);
166 	ret = wait_for_completion_interruptible(&epf_test->transfer_complete);
167 	if (ret < 0) {
168 		dmaengine_terminate_sync(chan);
169 		dev_err(dev, "DMA wait_for_completion_timeout\n");
170 		return -ETIMEDOUT;
171 	}
172 
173 	return 0;
174 }
175 
176 struct epf_dma_filter {
177 	struct device *dev;
178 	u32 dma_mask;
179 };
180 
181 static bool epf_dma_filter_fn(struct dma_chan *chan, void *node)
182 {
183 	struct epf_dma_filter *filter = node;
184 	struct dma_slave_caps caps;
185 
186 	memset(&caps, 0, sizeof(caps));
187 	dma_get_slave_caps(chan, &caps);
188 
189 	return chan->device->dev == filter->dev
190 		&& (filter->dma_mask & caps.directions);
191 }
192 
193 /**
194  * pci_epf_test_init_dma_chan() - Function to initialize EPF test DMA channel
195  * @epf_test: the EPF test device that performs data transfer operation
196  *
197  * Function to initialize EPF test DMA channel.
198  */
199 static int pci_epf_test_init_dma_chan(struct pci_epf_test *epf_test)
200 {
201 	struct pci_epf *epf = epf_test->epf;
202 	struct device *dev = &epf->dev;
203 	struct epf_dma_filter filter;
204 	struct dma_chan *dma_chan;
205 	dma_cap_mask_t mask;
206 	int ret;
207 
208 	filter.dev = epf->epc->dev.parent;
209 	filter.dma_mask = BIT(DMA_DEV_TO_MEM);
210 
211 	dma_cap_zero(mask);
212 	dma_cap_set(DMA_SLAVE, mask);
213 	dma_chan = dma_request_channel(mask, epf_dma_filter_fn, &filter);
214 	if (!dma_chan) {
215 		dev_info(dev, "Failed to get private DMA rx channel. Falling back to generic one\n");
216 		goto fail_back_tx;
217 	}
218 
219 	epf_test->dma_chan_rx = dma_chan;
220 
221 	filter.dma_mask = BIT(DMA_MEM_TO_DEV);
222 	dma_chan = dma_request_channel(mask, epf_dma_filter_fn, &filter);
223 
224 	if (!dma_chan) {
225 		dev_info(dev, "Failed to get private DMA tx channel. Falling back to generic one\n");
226 		goto fail_back_rx;
227 	}
228 
229 	epf_test->dma_chan_tx = dma_chan;
230 	epf_test->dma_private = true;
231 
232 	init_completion(&epf_test->transfer_complete);
233 
234 	return 0;
235 
236 fail_back_rx:
237 	dma_release_channel(epf_test->dma_chan_rx);
238 	epf_test->dma_chan_tx = NULL;
239 
240 fail_back_tx:
241 	dma_cap_zero(mask);
242 	dma_cap_set(DMA_MEMCPY, mask);
243 
244 	dma_chan = dma_request_chan_by_mask(&mask);
245 	if (IS_ERR(dma_chan)) {
246 		ret = PTR_ERR(dma_chan);
247 		if (ret != -EPROBE_DEFER)
248 			dev_err(dev, "Failed to get DMA channel\n");
249 		return ret;
250 	}
251 	init_completion(&epf_test->transfer_complete);
252 
253 	epf_test->dma_chan_tx = epf_test->dma_chan_rx = dma_chan;
254 
255 	return 0;
256 }
257 
258 /**
259  * pci_epf_test_clean_dma_chan() - Function to cleanup EPF test DMA channel
260  * @epf_test: the EPF test device that performs data transfer operation
261  *
262  * Helper to cleanup EPF test DMA channel.
263  */
264 static void pci_epf_test_clean_dma_chan(struct pci_epf_test *epf_test)
265 {
266 	if (!epf_test->dma_supported)
267 		return;
268 
269 	dma_release_channel(epf_test->dma_chan_tx);
270 	if (epf_test->dma_chan_tx == epf_test->dma_chan_rx) {
271 		epf_test->dma_chan_tx = NULL;
272 		epf_test->dma_chan_rx = NULL;
273 		return;
274 	}
275 
276 	dma_release_channel(epf_test->dma_chan_rx);
277 	epf_test->dma_chan_rx = NULL;
278 
279 	return;
280 }
281 
282 static void pci_epf_test_print_rate(const char *ops, u64 size,
283 				    struct timespec64 *start,
284 				    struct timespec64 *end, bool dma)
285 {
286 	struct timespec64 ts;
287 	u64 rate, ns;
288 
289 	ts = timespec64_sub(*end, *start);
290 
291 	/* convert both size (stored in 'rate') and time in terms of 'ns' */
292 	ns = timespec64_to_ns(&ts);
293 	rate = size * NSEC_PER_SEC;
294 
295 	/* Divide both size (stored in 'rate') and ns by a common factor */
296 	while (ns > UINT_MAX) {
297 		rate >>= 1;
298 		ns >>= 1;
299 	}
300 
301 	if (!ns)
302 		return;
303 
304 	/* calculate the rate */
305 	do_div(rate, (uint32_t)ns);
306 
307 	pr_info("\n%s => Size: %llu bytes\t DMA: %s\t Time: %llu.%09u seconds\t"
308 		"Rate: %llu KB/s\n", ops, size, dma ? "YES" : "NO",
309 		(u64)ts.tv_sec, (u32)ts.tv_nsec, rate / 1024);
310 }
311 
312 static int pci_epf_test_copy(struct pci_epf_test *epf_test)
313 {
314 	int ret;
315 	bool use_dma;
316 	void __iomem *src_addr;
317 	void __iomem *dst_addr;
318 	phys_addr_t src_phys_addr;
319 	phys_addr_t dst_phys_addr;
320 	struct timespec64 start, end;
321 	struct pci_epf *epf = epf_test->epf;
322 	struct device *dev = &epf->dev;
323 	struct pci_epc *epc = epf->epc;
324 	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
325 	struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar];
326 
327 	src_addr = pci_epc_mem_alloc_addr(epc, &src_phys_addr, reg->size);
328 	if (!src_addr) {
329 		dev_err(dev, "Failed to allocate source address\n");
330 		reg->status = STATUS_SRC_ADDR_INVALID;
331 		ret = -ENOMEM;
332 		goto err;
333 	}
334 
335 	ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, src_phys_addr,
336 			       reg->src_addr, reg->size);
337 	if (ret) {
338 		dev_err(dev, "Failed to map source address\n");
339 		reg->status = STATUS_SRC_ADDR_INVALID;
340 		goto err_src_addr;
341 	}
342 
343 	dst_addr = pci_epc_mem_alloc_addr(epc, &dst_phys_addr, reg->size);
344 	if (!dst_addr) {
345 		dev_err(dev, "Failed to allocate destination address\n");
346 		reg->status = STATUS_DST_ADDR_INVALID;
347 		ret = -ENOMEM;
348 		goto err_src_map_addr;
349 	}
350 
351 	ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, dst_phys_addr,
352 			       reg->dst_addr, reg->size);
353 	if (ret) {
354 		dev_err(dev, "Failed to map destination address\n");
355 		reg->status = STATUS_DST_ADDR_INVALID;
356 		goto err_dst_addr;
357 	}
358 
359 	ktime_get_ts64(&start);
360 	use_dma = !!(reg->flags & FLAG_USE_DMA);
361 	if (use_dma) {
362 		if (!epf_test->dma_supported) {
363 			dev_err(dev, "Cannot transfer data using DMA\n");
364 			ret = -EINVAL;
365 			goto err_map_addr;
366 		}
367 
368 		if (epf_test->dma_private) {
369 			dev_err(dev, "Cannot transfer data using DMA\n");
370 			ret = -EINVAL;
371 			goto err_map_addr;
372 		}
373 
374 		ret = pci_epf_test_data_transfer(epf_test, dst_phys_addr,
375 						 src_phys_addr, reg->size, 0,
376 						 DMA_MEM_TO_MEM);
377 		if (ret)
378 			dev_err(dev, "Data transfer failed\n");
379 	} else {
380 		void *buf;
381 
382 		buf = kzalloc(reg->size, GFP_KERNEL);
383 		if (!buf) {
384 			ret = -ENOMEM;
385 			goto err_map_addr;
386 		}
387 
388 		memcpy_fromio(buf, src_addr, reg->size);
389 		memcpy_toio(dst_addr, buf, reg->size);
390 		kfree(buf);
391 	}
392 	ktime_get_ts64(&end);
393 	pci_epf_test_print_rate("COPY", reg->size, &start, &end, use_dma);
394 
395 err_map_addr:
396 	pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, dst_phys_addr);
397 
398 err_dst_addr:
399 	pci_epc_mem_free_addr(epc, dst_phys_addr, dst_addr, reg->size);
400 
401 err_src_map_addr:
402 	pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, src_phys_addr);
403 
404 err_src_addr:
405 	pci_epc_mem_free_addr(epc, src_phys_addr, src_addr, reg->size);
406 
407 err:
408 	return ret;
409 }
410 
411 static int pci_epf_test_read(struct pci_epf_test *epf_test)
412 {
413 	int ret;
414 	void __iomem *src_addr;
415 	void *buf;
416 	u32 crc32;
417 	bool use_dma;
418 	phys_addr_t phys_addr;
419 	phys_addr_t dst_phys_addr;
420 	struct timespec64 start, end;
421 	struct pci_epf *epf = epf_test->epf;
422 	struct device *dev = &epf->dev;
423 	struct pci_epc *epc = epf->epc;
424 	struct device *dma_dev = epf->epc->dev.parent;
425 	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
426 	struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar];
427 
428 	src_addr = pci_epc_mem_alloc_addr(epc, &phys_addr, reg->size);
429 	if (!src_addr) {
430 		dev_err(dev, "Failed to allocate address\n");
431 		reg->status = STATUS_SRC_ADDR_INVALID;
432 		ret = -ENOMEM;
433 		goto err;
434 	}
435 
436 	ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, phys_addr,
437 			       reg->src_addr, reg->size);
438 	if (ret) {
439 		dev_err(dev, "Failed to map address\n");
440 		reg->status = STATUS_SRC_ADDR_INVALID;
441 		goto err_addr;
442 	}
443 
444 	buf = kzalloc(reg->size, GFP_KERNEL);
445 	if (!buf) {
446 		ret = -ENOMEM;
447 		goto err_map_addr;
448 	}
449 
450 	use_dma = !!(reg->flags & FLAG_USE_DMA);
451 	if (use_dma) {
452 		if (!epf_test->dma_supported) {
453 			dev_err(dev, "Cannot transfer data using DMA\n");
454 			ret = -EINVAL;
455 			goto err_dma_map;
456 		}
457 
458 		dst_phys_addr = dma_map_single(dma_dev, buf, reg->size,
459 					       DMA_FROM_DEVICE);
460 		if (dma_mapping_error(dma_dev, dst_phys_addr)) {
461 			dev_err(dev, "Failed to map destination buffer addr\n");
462 			ret = -ENOMEM;
463 			goto err_dma_map;
464 		}
465 
466 		ktime_get_ts64(&start);
467 		ret = pci_epf_test_data_transfer(epf_test, dst_phys_addr,
468 						 phys_addr, reg->size,
469 						 reg->src_addr, DMA_DEV_TO_MEM);
470 		if (ret)
471 			dev_err(dev, "Data transfer failed\n");
472 		ktime_get_ts64(&end);
473 
474 		dma_unmap_single(dma_dev, dst_phys_addr, reg->size,
475 				 DMA_FROM_DEVICE);
476 	} else {
477 		ktime_get_ts64(&start);
478 		memcpy_fromio(buf, src_addr, reg->size);
479 		ktime_get_ts64(&end);
480 	}
481 
482 	pci_epf_test_print_rate("READ", reg->size, &start, &end, use_dma);
483 
484 	crc32 = crc32_le(~0, buf, reg->size);
485 	if (crc32 != reg->checksum)
486 		ret = -EIO;
487 
488 err_dma_map:
489 	kfree(buf);
490 
491 err_map_addr:
492 	pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, phys_addr);
493 
494 err_addr:
495 	pci_epc_mem_free_addr(epc, phys_addr, src_addr, reg->size);
496 
497 err:
498 	return ret;
499 }
500 
501 static int pci_epf_test_write(struct pci_epf_test *epf_test)
502 {
503 	int ret;
504 	void __iomem *dst_addr;
505 	void *buf;
506 	bool use_dma;
507 	phys_addr_t phys_addr;
508 	phys_addr_t src_phys_addr;
509 	struct timespec64 start, end;
510 	struct pci_epf *epf = epf_test->epf;
511 	struct device *dev = &epf->dev;
512 	struct pci_epc *epc = epf->epc;
513 	struct device *dma_dev = epf->epc->dev.parent;
514 	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
515 	struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar];
516 
517 	dst_addr = pci_epc_mem_alloc_addr(epc, &phys_addr, reg->size);
518 	if (!dst_addr) {
519 		dev_err(dev, "Failed to allocate address\n");
520 		reg->status = STATUS_DST_ADDR_INVALID;
521 		ret = -ENOMEM;
522 		goto err;
523 	}
524 
525 	ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, phys_addr,
526 			       reg->dst_addr, reg->size);
527 	if (ret) {
528 		dev_err(dev, "Failed to map address\n");
529 		reg->status = STATUS_DST_ADDR_INVALID;
530 		goto err_addr;
531 	}
532 
533 	buf = kzalloc(reg->size, GFP_KERNEL);
534 	if (!buf) {
535 		ret = -ENOMEM;
536 		goto err_map_addr;
537 	}
538 
539 	get_random_bytes(buf, reg->size);
540 	reg->checksum = crc32_le(~0, buf, reg->size);
541 
542 	use_dma = !!(reg->flags & FLAG_USE_DMA);
543 	if (use_dma) {
544 		if (!epf_test->dma_supported) {
545 			dev_err(dev, "Cannot transfer data using DMA\n");
546 			ret = -EINVAL;
547 			goto err_dma_map;
548 		}
549 
550 		src_phys_addr = dma_map_single(dma_dev, buf, reg->size,
551 					       DMA_TO_DEVICE);
552 		if (dma_mapping_error(dma_dev, src_phys_addr)) {
553 			dev_err(dev, "Failed to map source buffer addr\n");
554 			ret = -ENOMEM;
555 			goto err_dma_map;
556 		}
557 
558 		ktime_get_ts64(&start);
559 
560 		ret = pci_epf_test_data_transfer(epf_test, phys_addr,
561 						 src_phys_addr, reg->size,
562 						 reg->dst_addr,
563 						 DMA_MEM_TO_DEV);
564 		if (ret)
565 			dev_err(dev, "Data transfer failed\n");
566 		ktime_get_ts64(&end);
567 
568 		dma_unmap_single(dma_dev, src_phys_addr, reg->size,
569 				 DMA_TO_DEVICE);
570 	} else {
571 		ktime_get_ts64(&start);
572 		memcpy_toio(dst_addr, buf, reg->size);
573 		ktime_get_ts64(&end);
574 	}
575 
576 	pci_epf_test_print_rate("WRITE", reg->size, &start, &end, use_dma);
577 
578 	/*
579 	 * wait 1ms inorder for the write to complete. Without this delay L3
580 	 * error in observed in the host system.
581 	 */
582 	usleep_range(1000, 2000);
583 
584 err_dma_map:
585 	kfree(buf);
586 
587 err_map_addr:
588 	pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, phys_addr);
589 
590 err_addr:
591 	pci_epc_mem_free_addr(epc, phys_addr, dst_addr, reg->size);
592 
593 err:
594 	return ret;
595 }
596 
597 static void pci_epf_test_raise_irq(struct pci_epf_test *epf_test, u8 irq_type,
598 				   u16 irq)
599 {
600 	struct pci_epf *epf = epf_test->epf;
601 	struct device *dev = &epf->dev;
602 	struct pci_epc *epc = epf->epc;
603 	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
604 	struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar];
605 
606 	reg->status |= STATUS_IRQ_RAISED;
607 
608 	switch (irq_type) {
609 	case IRQ_TYPE_LEGACY:
610 		pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no,
611 				  PCI_EPC_IRQ_LEGACY, 0);
612 		break;
613 	case IRQ_TYPE_MSI:
614 		pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no,
615 				  PCI_EPC_IRQ_MSI, irq);
616 		break;
617 	case IRQ_TYPE_MSIX:
618 		pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no,
619 				  PCI_EPC_IRQ_MSIX, irq);
620 		break;
621 	default:
622 		dev_err(dev, "Failed to raise IRQ, unknown type\n");
623 		break;
624 	}
625 }
626 
627 static void pci_epf_test_cmd_handler(struct work_struct *work)
628 {
629 	int ret;
630 	int count;
631 	u32 command;
632 	struct pci_epf_test *epf_test = container_of(work, struct pci_epf_test,
633 						     cmd_handler.work);
634 	struct pci_epf *epf = epf_test->epf;
635 	struct device *dev = &epf->dev;
636 	struct pci_epc *epc = epf->epc;
637 	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
638 	struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar];
639 
640 	command = reg->command;
641 	if (!command)
642 		goto reset_handler;
643 
644 	reg->command = 0;
645 	reg->status = 0;
646 
647 	if (reg->irq_type > IRQ_TYPE_MSIX) {
648 		dev_err(dev, "Failed to detect IRQ type\n");
649 		goto reset_handler;
650 	}
651 
652 	if (command & COMMAND_RAISE_LEGACY_IRQ) {
653 		reg->status = STATUS_IRQ_RAISED;
654 		pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no,
655 				  PCI_EPC_IRQ_LEGACY, 0);
656 		goto reset_handler;
657 	}
658 
659 	if (command & COMMAND_WRITE) {
660 		ret = pci_epf_test_write(epf_test);
661 		if (ret)
662 			reg->status |= STATUS_WRITE_FAIL;
663 		else
664 			reg->status |= STATUS_WRITE_SUCCESS;
665 		pci_epf_test_raise_irq(epf_test, reg->irq_type,
666 				       reg->irq_number);
667 		goto reset_handler;
668 	}
669 
670 	if (command & COMMAND_READ) {
671 		ret = pci_epf_test_read(epf_test);
672 		if (!ret)
673 			reg->status |= STATUS_READ_SUCCESS;
674 		else
675 			reg->status |= STATUS_READ_FAIL;
676 		pci_epf_test_raise_irq(epf_test, reg->irq_type,
677 				       reg->irq_number);
678 		goto reset_handler;
679 	}
680 
681 	if (command & COMMAND_COPY) {
682 		ret = pci_epf_test_copy(epf_test);
683 		if (!ret)
684 			reg->status |= STATUS_COPY_SUCCESS;
685 		else
686 			reg->status |= STATUS_COPY_FAIL;
687 		pci_epf_test_raise_irq(epf_test, reg->irq_type,
688 				       reg->irq_number);
689 		goto reset_handler;
690 	}
691 
692 	if (command & COMMAND_RAISE_MSI_IRQ) {
693 		count = pci_epc_get_msi(epc, epf->func_no, epf->vfunc_no);
694 		if (reg->irq_number > count || count <= 0)
695 			goto reset_handler;
696 		reg->status = STATUS_IRQ_RAISED;
697 		pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no,
698 				  PCI_EPC_IRQ_MSI, reg->irq_number);
699 		goto reset_handler;
700 	}
701 
702 	if (command & COMMAND_RAISE_MSIX_IRQ) {
703 		count = pci_epc_get_msix(epc, epf->func_no, epf->vfunc_no);
704 		if (reg->irq_number > count || count <= 0)
705 			goto reset_handler;
706 		reg->status = STATUS_IRQ_RAISED;
707 		pci_epc_raise_irq(epc, epf->func_no, epf->vfunc_no,
708 				  PCI_EPC_IRQ_MSIX, reg->irq_number);
709 		goto reset_handler;
710 	}
711 
712 reset_handler:
713 	queue_delayed_work(kpcitest_workqueue, &epf_test->cmd_handler,
714 			   msecs_to_jiffies(1));
715 }
716 
717 static void pci_epf_test_unbind(struct pci_epf *epf)
718 {
719 	struct pci_epf_test *epf_test = epf_get_drvdata(epf);
720 	struct pci_epc *epc = epf->epc;
721 	struct pci_epf_bar *epf_bar;
722 	int bar;
723 
724 	cancel_delayed_work(&epf_test->cmd_handler);
725 	pci_epf_test_clean_dma_chan(epf_test);
726 	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
727 		epf_bar = &epf->bar[bar];
728 
729 		if (epf_test->reg[bar]) {
730 			pci_epc_clear_bar(epc, epf->func_no, epf->vfunc_no,
731 					  epf_bar);
732 			pci_epf_free_space(epf, epf_test->reg[bar], bar,
733 					   PRIMARY_INTERFACE);
734 		}
735 	}
736 }
737 
738 static int pci_epf_test_set_bar(struct pci_epf *epf)
739 {
740 	int bar, add;
741 	int ret;
742 	struct pci_epf_bar *epf_bar;
743 	struct pci_epc *epc = epf->epc;
744 	struct device *dev = &epf->dev;
745 	struct pci_epf_test *epf_test = epf_get_drvdata(epf);
746 	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
747 	const struct pci_epc_features *epc_features;
748 
749 	epc_features = epf_test->epc_features;
750 
751 	for (bar = 0; bar < PCI_STD_NUM_BARS; bar += add) {
752 		epf_bar = &epf->bar[bar];
753 		/*
754 		 * pci_epc_set_bar() sets PCI_BASE_ADDRESS_MEM_TYPE_64
755 		 * if the specific implementation required a 64-bit BAR,
756 		 * even if we only requested a 32-bit BAR.
757 		 */
758 		add = (epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64) ? 2 : 1;
759 
760 		if (!!(epc_features->reserved_bar & (1 << bar)))
761 			continue;
762 
763 		ret = pci_epc_set_bar(epc, epf->func_no, epf->vfunc_no,
764 				      epf_bar);
765 		if (ret) {
766 			pci_epf_free_space(epf, epf_test->reg[bar], bar,
767 					   PRIMARY_INTERFACE);
768 			dev_err(dev, "Failed to set BAR%d\n", bar);
769 			if (bar == test_reg_bar)
770 				return ret;
771 		}
772 	}
773 
774 	return 0;
775 }
776 
777 static int pci_epf_test_core_init(struct pci_epf *epf)
778 {
779 	struct pci_epf_test *epf_test = epf_get_drvdata(epf);
780 	struct pci_epf_header *header = epf->header;
781 	const struct pci_epc_features *epc_features;
782 	struct pci_epc *epc = epf->epc;
783 	struct device *dev = &epf->dev;
784 	bool msix_capable = false;
785 	bool msi_capable = true;
786 	int ret;
787 
788 	epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no);
789 	if (epc_features) {
790 		msix_capable = epc_features->msix_capable;
791 		msi_capable = epc_features->msi_capable;
792 	}
793 
794 	if (epf->vfunc_no <= 1) {
795 		ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no, header);
796 		if (ret) {
797 			dev_err(dev, "Configuration header write failed\n");
798 			return ret;
799 		}
800 	}
801 
802 	ret = pci_epf_test_set_bar(epf);
803 	if (ret)
804 		return ret;
805 
806 	if (msi_capable) {
807 		ret = pci_epc_set_msi(epc, epf->func_no, epf->vfunc_no,
808 				      epf->msi_interrupts);
809 		if (ret) {
810 			dev_err(dev, "MSI configuration failed\n");
811 			return ret;
812 		}
813 	}
814 
815 	if (msix_capable) {
816 		ret = pci_epc_set_msix(epc, epf->func_no, epf->vfunc_no,
817 				       epf->msix_interrupts,
818 				       epf_test->test_reg_bar,
819 				       epf_test->msix_table_offset);
820 		if (ret) {
821 			dev_err(dev, "MSI-X configuration failed\n");
822 			return ret;
823 		}
824 	}
825 
826 	return 0;
827 }
828 
829 static int pci_epf_test_notifier(struct notifier_block *nb, unsigned long val,
830 				 void *data)
831 {
832 	struct pci_epf *epf = container_of(nb, struct pci_epf, nb);
833 	struct pci_epf_test *epf_test = epf_get_drvdata(epf);
834 	int ret;
835 
836 	switch (val) {
837 	case CORE_INIT:
838 		ret = pci_epf_test_core_init(epf);
839 		if (ret)
840 			return NOTIFY_BAD;
841 		break;
842 
843 	case LINK_UP:
844 		queue_delayed_work(kpcitest_workqueue, &epf_test->cmd_handler,
845 				   msecs_to_jiffies(1));
846 		break;
847 
848 	default:
849 		dev_err(&epf->dev, "Invalid EPF test notifier event\n");
850 		return NOTIFY_BAD;
851 	}
852 
853 	return NOTIFY_OK;
854 }
855 
856 static int pci_epf_test_alloc_space(struct pci_epf *epf)
857 {
858 	struct pci_epf_test *epf_test = epf_get_drvdata(epf);
859 	struct device *dev = &epf->dev;
860 	struct pci_epf_bar *epf_bar;
861 	size_t msix_table_size = 0;
862 	size_t test_reg_bar_size;
863 	size_t pba_size = 0;
864 	bool msix_capable;
865 	void *base;
866 	int bar, add;
867 	enum pci_barno test_reg_bar = epf_test->test_reg_bar;
868 	const struct pci_epc_features *epc_features;
869 	size_t test_reg_size;
870 
871 	epc_features = epf_test->epc_features;
872 
873 	test_reg_bar_size = ALIGN(sizeof(struct pci_epf_test_reg), 128);
874 
875 	msix_capable = epc_features->msix_capable;
876 	if (msix_capable) {
877 		msix_table_size = PCI_MSIX_ENTRY_SIZE * epf->msix_interrupts;
878 		epf_test->msix_table_offset = test_reg_bar_size;
879 		/* Align to QWORD or 8 Bytes */
880 		pba_size = ALIGN(DIV_ROUND_UP(epf->msix_interrupts, 8), 8);
881 	}
882 	test_reg_size = test_reg_bar_size + msix_table_size + pba_size;
883 
884 	if (epc_features->bar_fixed_size[test_reg_bar]) {
885 		if (test_reg_size > bar_size[test_reg_bar])
886 			return -ENOMEM;
887 		test_reg_size = bar_size[test_reg_bar];
888 	}
889 
890 	base = pci_epf_alloc_space(epf, test_reg_size, test_reg_bar,
891 				   epc_features->align, PRIMARY_INTERFACE);
892 	if (!base) {
893 		dev_err(dev, "Failed to allocated register space\n");
894 		return -ENOMEM;
895 	}
896 	epf_test->reg[test_reg_bar] = base;
897 
898 	for (bar = 0; bar < PCI_STD_NUM_BARS; bar += add) {
899 		epf_bar = &epf->bar[bar];
900 		add = (epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64) ? 2 : 1;
901 
902 		if (bar == test_reg_bar)
903 			continue;
904 
905 		if (!!(epc_features->reserved_bar & (1 << bar)))
906 			continue;
907 
908 		base = pci_epf_alloc_space(epf, bar_size[bar], bar,
909 					   epc_features->align,
910 					   PRIMARY_INTERFACE);
911 		if (!base)
912 			dev_err(dev, "Failed to allocate space for BAR%d\n",
913 				bar);
914 		epf_test->reg[bar] = base;
915 	}
916 
917 	return 0;
918 }
919 
920 static void pci_epf_configure_bar(struct pci_epf *epf,
921 				  const struct pci_epc_features *epc_features)
922 {
923 	struct pci_epf_bar *epf_bar;
924 	bool bar_fixed_64bit;
925 	int i;
926 
927 	for (i = 0; i < PCI_STD_NUM_BARS; i++) {
928 		epf_bar = &epf->bar[i];
929 		bar_fixed_64bit = !!(epc_features->bar_fixed_64bit & (1 << i));
930 		if (bar_fixed_64bit)
931 			epf_bar->flags |= PCI_BASE_ADDRESS_MEM_TYPE_64;
932 		if (epc_features->bar_fixed_size[i])
933 			bar_size[i] = epc_features->bar_fixed_size[i];
934 	}
935 }
936 
937 static int pci_epf_test_bind(struct pci_epf *epf)
938 {
939 	int ret;
940 	struct pci_epf_test *epf_test = epf_get_drvdata(epf);
941 	const struct pci_epc_features *epc_features;
942 	enum pci_barno test_reg_bar = BAR_0;
943 	struct pci_epc *epc = epf->epc;
944 	bool linkup_notifier = false;
945 	bool core_init_notifier = false;
946 
947 	if (WARN_ON_ONCE(!epc))
948 		return -EINVAL;
949 
950 	epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no);
951 	if (!epc_features) {
952 		dev_err(&epf->dev, "epc_features not implemented\n");
953 		return -EOPNOTSUPP;
954 	}
955 
956 	linkup_notifier = epc_features->linkup_notifier;
957 	core_init_notifier = epc_features->core_init_notifier;
958 	test_reg_bar = pci_epc_get_first_free_bar(epc_features);
959 	if (test_reg_bar < 0)
960 		return -EINVAL;
961 	pci_epf_configure_bar(epf, epc_features);
962 
963 	epf_test->test_reg_bar = test_reg_bar;
964 	epf_test->epc_features = epc_features;
965 
966 	ret = pci_epf_test_alloc_space(epf);
967 	if (ret)
968 		return ret;
969 
970 	if (!core_init_notifier) {
971 		ret = pci_epf_test_core_init(epf);
972 		if (ret)
973 			return ret;
974 	}
975 
976 	epf_test->dma_supported = true;
977 
978 	ret = pci_epf_test_init_dma_chan(epf_test);
979 	if (ret)
980 		epf_test->dma_supported = false;
981 
982 	if (linkup_notifier || core_init_notifier) {
983 		epf->nb.notifier_call = pci_epf_test_notifier;
984 		pci_epc_register_notifier(epc, &epf->nb);
985 	} else {
986 		queue_work(kpcitest_workqueue, &epf_test->cmd_handler.work);
987 	}
988 
989 	return 0;
990 }
991 
992 static const struct pci_epf_device_id pci_epf_test_ids[] = {
993 	{
994 		.name = "pci_epf_test",
995 	},
996 	{},
997 };
998 
999 static int pci_epf_test_probe(struct pci_epf *epf)
1000 {
1001 	struct pci_epf_test *epf_test;
1002 	struct device *dev = &epf->dev;
1003 
1004 	epf_test = devm_kzalloc(dev, sizeof(*epf_test), GFP_KERNEL);
1005 	if (!epf_test)
1006 		return -ENOMEM;
1007 
1008 	epf->header = &test_header;
1009 	epf_test->epf = epf;
1010 
1011 	INIT_DELAYED_WORK(&epf_test->cmd_handler, pci_epf_test_cmd_handler);
1012 
1013 	epf_set_drvdata(epf, epf_test);
1014 	return 0;
1015 }
1016 
1017 static struct pci_epf_ops ops = {
1018 	.unbind	= pci_epf_test_unbind,
1019 	.bind	= pci_epf_test_bind,
1020 };
1021 
1022 static struct pci_epf_driver test_driver = {
1023 	.driver.name	= "pci_epf_test",
1024 	.probe		= pci_epf_test_probe,
1025 	.id_table	= pci_epf_test_ids,
1026 	.ops		= &ops,
1027 	.owner		= THIS_MODULE,
1028 };
1029 
1030 static int __init pci_epf_test_init(void)
1031 {
1032 	int ret;
1033 
1034 	kpcitest_workqueue = alloc_workqueue("kpcitest",
1035 					     WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
1036 	if (!kpcitest_workqueue) {
1037 		pr_err("Failed to allocate the kpcitest work queue\n");
1038 		return -ENOMEM;
1039 	}
1040 
1041 	ret = pci_epf_register_driver(&test_driver);
1042 	if (ret) {
1043 		destroy_workqueue(kpcitest_workqueue);
1044 		pr_err("Failed to register pci epf test driver --> %d\n", ret);
1045 		return ret;
1046 	}
1047 
1048 	return 0;
1049 }
1050 module_init(pci_epf_test_init);
1051 
1052 static void __exit pci_epf_test_exit(void)
1053 {
1054 	if (kpcitest_workqueue)
1055 		destroy_workqueue(kpcitest_workqueue);
1056 	pci_epf_unregister_driver(&test_driver);
1057 }
1058 module_exit(pci_epf_test_exit);
1059 
1060 MODULE_DESCRIPTION("PCI EPF TEST DRIVER");
1061 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>");
1062 MODULE_LICENSE("GPL v2");
1063