1 // SPDX-License-Identifier: GPL-2.0-only
2 /**
3  * Host side test driver to test endpoint functionality
4  *
5  * Copyright (C) 2017 Texas Instruments
6  * Author: Kishon Vijay Abraham I <kishon@ti.com>
7  */
8 
9 #include <linux/crc32.h>
10 #include <linux/delay.h>
11 #include <linux/fs.h>
12 #include <linux/io.h>
13 #include <linux/interrupt.h>
14 #include <linux/irq.h>
15 #include <linux/miscdevice.h>
16 #include <linux/module.h>
17 #include <linux/mutex.h>
18 #include <linux/random.h>
19 #include <linux/slab.h>
20 #include <linux/uaccess.h>
21 #include <linux/pci.h>
22 #include <linux/pci_ids.h>
23 
24 #include <linux/pci_regs.h>
25 
26 #include <uapi/linux/pcitest.h>
27 
28 #define DRV_MODULE_NAME				"pci-endpoint-test"
29 
30 #define IRQ_TYPE_UNDEFINED			-1
31 #define IRQ_TYPE_LEGACY				0
32 #define IRQ_TYPE_MSI				1
33 #define IRQ_TYPE_MSIX				2
34 
35 #define PCI_ENDPOINT_TEST_MAGIC			0x0
36 
37 #define PCI_ENDPOINT_TEST_COMMAND		0x4
38 #define COMMAND_RAISE_LEGACY_IRQ		BIT(0)
39 #define COMMAND_RAISE_MSI_IRQ			BIT(1)
40 #define COMMAND_RAISE_MSIX_IRQ			BIT(2)
41 #define COMMAND_READ				BIT(3)
42 #define COMMAND_WRITE				BIT(4)
43 #define COMMAND_COPY				BIT(5)
44 
45 #define PCI_ENDPOINT_TEST_STATUS		0x8
46 #define STATUS_READ_SUCCESS			BIT(0)
47 #define STATUS_READ_FAIL			BIT(1)
48 #define STATUS_WRITE_SUCCESS			BIT(2)
49 #define STATUS_WRITE_FAIL			BIT(3)
50 #define STATUS_COPY_SUCCESS			BIT(4)
51 #define STATUS_COPY_FAIL			BIT(5)
52 #define STATUS_IRQ_RAISED			BIT(6)
53 #define STATUS_SRC_ADDR_INVALID			BIT(7)
54 #define STATUS_DST_ADDR_INVALID			BIT(8)
55 
56 #define PCI_ENDPOINT_TEST_LOWER_SRC_ADDR	0x0c
57 #define PCI_ENDPOINT_TEST_UPPER_SRC_ADDR	0x10
58 
59 #define PCI_ENDPOINT_TEST_LOWER_DST_ADDR	0x14
60 #define PCI_ENDPOINT_TEST_UPPER_DST_ADDR	0x18
61 
62 #define PCI_ENDPOINT_TEST_SIZE			0x1c
63 #define PCI_ENDPOINT_TEST_CHECKSUM		0x20
64 
65 #define PCI_ENDPOINT_TEST_IRQ_TYPE		0x24
66 #define PCI_ENDPOINT_TEST_IRQ_NUMBER		0x28
67 
68 #define PCI_ENDPOINT_TEST_FLAGS			0x2c
69 #define FLAG_USE_DMA				BIT(0)
70 
71 #define PCI_DEVICE_ID_TI_AM654			0xb00c
72 #define PCI_DEVICE_ID_TI_J7200			0xb00f
73 #define PCI_DEVICE_ID_TI_AM64			0xb010
74 #define PCI_DEVICE_ID_LS1088A			0x80c0
75 
76 #define is_am654_pci_dev(pdev)		\
77 		((pdev)->device == PCI_DEVICE_ID_TI_AM654)
78 
79 #define PCI_DEVICE_ID_RENESAS_R8A774A1		0x0028
80 #define PCI_DEVICE_ID_RENESAS_R8A774B1		0x002b
81 #define PCI_DEVICE_ID_RENESAS_R8A774C0		0x002d
82 #define PCI_DEVICE_ID_RENESAS_R8A774E1		0x0025
83 
84 static DEFINE_IDA(pci_endpoint_test_ida);
85 
86 #define to_endpoint_test(priv) container_of((priv), struct pci_endpoint_test, \
87 					    miscdev)
88 
89 static bool no_msi;
90 module_param(no_msi, bool, 0444);
91 MODULE_PARM_DESC(no_msi, "Disable MSI interrupt in pci_endpoint_test");
92 
93 static int irq_type = IRQ_TYPE_MSI;
94 module_param(irq_type, int, 0444);
95 MODULE_PARM_DESC(irq_type, "IRQ mode selection in pci_endpoint_test (0 - Legacy, 1 - MSI, 2 - MSI-X)");
96 
97 enum pci_barno {
98 	BAR_0,
99 	BAR_1,
100 	BAR_2,
101 	BAR_3,
102 	BAR_4,
103 	BAR_5,
104 };
105 
106 struct pci_endpoint_test {
107 	struct pci_dev	*pdev;
108 	void __iomem	*base;
109 	void __iomem	*bar[PCI_STD_NUM_BARS];
110 	struct completion irq_raised;
111 	int		last_irq;
112 	int		num_irqs;
113 	int		irq_type;
114 	/* mutex to protect the ioctls */
115 	struct mutex	mutex;
116 	struct miscdevice miscdev;
117 	enum pci_barno test_reg_bar;
118 	size_t alignment;
119 	const char *name;
120 };
121 
122 struct pci_endpoint_test_data {
123 	enum pci_barno test_reg_bar;
124 	size_t alignment;
125 	int irq_type;
126 };
127 
128 static inline u32 pci_endpoint_test_readl(struct pci_endpoint_test *test,
129 					  u32 offset)
130 {
131 	return readl(test->base + offset);
132 }
133 
134 static inline void pci_endpoint_test_writel(struct pci_endpoint_test *test,
135 					    u32 offset, u32 value)
136 {
137 	writel(value, test->base + offset);
138 }
139 
140 static inline u32 pci_endpoint_test_bar_readl(struct pci_endpoint_test *test,
141 					      int bar, int offset)
142 {
143 	return readl(test->bar[bar] + offset);
144 }
145 
146 static inline void pci_endpoint_test_bar_writel(struct pci_endpoint_test *test,
147 						int bar, u32 offset, u32 value)
148 {
149 	writel(value, test->bar[bar] + offset);
150 }
151 
152 static irqreturn_t pci_endpoint_test_irqhandler(int irq, void *dev_id)
153 {
154 	struct pci_endpoint_test *test = dev_id;
155 	u32 reg;
156 
157 	reg = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS);
158 	if (reg & STATUS_IRQ_RAISED) {
159 		test->last_irq = irq;
160 		complete(&test->irq_raised);
161 		reg &= ~STATUS_IRQ_RAISED;
162 	}
163 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_STATUS,
164 				 reg);
165 
166 	return IRQ_HANDLED;
167 }
168 
169 static void pci_endpoint_test_free_irq_vectors(struct pci_endpoint_test *test)
170 {
171 	struct pci_dev *pdev = test->pdev;
172 
173 	pci_free_irq_vectors(pdev);
174 	test->irq_type = IRQ_TYPE_UNDEFINED;
175 }
176 
177 static bool pci_endpoint_test_alloc_irq_vectors(struct pci_endpoint_test *test,
178 						int type)
179 {
180 	int irq = -1;
181 	struct pci_dev *pdev = test->pdev;
182 	struct device *dev = &pdev->dev;
183 	bool res = true;
184 
185 	switch (type) {
186 	case IRQ_TYPE_LEGACY:
187 		irq = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_LEGACY);
188 		if (irq < 0)
189 			dev_err(dev, "Failed to get Legacy interrupt\n");
190 		break;
191 	case IRQ_TYPE_MSI:
192 		irq = pci_alloc_irq_vectors(pdev, 1, 32, PCI_IRQ_MSI);
193 		if (irq < 0)
194 			dev_err(dev, "Failed to get MSI interrupts\n");
195 		break;
196 	case IRQ_TYPE_MSIX:
197 		irq = pci_alloc_irq_vectors(pdev, 1, 2048, PCI_IRQ_MSIX);
198 		if (irq < 0)
199 			dev_err(dev, "Failed to get MSI-X interrupts\n");
200 		break;
201 	default:
202 		dev_err(dev, "Invalid IRQ type selected\n");
203 	}
204 
205 	if (irq < 0) {
206 		irq = 0;
207 		res = false;
208 	}
209 
210 	test->irq_type = type;
211 	test->num_irqs = irq;
212 
213 	return res;
214 }
215 
216 static void pci_endpoint_test_release_irq(struct pci_endpoint_test *test)
217 {
218 	int i;
219 	struct pci_dev *pdev = test->pdev;
220 	struct device *dev = &pdev->dev;
221 
222 	for (i = 0; i < test->num_irqs; i++)
223 		devm_free_irq(dev, pci_irq_vector(pdev, i), test);
224 
225 	test->num_irqs = 0;
226 }
227 
228 static bool pci_endpoint_test_request_irq(struct pci_endpoint_test *test)
229 {
230 	int i;
231 	int err;
232 	struct pci_dev *pdev = test->pdev;
233 	struct device *dev = &pdev->dev;
234 
235 	for (i = 0; i < test->num_irqs; i++) {
236 		err = devm_request_irq(dev, pci_irq_vector(pdev, i),
237 				       pci_endpoint_test_irqhandler,
238 				       IRQF_SHARED, test->name, test);
239 		if (err)
240 			goto fail;
241 	}
242 
243 	return true;
244 
245 fail:
246 	switch (irq_type) {
247 	case IRQ_TYPE_LEGACY:
248 		dev_err(dev, "Failed to request IRQ %d for Legacy\n",
249 			pci_irq_vector(pdev, i));
250 		break;
251 	case IRQ_TYPE_MSI:
252 		dev_err(dev, "Failed to request IRQ %d for MSI %d\n",
253 			pci_irq_vector(pdev, i),
254 			i + 1);
255 		break;
256 	case IRQ_TYPE_MSIX:
257 		dev_err(dev, "Failed to request IRQ %d for MSI-X %d\n",
258 			pci_irq_vector(pdev, i),
259 			i + 1);
260 		break;
261 	}
262 
263 	return false;
264 }
265 
266 static bool pci_endpoint_test_bar(struct pci_endpoint_test *test,
267 				  enum pci_barno barno)
268 {
269 	int j;
270 	u32 val;
271 	int size;
272 	struct pci_dev *pdev = test->pdev;
273 
274 	if (!test->bar[barno])
275 		return false;
276 
277 	size = pci_resource_len(pdev, barno);
278 
279 	if (barno == test->test_reg_bar)
280 		size = 0x4;
281 
282 	for (j = 0; j < size; j += 4)
283 		pci_endpoint_test_bar_writel(test, barno, j, 0xA0A0A0A0);
284 
285 	for (j = 0; j < size; j += 4) {
286 		val = pci_endpoint_test_bar_readl(test, barno, j);
287 		if (val != 0xA0A0A0A0)
288 			return false;
289 	}
290 
291 	return true;
292 }
293 
294 static bool pci_endpoint_test_legacy_irq(struct pci_endpoint_test *test)
295 {
296 	u32 val;
297 
298 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE,
299 				 IRQ_TYPE_LEGACY);
300 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 0);
301 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
302 				 COMMAND_RAISE_LEGACY_IRQ);
303 	val = wait_for_completion_timeout(&test->irq_raised,
304 					  msecs_to_jiffies(1000));
305 	if (!val)
306 		return false;
307 
308 	return true;
309 }
310 
311 static bool pci_endpoint_test_msi_irq(struct pci_endpoint_test *test,
312 				       u16 msi_num, bool msix)
313 {
314 	u32 val;
315 	struct pci_dev *pdev = test->pdev;
316 
317 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE,
318 				 msix == false ? IRQ_TYPE_MSI :
319 				 IRQ_TYPE_MSIX);
320 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, msi_num);
321 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
322 				 msix == false ? COMMAND_RAISE_MSI_IRQ :
323 				 COMMAND_RAISE_MSIX_IRQ);
324 	val = wait_for_completion_timeout(&test->irq_raised,
325 					  msecs_to_jiffies(1000));
326 	if (!val)
327 		return false;
328 
329 	if (pci_irq_vector(pdev, msi_num - 1) == test->last_irq)
330 		return true;
331 
332 	return false;
333 }
334 
335 static bool pci_endpoint_test_copy(struct pci_endpoint_test *test,
336 				   unsigned long arg)
337 {
338 	struct pci_endpoint_test_xfer_param param;
339 	bool ret = false;
340 	void *src_addr;
341 	void *dst_addr;
342 	u32 flags = 0;
343 	bool use_dma;
344 	size_t size;
345 	dma_addr_t src_phys_addr;
346 	dma_addr_t dst_phys_addr;
347 	struct pci_dev *pdev = test->pdev;
348 	struct device *dev = &pdev->dev;
349 	void *orig_src_addr;
350 	dma_addr_t orig_src_phys_addr;
351 	void *orig_dst_addr;
352 	dma_addr_t orig_dst_phys_addr;
353 	size_t offset;
354 	size_t alignment = test->alignment;
355 	int irq_type = test->irq_type;
356 	u32 src_crc32;
357 	u32 dst_crc32;
358 	int err;
359 
360 	err = copy_from_user(&param, (void __user *)arg, sizeof(param));
361 	if (err) {
362 		dev_err(dev, "Failed to get transfer param\n");
363 		return false;
364 	}
365 
366 	size = param.size;
367 	if (size > SIZE_MAX - alignment)
368 		goto err;
369 
370 	use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA);
371 	if (use_dma)
372 		flags |= FLAG_USE_DMA;
373 
374 	if (irq_type < IRQ_TYPE_LEGACY || irq_type > IRQ_TYPE_MSIX) {
375 		dev_err(dev, "Invalid IRQ type option\n");
376 		goto err;
377 	}
378 
379 	orig_src_addr = kzalloc(size + alignment, GFP_KERNEL);
380 	if (!orig_src_addr) {
381 		dev_err(dev, "Failed to allocate source buffer\n");
382 		ret = false;
383 		goto err;
384 	}
385 
386 	get_random_bytes(orig_src_addr, size + alignment);
387 	orig_src_phys_addr = dma_map_single(dev, orig_src_addr,
388 					    size + alignment, DMA_TO_DEVICE);
389 	if (dma_mapping_error(dev, orig_src_phys_addr)) {
390 		dev_err(dev, "failed to map source buffer address\n");
391 		ret = false;
392 		goto err_src_phys_addr;
393 	}
394 
395 	if (alignment && !IS_ALIGNED(orig_src_phys_addr, alignment)) {
396 		src_phys_addr = PTR_ALIGN(orig_src_phys_addr, alignment);
397 		offset = src_phys_addr - orig_src_phys_addr;
398 		src_addr = orig_src_addr + offset;
399 	} else {
400 		src_phys_addr = orig_src_phys_addr;
401 		src_addr = orig_src_addr;
402 	}
403 
404 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_SRC_ADDR,
405 				 lower_32_bits(src_phys_addr));
406 
407 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_SRC_ADDR,
408 				 upper_32_bits(src_phys_addr));
409 
410 	src_crc32 = crc32_le(~0, src_addr, size);
411 
412 	orig_dst_addr = kzalloc(size + alignment, GFP_KERNEL);
413 	if (!orig_dst_addr) {
414 		dev_err(dev, "Failed to allocate destination address\n");
415 		ret = false;
416 		goto err_dst_addr;
417 	}
418 
419 	orig_dst_phys_addr = dma_map_single(dev, orig_dst_addr,
420 					    size + alignment, DMA_FROM_DEVICE);
421 	if (dma_mapping_error(dev, orig_dst_phys_addr)) {
422 		dev_err(dev, "failed to map destination buffer address\n");
423 		ret = false;
424 		goto err_dst_phys_addr;
425 	}
426 
427 	if (alignment && !IS_ALIGNED(orig_dst_phys_addr, alignment)) {
428 		dst_phys_addr = PTR_ALIGN(orig_dst_phys_addr, alignment);
429 		offset = dst_phys_addr - orig_dst_phys_addr;
430 		dst_addr = orig_dst_addr + offset;
431 	} else {
432 		dst_phys_addr = orig_dst_phys_addr;
433 		dst_addr = orig_dst_addr;
434 	}
435 
436 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_DST_ADDR,
437 				 lower_32_bits(dst_phys_addr));
438 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_DST_ADDR,
439 				 upper_32_bits(dst_phys_addr));
440 
441 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE,
442 				 size);
443 
444 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_FLAGS, flags);
445 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
446 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
447 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
448 				 COMMAND_COPY);
449 
450 	wait_for_completion(&test->irq_raised);
451 
452 	dma_unmap_single(dev, orig_dst_phys_addr, size + alignment,
453 			 DMA_FROM_DEVICE);
454 
455 	dst_crc32 = crc32_le(~0, dst_addr, size);
456 	if (dst_crc32 == src_crc32)
457 		ret = true;
458 
459 err_dst_phys_addr:
460 	kfree(orig_dst_addr);
461 
462 err_dst_addr:
463 	dma_unmap_single(dev, orig_src_phys_addr, size + alignment,
464 			 DMA_TO_DEVICE);
465 
466 err_src_phys_addr:
467 	kfree(orig_src_addr);
468 
469 err:
470 	return ret;
471 }
472 
473 static bool pci_endpoint_test_write(struct pci_endpoint_test *test,
474 				    unsigned long arg)
475 {
476 	struct pci_endpoint_test_xfer_param param;
477 	bool ret = false;
478 	u32 flags = 0;
479 	bool use_dma;
480 	u32 reg;
481 	void *addr;
482 	dma_addr_t phys_addr;
483 	struct pci_dev *pdev = test->pdev;
484 	struct device *dev = &pdev->dev;
485 	void *orig_addr;
486 	dma_addr_t orig_phys_addr;
487 	size_t offset;
488 	size_t alignment = test->alignment;
489 	int irq_type = test->irq_type;
490 	size_t size;
491 	u32 crc32;
492 	int err;
493 
494 	err = copy_from_user(&param, (void __user *)arg, sizeof(param));
495 	if (err != 0) {
496 		dev_err(dev, "Failed to get transfer param\n");
497 		return false;
498 	}
499 
500 	size = param.size;
501 	if (size > SIZE_MAX - alignment)
502 		goto err;
503 
504 	use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA);
505 	if (use_dma)
506 		flags |= FLAG_USE_DMA;
507 
508 	if (irq_type < IRQ_TYPE_LEGACY || irq_type > IRQ_TYPE_MSIX) {
509 		dev_err(dev, "Invalid IRQ type option\n");
510 		goto err;
511 	}
512 
513 	orig_addr = kzalloc(size + alignment, GFP_KERNEL);
514 	if (!orig_addr) {
515 		dev_err(dev, "Failed to allocate address\n");
516 		ret = false;
517 		goto err;
518 	}
519 
520 	get_random_bytes(orig_addr, size + alignment);
521 
522 	orig_phys_addr = dma_map_single(dev, orig_addr, size + alignment,
523 					DMA_TO_DEVICE);
524 	if (dma_mapping_error(dev, orig_phys_addr)) {
525 		dev_err(dev, "failed to map source buffer address\n");
526 		ret = false;
527 		goto err_phys_addr;
528 	}
529 
530 	if (alignment && !IS_ALIGNED(orig_phys_addr, alignment)) {
531 		phys_addr =  PTR_ALIGN(orig_phys_addr, alignment);
532 		offset = phys_addr - orig_phys_addr;
533 		addr = orig_addr + offset;
534 	} else {
535 		phys_addr = orig_phys_addr;
536 		addr = orig_addr;
537 	}
538 
539 	crc32 = crc32_le(~0, addr, size);
540 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_CHECKSUM,
541 				 crc32);
542 
543 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_SRC_ADDR,
544 				 lower_32_bits(phys_addr));
545 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_SRC_ADDR,
546 				 upper_32_bits(phys_addr));
547 
548 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE, size);
549 
550 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_FLAGS, flags);
551 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
552 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
553 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
554 				 COMMAND_READ);
555 
556 	wait_for_completion(&test->irq_raised);
557 
558 	reg = pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_STATUS);
559 	if (reg & STATUS_READ_SUCCESS)
560 		ret = true;
561 
562 	dma_unmap_single(dev, orig_phys_addr, size + alignment,
563 			 DMA_TO_DEVICE);
564 
565 err_phys_addr:
566 	kfree(orig_addr);
567 
568 err:
569 	return ret;
570 }
571 
572 static bool pci_endpoint_test_read(struct pci_endpoint_test *test,
573 				   unsigned long arg)
574 {
575 	struct pci_endpoint_test_xfer_param param;
576 	bool ret = false;
577 	u32 flags = 0;
578 	bool use_dma;
579 	size_t size;
580 	void *addr;
581 	dma_addr_t phys_addr;
582 	struct pci_dev *pdev = test->pdev;
583 	struct device *dev = &pdev->dev;
584 	void *orig_addr;
585 	dma_addr_t orig_phys_addr;
586 	size_t offset;
587 	size_t alignment = test->alignment;
588 	int irq_type = test->irq_type;
589 	u32 crc32;
590 	int err;
591 
592 	err = copy_from_user(&param, (void __user *)arg, sizeof(param));
593 	if (err) {
594 		dev_err(dev, "Failed to get transfer param\n");
595 		return false;
596 	}
597 
598 	size = param.size;
599 	if (size > SIZE_MAX - alignment)
600 		goto err;
601 
602 	use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA);
603 	if (use_dma)
604 		flags |= FLAG_USE_DMA;
605 
606 	if (irq_type < IRQ_TYPE_LEGACY || irq_type > IRQ_TYPE_MSIX) {
607 		dev_err(dev, "Invalid IRQ type option\n");
608 		goto err;
609 	}
610 
611 	orig_addr = kzalloc(size + alignment, GFP_KERNEL);
612 	if (!orig_addr) {
613 		dev_err(dev, "Failed to allocate destination address\n");
614 		ret = false;
615 		goto err;
616 	}
617 
618 	orig_phys_addr = dma_map_single(dev, orig_addr, size + alignment,
619 					DMA_FROM_DEVICE);
620 	if (dma_mapping_error(dev, orig_phys_addr)) {
621 		dev_err(dev, "failed to map source buffer address\n");
622 		ret = false;
623 		goto err_phys_addr;
624 	}
625 
626 	if (alignment && !IS_ALIGNED(orig_phys_addr, alignment)) {
627 		phys_addr = PTR_ALIGN(orig_phys_addr, alignment);
628 		offset = phys_addr - orig_phys_addr;
629 		addr = orig_addr + offset;
630 	} else {
631 		phys_addr = orig_phys_addr;
632 		addr = orig_addr;
633 	}
634 
635 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_DST_ADDR,
636 				 lower_32_bits(phys_addr));
637 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_DST_ADDR,
638 				 upper_32_bits(phys_addr));
639 
640 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE, size);
641 
642 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_FLAGS, flags);
643 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
644 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
645 	pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
646 				 COMMAND_WRITE);
647 
648 	wait_for_completion(&test->irq_raised);
649 
650 	dma_unmap_single(dev, orig_phys_addr, size + alignment,
651 			 DMA_FROM_DEVICE);
652 
653 	crc32 = crc32_le(~0, addr, size);
654 	if (crc32 == pci_endpoint_test_readl(test, PCI_ENDPOINT_TEST_CHECKSUM))
655 		ret = true;
656 
657 err_phys_addr:
658 	kfree(orig_addr);
659 err:
660 	return ret;
661 }
662 
663 static bool pci_endpoint_test_clear_irq(struct pci_endpoint_test *test)
664 {
665 	pci_endpoint_test_release_irq(test);
666 	pci_endpoint_test_free_irq_vectors(test);
667 	return true;
668 }
669 
670 static bool pci_endpoint_test_set_irq(struct pci_endpoint_test *test,
671 				      int req_irq_type)
672 {
673 	struct pci_dev *pdev = test->pdev;
674 	struct device *dev = &pdev->dev;
675 
676 	if (req_irq_type < IRQ_TYPE_LEGACY || req_irq_type > IRQ_TYPE_MSIX) {
677 		dev_err(dev, "Invalid IRQ type option\n");
678 		return false;
679 	}
680 
681 	if (test->irq_type == req_irq_type)
682 		return true;
683 
684 	pci_endpoint_test_release_irq(test);
685 	pci_endpoint_test_free_irq_vectors(test);
686 
687 	if (!pci_endpoint_test_alloc_irq_vectors(test, req_irq_type))
688 		goto err;
689 
690 	if (!pci_endpoint_test_request_irq(test))
691 		goto err;
692 
693 	return true;
694 
695 err:
696 	pci_endpoint_test_free_irq_vectors(test);
697 	return false;
698 }
699 
700 static long pci_endpoint_test_ioctl(struct file *file, unsigned int cmd,
701 				    unsigned long arg)
702 {
703 	int ret = -EINVAL;
704 	enum pci_barno bar;
705 	struct pci_endpoint_test *test = to_endpoint_test(file->private_data);
706 	struct pci_dev *pdev = test->pdev;
707 
708 	mutex_lock(&test->mutex);
709 	switch (cmd) {
710 	case PCITEST_BAR:
711 		bar = arg;
712 		if (bar > BAR_5)
713 			goto ret;
714 		if (is_am654_pci_dev(pdev) && bar == BAR_0)
715 			goto ret;
716 		ret = pci_endpoint_test_bar(test, bar);
717 		break;
718 	case PCITEST_LEGACY_IRQ:
719 		ret = pci_endpoint_test_legacy_irq(test);
720 		break;
721 	case PCITEST_MSI:
722 	case PCITEST_MSIX:
723 		ret = pci_endpoint_test_msi_irq(test, arg, cmd == PCITEST_MSIX);
724 		break;
725 	case PCITEST_WRITE:
726 		ret = pci_endpoint_test_write(test, arg);
727 		break;
728 	case PCITEST_READ:
729 		ret = pci_endpoint_test_read(test, arg);
730 		break;
731 	case PCITEST_COPY:
732 		ret = pci_endpoint_test_copy(test, arg);
733 		break;
734 	case PCITEST_SET_IRQTYPE:
735 		ret = pci_endpoint_test_set_irq(test, arg);
736 		break;
737 	case PCITEST_GET_IRQTYPE:
738 		ret = irq_type;
739 		break;
740 	case PCITEST_CLEAR_IRQ:
741 		ret = pci_endpoint_test_clear_irq(test);
742 		break;
743 	}
744 
745 ret:
746 	mutex_unlock(&test->mutex);
747 	return ret;
748 }
749 
750 static const struct file_operations pci_endpoint_test_fops = {
751 	.owner = THIS_MODULE,
752 	.unlocked_ioctl = pci_endpoint_test_ioctl,
753 };
754 
755 static int pci_endpoint_test_probe(struct pci_dev *pdev,
756 				   const struct pci_device_id *ent)
757 {
758 	int err;
759 	int id;
760 	char name[24];
761 	enum pci_barno bar;
762 	void __iomem *base;
763 	struct device *dev = &pdev->dev;
764 	struct pci_endpoint_test *test;
765 	struct pci_endpoint_test_data *data;
766 	enum pci_barno test_reg_bar = BAR_0;
767 	struct miscdevice *misc_device;
768 
769 	if (pci_is_bridge(pdev))
770 		return -ENODEV;
771 
772 	test = devm_kzalloc(dev, sizeof(*test), GFP_KERNEL);
773 	if (!test)
774 		return -ENOMEM;
775 
776 	test->test_reg_bar = 0;
777 	test->alignment = 0;
778 	test->pdev = pdev;
779 	test->irq_type = IRQ_TYPE_UNDEFINED;
780 
781 	if (no_msi)
782 		irq_type = IRQ_TYPE_LEGACY;
783 
784 	data = (struct pci_endpoint_test_data *)ent->driver_data;
785 	if (data) {
786 		test_reg_bar = data->test_reg_bar;
787 		test->test_reg_bar = test_reg_bar;
788 		test->alignment = data->alignment;
789 		irq_type = data->irq_type;
790 	}
791 
792 	init_completion(&test->irq_raised);
793 	mutex_init(&test->mutex);
794 
795 	if ((dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48)) != 0) &&
796 	    dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)) != 0) {
797 		dev_err(dev, "Cannot set DMA mask\n");
798 		return -EINVAL;
799 	}
800 
801 	err = pci_enable_device(pdev);
802 	if (err) {
803 		dev_err(dev, "Cannot enable PCI device\n");
804 		return err;
805 	}
806 
807 	err = pci_request_regions(pdev, DRV_MODULE_NAME);
808 	if (err) {
809 		dev_err(dev, "Cannot obtain PCI resources\n");
810 		goto err_disable_pdev;
811 	}
812 
813 	pci_set_master(pdev);
814 
815 	if (!pci_endpoint_test_alloc_irq_vectors(test, irq_type)) {
816 		err = -EINVAL;
817 		goto err_disable_irq;
818 	}
819 
820 	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
821 		if (pci_resource_flags(pdev, bar) & IORESOURCE_MEM) {
822 			base = pci_ioremap_bar(pdev, bar);
823 			if (!base) {
824 				dev_err(dev, "Failed to read BAR%d\n", bar);
825 				WARN_ON(bar == test_reg_bar);
826 			}
827 			test->bar[bar] = base;
828 		}
829 	}
830 
831 	test->base = test->bar[test_reg_bar];
832 	if (!test->base) {
833 		err = -ENOMEM;
834 		dev_err(dev, "Cannot perform PCI test without BAR%d\n",
835 			test_reg_bar);
836 		goto err_iounmap;
837 	}
838 
839 	pci_set_drvdata(pdev, test);
840 
841 	id = ida_simple_get(&pci_endpoint_test_ida, 0, 0, GFP_KERNEL);
842 	if (id < 0) {
843 		err = id;
844 		dev_err(dev, "Unable to get id\n");
845 		goto err_iounmap;
846 	}
847 
848 	snprintf(name, sizeof(name), DRV_MODULE_NAME ".%d", id);
849 	test->name = kstrdup(name, GFP_KERNEL);
850 	if (!test->name) {
851 		err = -ENOMEM;
852 		goto err_ida_remove;
853 	}
854 
855 	if (!pci_endpoint_test_request_irq(test)) {
856 		err = -EINVAL;
857 		goto err_kfree_test_name;
858 	}
859 
860 	misc_device = &test->miscdev;
861 	misc_device->minor = MISC_DYNAMIC_MINOR;
862 	misc_device->name = kstrdup(name, GFP_KERNEL);
863 	if (!misc_device->name) {
864 		err = -ENOMEM;
865 		goto err_release_irq;
866 	}
867 	misc_device->parent = &pdev->dev;
868 	misc_device->fops = &pci_endpoint_test_fops,
869 
870 	err = misc_register(misc_device);
871 	if (err) {
872 		dev_err(dev, "Failed to register device\n");
873 		goto err_kfree_name;
874 	}
875 
876 	return 0;
877 
878 err_kfree_name:
879 	kfree(misc_device->name);
880 
881 err_release_irq:
882 	pci_endpoint_test_release_irq(test);
883 
884 err_kfree_test_name:
885 	kfree(test->name);
886 
887 err_ida_remove:
888 	ida_simple_remove(&pci_endpoint_test_ida, id);
889 
890 err_iounmap:
891 	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
892 		if (test->bar[bar])
893 			pci_iounmap(pdev, test->bar[bar]);
894 	}
895 
896 err_disable_irq:
897 	pci_endpoint_test_free_irq_vectors(test);
898 	pci_release_regions(pdev);
899 
900 err_disable_pdev:
901 	pci_disable_device(pdev);
902 
903 	return err;
904 }
905 
906 static void pci_endpoint_test_remove(struct pci_dev *pdev)
907 {
908 	int id;
909 	enum pci_barno bar;
910 	struct pci_endpoint_test *test = pci_get_drvdata(pdev);
911 	struct miscdevice *misc_device = &test->miscdev;
912 
913 	if (sscanf(misc_device->name, DRV_MODULE_NAME ".%d", &id) != 1)
914 		return;
915 	if (id < 0)
916 		return;
917 
918 	misc_deregister(&test->miscdev);
919 	kfree(misc_device->name);
920 	kfree(test->name);
921 	ida_simple_remove(&pci_endpoint_test_ida, id);
922 	for (bar = 0; bar < PCI_STD_NUM_BARS; bar++) {
923 		if (test->bar[bar])
924 			pci_iounmap(pdev, test->bar[bar]);
925 	}
926 
927 	pci_endpoint_test_release_irq(test);
928 	pci_endpoint_test_free_irq_vectors(test);
929 
930 	pci_release_regions(pdev);
931 	pci_disable_device(pdev);
932 }
933 
934 static const struct pci_endpoint_test_data default_data = {
935 	.test_reg_bar = BAR_0,
936 	.alignment = SZ_4K,
937 	.irq_type = IRQ_TYPE_MSI,
938 };
939 
940 static const struct pci_endpoint_test_data am654_data = {
941 	.test_reg_bar = BAR_2,
942 	.alignment = SZ_64K,
943 	.irq_type = IRQ_TYPE_MSI,
944 };
945 
946 static const struct pci_endpoint_test_data j721e_data = {
947 	.alignment = 256,
948 	.irq_type = IRQ_TYPE_MSI,
949 };
950 
951 static const struct pci_device_id pci_endpoint_test_tbl[] = {
952 	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA74x),
953 	  .driver_data = (kernel_ulong_t)&default_data,
954 	},
955 	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_DRA72x),
956 	  .driver_data = (kernel_ulong_t)&default_data,
957 	},
958 	{ PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, 0x81c0),
959 	  .driver_data = (kernel_ulong_t)&default_data,
960 	},
961 	{ PCI_DEVICE(PCI_VENDOR_ID_FREESCALE, PCI_DEVICE_ID_LS1088A),
962 	  .driver_data = (kernel_ulong_t)&default_data,
963 	},
964 	{ PCI_DEVICE_DATA(SYNOPSYS, EDDA, NULL) },
965 	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM654),
966 	  .driver_data = (kernel_ulong_t)&am654_data
967 	},
968 	{ PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774A1),},
969 	{ PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774B1),},
970 	{ PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774C0),},
971 	{ PCI_DEVICE(PCI_VENDOR_ID_RENESAS, PCI_DEVICE_ID_RENESAS_R8A774E1),},
972 	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J721E),
973 	  .driver_data = (kernel_ulong_t)&j721e_data,
974 	},
975 	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_J7200),
976 	  .driver_data = (kernel_ulong_t)&j721e_data,
977 	},
978 	{ PCI_DEVICE(PCI_VENDOR_ID_TI, PCI_DEVICE_ID_TI_AM64),
979 	  .driver_data = (kernel_ulong_t)&j721e_data,
980 	},
981 	{ }
982 };
983 MODULE_DEVICE_TABLE(pci, pci_endpoint_test_tbl);
984 
985 static struct pci_driver pci_endpoint_test_driver = {
986 	.name		= DRV_MODULE_NAME,
987 	.id_table	= pci_endpoint_test_tbl,
988 	.probe		= pci_endpoint_test_probe,
989 	.remove		= pci_endpoint_test_remove,
990 	.sriov_configure = pci_sriov_configure_simple,
991 };
992 module_pci_driver(pci_endpoint_test_driver);
993 
994 MODULE_DESCRIPTION("PCI ENDPOINT TEST HOST DRIVER");
995 MODULE_AUTHOR("Kishon Vijay Abraham I <kishon@ti.com>");
996 MODULE_LICENSE("GPL v2");
997