p2pdma.c (3c53c6255d598db7084c5c3d7553d7200e857818) p2pdma.c (f6b6aefee70aa5261deec7feab80c249bf58397f)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * PCI Peer 2 Peer DMA support.
4 *
5 * Copyright (c) 2016-2018, Logan Gunthorpe
6 * Copyright (c) 2016-2017, Microsemi Corporation
7 * Copyright (c) 2017, Christoph Hellwig
8 * Copyright (c) 2018, Eideticom Inc.

--- 4 unchanged lines hidden (view full) ---

13#include <linux/pci-p2pdma.h>
14#include <linux/module.h>
15#include <linux/slab.h>
16#include <linux/genalloc.h>
17#include <linux/memremap.h>
18#include <linux/percpu-refcount.h>
19#include <linux/random.h>
20#include <linux/seq_buf.h>
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * PCI Peer 2 Peer DMA support.
4 *
5 * Copyright (c) 2016-2018, Logan Gunthorpe
6 * Copyright (c) 2016-2017, Microsemi Corporation
7 * Copyright (c) 2017, Christoph Hellwig
8 * Copyright (c) 2018, Eideticom Inc.

--- 4 unchanged lines hidden (view full) ---

13#include <linux/pci-p2pdma.h>
14#include <linux/module.h>
15#include <linux/slab.h>
16#include <linux/genalloc.h>
17#include <linux/memremap.h>
18#include <linux/percpu-refcount.h>
19#include <linux/random.h>
20#include <linux/seq_buf.h>
21#include <linux/iommu.h>
22
23struct pci_p2pdma {
21
22struct pci_p2pdma {
23 struct percpu_ref devmap_ref;
24 struct completion devmap_ref_done;
24 struct gen_pool *pool;
25 bool p2pmem_published;
26};
27
25 struct gen_pool *pool;
26 bool p2pmem_published;
27};
28
28struct p2pdma_pagemap {
29 struct dev_pagemap pgmap;
30 struct percpu_ref ref;
31 struct completion ref_done;
32};
33
34static ssize_t size_show(struct device *dev, struct device_attribute *attr,
35 char *buf)
36{
37 struct pci_dev *pdev = to_pci_dev(dev);
38 size_t size = 0;
39
40 if (pdev->p2pdma->pool)
41 size = gen_pool_size(pdev->p2pdma->pool);

--- 32 unchanged lines hidden (view full) ---

74 NULL,
75};
76
77static const struct attribute_group p2pmem_group = {
78 .attrs = p2pmem_attrs,
79 .name = "p2pmem",
80};
81
29static ssize_t size_show(struct device *dev, struct device_attribute *attr,
30 char *buf)
31{
32 struct pci_dev *pdev = to_pci_dev(dev);
33 size_t size = 0;
34
35 if (pdev->p2pdma->pool)
36 size = gen_pool_size(pdev->p2pdma->pool);

--- 32 unchanged lines hidden (view full) ---

69 NULL,
70};
71
72static const struct attribute_group p2pmem_group = {
73 .attrs = p2pmem_attrs,
74 .name = "p2pmem",
75};
76
82static struct p2pdma_pagemap *to_p2p_pgmap(struct percpu_ref *ref)
83{
84 return container_of(ref, struct p2pdma_pagemap, ref);
85}
86
87static void pci_p2pdma_percpu_release(struct percpu_ref *ref)
88{
77static void pci_p2pdma_percpu_release(struct percpu_ref *ref)
78{
89 struct p2pdma_pagemap *p2p_pgmap = to_p2p_pgmap(ref);
79 struct pci_p2pdma *p2p =
80 container_of(ref, struct pci_p2pdma, devmap_ref);
90
81
91 complete(&p2p_pgmap->ref_done);
82 complete_all(&p2p->devmap_ref_done);
92}
93
94static void pci_p2pdma_percpu_kill(struct percpu_ref *ref)
95{
83}
84
85static void pci_p2pdma_percpu_kill(struct percpu_ref *ref)
86{
87 /*
88 * pci_p2pdma_add_resource() may be called multiple times
89 * by a driver and may register the percpu_kill devm action multiple
90 * times. We only want the first action to actually kill the
91 * percpu_ref.
92 */
93 if (percpu_ref_is_dying(ref))
94 return;
95
96 percpu_ref_kill(ref);
97}
98
96 percpu_ref_kill(ref);
97}
98
99static void pci_p2pdma_percpu_cleanup(struct percpu_ref *ref)
100{
101 struct p2pdma_pagemap *p2p_pgmap = to_p2p_pgmap(ref);
102
103 wait_for_completion(&p2p_pgmap->ref_done);
104 percpu_ref_exit(&p2p_pgmap->ref);
105}
106
107static void pci_p2pdma_release(void *data)
108{
109 struct pci_dev *pdev = data;
99static void pci_p2pdma_release(void *data)
100{
101 struct pci_dev *pdev = data;
110 struct pci_p2pdma *p2pdma = pdev->p2pdma;
111
102
112 if (!p2pdma)
103 if (!pdev->p2pdma)
113 return;
114
104 return;
105
115 /* Flush and disable pci_alloc_p2p_mem() */
116 pdev->p2pdma = NULL;
117 synchronize_rcu();
106 wait_for_completion(&pdev->p2pdma->devmap_ref_done);
107 percpu_ref_exit(&pdev->p2pdma->devmap_ref);
118
108
119 gen_pool_destroy(p2pdma->pool);
109 gen_pool_destroy(pdev->p2pdma->pool);
120 sysfs_remove_group(&pdev->dev.kobj, &p2pmem_group);
110 sysfs_remove_group(&pdev->dev.kobj, &p2pmem_group);
111 pdev->p2pdma = NULL;
121}
122
123static int pci_p2pdma_setup(struct pci_dev *pdev)
124{
125 int error = -ENOMEM;
126 struct pci_p2pdma *p2p;
127
128 p2p = devm_kzalloc(&pdev->dev, sizeof(*p2p), GFP_KERNEL);
129 if (!p2p)
130 return -ENOMEM;
131
132 p2p->pool = gen_pool_create(PAGE_SHIFT, dev_to_node(&pdev->dev));
133 if (!p2p->pool)
134 goto out;
135
112}
113
114static int pci_p2pdma_setup(struct pci_dev *pdev)
115{
116 int error = -ENOMEM;
117 struct pci_p2pdma *p2p;
118
119 p2p = devm_kzalloc(&pdev->dev, sizeof(*p2p), GFP_KERNEL);
120 if (!p2p)
121 return -ENOMEM;
122
123 p2p->pool = gen_pool_create(PAGE_SHIFT, dev_to_node(&pdev->dev));
124 if (!p2p->pool)
125 goto out;
126
127 init_completion(&p2p->devmap_ref_done);
128 error = percpu_ref_init(&p2p->devmap_ref,
129 pci_p2pdma_percpu_release, 0, GFP_KERNEL);
130 if (error)
131 goto out_pool_destroy;
132
136 error = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_release, pdev);
137 if (error)
138 goto out_pool_destroy;
139
140 pdev->p2pdma = p2p;
141
142 error = sysfs_create_group(&pdev->dev.kobj, &p2pmem_group);
143 if (error)

--- 17 unchanged lines hidden (view full) ---

161 * @offset: offset into the PCI BAR
162 *
163 * The memory will be given ZONE_DEVICE struct pages so that it may
164 * be used with any DMA request.
165 */
166int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size,
167 u64 offset)
168{
133 error = devm_add_action_or_reset(&pdev->dev, pci_p2pdma_release, pdev);
134 if (error)
135 goto out_pool_destroy;
136
137 pdev->p2pdma = p2p;
138
139 error = sysfs_create_group(&pdev->dev.kobj, &p2pmem_group);
140 if (error)

--- 17 unchanged lines hidden (view full) ---

158 * @offset: offset into the PCI BAR
159 *
160 * The memory will be given ZONE_DEVICE struct pages so that it may
161 * be used with any DMA request.
162 */
163int pci_p2pdma_add_resource(struct pci_dev *pdev, int bar, size_t size,
164 u64 offset)
165{
169 struct p2pdma_pagemap *p2p_pgmap;
170 struct dev_pagemap *pgmap;
171 void *addr;
172 int error;
173
174 if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM))
175 return -EINVAL;
176
177 if (offset >= pci_resource_len(pdev, bar))

--- 6 unchanged lines hidden (view full) ---

184 return -EINVAL;
185
186 if (!pdev->p2pdma) {
187 error = pci_p2pdma_setup(pdev);
188 if (error)
189 return error;
190 }
191
166 struct dev_pagemap *pgmap;
167 void *addr;
168 int error;
169
170 if (!(pci_resource_flags(pdev, bar) & IORESOURCE_MEM))
171 return -EINVAL;
172
173 if (offset >= pci_resource_len(pdev, bar))

--- 6 unchanged lines hidden (view full) ---

180 return -EINVAL;
181
182 if (!pdev->p2pdma) {
183 error = pci_p2pdma_setup(pdev);
184 if (error)
185 return error;
186 }
187
192 p2p_pgmap = devm_kzalloc(&pdev->dev, sizeof(*p2p_pgmap), GFP_KERNEL);
193 if (!p2p_pgmap)
188 pgmap = devm_kzalloc(&pdev->dev, sizeof(*pgmap), GFP_KERNEL);
189 if (!pgmap)
194 return -ENOMEM;
195
190 return -ENOMEM;
191
196 init_completion(&p2p_pgmap->ref_done);
197 error = percpu_ref_init(&p2p_pgmap->ref,
198 pci_p2pdma_percpu_release, 0, GFP_KERNEL);
199 if (error)
200 goto pgmap_free;
201
202 pgmap = &p2p_pgmap->pgmap;
203
204 pgmap->res.start = pci_resource_start(pdev, bar) + offset;
205 pgmap->res.end = pgmap->res.start + size - 1;
206 pgmap->res.flags = pci_resource_flags(pdev, bar);
192 pgmap->res.start = pci_resource_start(pdev, bar) + offset;
193 pgmap->res.end = pgmap->res.start + size - 1;
194 pgmap->res.flags = pci_resource_flags(pdev, bar);
207 pgmap->ref = &p2p_pgmap->ref;
195 pgmap->ref = &pdev->p2pdma->devmap_ref;
208 pgmap->type = MEMORY_DEVICE_PCI_P2PDMA;
209 pgmap->pci_p2pdma_bus_offset = pci_bus_address(pdev, bar) -
210 pci_resource_start(pdev, bar);
211 pgmap->kill = pci_p2pdma_percpu_kill;
196 pgmap->type = MEMORY_DEVICE_PCI_P2PDMA;
197 pgmap->pci_p2pdma_bus_offset = pci_bus_address(pdev, bar) -
198 pci_resource_start(pdev, bar);
199 pgmap->kill = pci_p2pdma_percpu_kill;
212 pgmap->cleanup = pci_p2pdma_percpu_cleanup;
213
214 addr = devm_memremap_pages(&pdev->dev, pgmap);
215 if (IS_ERR(addr)) {
216 error = PTR_ERR(addr);
217 goto pgmap_free;
218 }
219
200
201 addr = devm_memremap_pages(&pdev->dev, pgmap);
202 if (IS_ERR(addr)) {
203 error = PTR_ERR(addr);
204 goto pgmap_free;
205 }
206
220 error = gen_pool_add_owner(pdev->p2pdma->pool, (unsigned long)addr,
207 error = gen_pool_add_virt(pdev->p2pdma->pool, (unsigned long)addr,
221 pci_bus_address(pdev, bar) + offset,
208 pci_bus_address(pdev, bar) + offset,
222 resource_size(&pgmap->res), dev_to_node(&pdev->dev),
223 &p2p_pgmap->ref);
209 resource_size(&pgmap->res), dev_to_node(&pdev->dev));
224 if (error)
210 if (error)
225 goto pages_free;
211 goto pgmap_free;
226
227 pci_info(pdev, "added peer-to-peer DMA memory %pR\n",
228 &pgmap->res);
229
230 return 0;
231
212
213 pci_info(pdev, "added peer-to-peer DMA memory %pR\n",
214 &pgmap->res);
215
216 return 0;
217
232pages_free:
233 devm_memunmap_pages(&pdev->dev, pgmap);
234pgmap_free:
218pgmap_free:
235 devm_kfree(&pdev->dev, p2p_pgmap);
219 devm_kfree(&pdev->dev, pgmap);
236 return error;
237}
238EXPORT_SYMBOL_GPL(pci_p2pdma_add_resource);
239
240/*
241 * Note this function returns the parent PCI device with a
220 return error;
221}
222EXPORT_SYMBOL_GPL(pci_p2pdma_add_resource);
223
224/*
225 * Note this function returns the parent PCI device with a
242 * reference taken. It is the caller's responsibily to drop
226 * reference taken. It is the caller's responsibility to drop
243 * the reference.
244 */
245static struct pci_dev *find_parent_pci_dev(struct device *dev)
246{
247 struct device *parent;
248
249 dev = get_device(dev);
250

--- 44 unchanged lines hidden (view full) ---

295 * complex and compare it to a whitelist of known good hardware.
296 */
297static bool root_complex_whitelist(struct pci_dev *dev)
298{
299 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
300 struct pci_dev *root = pci_get_slot(host->bus, PCI_DEVFN(0, 0));
301 unsigned short vendor, device;
302
227 * the reference.
228 */
229static struct pci_dev *find_parent_pci_dev(struct device *dev)
230{
231 struct device *parent;
232
233 dev = get_device(dev);
234

--- 44 unchanged lines hidden (view full) ---

279 * complex and compare it to a whitelist of known good hardware.
280 */
281static bool root_complex_whitelist(struct pci_dev *dev)
282{
283 struct pci_host_bridge *host = pci_find_host_bridge(dev->bus);
284 struct pci_dev *root = pci_get_slot(host->bus, PCI_DEVFN(0, 0));
285 unsigned short vendor, device;
286
303 if (iommu_present(dev->dev.bus))
304 return false;
305
306 if (!root)
307 return false;
308
309 vendor = root->vendor;
310 device = root->device;
311 pci_dev_put(root);
312
313 /* AMD ZEN host bridges can do peer to peer */

--- 80 unchanged lines hidden (view full) ---

394 }
395
396 a = pci_upstream_bridge(a);
397 dist_a++;
398 }
399
400 /*
401 * Allow the connection if both devices are on a whitelisted root
287 if (!root)
288 return false;
289
290 vendor = root->vendor;
291 device = root->device;
292 pci_dev_put(root);
293
294 /* AMD ZEN host bridges can do peer to peer */

--- 80 unchanged lines hidden (view full) ---

375 }
376
377 a = pci_upstream_bridge(a);
378 dist_a++;
379 }
380
381 /*
382 * Allow the connection if both devices are on a whitelisted root
402 * complex, but add an arbitary large value to the distance.
383 * complex, but add an arbitrary large value to the distance.
403 */
404 if (root_complex_whitelist(provider) &&
405 root_complex_whitelist(client))
406 return 0x1000 + dist_a + dist_b;
407
408 return -1;
409
410check_b_path_acs:

--- 42 unchanged lines hidden (view full) ---

453 }
454
455 kfree(acs_list.buffer);
456
457 return ret;
458}
459
460/**
384 */
385 if (root_complex_whitelist(provider) &&
386 root_complex_whitelist(client))
387 return 0x1000 + dist_a + dist_b;
388
389 return -1;
390
391check_b_path_acs:

--- 42 unchanged lines hidden (view full) ---

434 }
435
436 kfree(acs_list.buffer);
437
438 return ret;
439}
440
441/**
461 * pci_p2pdma_distance_many - Determive the cumulative distance between
442 * pci_p2pdma_distance_many - Determine the cumulative distance between
462 * a p2pdma provider and the clients in use.
463 * @provider: p2pdma provider to check against the client list
464 * @clients: array of devices to check (NULL-terminated)
465 * @num_clients: number of clients in the array
466 * @verbose: if true, print warnings for devices when we return -1
467 *
468 * Returns -1 if any of the clients are not compatible (behind the same
469 * root port as the provider), otherwise returns a positive number where

--- 129 unchanged lines hidden (view full) ---

599 * pci_alloc_p2p_mem - allocate peer-to-peer DMA memory
600 * @pdev: the device to allocate memory from
601 * @size: number of bytes to allocate
602 *
603 * Returns the allocated memory or NULL on error.
604 */
605void *pci_alloc_p2pmem(struct pci_dev *pdev, size_t size)
606{
443 * a p2pdma provider and the clients in use.
444 * @provider: p2pdma provider to check against the client list
445 * @clients: array of devices to check (NULL-terminated)
446 * @num_clients: number of clients in the array
447 * @verbose: if true, print warnings for devices when we return -1
448 *
449 * Returns -1 if any of the clients are not compatible (behind the same
450 * root port as the provider), otherwise returns a positive number where

--- 129 unchanged lines hidden (view full) ---

580 * pci_alloc_p2p_mem - allocate peer-to-peer DMA memory
581 * @pdev: the device to allocate memory from
582 * @size: number of bytes to allocate
583 *
584 * Returns the allocated memory or NULL on error.
585 */
586void *pci_alloc_p2pmem(struct pci_dev *pdev, size_t size)
587{
607 void *ret = NULL;
608 struct percpu_ref *ref;
588 void *ret;
609
589
610 /*
611 * Pairs with synchronize_rcu() in pci_p2pdma_release() to
612 * ensure pdev->p2pdma is non-NULL for the duration of the
613 * read-lock.
614 */
615 rcu_read_lock();
616 if (unlikely(!pdev->p2pdma))
590 if (unlikely(!pdev->p2pdma))
617 goto out;
591 return NULL;
618
592
619 ret = (void *)gen_pool_alloc_owner(pdev->p2pdma->pool, size,
620 (void **) &ref);
621 if (!ret)
622 goto out;
593 if (unlikely(!percpu_ref_tryget_live(&pdev->p2pdma->devmap_ref)))
594 return NULL;
623
595
624 if (unlikely(!percpu_ref_tryget_live(ref))) {
625 gen_pool_free(pdev->p2pdma->pool, (unsigned long) ret, size);
626 ret = NULL;
627 goto out;
628 }
629out:
630 rcu_read_unlock();
596 ret = (void *)gen_pool_alloc(pdev->p2pdma->pool, size);
597
598 if (unlikely(!ret))
599 percpu_ref_put(&pdev->p2pdma->devmap_ref);
600
631 return ret;
632}
633EXPORT_SYMBOL_GPL(pci_alloc_p2pmem);
634
635/**
636 * pci_free_p2pmem - free peer-to-peer DMA memory
637 * @pdev: the device the memory was allocated from
638 * @addr: address of the memory that was allocated
639 * @size: number of bytes that were allocated
640 */
641void pci_free_p2pmem(struct pci_dev *pdev, void *addr, size_t size)
642{
601 return ret;
602}
603EXPORT_SYMBOL_GPL(pci_alloc_p2pmem);
604
605/**
606 * pci_free_p2pmem - free peer-to-peer DMA memory
607 * @pdev: the device the memory was allocated from
608 * @addr: address of the memory that was allocated
609 * @size: number of bytes that were allocated
610 */
611void pci_free_p2pmem(struct pci_dev *pdev, void *addr, size_t size)
612{
643 struct percpu_ref *ref;
644
645 gen_pool_free_owner(pdev->p2pdma->pool, (uintptr_t)addr, size,
646 (void **) &ref);
647 percpu_ref_put(ref);
613 gen_pool_free(pdev->p2pdma->pool, (uintptr_t)addr, size);
614 percpu_ref_put(&pdev->p2pdma->devmap_ref);
648}
649EXPORT_SYMBOL_GPL(pci_free_p2pmem);
650
651/**
652 * pci_virt_to_bus - return the PCI bus address for a given virtual
653 * address obtained with pci_alloc_p2pmem()
654 * @pdev: the device the memory was allocated from
655 * @addr: address of the memory that was allocated

--- 209 unchanged lines hidden --- 		615}
616EXPORT_SYMBOL_GPL(pci_free_p2pmem);
617
618/**
619 * pci_virt_to_bus - return the PCI bus address for a given virtual
620 * address obtained with pci_alloc_p2pmem()
621 * @pdev: the device the memory was allocated from
622 * @addr: address of the memory that was allocated

--- 209 unchanged lines hidden ---