/openbmc/linux/mm/ |
H A D | memremap.c | 107 pfn_first(pgmap, range_id)) >> pgmap->vmemmap_shift; in pfn_len() 144 percpu_ref_put_many(&pgmap->ref, pfn_len(pgmap, i)); in memunmap_pages() 272 percpu_ref_get_many(&pgmap->ref, pfn_len(pgmap, range_id)); in pagemap_range() 295 .pgmap = pgmap, in memremap_pages() 310 if (!pgmap->ops || !pgmap->ops->migrate_to_ram) { in memremap_pages() 412 pgmap); in devm_memremap_pages() 454 if (pgmap) { in get_dev_pagemap() 455 if (phys >= pgmap->range.start && phys <= pgmap->range.end) in get_dev_pagemap() 463 if (pgmap && !percpu_ref_tryget_live_rcu(&pgmap->ref)) in get_dev_pagemap() 464 pgmap = NULL; in get_dev_pagemap() [all …]
|
H A D | sparse-vmemmap.c | 373 struct dev_pagemap *pgmap) in reuse_compound_section() argument 375 unsigned long nr_pages = pgmap_vmemmap_nr(pgmap); in reuse_compound_section() 377 PHYS_PFN(pgmap->ranges[pgmap->nr_range].start); in reuse_compound_section() 402 struct dev_pagemap *pgmap) in vmemmap_populate_compound_pages() argument 408 if (reuse_compound_section(start_pfn, pgmap)) { in vmemmap_populate_compound_pages() 421 size = min(end - start, pgmap_vmemmap_nr(pgmap) * sizeof(struct page)); in vmemmap_populate_compound_pages() 454 struct dev_pagemap *pgmap) in __populate_section_memmap() argument 464 if (vmemmap_can_optimize(altmap, pgmap)) in __populate_section_memmap() 465 r = vmemmap_populate_compound_pages(pfn, start, end, nid, pgmap); in __populate_section_memmap()
|
H A D | gup.c | 29 struct dev_pagemap *pgmap; member 621 *pgmap = get_dev_pagemap(pte_pfn(pte), *pgmap); in follow_page_pte() 622 if (*pgmap) in follow_page_pte() 852 if (ctx.pgmap) in follow_page() 1338 if (ctx.pgmap) in __get_user_pages() 2597 pgmap = get_dev_pagemap(pte_pfn(pte), pgmap); in gup_pte_range() 2654 if (pgmap) in gup_pte_range() 2655 put_dev_pagemap(pgmap); in gup_pte_range() 2689 pgmap = get_dev_pagemap(pfn, pgmap); in __gup_device_huge() 2690 if (unlikely(!pgmap)) { in __gup_device_huge() [all …]
|
H A D | sparse.c | 430 struct dev_pagemap *pgmap) in __populate_section_memmap() argument 633 struct dev_pagemap *pgmap) in populate_section_memmap() argument 635 return __populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap); in populate_section_memmap() 705 struct dev_pagemap *pgmap) in populate_section_memmap() argument 830 struct dev_pagemap *pgmap) in section_activate() argument 862 memmap = populate_section_memmap(pfn, nr_pages, nid, altmap, pgmap); in section_activate() 892 struct dev_pagemap *pgmap) in sparse_add_section() argument 903 memmap = section_activate(nid, start_pfn, nr_pages, altmap, pgmap); in sparse_add_section()
|
H A D | mm_init.c | 973 struct dev_pagemap *pgmap) in __init_zone_device_page() argument 992 page->pgmap = pgmap; in __init_zone_device_page() 1014 if (pgmap->type == MEMORY_DEVICE_PRIVATE || in __init_zone_device_page() 1015 pgmap->type == MEMORY_DEVICE_COHERENT) in __init_zone_device_page() 1028 struct dev_pagemap *pgmap) in compound_nr_pages() argument 1030 if (!vmemmap_can_optimize(altmap, pgmap)) in compound_nr_pages() 1031 return pgmap_vmemmap_nr(pgmap); in compound_nr_pages() 1039 struct dev_pagemap *pgmap, in memmap_init_compound() argument 1043 unsigned int order = pgmap->vmemmap_shift; in memmap_init_compound() 1066 struct dev_pagemap *pgmap) in memmap_init_zone_device() argument [all …]
|
H A D | memory-failure.c | 1737 struct dev_pagemap *pgmap) in mf_generic_kill_procs() argument 1760 switch (pgmap->type) { in mf_generic_kill_procs() 2130 struct dev_pagemap *pgmap) in memory_failure_dev_pagemap() argument 2135 if (!pgmap_pfn_valid(pgmap, pfn)) in memory_failure_dev_pagemap() 2142 if (pgmap_has_memory_failure(pgmap)) { in memory_failure_dev_pagemap() 2143 rc = pgmap->ops->memory_failure(pgmap, pfn, 1, flags); in memory_failure_dev_pagemap() 2155 put_dev_pagemap(pgmap); in memory_failure_dev_pagemap() 2186 struct dev_pagemap *pgmap; in memory_failure() local 2207 pgmap = get_dev_pagemap(pfn, NULL); in memory_failure() 2209 if (pgmap) { in memory_failure() [all …]
|
H A D | memory_hotplug.c | 329 struct dev_pagemap *pgmap; in pfn_to_online_page() local 358 pgmap = get_dev_pagemap(pfn, NULL); in pfn_to_online_page() 359 put_dev_pagemap(pgmap); in pfn_to_online_page() 362 if (pgmap) in pfn_to_online_page() 404 params->pgmap); in __add_pages()
|
/openbmc/linux/include/linux/ |
H A D | memremap.h | 144 return pgmap->ops && pgmap->ops->memory_failure; in pgmap_has_memory_failure() 149 if (pgmap->flags & PGMAP_ALTMAP_VALID) in pgmap_altmap() 150 return &pgmap->altmap; in pgmap_altmap() 156 return 1 << pgmap->vmemmap_shift; in pgmap_vmemmap_nr() 163 page->pgmap->type == MEMORY_DEVICE_PRIVATE; in is_device_private_page() 196 struct dev_pagemap *pgmap); 204 struct dev_pagemap *pgmap) in devm_memremap_pages() argument 216 struct dev_pagemap *pgmap) in devm_memunmap_pages() argument 221 struct dev_pagemap *pgmap) in get_dev_pagemap() argument 250 if (pgmap) in put_dev_pagemap() [all …]
|
H A D | huge_mm.h | 222 pmd_t *pmd, int flags, struct dev_pagemap **pgmap); 224 pud_t *pud, int flags, struct dev_pagemap **pgmap); 375 unsigned long addr, pmd_t *pmd, int flags, struct dev_pagemap **pgmap) in follow_devmap_pmd() argument 381 unsigned long addr, pud_t *pud, int flags, struct dev_pagemap **pgmap) in follow_devmap_pud() argument
|
H A D | memory_hotplug.h | 119 struct dev_pagemap *pgmap; member 348 struct dev_pagemap *pgmap);
|
/openbmc/linux/drivers/pci/ |
H A D | p2pdma.c | 31 struct dev_pagemap pgmap; member 38 return container_of(pgmap, struct pci_p2pdma_pagemap, pgmap); in to_p2p_pgmap() 196 struct pci_p2pdma_pagemap *pgmap = to_p2p_pgmap(page->pgmap); in p2pdma_page_free() local 289 struct dev_pagemap *pgmap; in pci_p2pdma_add_resource() local 316 pgmap = &p2p_pgmap->pgmap; in pci_p2pdma_add_resource() 318 pgmap->range.end = pgmap->range.start + size - 1; in pci_p2pdma_add_resource() 319 pgmap->nr_range = 1; in pci_p2pdma_add_resource() 342 &pgmap->ref); in pci_p2pdma_add_resource() 347 pgmap->range.start, pgmap->range.end); in pci_p2pdma_add_resource() 1020 if (state->pgmap != sg_page(sg)->pgmap) { in pci_p2pdma_map_segment() [all …]
|
/openbmc/linux/drivers/xen/ |
H A D | unpopulated-alloc.c | 36 struct dev_pagemap *pgmap; in fill_list() local 84 pgmap = kzalloc(sizeof(*pgmap), GFP_KERNEL); in fill_list() 85 if (!pgmap) { in fill_list() 90 pgmap->type = MEMORY_DEVICE_GENERIC; in fill_list() 91 pgmap->range = (struct range) { in fill_list() 95 pgmap->nr_range = 1; in fill_list() 96 pgmap->owner = res; in fill_list() 121 vaddr = memremap_pages(pgmap, NUMA_NO_NODE); in fill_list() 139 kfree(pgmap); in fill_list()
|
/openbmc/linux/drivers/dax/ |
H A D | pmem.c | 19 struct dev_pagemap pgmap = { }; in __dax_pmem_probe() local 33 rc = nvdimm_setup_pfn(nd_pfn, &pgmap); in __dax_pmem_probe() 53 range = pgmap.range; in __dax_pmem_probe() 64 .pgmap = &pgmap, in __dax_pmem_probe()
|
H A D | device.c | 85 if (dev_dax->pgmap->vmemmap_shift) in dax_set_mapping() 397 struct dev_pagemap *pgmap; in dev_dax_probe() local 410 pgmap = dev_dax->pgmap; in dev_dax_probe() 412 if (dev_dax->pgmap) { in dev_dax_probe() 418 pgmap = devm_kzalloc(dev, in dev_dax_probe() 421 if (!pgmap) in dev_dax_probe() 424 pgmap->nr_range = dev_dax->nr_range; in dev_dax_probe() 425 dev_dax->pgmap = pgmap; in dev_dax_probe() 429 pgmap->ranges[i] = *range; in dev_dax_probe() 444 pgmap->type = MEMORY_DEVICE_GENERIC; in dev_dax_probe() [all …]
|
H A D | bus.h | 23 struct dev_pagemap *pgmap; member
|
H A D | dax-private.h | 72 struct dev_pagemap *pgmap; member
|
H A D | bus.c | 409 dev_dax->pgmap = NULL; in kill_dev_dax() 1318 kfree(dev_dax->pgmap); in dev_dax_release() 1370 if (data->pgmap) { in devm_create_dev_dax() 1374 dev_dax->pgmap = kmemdup(data->pgmap, in devm_create_dev_dax() 1376 if (!dev_dax->pgmap) { in devm_create_dev_dax() 1430 kfree(dev_dax->pgmap); in devm_create_dev_dax()
|
/openbmc/linux/tools/testing/nvdimm/test/ |
H A D | iomap.c | 99 struct dev_pagemap *pgmap = _pgmap; in nfit_test_kill() local 101 WARN_ON(!pgmap); in nfit_test_kill() 103 percpu_ref_kill(&pgmap->ref); in nfit_test_kill() 105 wait_for_completion(&pgmap->done); in nfit_test_kill() 106 percpu_ref_exit(&pgmap->ref); in nfit_test_kill() 113 complete(&pgmap->done); in dev_pagemap_percpu_release() 119 resource_size_t offset = pgmap->range.start; in __wrap_devm_memremap_pages() 123 return devm_memremap_pages(dev, pgmap); in __wrap_devm_memremap_pages() 125 init_completion(&pgmap->done); in __wrap_devm_memremap_pages() 126 error = percpu_ref_init(&pgmap->ref, dev_pagemap_percpu_release, 0, in __wrap_devm_memremap_pages() [all …]
|
/openbmc/linux/drivers/nvdimm/ |
H A D | pmem.c | 438 container_of(pgmap, struct pmem_device, pgmap); in pmem_pagemap_memory_failure() 506 pmem->pgmap.owner = pmem; in pmem_attach_disk() 509 pmem->pgmap.type = MEMORY_DEVICE_FS_DAX; in pmem_attach_disk() 510 pmem->pgmap.ops = &fsdax_pagemap_ops; in pmem_attach_disk() 515 range_len(&pmem->pgmap.range); in pmem_attach_disk() 517 bb_range = pmem->pgmap.range; in pmem_attach_disk() 520 pmem->pgmap.range.start = res->start; in pmem_attach_disk() 521 pmem->pgmap.range.end = res->end; in pmem_attach_disk() 522 pmem->pgmap.nr_range = 1; in pmem_attach_disk() 524 pmem->pgmap.ops = &fsdax_pagemap_ops; in pmem_attach_disk() [all …]
|
H A D | pfn_devs.c | 682 static int __nvdimm_setup_pfn(struct nd_pfn *nd_pfn, struct dev_pagemap *pgmap) in __nvdimm_setup_pfn() argument 684 struct range *range = &pgmap->range; in __nvdimm_setup_pfn() 685 struct vmem_altmap *altmap = &pgmap->altmap; in __nvdimm_setup_pfn() 705 pgmap->nr_range = 1; in __nvdimm_setup_pfn() 720 pgmap->flags |= PGMAP_ALTMAP_VALID; in __nvdimm_setup_pfn() 855 int nvdimm_setup_pfn(struct nd_pfn *nd_pfn, struct dev_pagemap *pgmap) in nvdimm_setup_pfn() argument 867 return __nvdimm_setup_pfn(nd_pfn, pgmap); in nvdimm_setup_pfn()
|
H A D | pmem.h | 29 struct dev_pagemap pgmap; member
|
/openbmc/linux/drivers/gpu/drm/amd/amdkfd/ |
H A D | kfd_migrate.c | 1012 struct dev_pagemap *pgmap; in kgd2kfd_init_zone_device() local 1024 pgmap = &kfddev->pgmap; in kgd2kfd_init_zone_device() 1025 memset(pgmap, 0, sizeof(*pgmap)); in kgd2kfd_init_zone_device() 1034 pgmap->type = MEMORY_DEVICE_COHERENT; in kgd2kfd_init_zone_device() 1039 pgmap->range.start = res->start; in kgd2kfd_init_zone_device() 1040 pgmap->range.end = res->end; in kgd2kfd_init_zone_device() 1041 pgmap->type = MEMORY_DEVICE_PRIVATE; in kgd2kfd_init_zone_device() 1044 pgmap->nr_range = 1; in kgd2kfd_init_zone_device() 1045 pgmap->ops = &svm_migrate_pgmap_ops; in kgd2kfd_init_zone_device() 1047 pgmap->flags = 0; in kgd2kfd_init_zone_device() [all …]
|
H A D | kfd_svm.h | 202 #define KFD_IS_SVM_API_SUPPORTED(adev) ((adev)->kfd.pgmap.type != 0 ||\
|
/openbmc/linux/arch/powerpc/include/asm/book3s/64/ |
H A D | radix.h | 369 bool vmemmap_can_optimize(struct vmem_altmap *altmap, struct dev_pagemap *pgmap); 376 struct dev_pagemap *pgmap);
|
/openbmc/linux/fs/fuse/ |
H A D | virtio_fs.c | 801 struct dev_pagemap *pgmap; in virtio_fs_setup_dax() local 825 pgmap = devm_kzalloc(&vdev->dev, sizeof(*pgmap), GFP_KERNEL); in virtio_fs_setup_dax() 826 if (!pgmap) in virtio_fs_setup_dax() 829 pgmap->type = MEMORY_DEVICE_FS_DAX; in virtio_fs_setup_dax() 836 pgmap->range = (struct range) { in virtio_fs_setup_dax() 840 pgmap->nr_range = 1; in virtio_fs_setup_dax() 842 fs->window_kaddr = devm_memremap_pages(&vdev->dev, pgmap); in virtio_fs_setup_dax()
|