1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */
29476df7dSDan Williams #ifndef _LINUX_MEMREMAP_H_
39476df7dSDan Williams #define _LINUX_MEMREMAP_H_
4dc90f084SChristoph Hellwig
55bb88dc5SAlex Sierra #include <linux/mmzone.h>
6a4574f63SDan Williams #include <linux/range.h>
75c2c2587SDan Williams #include <linux/ioport.h>
85c2c2587SDan Williams #include <linux/percpu-refcount.h>
99476df7dSDan Williams
109476df7dSDan Williams struct resource;
119476df7dSDan Williams struct device;
124b94ffdcSDan Williams
134b94ffdcSDan Williams /**
144b94ffdcSDan Williams * struct vmem_altmap - pre-allocated storage for vmemmap_populate
154b94ffdcSDan Williams * @base_pfn: base of the entire dev_pagemap mapping
164b94ffdcSDan Williams * @reserve: pages mapped, but reserved for driver use (relative to @base)
174b94ffdcSDan Williams * @free: free pages set aside in the mapping for memmap storage
184b94ffdcSDan Williams * @align: pages reserved to meet allocation alignments
194b94ffdcSDan Williams * @alloc: track pages consumed, private to vmemmap_populate()
204b94ffdcSDan Williams */
214b94ffdcSDan Williams struct vmem_altmap {
22a08a2ae3SOscar Salvador unsigned long base_pfn;
23cf387d96SAneesh Kumar K.V const unsigned long end_pfn;
244b94ffdcSDan Williams const unsigned long reserve;
254b94ffdcSDan Williams unsigned long free;
264b94ffdcSDan Williams unsigned long align;
274b94ffdcSDan Williams unsigned long alloc;
284b94ffdcSDan Williams };
294b94ffdcSDan Williams
305042db43SJérôme Glisse /*
31041711ceSZhen Lei * Specialize ZONE_DEVICE memory into multiple types each has a different
325042db43SJérôme Glisse * usage.
335042db43SJérôme Glisse *
345042db43SJérôme Glisse * MEMORY_DEVICE_PRIVATE:
355042db43SJérôme Glisse * Device memory that is not directly addressable by the CPU: CPU can neither
365042db43SJérôme Glisse * read nor write private memory. In this case, we do still have struct pages
375042db43SJérôme Glisse * backing the device memory. Doing so simplifies the implementation, but it is
385042db43SJérôme Glisse * important to remember that there are certain points at which the struct page
395042db43SJérôme Glisse * must be treated as an opaque object, rather than a "normal" struct page.
405042db43SJérôme Glisse *
415042db43SJérôme Glisse * A more complete discussion of unaddressable memory may be found in
42ee65728eSMike Rapoport * include/linux/hmm.h and Documentation/mm/hmm.rst.
43df6ad698SJérôme Glisse *
44f25cbb7aSAlex Sierra * MEMORY_DEVICE_COHERENT:
45f25cbb7aSAlex Sierra * Device memory that is cache coherent from device and CPU point of view. This
46f25cbb7aSAlex Sierra * is used on platforms that have an advanced system bus (like CAPI or CXL). A
47f25cbb7aSAlex Sierra * driver can hotplug the device memory using ZONE_DEVICE and with that memory
48f25cbb7aSAlex Sierra * type. Any page of a process can be migrated to such memory. However no one
49f25cbb7aSAlex Sierra * should be allowed to pin such memory so that it can always be evicted.
50f25cbb7aSAlex Sierra *
51e7638488SDan Williams * MEMORY_DEVICE_FS_DAX:
52e7638488SDan Williams * Host memory that has similar access semantics as System RAM i.e. DMA
53e7638488SDan Williams * coherent and supports page pinning. In support of coordinating page
54e7638488SDan Williams * pinning vs other operations MEMORY_DEVICE_FS_DAX arranges for a
55e7638488SDan Williams * wakeup event whenever a page is unpinned and becomes idle. This
56e7638488SDan Williams * wakeup is used to coordinate physical address space management (ex:
57e7638488SDan Williams * fs truncate/hole punch) vs pinned pages (ex: device dma).
5852916982SLogan Gunthorpe *
594533d3aeSRoger Pau Monne * MEMORY_DEVICE_GENERIC:
603ed2dcdfSChristoph Hellwig * Host memory that has similar access semantics as System RAM i.e. DMA
614533d3aeSRoger Pau Monne * coherent and supports page pinning. This is for example used by DAX devices
624533d3aeSRoger Pau Monne * that expose memory using a character device.
633ed2dcdfSChristoph Hellwig *
6452916982SLogan Gunthorpe * MEMORY_DEVICE_PCI_P2PDMA:
6552916982SLogan Gunthorpe * Device memory residing in a PCI BAR intended for use with Peer-to-Peer
6652916982SLogan Gunthorpe * transactions.
675042db43SJérôme Glisse */
685042db43SJérôme Glisse enum memory_type {
693ed2dcdfSChristoph Hellwig /* 0 is reserved to catch uninitialized type fields */
70e7638488SDan Williams MEMORY_DEVICE_PRIVATE = 1,
71f25cbb7aSAlex Sierra MEMORY_DEVICE_COHERENT,
72e7638488SDan Williams MEMORY_DEVICE_FS_DAX,
734533d3aeSRoger Pau Monne MEMORY_DEVICE_GENERIC,
7452916982SLogan Gunthorpe MEMORY_DEVICE_PCI_P2PDMA,
755042db43SJérôme Glisse };
765042db43SJérôme Glisse
771e240e8dSChristoph Hellwig struct dev_pagemap_ops {
785042db43SJérôme Glisse /*
7927674ef6SChristoph Hellwig * Called once the page refcount reaches 0. The reference count will be
8027674ef6SChristoph Hellwig * reset to one by the core code after the method is called to prepare
8127674ef6SChristoph Hellwig * for handing out the page again.
825042db43SJérôme Glisse */
8380a72d0aSChristoph Hellwig void (*page_free)(struct page *page);
841e240e8dSChristoph Hellwig
851e240e8dSChristoph Hellwig /*
86897e6365SChristoph Hellwig * Used for private (un-addressable) device memory only. Must migrate
87897e6365SChristoph Hellwig * the page back to a CPU accessible page.
88897e6365SChristoph Hellwig */
89897e6365SChristoph Hellwig vm_fault_t (*migrate_to_ram)(struct vm_fault *vmf);
9033a8f7f2SShiyang Ruan
9133a8f7f2SShiyang Ruan /*
9233a8f7f2SShiyang Ruan * Handle the memory failure happens on a range of pfns. Notify the
9333a8f7f2SShiyang Ruan * processes who are using these pfns, and try to recover the data on
9433a8f7f2SShiyang Ruan * them if necessary. The mf_flags is finally passed to the recover
9533a8f7f2SShiyang Ruan * function through the whole notify routine.
9633a8f7f2SShiyang Ruan *
9733a8f7f2SShiyang Ruan * When this is not implemented, or it returns -EOPNOTSUPP, the caller
9833a8f7f2SShiyang Ruan * will fall back to a common handler called mf_generic_kill_procs().
9933a8f7f2SShiyang Ruan */
10033a8f7f2SShiyang Ruan int (*memory_failure)(struct dev_pagemap *pgmap, unsigned long pfn,
10133a8f7f2SShiyang Ruan unsigned long nr_pages, int mf_flags);
1021e240e8dSChristoph Hellwig };
1035042db43SJérôme Glisse
104514caf23SChristoph Hellwig #define PGMAP_ALTMAP_VALID (1 << 0)
105514caf23SChristoph Hellwig
1069476df7dSDan Williams /**
1079476df7dSDan Williams * struct dev_pagemap - metadata for ZONE_DEVICE mappings
1084b94ffdcSDan Williams * @altmap: pre-allocated/reserved memory for vmemmap allocations
1095c2c2587SDan Williams * @ref: reference count that pins the devm_memremap_pages() mapping
110b80892caSChristoph Hellwig * @done: completion for @ref
1115042db43SJérôme Glisse * @type: memory type: see MEMORY_* in memory_hotplug.h
112514caf23SChristoph Hellwig * @flags: PGMAP_* flags to specify defailed behavior
113c4386bd8SJoao Martins * @vmemmap_shift: structural definition of how the vmemmap page metadata
114c4386bd8SJoao Martins * is populated, specifically the metadata page order.
115c4386bd8SJoao Martins * A zero value (default) uses base pages as the vmemmap metadata
116c4386bd8SJoao Martins * representation. A bigger value will set up compound struct pages
117c4386bd8SJoao Martins * of the requested order value.
1181e240e8dSChristoph Hellwig * @ops: method table
119f894ddd5SChristoph Hellwig * @owner: an opaque pointer identifying the entity that manages this
120f894ddd5SChristoph Hellwig * instance. Used by various helpers to make sure that no
121f894ddd5SChristoph Hellwig * foreign ZONE_DEVICE memory is accessed.
122b7b3c01bSDan Williams * @nr_range: number of ranges to be mapped
123b7b3c01bSDan Williams * @range: range to be mapped when nr_range == 1
124b7b3c01bSDan Williams * @ranges: array of ranges to be mapped when nr_range > 1
1259476df7dSDan Williams */
1269476df7dSDan Williams struct dev_pagemap {
127e7744aa2SLogan Gunthorpe struct vmem_altmap altmap;
128b80892caSChristoph Hellwig struct percpu_ref ref;
12924917f6bSChristoph Hellwig struct completion done;
1305042db43SJérôme Glisse enum memory_type type;
131514caf23SChristoph Hellwig unsigned int flags;
132c4386bd8SJoao Martins unsigned long vmemmap_shift;
1331e240e8dSChristoph Hellwig const struct dev_pagemap_ops *ops;
134f894ddd5SChristoph Hellwig void *owner;
135b7b3c01bSDan Williams int nr_range;
136b7b3c01bSDan Williams union {
137b7b3c01bSDan Williams struct range range;
138*06919d22SGustavo A. R. Silva DECLARE_FLEX_ARRAY(struct range, ranges);
139b7b3c01bSDan Williams };
1409476df7dSDan Williams };
1419476df7dSDan Williams
pgmap_has_memory_failure(struct dev_pagemap * pgmap)14265d3440eSDan Williams static inline bool pgmap_has_memory_failure(struct dev_pagemap *pgmap)
14365d3440eSDan Williams {
14465d3440eSDan Williams return pgmap->ops && pgmap->ops->memory_failure;
14565d3440eSDan Williams }
14665d3440eSDan Williams
pgmap_altmap(struct dev_pagemap * pgmap)147514caf23SChristoph Hellwig static inline struct vmem_altmap *pgmap_altmap(struct dev_pagemap *pgmap)
148514caf23SChristoph Hellwig {
149514caf23SChristoph Hellwig if (pgmap->flags & PGMAP_ALTMAP_VALID)
150514caf23SChristoph Hellwig return &pgmap->altmap;
151514caf23SChristoph Hellwig return NULL;
152514caf23SChristoph Hellwig }
153514caf23SChristoph Hellwig
pgmap_vmemmap_nr(struct dev_pagemap * pgmap)154c4386bd8SJoao Martins static inline unsigned long pgmap_vmemmap_nr(struct dev_pagemap *pgmap)
155c4386bd8SJoao Martins {
156c4386bd8SJoao Martins return 1 << pgmap->vmemmap_shift;
157c4386bd8SJoao Martins }
158c4386bd8SJoao Martins
is_device_private_page(const struct page * page)159dc90f084SChristoph Hellwig static inline bool is_device_private_page(const struct page *page)
160dc90f084SChristoph Hellwig {
16127674ef6SChristoph Hellwig return IS_ENABLED(CONFIG_DEVICE_PRIVATE) &&
162dc90f084SChristoph Hellwig is_zone_device_page(page) &&
163dc90f084SChristoph Hellwig page->pgmap->type == MEMORY_DEVICE_PRIVATE;
164dc90f084SChristoph Hellwig }
165dc90f084SChristoph Hellwig
folio_is_device_private(const struct folio * folio)166536939ffSMatthew Wilcox (Oracle) static inline bool folio_is_device_private(const struct folio *folio)
167536939ffSMatthew Wilcox (Oracle) {
168536939ffSMatthew Wilcox (Oracle) return is_device_private_page(&folio->page);
169536939ffSMatthew Wilcox (Oracle) }
170536939ffSMatthew Wilcox (Oracle)
is_pci_p2pdma_page(const struct page * page)171dc90f084SChristoph Hellwig static inline bool is_pci_p2pdma_page(const struct page *page)
172dc90f084SChristoph Hellwig {
17327674ef6SChristoph Hellwig return IS_ENABLED(CONFIG_PCI_P2PDMA) &&
174dc90f084SChristoph Hellwig is_zone_device_page(page) &&
175dc90f084SChristoph Hellwig page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA;
176dc90f084SChristoph Hellwig }
177dc90f084SChristoph Hellwig
is_device_coherent_page(const struct page * page)178f25cbb7aSAlex Sierra static inline bool is_device_coherent_page(const struct page *page)
179f25cbb7aSAlex Sierra {
180f25cbb7aSAlex Sierra return is_zone_device_page(page) &&
181f25cbb7aSAlex Sierra page->pgmap->type == MEMORY_DEVICE_COHERENT;
182f25cbb7aSAlex Sierra }
183f25cbb7aSAlex Sierra
folio_is_device_coherent(const struct folio * folio)184f25cbb7aSAlex Sierra static inline bool folio_is_device_coherent(const struct folio *folio)
185f25cbb7aSAlex Sierra {
186f25cbb7aSAlex Sierra return is_device_coherent_page(&folio->page);
187f25cbb7aSAlex Sierra }
188f25cbb7aSAlex Sierra
1899476df7dSDan Williams #ifdef CONFIG_ZONE_DEVICE
190ef233450SAlistair Popple void zone_device_page_init(struct page *page);
1916869b7b2SChristoph Hellwig void *memremap_pages(struct dev_pagemap *pgmap, int nid);
1926869b7b2SChristoph Hellwig void memunmap_pages(struct dev_pagemap *pgmap);
193e8d51348SChristoph Hellwig void *devm_memremap_pages(struct device *dev, struct dev_pagemap *pgmap);
1942e3f139eSDan Williams void devm_memunmap_pages(struct device *dev, struct dev_pagemap *pgmap);
1950822acb8SChristoph Hellwig struct dev_pagemap *get_dev_pagemap(unsigned long pfn,
1960822acb8SChristoph Hellwig struct dev_pagemap *pgmap);
19734dc45beSDan Williams bool pgmap_pfn_valid(struct dev_pagemap *pgmap, unsigned long pfn);
1987b2d55d2SJérôme Glisse
1998e37d00aSChristoph Hellwig unsigned long vmem_altmap_offset(struct vmem_altmap *altmap);
2008e37d00aSChristoph Hellwig void vmem_altmap_free(struct vmem_altmap *altmap, unsigned long nr_pfns);
2019ffc1d19SDan Williams unsigned long memremap_compat_align(void);
2029476df7dSDan Williams #else
devm_memremap_pages(struct device * dev,struct dev_pagemap * pgmap)2039476df7dSDan Williams static inline void *devm_memremap_pages(struct device *dev,
204e8d51348SChristoph Hellwig struct dev_pagemap *pgmap)
2059476df7dSDan Williams {
2069476df7dSDan Williams /*
2079476df7dSDan Williams * Fail attempts to call devm_memremap_pages() without
2089476df7dSDan Williams * ZONE_DEVICE support enabled, this requires callers to fall
2099476df7dSDan Williams * back to plain devm_memremap() based on config
2109476df7dSDan Williams */
2119476df7dSDan Williams WARN_ON_ONCE(1);
2129476df7dSDan Williams return ERR_PTR(-ENXIO);
2139476df7dSDan Williams }
2149476df7dSDan Williams
devm_memunmap_pages(struct device * dev,struct dev_pagemap * pgmap)2152e3f139eSDan Williams static inline void devm_memunmap_pages(struct device *dev,
2162e3f139eSDan Williams struct dev_pagemap *pgmap)
2172e3f139eSDan Williams {
2182e3f139eSDan Williams }
2192e3f139eSDan Williams
get_dev_pagemap(unsigned long pfn,struct dev_pagemap * pgmap)2200822acb8SChristoph Hellwig static inline struct dev_pagemap *get_dev_pagemap(unsigned long pfn,
2210822acb8SChristoph Hellwig struct dev_pagemap *pgmap)
2229476df7dSDan Williams {
2239476df7dSDan Williams return NULL;
2249476df7dSDan Williams }
2258e37d00aSChristoph Hellwig
pgmap_pfn_valid(struct dev_pagemap * pgmap,unsigned long pfn)22634dc45beSDan Williams static inline bool pgmap_pfn_valid(struct dev_pagemap *pgmap, unsigned long pfn)
22734dc45beSDan Williams {
22834dc45beSDan Williams return false;
22934dc45beSDan Williams }
23034dc45beSDan Williams
vmem_altmap_offset(struct vmem_altmap * altmap)2318e37d00aSChristoph Hellwig static inline unsigned long vmem_altmap_offset(struct vmem_altmap *altmap)
2328e37d00aSChristoph Hellwig {
2338e37d00aSChristoph Hellwig return 0;
2348e37d00aSChristoph Hellwig }
2358e37d00aSChristoph Hellwig
vmem_altmap_free(struct vmem_altmap * altmap,unsigned long nr_pfns)2368e37d00aSChristoph Hellwig static inline void vmem_altmap_free(struct vmem_altmap *altmap,
2378e37d00aSChristoph Hellwig unsigned long nr_pfns)
2388e37d00aSChristoph Hellwig {
2398e37d00aSChristoph Hellwig }
2409ffc1d19SDan Williams
2419ffc1d19SDan Williams /* when memremap_pages() is disabled all archs can remap a single page */
memremap_compat_align(void)2429ffc1d19SDan Williams static inline unsigned long memremap_compat_align(void)
2439ffc1d19SDan Williams {
2449ffc1d19SDan Williams return PAGE_SIZE;
2459ffc1d19SDan Williams }
2468e37d00aSChristoph Hellwig #endif /* CONFIG_ZONE_DEVICE */
2477b2d55d2SJérôme Glisse
put_dev_pagemap(struct dev_pagemap * pgmap)2485c2c2587SDan Williams static inline void put_dev_pagemap(struct dev_pagemap *pgmap)
2495c2c2587SDan Williams {
2505c2c2587SDan Williams if (pgmap)
251b80892caSChristoph Hellwig percpu_ref_put(&pgmap->ref);
2525c2c2587SDan Williams }
2539ffc1d19SDan Williams
2549476df7dSDan Williams #endif /* _LINUX_MEMREMAP_H_ */
255