1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Copyright 2013 Red Hat Inc. 4 * 5 * Authors: Jérôme Glisse <jglisse@redhat.com> 6 */ 7 /* 8 * Heterogeneous Memory Management (HMM) 9 * 10 * See Documentation/vm/hmm.rst for reasons and overview of what HMM is and it 11 * is for. Here we focus on the HMM API description, with some explanation of 12 * the underlying implementation. 13 * 14 * Short description: HMM provides a set of helpers to share a virtual address 15 * space between CPU and a device, so that the device can access any valid 16 * address of the process (while still obeying memory protection). HMM also 17 * provides helpers to migrate process memory to device memory, and back. Each 18 * set of functionality (address space mirroring, and migration to and from 19 * device memory) can be used independently of the other. 20 * 21 * 22 * HMM address space mirroring API: 23 * 24 * Use HMM address space mirroring if you want to mirror a range of the CPU 25 * page tables of a process into a device page table. Here, "mirror" means "keep 26 * synchronized". Prerequisites: the device must provide the ability to write- 27 * protect its page tables (at PAGE_SIZE granularity), and must be able to 28 * recover from the resulting potential page faults. 29 * 30 * HMM guarantees that at any point in time, a given virtual address points to 31 * either the same memory in both CPU and device page tables (that is: CPU and 32 * device page tables each point to the same pages), or that one page table (CPU 33 * or device) points to no entry, while the other still points to the old page 34 * for the address. The latter case happens when the CPU page table update 35 * happens first, and then the update is mirrored over to the device page table. 36 * This does not cause any issue, because the CPU page table cannot start 37 * pointing to a new page until the device page table is invalidated. 38 * 39 * HMM uses mmu_notifiers to monitor the CPU page tables, and forwards any 40 * updates to each device driver that has registered a mirror. It also provides 41 * some API calls to help with taking a snapshot of the CPU page table, and to 42 * synchronize with any updates that might happen concurrently. 43 * 44 * 45 * HMM migration to and from device memory: 46 * 47 * HMM provides a set of helpers to hotplug device memory as ZONE_DEVICE, with 48 * a new MEMORY_DEVICE_PRIVATE type. This provides a struct page for each page 49 * of the device memory, and allows the device driver to manage its memory 50 * using those struct pages. Having struct pages for device memory makes 51 * migration easier. Because that memory is not addressable by the CPU it must 52 * never be pinned to the device; in other words, any CPU page fault can always 53 * cause the device memory to be migrated (copied/moved) back to regular memory. 54 * 55 * A new migrate helper (migrate_vma()) has been added (see mm/migrate.c) that 56 * allows use of a device DMA engine to perform the copy operation between 57 * regular system memory and device memory. 58 */ 59 #ifndef LINUX_HMM_H 60 #define LINUX_HMM_H 61 62 #include <linux/kconfig.h> 63 #include <asm/pgtable.h> 64 65 #include <linux/device.h> 66 #include <linux/migrate.h> 67 #include <linux/memremap.h> 68 #include <linux/completion.h> 69 #include <linux/mmu_notifier.h> 70 71 /* 72 * hmm_pfn_flag_e - HMM flag enums 73 * 74 * Flags: 75 * HMM_PFN_VALID: pfn is valid. It has, at least, read permission. 76 * HMM_PFN_WRITE: CPU page table has write permission set 77 * HMM_PFN_DEVICE_PRIVATE: private device memory (ZONE_DEVICE) 78 * 79 * The driver provides a flags array for mapping page protections to device 80 * PTE bits. If the driver valid bit for an entry is bit 3, 81 * i.e., (entry & (1 << 3)), then the driver must provide 82 * an array in hmm_range.flags with hmm_range.flags[HMM_PFN_VALID] == 1 << 3. 83 * Same logic apply to all flags. This is the same idea as vm_page_prot in vma 84 * except that this is per device driver rather than per architecture. 85 */ 86 enum hmm_pfn_flag_e { 87 HMM_PFN_VALID = 0, 88 HMM_PFN_WRITE, 89 HMM_PFN_DEVICE_PRIVATE, 90 HMM_PFN_FLAG_MAX 91 }; 92 93 /* 94 * hmm_pfn_value_e - HMM pfn special value 95 * 96 * Flags: 97 * HMM_PFN_ERROR: corresponding CPU page table entry points to poisoned memory 98 * HMM_PFN_NONE: corresponding CPU page table entry is pte_none() 99 * HMM_PFN_SPECIAL: corresponding CPU page table entry is special; i.e., the 100 * result of vmf_insert_pfn() or vm_insert_page(). Therefore, it should not 101 * be mirrored by a device, because the entry will never have HMM_PFN_VALID 102 * set and the pfn value is undefined. 103 * 104 * Driver provides values for none entry, error entry, and special entry. 105 * Driver can alias (i.e., use same value) error and special, but 106 * it should not alias none with error or special. 107 * 108 * HMM pfn value returned by hmm_vma_get_pfns() or hmm_vma_fault() will be: 109 * hmm_range.values[HMM_PFN_ERROR] if CPU page table entry is poisonous, 110 * hmm_range.values[HMM_PFN_NONE] if there is no CPU page table entry, 111 * hmm_range.values[HMM_PFN_SPECIAL] if CPU page table entry is a special one 112 */ 113 enum hmm_pfn_value_e { 114 HMM_PFN_ERROR, 115 HMM_PFN_NONE, 116 HMM_PFN_SPECIAL, 117 HMM_PFN_VALUE_MAX 118 }; 119 120 /* 121 * struct hmm_range - track invalidation lock on virtual address range 122 * 123 * @notifier: a mmu_interval_notifier that includes the start/end 124 * @notifier_seq: result of mmu_interval_read_begin() 125 * @hmm: the core HMM structure this range is active against 126 * @vma: the vm area struct for the range 127 * @list: all range lock are on a list 128 * @start: range virtual start address (inclusive) 129 * @end: range virtual end address (exclusive) 130 * @pfns: array of pfns (big enough for the range) 131 * @flags: pfn flags to match device driver page table 132 * @values: pfn value for some special case (none, special, error, ...) 133 * @default_flags: default flags for the range (write, read, ... see hmm doc) 134 * @pfn_flags_mask: allows to mask pfn flags so that only default_flags matter 135 * @pfn_shifts: pfn shift value (should be <= PAGE_SHIFT) 136 * @valid: pfns array did not change since it has been fill by an HMM function 137 */ 138 struct hmm_range { 139 struct mmu_interval_notifier *notifier; 140 unsigned long notifier_seq; 141 unsigned long start; 142 unsigned long end; 143 uint64_t *pfns; 144 const uint64_t *flags; 145 const uint64_t *values; 146 uint64_t default_flags; 147 uint64_t pfn_flags_mask; 148 uint8_t pfn_shift; 149 }; 150 151 /* 152 * hmm_device_entry_to_page() - return struct page pointed to by a device entry 153 * @range: range use to decode device entry value 154 * @entry: device entry value to get corresponding struct page from 155 * Return: struct page pointer if entry is a valid, NULL otherwise 156 * 157 * If the device entry is valid (ie valid flag set) then return the struct page 158 * matching the entry value. Otherwise return NULL. 159 */ 160 static inline struct page *hmm_device_entry_to_page(const struct hmm_range *range, 161 uint64_t entry) 162 { 163 if (entry == range->values[HMM_PFN_NONE]) 164 return NULL; 165 if (entry == range->values[HMM_PFN_ERROR]) 166 return NULL; 167 if (entry == range->values[HMM_PFN_SPECIAL]) 168 return NULL; 169 if (!(entry & range->flags[HMM_PFN_VALID])) 170 return NULL; 171 return pfn_to_page(entry >> range->pfn_shift); 172 } 173 174 /* 175 * hmm_device_entry_to_pfn() - return pfn value store in a device entry 176 * @range: range use to decode device entry value 177 * @entry: device entry to extract pfn from 178 * Return: pfn value if device entry is valid, -1UL otherwise 179 */ 180 static inline unsigned long 181 hmm_device_entry_to_pfn(const struct hmm_range *range, uint64_t pfn) 182 { 183 if (pfn == range->values[HMM_PFN_NONE]) 184 return -1UL; 185 if (pfn == range->values[HMM_PFN_ERROR]) 186 return -1UL; 187 if (pfn == range->values[HMM_PFN_SPECIAL]) 188 return -1UL; 189 if (!(pfn & range->flags[HMM_PFN_VALID])) 190 return -1UL; 191 return (pfn >> range->pfn_shift); 192 } 193 194 /* 195 * hmm_device_entry_from_page() - create a valid device entry for a page 196 * @range: range use to encode HMM pfn value 197 * @page: page for which to create the device entry 198 * Return: valid device entry for the page 199 */ 200 static inline uint64_t hmm_device_entry_from_page(const struct hmm_range *range, 201 struct page *page) 202 { 203 return (page_to_pfn(page) << range->pfn_shift) | 204 range->flags[HMM_PFN_VALID]; 205 } 206 207 /* 208 * hmm_device_entry_from_pfn() - create a valid device entry value from pfn 209 * @range: range use to encode HMM pfn value 210 * @pfn: pfn value for which to create the device entry 211 * Return: valid device entry for the pfn 212 */ 213 static inline uint64_t hmm_device_entry_from_pfn(const struct hmm_range *range, 214 unsigned long pfn) 215 { 216 return (pfn << range->pfn_shift) | 217 range->flags[HMM_PFN_VALID]; 218 } 219 220 /* 221 * Retry fault if non-blocking, drop mmap_sem and return -EAGAIN in that case. 222 */ 223 #define HMM_FAULT_ALLOW_RETRY (1 << 0) 224 225 /* Don't fault in missing PTEs, just snapshot the current state. */ 226 #define HMM_FAULT_SNAPSHOT (1 << 1) 227 228 #ifdef CONFIG_HMM_MIRROR 229 /* 230 * Please see Documentation/vm/hmm.rst for how to use the range API. 231 */ 232 long hmm_range_fault(struct hmm_range *range, unsigned int flags); 233 #else 234 static inline long hmm_range_fault(struct hmm_range *range, unsigned int flags) 235 { 236 return -EOPNOTSUPP; 237 } 238 #endif 239 240 /* 241 * HMM_RANGE_DEFAULT_TIMEOUT - default timeout (ms) when waiting for a range 242 * 243 * When waiting for mmu notifiers we need some kind of time out otherwise we 244 * could potentialy wait for ever, 1000ms ie 1s sounds like a long time to 245 * wait already. 246 */ 247 #define HMM_RANGE_DEFAULT_TIMEOUT 1000 248 249 #endif /* LINUX_HMM_H */ 250