1 #ifndef _ASM_X86_PGTABLE_2LEVEL_H 2 #define _ASM_X86_PGTABLE_2LEVEL_H 3 4 #define pte_ERROR(e) \ 5 pr_err("%s:%d: bad pte %08lx\n", __FILE__, __LINE__, (e).pte_low) 6 #define pgd_ERROR(e) \ 7 pr_err("%s:%d: bad pgd %08lx\n", __FILE__, __LINE__, pgd_val(e)) 8 9 /* 10 * Certain architectures need to do special things when PTEs 11 * within a page table are directly modified. Thus, the following 12 * hook is made available. 13 */ 14 static inline void native_set_pte(pte_t *ptep , pte_t pte) 15 { 16 *ptep = pte; 17 } 18 19 static inline void native_set_pmd(pmd_t *pmdp, pmd_t pmd) 20 { 21 *pmdp = pmd; 22 } 23 24 static inline void native_set_pte_atomic(pte_t *ptep, pte_t pte) 25 { 26 native_set_pte(ptep, pte); 27 } 28 29 static inline void native_pmd_clear(pmd_t *pmdp) 30 { 31 native_set_pmd(pmdp, __pmd(0)); 32 } 33 34 static inline void native_pte_clear(struct mm_struct *mm, 35 unsigned long addr, pte_t *xp) 36 { 37 *xp = native_make_pte(0); 38 } 39 40 #ifdef CONFIG_SMP 41 static inline pte_t native_ptep_get_and_clear(pte_t *xp) 42 { 43 return __pte(xchg(&xp->pte_low, 0)); 44 } 45 #else 46 #define native_ptep_get_and_clear(xp) native_local_ptep_get_and_clear(xp) 47 #endif 48 49 #ifdef CONFIG_SMP 50 static inline pmd_t native_pmdp_get_and_clear(pmd_t *xp) 51 { 52 return __pmd(xchg((pmdval_t *)xp, 0)); 53 } 54 #else 55 #define native_pmdp_get_and_clear(xp) native_local_pmdp_get_and_clear(xp) 56 #endif 57 58 /* Bit manipulation helper on pte/pgoff entry */ 59 static inline unsigned long pte_bitop(unsigned long value, unsigned int rightshift, 60 unsigned long mask, unsigned int leftshift) 61 { 62 return ((value >> rightshift) & mask) << leftshift; 63 } 64 65 #ifdef CONFIG_MEM_SOFT_DIRTY 66 67 /* 68 * Bits _PAGE_BIT_PRESENT, _PAGE_BIT_FILE, _PAGE_BIT_SOFT_DIRTY and 69 * _PAGE_BIT_PROTNONE are taken, split up the 28 bits of offset 70 * into this range. 71 */ 72 #define PTE_FILE_MAX_BITS 28 73 #define PTE_FILE_SHIFT1 (_PAGE_BIT_PRESENT + 1) 74 #define PTE_FILE_SHIFT2 (_PAGE_BIT_FILE + 1) 75 #define PTE_FILE_SHIFT3 (_PAGE_BIT_PROTNONE + 1) 76 #define PTE_FILE_SHIFT4 (_PAGE_BIT_SOFT_DIRTY + 1) 77 #define PTE_FILE_BITS1 (PTE_FILE_SHIFT2 - PTE_FILE_SHIFT1 - 1) 78 #define PTE_FILE_BITS2 (PTE_FILE_SHIFT3 - PTE_FILE_SHIFT2 - 1) 79 #define PTE_FILE_BITS3 (PTE_FILE_SHIFT4 - PTE_FILE_SHIFT3 - 1) 80 81 #define PTE_FILE_MASK1 ((1U << PTE_FILE_BITS1) - 1) 82 #define PTE_FILE_MASK2 ((1U << PTE_FILE_BITS2) - 1) 83 #define PTE_FILE_MASK3 ((1U << PTE_FILE_BITS3) - 1) 84 85 #define PTE_FILE_LSHIFT2 (PTE_FILE_BITS1) 86 #define PTE_FILE_LSHIFT3 (PTE_FILE_BITS1 + PTE_FILE_BITS2) 87 #define PTE_FILE_LSHIFT4 (PTE_FILE_BITS1 + PTE_FILE_BITS2 + PTE_FILE_BITS3) 88 89 static __always_inline pgoff_t pte_to_pgoff(pte_t pte) 90 { 91 return (pgoff_t) 92 (pte_bitop(pte.pte_low, PTE_FILE_SHIFT1, PTE_FILE_MASK1, 0) + 93 pte_bitop(pte.pte_low, PTE_FILE_SHIFT2, PTE_FILE_MASK2, PTE_FILE_LSHIFT2) + 94 pte_bitop(pte.pte_low, PTE_FILE_SHIFT3, PTE_FILE_MASK3, PTE_FILE_LSHIFT3) + 95 pte_bitop(pte.pte_low, PTE_FILE_SHIFT4, -1UL, PTE_FILE_LSHIFT4)); 96 } 97 98 static __always_inline pte_t pgoff_to_pte(pgoff_t off) 99 { 100 return (pte_t){ 101 .pte_low = 102 pte_bitop(off, 0, PTE_FILE_MASK1, PTE_FILE_SHIFT1) + 103 pte_bitop(off, PTE_FILE_LSHIFT2, PTE_FILE_MASK2, PTE_FILE_SHIFT2) + 104 pte_bitop(off, PTE_FILE_LSHIFT3, PTE_FILE_MASK3, PTE_FILE_SHIFT3) + 105 pte_bitop(off, PTE_FILE_LSHIFT4, -1UL, PTE_FILE_SHIFT4) + 106 _PAGE_FILE, 107 }; 108 } 109 110 #else /* CONFIG_MEM_SOFT_DIRTY */ 111 112 /* 113 * Bits _PAGE_BIT_PRESENT, _PAGE_BIT_FILE and _PAGE_BIT_PROTNONE are taken, 114 * split up the 29 bits of offset into this range. 115 */ 116 #define PTE_FILE_MAX_BITS 29 117 #define PTE_FILE_SHIFT1 (_PAGE_BIT_PRESENT + 1) 118 #if _PAGE_BIT_FILE < _PAGE_BIT_PROTNONE 119 #define PTE_FILE_SHIFT2 (_PAGE_BIT_FILE + 1) 120 #define PTE_FILE_SHIFT3 (_PAGE_BIT_PROTNONE + 1) 121 #else 122 #define PTE_FILE_SHIFT2 (_PAGE_BIT_PROTNONE + 1) 123 #define PTE_FILE_SHIFT3 (_PAGE_BIT_FILE + 1) 124 #endif 125 #define PTE_FILE_BITS1 (PTE_FILE_SHIFT2 - PTE_FILE_SHIFT1 - 1) 126 #define PTE_FILE_BITS2 (PTE_FILE_SHIFT3 - PTE_FILE_SHIFT2 - 1) 127 128 #define PTE_FILE_MASK1 ((1U << PTE_FILE_BITS1) - 1) 129 #define PTE_FILE_MASK2 ((1U << PTE_FILE_BITS2) - 1) 130 131 #define PTE_FILE_LSHIFT2 (PTE_FILE_BITS1) 132 #define PTE_FILE_LSHIFT3 (PTE_FILE_BITS1 + PTE_FILE_BITS2) 133 134 static __always_inline pgoff_t pte_to_pgoff(pte_t pte) 135 { 136 return (pgoff_t) 137 (pte_bitop(pte.pte_low, PTE_FILE_SHIFT1, PTE_FILE_MASK1, 0) + 138 pte_bitop(pte.pte_low, PTE_FILE_SHIFT2, PTE_FILE_MASK2, PTE_FILE_LSHIFT2) + 139 pte_bitop(pte.pte_low, PTE_FILE_SHIFT3, -1UL, PTE_FILE_LSHIFT3)); 140 } 141 142 static __always_inline pte_t pgoff_to_pte(pgoff_t off) 143 { 144 return (pte_t){ 145 .pte_low = 146 pte_bitop(off, 0, PTE_FILE_MASK1, PTE_FILE_SHIFT1) + 147 pte_bitop(off, PTE_FILE_LSHIFT2, PTE_FILE_MASK2, PTE_FILE_SHIFT2) + 148 pte_bitop(off, PTE_FILE_LSHIFT3, -1UL, PTE_FILE_SHIFT3) + 149 _PAGE_FILE, 150 }; 151 } 152 153 #endif /* CONFIG_MEM_SOFT_DIRTY */ 154 155 /* Encode and de-code a swap entry */ 156 #if _PAGE_BIT_FILE < _PAGE_BIT_PROTNONE 157 #define SWP_TYPE_BITS (_PAGE_BIT_FILE - _PAGE_BIT_PRESENT - 1) 158 #define SWP_OFFSET_SHIFT (_PAGE_BIT_PROTNONE + 1) 159 #else 160 #define SWP_TYPE_BITS (_PAGE_BIT_PROTNONE - _PAGE_BIT_PRESENT - 1) 161 #define SWP_OFFSET_SHIFT (_PAGE_BIT_FILE + 1) 162 #endif 163 164 #define MAX_SWAPFILES_CHECK() BUILD_BUG_ON(MAX_SWAPFILES_SHIFT > SWP_TYPE_BITS) 165 166 #define __swp_type(x) (((x).val >> (_PAGE_BIT_PRESENT + 1)) \ 167 & ((1U << SWP_TYPE_BITS) - 1)) 168 #define __swp_offset(x) ((x).val >> SWP_OFFSET_SHIFT) 169 #define __swp_entry(type, offset) ((swp_entry_t) { \ 170 ((type) << (_PAGE_BIT_PRESENT + 1)) \ 171 | ((offset) << SWP_OFFSET_SHIFT) }) 172 #define __pte_to_swp_entry(pte) ((swp_entry_t) { (pte).pte_low }) 173 #define __swp_entry_to_pte(x) ((pte_t) { .pte = (x).val }) 174 175 #endif /* _ASM_X86_PGTABLE_2LEVEL_H */ 176