1 #include <linux/pfn.h> 2 #include <asm/xen/page.h> 3 #include <asm/xen/hypercall.h> 4 #include <xen/interface/memory.h> 5 6 #include "multicalls.h" 7 #include "mmu.h" 8 9 /* 10 * Protects atomic reservation decrease/increase against concurrent increases. 11 * Also protects non-atomic updates of current_pages and balloon lists. 12 */ 13 DEFINE_SPINLOCK(xen_reservation_lock); 14 15 unsigned long arbitrary_virt_to_mfn(void *vaddr) 16 { 17 xmaddr_t maddr = arbitrary_virt_to_machine(vaddr); 18 19 return PFN_DOWN(maddr.maddr); 20 } 21 22 xmaddr_t arbitrary_virt_to_machine(void *vaddr) 23 { 24 unsigned long address = (unsigned long)vaddr; 25 unsigned int level; 26 pte_t *pte; 27 unsigned offset; 28 29 /* 30 * if the PFN is in the linear mapped vaddr range, we can just use 31 * the (quick) virt_to_machine() p2m lookup 32 */ 33 if (virt_addr_valid(vaddr)) 34 return virt_to_machine(vaddr); 35 36 /* otherwise we have to do a (slower) full page-table walk */ 37 38 pte = lookup_address(address, &level); 39 BUG_ON(pte == NULL); 40 offset = address & ~PAGE_MASK; 41 return XMADDR(((phys_addr_t)pte_mfn(*pte) << PAGE_SHIFT) + offset); 42 } 43 EXPORT_SYMBOL_GPL(arbitrary_virt_to_machine); 44 45 static noinline void xen_flush_tlb_all(void) 46 { 47 struct mmuext_op *op; 48 struct multicall_space mcs; 49 50 preempt_disable(); 51 52 mcs = xen_mc_entry(sizeof(*op)); 53 54 op = mcs.args; 55 op->cmd = MMUEXT_TLB_FLUSH_ALL; 56 MULTI_mmuext_op(mcs.mc, op, 1, NULL, DOMID_SELF); 57 58 xen_mc_issue(PARAVIRT_LAZY_MMU); 59 60 preempt_enable(); 61 } 62 63 #define REMAP_BATCH_SIZE 16 64 65 struct remap_data { 66 xen_pfn_t *pfn; 67 bool contiguous; 68 bool no_translate; 69 pgprot_t prot; 70 struct mmu_update *mmu_update; 71 }; 72 73 static int remap_area_pfn_pte_fn(pte_t *ptep, pgtable_t token, 74 unsigned long addr, void *data) 75 { 76 struct remap_data *rmd = data; 77 pte_t pte = pte_mkspecial(mfn_pte(*rmd->pfn, rmd->prot)); 78 79 /* 80 * If we have a contiguous range, just update the pfn itself, 81 * else update pointer to be "next pfn". 82 */ 83 if (rmd->contiguous) 84 (*rmd->pfn)++; 85 else 86 rmd->pfn++; 87 88 rmd->mmu_update->ptr = virt_to_machine(ptep).maddr; 89 rmd->mmu_update->ptr |= rmd->no_translate ? 90 MMU_PT_UPDATE_NO_TRANSLATE : 91 MMU_NORMAL_PT_UPDATE; 92 rmd->mmu_update->val = pte_val_ma(pte); 93 rmd->mmu_update++; 94 95 return 0; 96 } 97 98 static int do_remap_pfn(struct vm_area_struct *vma, 99 unsigned long addr, 100 xen_pfn_t *pfn, int nr, 101 int *err_ptr, pgprot_t prot, 102 unsigned int domid, 103 bool no_translate, 104 struct page **pages) 105 { 106 int err = 0; 107 struct remap_data rmd; 108 struct mmu_update mmu_update[REMAP_BATCH_SIZE]; 109 unsigned long range; 110 int mapped = 0; 111 112 BUG_ON(!((vma->vm_flags & (VM_PFNMAP | VM_IO)) == (VM_PFNMAP | VM_IO))); 113 114 rmd.pfn = pfn; 115 rmd.prot = prot; 116 /* 117 * We use the err_ptr to indicate if there we are doing a contiguous 118 * mapping or a discontigious mapping. 119 */ 120 rmd.contiguous = !err_ptr; 121 rmd.no_translate = no_translate; 122 123 while (nr) { 124 int index = 0; 125 int done = 0; 126 int batch = min(REMAP_BATCH_SIZE, nr); 127 int batch_left = batch; 128 range = (unsigned long)batch << PAGE_SHIFT; 129 130 rmd.mmu_update = mmu_update; 131 err = apply_to_page_range(vma->vm_mm, addr, range, 132 remap_area_pfn_pte_fn, &rmd); 133 if (err) 134 goto out; 135 136 /* We record the error for each page that gives an error, but 137 * continue mapping until the whole set is done */ 138 do { 139 int i; 140 141 err = HYPERVISOR_mmu_update(&mmu_update[index], 142 batch_left, &done, domid); 143 144 /* 145 * @err_ptr may be the same buffer as @gfn, so 146 * only clear it after each chunk of @gfn is 147 * used. 148 */ 149 if (err_ptr) { 150 for (i = index; i < index + done; i++) 151 err_ptr[i] = 0; 152 } 153 if (err < 0) { 154 if (!err_ptr) 155 goto out; 156 err_ptr[i] = err; 157 done++; /* Skip failed frame. */ 158 } else 159 mapped += done; 160 batch_left -= done; 161 index += done; 162 } while (batch_left); 163 164 nr -= batch; 165 addr += range; 166 if (err_ptr) 167 err_ptr += batch; 168 cond_resched(); 169 } 170 out: 171 172 xen_flush_tlb_all(); 173 174 return err < 0 ? err : mapped; 175 } 176 177 int xen_remap_domain_gfn_range(struct vm_area_struct *vma, 178 unsigned long addr, 179 xen_pfn_t gfn, int nr, 180 pgprot_t prot, unsigned domid, 181 struct page **pages) 182 { 183 if (xen_feature(XENFEAT_auto_translated_physmap)) 184 return -EOPNOTSUPP; 185 186 return do_remap_pfn(vma, addr, &gfn, nr, NULL, prot, domid, false, 187 pages); 188 } 189 EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_range); 190 191 int xen_remap_domain_gfn_array(struct vm_area_struct *vma, 192 unsigned long addr, 193 xen_pfn_t *gfn, int nr, 194 int *err_ptr, pgprot_t prot, 195 unsigned domid, struct page **pages) 196 { 197 if (xen_feature(XENFEAT_auto_translated_physmap)) 198 return xen_xlate_remap_gfn_array(vma, addr, gfn, nr, err_ptr, 199 prot, domid, pages); 200 201 /* We BUG_ON because it's a programmer error to pass a NULL err_ptr, 202 * and the consequences later is quite hard to detect what the actual 203 * cause of "wrong memory was mapped in". 204 */ 205 BUG_ON(err_ptr == NULL); 206 return do_remap_pfn(vma, addr, gfn, nr, err_ptr, prot, domid, 207 false, pages); 208 } 209 EXPORT_SYMBOL_GPL(xen_remap_domain_gfn_array); 210 211 int xen_remap_domain_mfn_array(struct vm_area_struct *vma, 212 unsigned long addr, 213 xen_pfn_t *mfn, int nr, 214 int *err_ptr, pgprot_t prot, 215 unsigned int domid, struct page **pages) 216 { 217 if (xen_feature(XENFEAT_auto_translated_physmap)) 218 return -EOPNOTSUPP; 219 220 return do_remap_pfn(vma, addr, mfn, nr, err_ptr, prot, domid, 221 true, pages); 222 } 223 EXPORT_SYMBOL_GPL(xen_remap_domain_mfn_array); 224 225 /* Returns: 0 success */ 226 int xen_unmap_domain_gfn_range(struct vm_area_struct *vma, 227 int nr, struct page **pages) 228 { 229 if (xen_feature(XENFEAT_auto_translated_physmap)) 230 return xen_xlate_unmap_gfn_range(vma, nr, pages); 231 232 if (!pages) 233 return 0; 234 235 return -EINVAL; 236 } 237 EXPORT_SYMBOL_GPL(xen_unmap_domain_gfn_range); 238