Lines Matching +full:gpa +full:- +full:1
1 // SPDX-License-Identifier: GPL-2.0-only
22 #include <asm/pte-walk.h>
43 /* Can't access quadrants 1 or 2 in non-HV mode, call the HV to do it */ in __kvmhv_copy_tofrom_guest_radix()
52 quadrant = 1; in __kvmhv_copy_tofrom_guest_radix()
68 if (quadrant == 1) { in __kvmhv_copy_tofrom_guest_radix()
85 if (quadrant == 1 && pid != old_pid) in __kvmhv_copy_tofrom_guest_radix()
99 int lpid = vcpu->kvm->arch.lpid; in kvmhv_copy_tofrom_guest_radix()
100 int pid = vcpu->arch.pid; in kvmhv_copy_tofrom_guest_radix()
104 return -EINVAL; in kvmhv_copy_tofrom_guest_radix()
107 if (vcpu->arch.nested) in kvmhv_copy_tofrom_guest_radix()
108 lpid = vcpu->arch.nested->shadow_lpid; in kvmhv_copy_tofrom_guest_radix()
126 memset(to + (n - ret), 0, ret); in kvmhv_copy_from_guest_radix()
141 struct kvm *kvm = vcpu->kvm; in kvmppc_mmu_walk_radix_tree()
144 u64 pte, base, gpa; in kvmppc_mmu_walk_radix_tree() local
147 rts = ((root & RTS1_MASK) >> (RTS1_SHIFT - 3)) | in kvmppc_mmu_walk_radix_tree()
154 /* Current implementations only support 52-bit space */ in kvmppc_mmu_walk_radix_tree()
156 return -EINVAL; in kvmppc_mmu_walk_radix_tree()
159 for (level = 3; level >= 0; --level) { in kvmppc_mmu_walk_radix_tree()
163 return -EINVAL; in kvmppc_mmu_walk_radix_tree()
165 return -EINVAL; in kvmppc_mmu_walk_radix_tree()
166 offset -= bits; in kvmppc_mmu_walk_radix_tree()
167 index = (eaddr >> offset) & ((1UL << bits) - 1); in kvmppc_mmu_walk_radix_tree()
169 if (base & ((1UL << (bits + 3)) - 1)) in kvmppc_mmu_walk_radix_tree()
170 return -EINVAL; in kvmppc_mmu_walk_radix_tree()
184 return -ENOENT; in kvmppc_mmu_walk_radix_tree()
195 return -EINVAL; in kvmppc_mmu_walk_radix_tree()
199 gpa = pte & 0x01fffffffffff000ul; in kvmppc_mmu_walk_radix_tree()
200 if (gpa & ((1ul << offset) - 1)) in kvmppc_mmu_walk_radix_tree()
201 return -EINVAL; in kvmppc_mmu_walk_radix_tree()
202 gpa |= eaddr & ((1ul << offset) - 1); in kvmppc_mmu_walk_radix_tree()
206 gpte->page_size = ps; in kvmppc_mmu_walk_radix_tree()
207 gpte->page_shift = offset; in kvmppc_mmu_walk_radix_tree()
209 gpte->eaddr = eaddr; in kvmppc_mmu_walk_radix_tree()
210 gpte->raddr = gpa; in kvmppc_mmu_walk_radix_tree()
213 gpte->may_read = !!(pte & _PAGE_READ); in kvmppc_mmu_walk_radix_tree()
214 gpte->may_write = !!(pte & _PAGE_WRITE); in kvmppc_mmu_walk_radix_tree()
215 gpte->may_execute = !!(pte & _PAGE_EXEC); in kvmppc_mmu_walk_radix_tree()
217 gpte->rc = pte & (_PAGE_ACCESSED | _PAGE_DIRTY); in kvmppc_mmu_walk_radix_tree()
228 * table have the same layout, a partition-scoped page table and a
229 * process-scoped page table have the same layout, and the 2nd
237 struct kvm *kvm = vcpu->kvm; in kvmppc_mmu_radix_translate_table()
243 return -EINVAL; in kvmppc_mmu_radix_translate_table()
244 size = 1ul << ((table & PRTS_MASK) + 12); in kvmppc_mmu_radix_translate_table()
248 return -EINVAL; in kvmppc_mmu_radix_translate_table()
274 pid = vcpu->arch.pid; in kvmppc_mmu_radix_xlate()
280 return -EINVAL; in kvmppc_mmu_radix_xlate()
284 vcpu->kvm->arch.process_table, pid, &pte); in kvmppc_mmu_radix_xlate()
291 gpte->may_read = 0; in kvmppc_mmu_radix_xlate()
292 gpte->may_write = 0; in kvmppc_mmu_radix_xlate()
293 gpte->may_execute = 0; in kvmppc_mmu_radix_xlate()
298 if (kvmppc_get_amr_hv(vcpu) & (1ul << 62)) in kvmppc_mmu_radix_xlate()
299 gpte->may_read = 0; in kvmppc_mmu_radix_xlate()
300 if (kvmppc_get_amr_hv(vcpu) & (1ul << 63)) in kvmppc_mmu_radix_xlate()
301 gpte->may_write = 0; in kvmppc_mmu_radix_xlate()
302 if (vcpu->arch.iamr & (1ul << 62)) in kvmppc_mmu_radix_xlate()
303 gpte->may_execute = 0; in kvmppc_mmu_radix_xlate()
319 psize = 1UL << pshift; in kvmppc_radix_tlbie_page()
323 addr &= ~(psize - 1); in kvmppc_radix_tlbie_page()
334 rc = plpar_hcall_norets(H_TLB_INVALIDATE, H_TLBIE_P1_ENC(0, 0, 1), in kvmppc_radix_tlbie_page()
358 rc = plpar_hcall_norets(H_TLB_INVALIDATE, H_TLBIE_P1_ENC(1, 0, 1), in kvmppc_radix_flush_pwc()
364 0, -1UL); in kvmppc_radix_flush_pwc()
379 radix__set_pte_at(kvm->mm, addr, ptep, pte, 0); in kvmppc_radix_set_pte_at()
417 /* Called with kvm->mmu_lock held */
418 void kvmppc_unmap_pte(struct kvm *kvm, pte_t *pte, unsigned long gpa, in kvmppc_unmap_pte() argument
425 unsigned long gfn = gpa >> PAGE_SHIFT; in kvmppc_unmap_pte()
429 old = kvmppc_radix_update_pte(kvm, pte, ~0UL, 0, gpa, shift); in kvmppc_unmap_pte()
430 kvmppc_radix_tlbie_page(kvm, gpa, shift, lpid); in kvmppc_unmap_pte()
433 if (lpid != kvm->arch.lpid) in kvmppc_unmap_pte()
441 if (shift) { /* 1GB or 2MB page */ in kvmppc_unmap_pte()
442 page_size = 1ul << shift; in kvmppc_unmap_pte()
444 kvm->stat.num_2M_pages--; in kvmppc_unmap_pte()
446 kvm->stat.num_1G_pages--; in kvmppc_unmap_pte()
449 gpa &= ~(page_size - 1); in kvmppc_unmap_pte()
451 kvmhv_remove_nest_rmap_range(kvm, memslot, gpa, hpa, page_size); in kvmppc_unmap_pte()
453 if ((old & _PAGE_DIRTY) && memslot->dirty_bitmap) in kvmppc_unmap_pte()
469 * (or 4kB) mappings (of sub-pages of the same 2MB page).
505 WARN_ON_ONCE(1); in kvmppc_unmap_free_pmd()
540 pud_free(kvm->mm, pud); in kvmppc_unmap_free_pud()
561 if (kvm->arch.pgtable) { in kvmppc_free_radix()
562 kvmppc_free_pgtable_radix(kvm, kvm->arch.pgtable, in kvmppc_free_radix()
563 kvm->arch.lpid); in kvmppc_free_radix()
564 pgd_free(kvm->mm, kvm->arch.pgtable); in kvmppc_free_radix()
565 kvm->arch.pgtable = NULL; in kvmppc_free_radix()
570 unsigned long gpa, unsigned int lpid) in kvmppc_unmap_free_pmd_entry_table() argument
586 unsigned long gpa, unsigned int lpid) in kvmppc_unmap_free_pud_entry_table() argument
611 unsigned long gpa, unsigned int level, in kvmppc_create_pte() argument
622 /* Traverse the guest's 2nd-level tree, allocate new levels needed */ in kvmppc_create_pte()
623 pgd = pgtable + pgd_index(gpa); in kvmppc_create_pte()
624 p4d = p4d_offset(pgd, gpa); in kvmppc_create_pte()
628 pud = pud_offset(p4d, gpa); in kvmppc_create_pte()
630 new_pud = pud_alloc_one(kvm->mm, gpa); in kvmppc_create_pte()
634 pmd = pmd_offset(pud, gpa); in kvmppc_create_pte()
635 else if (level <= 1) in kvmppc_create_pte()
642 spin_lock(&kvm->mmu_lock); in kvmppc_create_pte()
643 ret = -EAGAIN; in kvmppc_create_pte()
648 ret = -ENOMEM; in kvmppc_create_pte()
652 p4d_populate(kvm->mm, p4d, new_pud); in kvmppc_create_pte()
655 pud = pud_offset(p4d, gpa); in kvmppc_create_pte()
657 unsigned long hgpa = gpa & PUD_MASK; in kvmppc_create_pte()
665 /* Valid 1GB page here already, add our extra bits */ in kvmppc_create_pte()
675 * a 1GB pte in after we saw a pmd page, try again. in kvmppc_create_pte()
678 ret = -EAGAIN; in kvmppc_create_pte()
681 /* Valid 1GB page here already, remove it */ in kvmppc_create_pte()
692 kvmppc_unmap_free_pud_entry_table(kvm, pud, gpa, lpid); in kvmppc_create_pte()
694 kvmppc_radix_set_pte_at(kvm, gpa, (pte_t *)pud, pte); in kvmppc_create_pte()
703 pud_populate(kvm->mm, pud, new_pmd); in kvmppc_create_pte()
706 pmd = pmd_offset(pud, gpa); in kvmppc_create_pte()
708 unsigned long lgpa = gpa & PMD_MASK; in kvmppc_create_pte()
711 if (level == 1) { in kvmppc_create_pte()
730 ret = -EAGAIN; in kvmppc_create_pte()
737 if (level == 1) { in kvmppc_create_pte()
744 kvmppc_unmap_free_pmd_entry_table(kvm, pmd, gpa, lpid); in kvmppc_create_pte()
746 kvmppc_radix_set_pte_at(kvm, gpa, pmdp_ptep(pmd), pte); in kvmppc_create_pte()
755 pmd_populate(kvm->mm, pmd, new_ptep); in kvmppc_create_pte()
758 ptep = pte_offset_kernel(pmd, gpa); in kvmppc_create_pte()
768 kvmppc_radix_update_pte(kvm, ptep, 0, pte_val(pte), gpa, 0); in kvmppc_create_pte()
772 kvmppc_radix_set_pte_at(kvm, gpa, ptep, pte); in kvmppc_create_pte()
778 spin_unlock(&kvm->mmu_lock); in kvmppc_create_pte()
780 pud_free(kvm->mm, new_pud); in kvmppc_create_pte()
789 unsigned long gpa, unsigned int lpid) in kvmppc_hv_handle_set_rc() argument
796 * Need to set an R or C bit in the 2nd-level tables; in kvmppc_hv_handle_set_rc()
805 ptep = find_kvm_nested_guest_pte(kvm, lpid, gpa, &shift); in kvmppc_hv_handle_set_rc()
807 ptep = find_kvm_secondary_pte(kvm, gpa, &shift); in kvmppc_hv_handle_set_rc()
810 kvmppc_radix_update_pte(kvm, ptep, 0, pgflags, gpa, shift); in kvmppc_hv_handle_set_rc()
817 unsigned long gpa, in kvmppc_book3s_instantiate_page() argument
822 struct kvm *kvm = vcpu->kvm; in kvmppc_book3s_instantiate_page()
825 unsigned long hva, gfn = gpa >> PAGE_SHIFT; in kvmppc_book3s_instantiate_page()
834 mmu_seq = kvm->mmu_invalidate_seq; in kvmppc_book3s_instantiate_page()
849 /* Call KVM generic code to do the slow-path check */ in kvmppc_book3s_instantiate_page()
853 return -EFAULT; in kvmppc_book3s_instantiate_page()
866 spin_lock(&kvm->mmu_lock); in kvmppc_book3s_instantiate_page()
871 spin_unlock(&kvm->mmu_lock); in kvmppc_book3s_instantiate_page()
883 large_enable = !(memslot->flags & KVM_MEM_LOG_DIRTY_PAGES); in kvmppc_book3s_instantiate_page()
887 (gpa & (PUD_SIZE - PAGE_SIZE)) == in kvmppc_book3s_instantiate_page()
888 (hva & (PUD_SIZE - PAGE_SIZE))) { in kvmppc_book3s_instantiate_page()
891 (gpa & (PMD_SIZE - PAGE_SIZE)) == in kvmppc_book3s_instantiate_page()
892 (hva & (PMD_SIZE - PAGE_SIZE))) { in kvmppc_book3s_instantiate_page()
893 level = 1; in kvmppc_book3s_instantiate_page()
902 unsigned long rpnmask = (1ul << shift) - PAGE_SIZE; in kvmppc_book3s_instantiate_page()
916 ret = kvmppc_create_pte(kvm, kvm->arch.pgtable, pte, gpa, level, in kvmppc_book3s_instantiate_page()
917 mmu_seq, kvm->arch.lpid, NULL, NULL); in kvmppc_book3s_instantiate_page()
931 if (level == 1) in kvmppc_book3s_instantiate_page()
932 kvm->stat.num_2M_pages++; in kvmppc_book3s_instantiate_page()
934 kvm->stat.num_1G_pages++; in kvmppc_book3s_instantiate_page()
943 struct kvm *kvm = vcpu->kvm; in kvmppc_book3s_radix_page_fault()
944 unsigned long gpa, gfn; in kvmppc_book3s_radix_page_fault() local
953 return -EFAULT; in kvmppc_book3s_radix_page_fault()
965 gpa = vcpu->arch.fault_gpa & ~0xfffUL; in kvmppc_book3s_radix_page_fault()
966 gpa &= ~0xF000000000000000ul; in kvmppc_book3s_radix_page_fault()
967 gfn = gpa >> PAGE_SHIFT; in kvmppc_book3s_radix_page_fault()
969 gpa |= ea & 0xfff; in kvmppc_book3s_radix_page_fault()
971 if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE) in kvmppc_book3s_radix_page_fault()
978 if (!memslot || (memslot->flags & KVM_MEMSLOT_INVALID)) { in kvmppc_book3s_radix_page_fault()
983 * unusual error - reflect it to the guest as DSI. in kvmppc_book3s_radix_page_fault()
990 return kvmppc_hv_emulate_mmio(vcpu, gpa, ea, writing); in kvmppc_book3s_radix_page_fault()
993 if (memslot->flags & KVM_MEM_READONLY) { in kvmppc_book3s_radix_page_fault()
1006 spin_lock(&kvm->mmu_lock); in kvmppc_book3s_radix_page_fault()
1008 gpa, kvm->arch.lpid)) in kvmppc_book3s_radix_page_fault()
1010 spin_unlock(&kvm->mmu_lock); in kvmppc_book3s_radix_page_fault()
1018 ret = kvmppc_book3s_instantiate_page(vcpu, gpa, memslot, writing, in kvmppc_book3s_radix_page_fault()
1021 if (ret == 0 || ret == -EAGAIN) in kvmppc_book3s_radix_page_fault()
1026 /* Called with kvm->mmu_lock held */
1031 unsigned long gpa = gfn << PAGE_SHIFT; in kvm_unmap_radix() local
1034 if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE) { in kvm_unmap_radix()
1035 uv_page_inval(kvm->arch.lpid, gpa, PAGE_SHIFT); in kvm_unmap_radix()
1039 ptep = find_kvm_secondary_pte(kvm, gpa, &shift); in kvm_unmap_radix()
1041 kvmppc_unmap_pte(kvm, ptep, gpa, shift, memslot, in kvm_unmap_radix()
1042 kvm->arch.lpid); in kvm_unmap_radix()
1045 /* Called with kvm->mmu_lock held */
1050 unsigned long gpa = gfn << PAGE_SHIFT; in kvm_age_radix() local
1055 if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE) in kvm_age_radix()
1058 ptep = find_kvm_secondary_pte(kvm, gpa, &shift); in kvm_age_radix()
1061 gpa, shift); in kvm_age_radix()
1064 rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn]; in kvm_age_radix()
1067 1UL << shift); in kvm_age_radix()
1073 /* Called with kvm->mmu_lock held */
1079 unsigned long gpa = gfn << PAGE_SHIFT; in kvm_test_age_radix() local
1083 if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE) in kvm_test_age_radix()
1086 ptep = find_kvm_secondary_pte(kvm, gpa, &shift); in kvm_test_age_radix()
1096 unsigned long gfn = memslot->base_gfn + pagenum; in kvm_radix_test_clear_dirty()
1097 unsigned long gpa = gfn << PAGE_SHIFT; in kvm_radix_test_clear_dirty() local
1103 if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE) in kvm_radix_test_clear_dirty()
1107 * For performance reasons we don't hold kvm->mmu_lock while walking the in kvm_radix_test_clear_dirty()
1110 ptep = find_kvm_secondary_pte_unlocked(kvm, gpa, &shift); in kvm_radix_test_clear_dirty()
1116 spin_lock(&kvm->mmu_lock); in kvm_radix_test_clear_dirty()
1128 spin_unlock(&kvm->mmu_lock); in kvm_radix_test_clear_dirty()
1133 ret = 1; in kvm_radix_test_clear_dirty()
1136 gpa, shift); in kvm_radix_test_clear_dirty()
1137 kvmppc_radix_tlbie_page(kvm, gpa, shift, kvm->arch.lpid); in kvm_radix_test_clear_dirty()
1139 rmapp = &memslot->arch.rmap[gfn - memslot->base_gfn]; in kvm_radix_test_clear_dirty()
1142 1UL << shift); in kvm_radix_test_clear_dirty()
1143 spin_unlock(&kvm->mmu_lock); in kvm_radix_test_clear_dirty()
1154 for (i = 0; i < memslot->npages; i = j) { in kvmppc_hv_get_dirty_log_radix()
1162 * real address, if npages > 1 we can skip to i + npages. in kvmppc_hv_get_dirty_log_radix()
1164 j = i + 1; in kvmppc_hv_get_dirty_log_radix()
1178 unsigned long gpa; in kvmppc_radix_flush_memslot() local
1181 if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START) in kvmppc_radix_flush_memslot()
1184 if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE) in kvmppc_radix_flush_memslot()
1187 gpa = memslot->base_gfn << PAGE_SHIFT; in kvmppc_radix_flush_memslot()
1188 spin_lock(&kvm->mmu_lock); in kvmppc_radix_flush_memslot()
1189 for (n = memslot->npages; n; --n) { in kvmppc_radix_flush_memslot()
1190 ptep = find_kvm_secondary_pte(kvm, gpa, &shift); in kvmppc_radix_flush_memslot()
1192 kvmppc_unmap_pte(kvm, ptep, gpa, shift, memslot, in kvmppc_radix_flush_memslot()
1193 kvm->arch.lpid); in kvmppc_radix_flush_memslot()
1194 gpa += PAGE_SIZE; in kvmppc_radix_flush_memslot()
1200 kvm->mmu_invalidate_seq++; in kvmppc_radix_flush_memslot()
1201 spin_unlock(&kvm->mmu_lock); in kvmppc_radix_flush_memslot()
1209 info->ap_encodings[*indexp] = mmu_psize_defs[psize].shift | in add_rmmu_ap_encoding()
1219 return -EINVAL; in kvmhv_get_rmmu_info()
1223 info->geometries[0].page_shift = 12; in kvmhv_get_rmmu_info()
1224 info->geometries[0].level_bits[0] = 9; in kvmhv_get_rmmu_info()
1225 for (i = 1; i < 4; ++i) in kvmhv_get_rmmu_info()
1226 info->geometries[0].level_bits[i] = p9_supported_radix_bits[i]; in kvmhv_get_rmmu_info()
1228 info->geometries[1].page_shift = 16; in kvmhv_get_rmmu_info()
1230 info->geometries[1].level_bits[i] = p9_supported_radix_bits[i]; in kvmhv_get_rmmu_info()
1243 kvm->arch.pgtable = pgd_alloc(kvm->mm); in kvmppc_init_vm_radix()
1244 if (!kvm->arch.pgtable) in kvmppc_init_vm_radix()
1245 return -ENOMEM; in kvmppc_init_vm_radix()
1262 unsigned long gpa; member
1272 struct kvm *kvm = inode->i_private; in debugfs_radix_open()
1277 return -ENOMEM; in debugfs_radix_open()
1280 p->kvm = kvm; in debugfs_radix_open()
1281 mutex_init(&p->mutex); in debugfs_radix_open()
1282 file->private_data = p; in debugfs_radix_open()
1289 struct debugfs_radix_state *p = file->private_data; in debugfs_radix_release()
1291 kvm_put_kvm(p->kvm); in debugfs_radix_release()
1299 struct debugfs_radix_state *p = file->private_data; in debugfs_radix_read()
1303 unsigned long gpa; in debugfs_radix_read() local
1314 kvm = p->kvm; in debugfs_radix_read()
1318 ret = mutex_lock_interruptible(&p->mutex); in debugfs_radix_read()
1322 if (p->chars_left) { in debugfs_radix_read()
1323 n = p->chars_left; in debugfs_radix_read()
1326 r = copy_to_user(buf, p->buf + p->buf_index, n); in debugfs_radix_read()
1327 n -= r; in debugfs_radix_read()
1328 p->chars_left -= n; in debugfs_radix_read()
1329 p->buf_index += n; in debugfs_radix_read()
1331 len -= n; in debugfs_radix_read()
1335 ret = -EFAULT; in debugfs_radix_read()
1340 gpa = p->gpa; in debugfs_radix_read()
1343 while (len != 0 && p->lpid >= 0) { in debugfs_radix_read()
1344 if (gpa >= RADIX_PGTABLE_RANGE) { in debugfs_radix_read()
1345 gpa = 0; in debugfs_radix_read()
1351 p->lpid = kvmhv_nested_next_lpid(kvm, p->lpid); in debugfs_radix_read()
1352 p->hdr = 0; in debugfs_radix_read()
1353 if (p->lpid < 0) in debugfs_radix_read()
1357 if (p->lpid == 0) { in debugfs_radix_read()
1358 pgt = kvm->arch.pgtable; in debugfs_radix_read()
1360 nested = kvmhv_get_nested(kvm, p->lpid, false); in debugfs_radix_read()
1362 gpa = RADIX_PGTABLE_RANGE; in debugfs_radix_read()
1365 pgt = nested->shadow_pgtable; in debugfs_radix_read()
1369 if (!p->hdr) { in debugfs_radix_read()
1370 if (p->lpid > 0) in debugfs_radix_read()
1371 n = scnprintf(p->buf, sizeof(p->buf), in debugfs_radix_read()
1372 "\nNested LPID %d: ", p->lpid); in debugfs_radix_read()
1373 n += scnprintf(p->buf + n, sizeof(p->buf) - n, in debugfs_radix_read()
1375 p->hdr = 1; in debugfs_radix_read()
1379 pgdp = pgt + pgd_index(gpa); in debugfs_radix_read()
1380 p4dp = p4d_offset(pgdp, gpa); in debugfs_radix_read()
1383 gpa = (gpa & P4D_MASK) + P4D_SIZE; in debugfs_radix_read()
1387 pudp = pud_offset(&p4d, gpa); in debugfs_radix_read()
1390 gpa = (gpa & PUD_MASK) + PUD_SIZE; in debugfs_radix_read()
1399 pmdp = pmd_offset(&pud, gpa); in debugfs_radix_read()
1402 gpa = (gpa & PMD_MASK) + PMD_SIZE; in debugfs_radix_read()
1411 ptep = pte_offset_kernel(&pmd, gpa); in debugfs_radix_read()
1414 gpa += PAGE_SIZE; in debugfs_radix_read()
1419 n = scnprintf(p->buf, sizeof(p->buf), in debugfs_radix_read()
1420 " %lx: %lx %d\n", gpa, pte, shift); in debugfs_radix_read()
1421 gpa += 1ul << shift; in debugfs_radix_read()
1423 p->chars_left = n; in debugfs_radix_read()
1426 r = copy_to_user(buf, p->buf, n); in debugfs_radix_read()
1427 n -= r; in debugfs_radix_read()
1428 p->chars_left -= n; in debugfs_radix_read()
1429 p->buf_index = n; in debugfs_radix_read()
1431 len -= n; in debugfs_radix_read()
1435 ret = -EFAULT; in debugfs_radix_read()
1439 p->gpa = gpa; in debugfs_radix_read()
1444 mutex_unlock(&p->mutex); in debugfs_radix_read()
1451 return -EACCES; in debugfs_radix_write()
1465 debugfs_create_file("radix", 0400, kvm->debugfs_dentry, kvm, in kvmhv_radix_debugfs_init()
1473 kvm_pte_cache = kmem_cache_create("kvm-pte", size, size, 0, pte_ctor); in kvmppc_radix_init()
1475 return -ENOMEM; in kvmppc_radix_init()
1479 kvm_pmd_cache = kmem_cache_create("kvm-pmd", size, size, 0, pmd_ctor); in kvmppc_radix_init()
1482 return -ENOMEM; in kvmppc_radix_init()