47 static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,
53 	struct kvm *kvm;  member
118 void kvmppc_set_hpt(struct kvm *kvm, struct kvm_hpt_info *info)  in kvmppc_set_hpt()  argument
120 	atomic64_set(&kvm->arch.mmio_update, 0);  in kvmppc_set_hpt()
121 	kvm->arch.hpt = *info;  in kvmppc_set_hpt()
122 	kvm->arch.sdr1 = __pa(info->virt) | (info->order - 18);  in kvmppc_set_hpt()
125 		 info->virt, (long)info->order, kvm->arch.lpid);  in kvmppc_set_hpt()
128 int kvmppc_alloc_reset_hpt(struct kvm *kvm, int order)  in kvmppc_alloc_reset_hpt()  argument
133 	mutex_lock(&kvm->arch.mmu_setup_lock);  in kvmppc_alloc_reset_hpt()
134 	if (kvm->arch.mmu_ready) {  in kvmppc_alloc_reset_hpt()
135 		kvm->arch.mmu_ready = 0;  in kvmppc_alloc_reset_hpt()
138 		if (atomic_read(&kvm->arch.vcpus_running)) {  in kvmppc_alloc_reset_hpt()
139 			kvm->arch.mmu_ready = 1;  in kvmppc_alloc_reset_hpt()
143 	if (kvm_is_radix(kvm)) {  in kvmppc_alloc_reset_hpt()
144 		err = kvmppc_switch_mmu_to_hpt(kvm);  in kvmppc_alloc_reset_hpt()
149 	if (kvm->arch.hpt.order == order) {  in kvmppc_alloc_reset_hpt()
153 		memset((void *)kvm->arch.hpt.virt, 0, 1ul << order);  in kvmppc_alloc_reset_hpt()
157 		kvmppc_rmap_reset(kvm);  in kvmppc_alloc_reset_hpt()
162 	if (kvm->arch.hpt.virt) {  in kvmppc_alloc_reset_hpt()
163 		kvmppc_free_hpt(&kvm->arch.hpt);  in kvmppc_alloc_reset_hpt()
164 		kvmppc_rmap_reset(kvm);  in kvmppc_alloc_reset_hpt()
170 	kvmppc_set_hpt(kvm, &info);  in kvmppc_alloc_reset_hpt()
175 		cpumask_setall(&kvm->arch.need_tlb_flush);  in kvmppc_alloc_reset_hpt()
177 	mutex_unlock(&kvm->arch.mmu_setup_lock);  in kvmppc_alloc_reset_hpt()
217 	struct kvm *kvm = vcpu->kvm;  in kvmppc_map_vrma()  local
226 	if (npages > kvmppc_hpt_mask(&kvm->arch.hpt) + 1)  in kvmppc_map_vrma()
227 		npages = kvmppc_hpt_mask(&kvm->arch.hpt) + 1;  in kvmppc_map_vrma()
238 			& kvmppc_hpt_mask(&kvm->arch.hpt);  in kvmppc_map_vrma()
248 		ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, hash, hp_v, hp_r,  in kvmppc_map_vrma()
292 static long kvmppc_virtmode_do_h_enter(struct kvm *kvm, unsigned long flags,  in kvmppc_virtmode_do_h_enter()  argument
299 	ret = kvmppc_do_h_enter(kvm, flags, pte_index, pteh, ptel,  in kvmppc_virtmode_do_h_enter()
300 				kvm->mm->pgd, false, pte_idx_ret);  in kvmppc_virtmode_do_h_enter()
344 	struct kvm *kvm = vcpu->kvm;  in kvmppc_mmu_book3s_64_hv_xlate()  local
353 	if (kvm_is_radix(vcpu->kvm))  in kvmppc_mmu_book3s_64_hv_xlate()
364 		slb_v = vcpu->kvm->arch.vrma_slb_v;  in kvmppc_mmu_book3s_64_hv_xlate()
369 	index = kvmppc_hv_find_lock_hpte(kvm, eaddr, slb_v,  in kvmppc_mmu_book3s_64_hv_xlate()
375 	hptep = (__be64 *)(kvm->arch.hpt.virt + (index << 4));  in kvmppc_mmu_book3s_64_hv_xlate()
379 	gr = kvm->arch.hpt.rev[index].guest_rpte;  in kvmppc_mmu_book3s_64_hv_xlate()
447 		idx = srcu_read_lock(&vcpu->kvm->srcu);  in kvmppc_hv_emulate_mmio()
450 		srcu_read_unlock(&vcpu->kvm->srcu, idx);  in kvmppc_hv_emulate_mmio()
509 	struct kvm *kvm = vcpu->kvm;  in kvmppc_book3s_hv_page_fault()  local
528 	if (kvm_is_radix(kvm))  in kvmppc_book3s_hv_page_fault()
541 		mmio_update = atomic64_read(&kvm->arch.mmio_update);  in kvmppc_book3s_hv_page_fault()
554 	hptep = (__be64 *)(kvm->arch.hpt.virt + (index << 4));  in kvmppc_book3s_hv_page_fault()
555 	rev = &kvm->arch.hpt.rev[index];  in kvmppc_book3s_hv_page_fault()
579 	memslot = gfn_to_memslot(kvm, gfn);  in kvmppc_book3s_hv_page_fault()
596 	mmu_seq = kvm->mmu_invalidate_seq;  in kvmppc_book3s_hv_page_fault()
632 	spin_lock(&kvm->mmu_lock);  in kvmppc_book3s_hv_page_fault()
633 	ptep = find_kvm_host_pte(kvm, mmu_seq, hva, &shift);  in kvmppc_book3s_hv_page_fault()
637 	spin_unlock(&kvm->mmu_lock);  in kvmppc_book3s_hv_page_fault()
696 	if (!kvm->arch.mmu_ready)  in kvmppc_book3s_hv_page_fault()
711 	if (mmu_invalidate_retry(vcpu->kvm, mmu_seq)) {  in kvmppc_book3s_hv_page_fault()
724 		kvmppc_invalidate_hpte(kvm, hptep, index);  in kvmppc_book3s_hv_page_fault()
728 		kvmppc_add_revmap_chain(kvm, rev, rmap, index, 0);  in kvmppc_book3s_hv_page_fault()
756 void kvmppc_rmap_reset(struct kvm *kvm)  in kvmppc_rmap_reset()  argument
762 	srcu_idx = srcu_read_lock(&kvm->srcu);  in kvmppc_rmap_reset()
763 	slots = kvm_memslots(kvm);  in kvmppc_rmap_reset()
766 		spin_lock(&kvm->mmu_lock);  in kvmppc_rmap_reset()
773 		spin_unlock(&kvm->mmu_lock);  in kvmppc_rmap_reset()
775 	srcu_read_unlock(&kvm->srcu, srcu_idx);  in kvmppc_rmap_reset()
779 static void kvmppc_unmap_hpte(struct kvm *kvm, unsigned long i,  in kvmppc_unmap_hpte()  argument
783 	__be64 *hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));  in kvmppc_unmap_hpte()
784 	struct revmap_entry *rev = kvm->arch.hpt.rev;  in kvmppc_unmap_hpte()
807 		kvmppc_invalidate_hpte(kvm, hptep, i);  in kvmppc_unmap_hpte()
816 			note_hpte_modification(kvm, &rev[i]);  in kvmppc_unmap_hpte()
821 static void kvm_unmap_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,  in kvm_unmap_rmapp()  argument
842 		hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));  in kvm_unmap_rmapp()
851 		kvmppc_unmap_hpte(kvm, i, memslot, rmapp, gfn);  in kvm_unmap_rmapp()
857 bool kvm_unmap_gfn_range_hv(struct kvm *kvm, struct kvm_gfn_range *range)  in kvm_unmap_gfn_range_hv()  argument
861 	if (kvm_is_radix(kvm)) {  in kvm_unmap_gfn_range_hv()
863 			kvm_unmap_radix(kvm, range->slot, gfn);  in kvm_unmap_gfn_range_hv()
866 			kvm_unmap_rmapp(kvm, range->slot, gfn);  in kvm_unmap_gfn_range_hv()
872 void kvmppc_core_flush_memslot_hv(struct kvm *kvm,  in kvmppc_core_flush_memslot_hv()  argument
881 	if (kvm_is_radix(kvm)) {  in kvmppc_core_flush_memslot_hv()
882 		kvmppc_radix_flush_memslot(kvm, memslot);  in kvmppc_core_flush_memslot_hv()
894 			kvm_unmap_rmapp(kvm, memslot, gfn);  in kvmppc_core_flush_memslot_hv()
899 static bool kvm_age_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,  in kvm_age_rmapp()  argument
902 	struct revmap_entry *rev = kvm->arch.hpt.rev;  in kvm_age_rmapp()
922 		hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));  in kvm_age_rmapp()
940 			kvmppc_clear_ref_hpte(kvm, hptep, i);  in kvm_age_rmapp()
943 				note_hpte_modification(kvm, &rev[i]);  in kvm_age_rmapp()
954 bool kvm_age_gfn_hv(struct kvm *kvm, struct kvm_gfn_range *range)  in kvm_age_gfn_hv()  argument
959 	if (kvm_is_radix(kvm)) {  in kvm_age_gfn_hv()
961 			ret |= kvm_age_radix(kvm, range->slot, gfn);  in kvm_age_gfn_hv()
964 			ret |= kvm_age_rmapp(kvm, range->slot, gfn);  in kvm_age_gfn_hv()
970 static bool kvm_test_age_rmapp(struct kvm *kvm, struct kvm_memory_slot *memslot,  in kvm_test_age_rmapp()  argument
973 	struct revmap_entry *rev = kvm->arch.hpt.rev;  in kvm_test_age_rmapp()
990 			hp = (unsigned long *)(kvm->arch.hpt.virt + (i << 4));  in kvm_test_age_rmapp()
1003 bool kvm_test_age_gfn_hv(struct kvm *kvm, struct kvm_gfn_range *range)  in kvm_test_age_gfn_hv()  argument
1007 	if (kvm_is_radix(kvm))  in kvm_test_age_gfn_hv()
1008 		return kvm_test_age_radix(kvm, range->slot, range->start);  in kvm_test_age_gfn_hv()
1010 		return kvm_test_age_rmapp(kvm, range->slot, range->start);  in kvm_test_age_gfn_hv()
1013 bool kvm_set_spte_gfn_hv(struct kvm *kvm, struct kvm_gfn_range *range)  in kvm_set_spte_gfn_hv()  argument
1017 	if (kvm_is_radix(kvm))  in kvm_set_spte_gfn_hv()
1018 		kvm_unmap_radix(kvm, range->slot, range->start);  in kvm_set_spte_gfn_hv()
1020 		kvm_unmap_rmapp(kvm, range->slot, range->start);  in kvm_set_spte_gfn_hv()
1025 static int vcpus_running(struct kvm *kvm)  in vcpus_running()  argument
1027 	return atomic_read(&kvm->arch.vcpus_running) != 0;  in vcpus_running()
1034 static int kvm_test_clear_dirty_npages(struct kvm *kvm, unsigned long *rmapp)  in kvm_test_clear_dirty_npages()  argument
1036 	struct revmap_entry *rev = kvm->arch.hpt.rev;  in kvm_test_clear_dirty_npages()
1053 		hptep = (__be64 *) (kvm->arch.hpt.virt + (i << 4));  in kvm_test_clear_dirty_npages()
1072 		    (!hpte_is_writable(hptep1) || vcpus_running(kvm)))  in kvm_test_clear_dirty_npages()
1091 		kvmppc_invalidate_hpte(kvm, hptep, i);  in kvm_test_clear_dirty_npages()
1098 				note_hpte_modification(kvm, &rev[i]);  in kvm_test_clear_dirty_npages()
1133 long kvmppc_hv_get_dirty_log_hpt(struct kvm *kvm,  in kvmppc_hv_get_dirty_log_hpt()  argument
1142 		int npages = kvm_test_clear_dirty_npages(kvm, rmapp);  in kvmppc_hv_get_dirty_log_hpt()
1156 void *kvmppc_pin_guest_page(struct kvm *kvm, unsigned long gpa,  in kvmppc_pin_guest_page()  argument
1166 	srcu_idx = srcu_read_lock(&kvm->srcu);  in kvmppc_pin_guest_page()
1167 	memslot = gfn_to_memslot(kvm, gfn);  in kvmppc_pin_guest_page()
1175 	srcu_read_unlock(&kvm->srcu, srcu_idx);  in kvmppc_pin_guest_page()
1183 	srcu_read_unlock(&kvm->srcu, srcu_idx);  in kvmppc_pin_guest_page()
1187 void kvmppc_unpin_guest_page(struct kvm *kvm, void *va, unsigned long gpa,  in kvmppc_unpin_guest_page()  argument
1202 	srcu_idx = srcu_read_lock(&kvm->srcu);  in kvmppc_unpin_guest_page()
1203 	memslot = gfn_to_memslot(kvm, gfn);  in kvmppc_unpin_guest_page()
1206 	srcu_read_unlock(&kvm->srcu, srcu_idx);  in kvmppc_unpin_guest_page()
1229 	struct kvm *kvm = resize->kvm;  in resize_hpt_rehash_hpte()  local
1230 	struct kvm_hpt_info *old = &kvm->arch.hpt;  in resize_hpt_rehash_hpte()
1277 		int srcu_idx = srcu_read_lock(&kvm->srcu);  in resize_hpt_rehash_hpte()
1279 			__gfn_to_memslot(kvm_memslots(kvm), gfn);  in resize_hpt_rehash_hpte()
1286 			kvmppc_unmap_hpte(kvm, idx, memslot, rmapp, gfn);  in resize_hpt_rehash_hpte()
1290 		srcu_read_unlock(&kvm->srcu, srcu_idx);  in resize_hpt_rehash_hpte()
1383 	struct kvm *kvm = resize->kvm;  in resize_hpt_rehash()  local
1387 	for (i = 0; i < kvmppc_hpt_npte(&kvm->arch.hpt); i++) {  in resize_hpt_rehash()
1398 	struct kvm *kvm = resize->kvm;  in resize_hpt_pivot()  local
1406 	spin_lock(&kvm->mmu_lock);  in resize_hpt_pivot()
1409 	hpt_tmp = kvm->arch.hpt;  in resize_hpt_pivot()
1410 	kvmppc_set_hpt(kvm, &resize->hpt);  in resize_hpt_pivot()
1413 	spin_unlock(&kvm->mmu_lock);  in resize_hpt_pivot()
1415 	synchronize_srcu_expedited(&kvm->srcu);  in resize_hpt_pivot()
1418 		kvmppc_setup_partition_table(kvm);  in resize_hpt_pivot()
1423 static void resize_hpt_release(struct kvm *kvm, struct kvm_resize_hpt *resize)  in resize_hpt_release()  argument
1425 	if (WARN_ON(!mutex_is_locked(&kvm->arch.mmu_setup_lock)))  in resize_hpt_release()
1437 	if (kvm->arch.resize_hpt == resize)  in resize_hpt_release()
1438 		kvm->arch.resize_hpt = NULL;  in resize_hpt_release()
1446 	struct kvm *kvm = resize->kvm;  in resize_hpt_prepare_work()  local
1452 	mutex_lock(&kvm->arch.mmu_setup_lock);  in resize_hpt_prepare_work()
1455 	if (kvm->arch.resize_hpt == resize) {  in resize_hpt_prepare_work()
1459 		mutex_unlock(&kvm->arch.mmu_setup_lock);  in resize_hpt_prepare_work()
1472 		mutex_lock(&kvm->arch.mmu_setup_lock);  in resize_hpt_prepare_work()
1480 	if (kvm->arch.resize_hpt != resize)  in resize_hpt_prepare_work()
1481 		resize_hpt_release(kvm, resize);  in resize_hpt_prepare_work()
1483 	mutex_unlock(&kvm->arch.mmu_setup_lock);  in resize_hpt_prepare_work()
1486 int kvm_vm_ioctl_resize_hpt_prepare(struct kvm *kvm,  in kvm_vm_ioctl_resize_hpt_prepare()  argument
1494 	if (flags != 0 || kvm_is_radix(kvm))  in kvm_vm_ioctl_resize_hpt_prepare()
1500 	mutex_lock(&kvm->arch.mmu_setup_lock);  in kvm_vm_ioctl_resize_hpt_prepare()
1502 	resize = kvm->arch.resize_hpt;  in kvm_vm_ioctl_resize_hpt_prepare()
1511 				resize_hpt_release(kvm, resize);  in kvm_vm_ioctl_resize_hpt_prepare()
1517 		resize_hpt_release(kvm, resize);  in kvm_vm_ioctl_resize_hpt_prepare()
1534 	resize->kvm = kvm;  in kvm_vm_ioctl_resize_hpt_prepare()
1536 	kvm->arch.resize_hpt = resize;  in kvm_vm_ioctl_resize_hpt_prepare()
1543 	mutex_unlock(&kvm->arch.mmu_setup_lock);  in kvm_vm_ioctl_resize_hpt_prepare()
1552 int kvm_vm_ioctl_resize_hpt_commit(struct kvm *kvm,  in kvm_vm_ioctl_resize_hpt_commit()  argument
1560 	if (flags != 0 || kvm_is_radix(kvm))  in kvm_vm_ioctl_resize_hpt_commit()
1566 	mutex_lock(&kvm->arch.mmu_setup_lock);  in kvm_vm_ioctl_resize_hpt_commit()
1568 	resize = kvm->arch.resize_hpt;  in kvm_vm_ioctl_resize_hpt_commit()
1572 	if (WARN_ON(!kvm->arch.mmu_ready))  in kvm_vm_ioctl_resize_hpt_commit()
1576 	kvm->arch.mmu_ready = 0;  in kvm_vm_ioctl_resize_hpt_commit()
1599 	kvm->arch.mmu_ready = 1;  in kvm_vm_ioctl_resize_hpt_commit()
1602 	resize_hpt_release(kvm, resize);  in kvm_vm_ioctl_resize_hpt_commit()
1603 	mutex_unlock(&kvm->arch.mmu_setup_lock);  in kvm_vm_ioctl_resize_hpt_commit()
1626 	struct kvm	*kvm;  member
1729 	struct kvm *kvm = ctx->kvm;  in kvm_htab_read()  local
1742 	if (kvm_is_radix(kvm))  in kvm_htab_read()
1749 	hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE));  in kvm_htab_read()
1750 	revp = kvm->arch.hpt.rev + i;  in kvm_htab_read()
1765 			while (i < kvmppc_hpt_npte(&kvm->arch.hpt) &&  in kvm_htab_read()
1775 		while (i < kvmppc_hpt_npte(&kvm->arch.hpt) &&  in kvm_htab_read()
1791 		while (i < kvmppc_hpt_npte(&kvm->arch.hpt) &&  in kvm_htab_read()
1812 		if (i >= kvmppc_hpt_npte(&kvm->arch.hpt)) {  in kvm_htab_read()
1828 	struct kvm *kvm = ctx->kvm;  in kvm_htab_write()  local
1842 	if (kvm_is_radix(kvm))  in kvm_htab_write()
1846 	mutex_lock(&kvm->arch.mmu_setup_lock);  in kvm_htab_write()
1847 	mmu_ready = kvm->arch.mmu_ready;  in kvm_htab_write()
1849 		kvm->arch.mmu_ready = 0;	/* temporarily */  in kvm_htab_write()
1852 		if (atomic_read(&kvm->arch.vcpus_running)) {  in kvm_htab_write()
1853 			kvm->arch.mmu_ready = 1;  in kvm_htab_write()
1854 			mutex_unlock(&kvm->arch.mmu_setup_lock);  in kvm_htab_write()
1874 		if (i >= kvmppc_hpt_npte(&kvm->arch.hpt) ||  in kvm_htab_write()
1875 		    i + hdr.n_valid + hdr.n_invalid > kvmppc_hpt_npte(&kvm->arch.hpt))  in kvm_htab_write()
1878 		hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE));  in kvm_htab_write()
1900 				kvmppc_do_h_remove(kvm, 0, i, 0, tmp);  in kvm_htab_write()
1902 			ret = kvmppc_virtmode_do_h_enter(kvm, H_EXACT, i, v, r,  in kvm_htab_write()
1912 				kvm->arch.vrma_slb_v = senc | SLB_VSID_B_1T |  in kvm_htab_write()
1916 					kvmppc_update_lpcr(kvm, lpcr,  in kvm_htab_write()
1919 					kvmppc_setup_partition_table(kvm);  in kvm_htab_write()
1929 				kvmppc_do_h_remove(kvm, 0, i, 0, tmp);  in kvm_htab_write()
1939 	kvm->arch.mmu_ready = mmu_ready;  in kvm_htab_write()
1940 	mutex_unlock(&kvm->arch.mmu_setup_lock);  in kvm_htab_write()
1953 		atomic_dec(&ctx->kvm->arch.hpte_mod_interest);  in kvm_htab_release()
1954 	kvm_put_kvm(ctx->kvm);  in kvm_htab_release()
1966 int kvm_vm_ioctl_get_htab_fd(struct kvm *kvm, struct kvm_get_htab_fd *ghf)  in kvm_vm_ioctl_get_htab_fd()  argument
1978 	kvm_get_kvm(kvm);  in kvm_vm_ioctl_get_htab_fd()
1979 	ctx->kvm = kvm;  in kvm_vm_ioctl_get_htab_fd()
1988 		kvm_put_kvm_no_destroy(kvm);  in kvm_vm_ioctl_get_htab_fd()
1993 		mutex_lock(&kvm->slots_lock);  in kvm_vm_ioctl_get_htab_fd()
1994 		atomic_inc(&kvm->arch.hpte_mod_interest);  in kvm_vm_ioctl_get_htab_fd()
1996 		synchronize_srcu_expedited(&kvm->srcu);  in kvm_vm_ioctl_get_htab_fd()
1997 		mutex_unlock(&kvm->slots_lock);  in kvm_vm_ioctl_get_htab_fd()
2004 	struct kvm	*kvm;  member
2014 	struct kvm *kvm = inode->i_private;  in debugfs_htab_open()  local
2021 	kvm_get_kvm(kvm);  in debugfs_htab_open()
2022 	p->kvm = kvm;  in debugfs_htab_open()
2033 	kvm_put_kvm(p->kvm);  in debugfs_htab_release()
2045 	struct kvm *kvm;  in debugfs_htab_read()  local
2048 	kvm = p->kvm;  in debugfs_htab_read()
2049 	if (kvm_is_radix(kvm))  in debugfs_htab_read()
2075 	hptp = (__be64 *)(kvm->arch.hpt.virt + (i * HPTE_SIZE));  in debugfs_htab_read()
2076 	for (; len != 0 && i < kvmppc_hpt_npte(&kvm->arch.hpt);  in debugfs_htab_read()
2087 		gr = kvm->arch.hpt.rev[i].guest_rpte;  in debugfs_htab_read()
2135 void kvmppc_mmu_debugfs_init(struct kvm *kvm)  in kvmppc_mmu_debugfs_init()  argument
2137 	debugfs_create_file("htab", 0400, kvm->debugfs_dentry, kvm,  in kvmppc_mmu_debugfs_init()