kvm_main.c (ed922739c9199bf515a3e7fec3e319ce1edeef2a) kvm_main.c (a54d806688fe1e482350ce759a8a0fc9ebf814b0)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Kernel-based Virtual Machine driver for Linux
4 *
5 * This module enables machines with Intel VT-x extensions to run virtual
6 * machines without emulation or binary translation.
7 *
8 * Copyright (C) 2006 Qumranet, Inc.

--- 419 unchanged lines hidden (view full) ---

428 vcpu->pre_pcpu = -1;
429 INIT_LIST_HEAD(&vcpu->blocked_vcpu_list);
430
431 kvm_vcpu_set_in_spin_loop(vcpu, false);
432 kvm_vcpu_set_dy_eligible(vcpu, false);
433 vcpu->preempted = false;
434 vcpu->ready = false;
435 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Kernel-based Virtual Machine driver for Linux
4 *
5 * This module enables machines with Intel VT-x extensions to run virtual
6 * machines without emulation or binary translation.
7 *
8 * Copyright (C) 2006 Qumranet, Inc.

--- 419 unchanged lines hidden (view full) ---

428 vcpu->pre_pcpu = -1;
429 INIT_LIST_HEAD(&vcpu->blocked_vcpu_list);
430
431 kvm_vcpu_set_in_spin_loop(vcpu, false);
432 kvm_vcpu_set_dy_eligible(vcpu, false);
433 vcpu->preempted = false;
434 vcpu->ready = false;
435 preempt_notifier_init(&vcpu->preempt_notifier, &kvm_preempt_ops);
436 vcpu->last_used_slot = 0;
436 vcpu->last_used_slot = NULL;
437}
438
439static void kvm_vcpu_destroy(struct kvm_vcpu *vcpu)
440{
441 kvm_dirty_ring_free(&vcpu->dirty_ring);
442 kvm_arch_vcpu_destroy(vcpu);
443
444 /*

--- 95 unchanged lines hidden (view full) ---

540 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
541 struct interval_tree_node *node;
542
543 slots = __kvm_memslots(kvm, i);
544 kvm_for_each_memslot_in_hva_range(node, slots,
545 range->start, range->end - 1) {
546 unsigned long hva_start, hva_end;
547
437}
438
439static void kvm_vcpu_destroy(struct kvm_vcpu *vcpu)
440{
441 kvm_dirty_ring_free(&vcpu->dirty_ring);
442 kvm_arch_vcpu_destroy(vcpu);
443
444 /*

--- 95 unchanged lines hidden (view full) ---

540 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
541 struct interval_tree_node *node;
542
543 slots = __kvm_memslots(kvm, i);
544 kvm_for_each_memslot_in_hva_range(node, slots,
545 range->start, range->end - 1) {
546 unsigned long hva_start, hva_end;
547
548 slot = container_of(node, struct kvm_memory_slot, hva_node);
548 slot = container_of(node, struct kvm_memory_slot, hva_node[slots->node_idx]);
549 hva_start = max(range->start, slot->userspace_addr);
550 hva_end = min(range->end, slot->userspace_addr +
551 (slot->npages << PAGE_SHIFT));
552
553 /*
554 * To optimize for the likely case where the address
555 * range is covered by zero or one memslots, don't
556 * bother making these conditional (to avoid writes on

--- 314 unchanged lines hidden (view full) ---

871{
872}
873
874static void kvm_destroy_pm_notifier(struct kvm *kvm)
875{
876}
877#endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */
878
549 hva_start = max(range->start, slot->userspace_addr);
550 hva_end = min(range->end, slot->userspace_addr +
551 (slot->npages << PAGE_SHIFT));
552
553 /*
554 * To optimize for the likely case where the address
555 * range is covered by zero or one memslots, don't
556 * bother making these conditional (to avoid writes on

--- 314 unchanged lines hidden (view full) ---

871{
872}
873
874static void kvm_destroy_pm_notifier(struct kvm *kvm)
875{
876}
877#endif /* CONFIG_HAVE_KVM_PM_NOTIFIER */
878
879static struct kvm_memslots *kvm_alloc_memslots(void)
880{
881 struct kvm_memslots *slots;
882
883 slots = kvzalloc(sizeof(struct kvm_memslots), GFP_KERNEL_ACCOUNT);
884 if (!slots)
885 return NULL;
886
887 slots->hva_tree = RB_ROOT_CACHED;
888 hash_init(slots->id_hash);
889
890 return slots;
891}
892
893static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
894{
895 if (!memslot->dirty_bitmap)
896 return;
897
898 kvfree(memslot->dirty_bitmap);
899 memslot->dirty_bitmap = NULL;
900}
901
879static void kvm_destroy_dirty_bitmap(struct kvm_memory_slot *memslot)
880{
881 if (!memslot->dirty_bitmap)
882 return;
883
884 kvfree(memslot->dirty_bitmap);
885 memslot->dirty_bitmap = NULL;
886}
887
888/* This does not remove the slot from struct kvm_memslots data structures */
902static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
903{
904 kvm_destroy_dirty_bitmap(slot);
905
906 kvm_arch_free_memslot(kvm, slot);
907
889static void kvm_free_memslot(struct kvm *kvm, struct kvm_memory_slot *slot)
890{
891 kvm_destroy_dirty_bitmap(slot);
892
893 kvm_arch_free_memslot(kvm, slot);
894
908 slot->flags = 0;
909 slot->npages = 0;
895 kfree(slot);
910}
911
912static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
913{
896}
897
898static void kvm_free_memslots(struct kvm *kvm, struct kvm_memslots *slots)
899{
900 struct hlist_node *idnode;
914 struct kvm_memory_slot *memslot;
901 struct kvm_memory_slot *memslot;
902 int bkt;
915
903
916 if (!slots)
904 /*
905 * The same memslot objects live in both active and inactive sets,
906 * arbitrarily free using index '1' so the second invocation of this
907 * function isn't operating over a structure with dangling pointers
908 * (even though this function isn't actually touching them).
909 */
910 if (!slots->node_idx)
917 return;
918
911 return;
912
919 kvm_for_each_memslot(memslot, slots)
913 hash_for_each_safe(slots->id_hash, bkt, idnode, memslot, id_node[1])
920 kvm_free_memslot(kvm, memslot);
914 kvm_free_memslot(kvm, memslot);
921
922 kvfree(slots);
923}
924
925static umode_t kvm_stats_debugfs_mode(const struct _kvm_stats_desc *pdesc)
926{
927 switch (pdesc->desc.flags & KVM_STATS_TYPE_MASK) {
928 case KVM_STATS_TYPE_INSTANT:
929 return 0444;
930 case KVM_STATS_TYPE_CUMULATIVE:

--- 122 unchanged lines hidden (view full) ---

1053int __weak kvm_arch_create_vm_debugfs(struct kvm *kvm)
1054{
1055 return 0;
1056}
1057
1058static struct kvm *kvm_create_vm(unsigned long type)
1059{
1060 struct kvm *kvm = kvm_arch_alloc_vm();
915}
916
917static umode_t kvm_stats_debugfs_mode(const struct _kvm_stats_desc *pdesc)
918{
919 switch (pdesc->desc.flags & KVM_STATS_TYPE_MASK) {
920 case KVM_STATS_TYPE_INSTANT:
921 return 0444;
922 case KVM_STATS_TYPE_CUMULATIVE:

--- 122 unchanged lines hidden (view full) ---

1045int __weak kvm_arch_create_vm_debugfs(struct kvm *kvm)
1046{
1047 return 0;
1048}
1049
1050static struct kvm *kvm_create_vm(unsigned long type)
1051{
1052 struct kvm *kvm = kvm_arch_alloc_vm();
1053 struct kvm_memslots *slots;
1061 int r = -ENOMEM;
1054 int r = -ENOMEM;
1062 int i;
1055 int i, j;
1063
1064 if (!kvm)
1065 return ERR_PTR(-ENOMEM);
1066
1067 KVM_MMU_LOCK_INIT(kvm);
1068 mmgrab(current->mm);
1069 kvm->mm = current->mm;
1070 kvm_eventfd_init(kvm);

--- 11 unchanged lines hidden (view full) ---

1082
1083 if (init_srcu_struct(&kvm->srcu))
1084 goto out_err_no_srcu;
1085 if (init_srcu_struct(&kvm->irq_srcu))
1086 goto out_err_no_irq_srcu;
1087
1088 refcount_set(&kvm->users_count, 1);
1089 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
1056
1057 if (!kvm)
1058 return ERR_PTR(-ENOMEM);
1059
1060 KVM_MMU_LOCK_INIT(kvm);
1061 mmgrab(current->mm);
1062 kvm->mm = current->mm;
1063 kvm_eventfd_init(kvm);

--- 11 unchanged lines hidden (view full) ---

1075
1076 if (init_srcu_struct(&kvm->srcu))
1077 goto out_err_no_srcu;
1078 if (init_srcu_struct(&kvm->irq_srcu))
1079 goto out_err_no_irq_srcu;
1080
1081 refcount_set(&kvm->users_count, 1);
1082 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
1090 struct kvm_memslots *slots = kvm_alloc_memslots();
1083 for (j = 0; j < 2; j++) {
1084 slots = &kvm->__memslots[i][j];
1091
1085
1092 if (!slots)
1093 goto out_err_no_arch_destroy_vm;
1094 /* Generations must be different for each address space. */
1095 slots->generation = i;
1096 rcu_assign_pointer(kvm->memslots[i], slots);
1086 atomic_long_set(&slots->last_used_slot, (unsigned long)NULL);
1087 slots->hva_tree = RB_ROOT_CACHED;
1088 slots->gfn_tree = RB_ROOT;
1089 hash_init(slots->id_hash);
1090 slots->node_idx = j;
1091
1092 /* Generations must be different for each address space. */
1093 slots->generation = i;
1094 }
1095
1096 rcu_assign_pointer(kvm->memslots[i], &kvm->__memslots[i][0]);
1097 }
1098
1099 for (i = 0; i < KVM_NR_BUSES; i++) {
1100 rcu_assign_pointer(kvm->buses[i],
1101 kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL_ACCOUNT));
1102 if (!kvm->buses[i])
1103 goto out_err_no_arch_destroy_vm;
1104 }

--- 37 unchanged lines hidden (view full) ---

1142out_err_no_mmu_notifier:
1143 hardware_disable_all();
1144out_err_no_disable:
1145 kvm_arch_destroy_vm(kvm);
1146out_err_no_arch_destroy_vm:
1147 WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count));
1148 for (i = 0; i < KVM_NR_BUSES; i++)
1149 kfree(kvm_get_bus(kvm, i));
1097 }
1098
1099 for (i = 0; i < KVM_NR_BUSES; i++) {
1100 rcu_assign_pointer(kvm->buses[i],
1101 kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL_ACCOUNT));
1102 if (!kvm->buses[i])
1103 goto out_err_no_arch_destroy_vm;
1104 }

--- 37 unchanged lines hidden (view full) ---

1142out_err_no_mmu_notifier:
1143 hardware_disable_all();
1144out_err_no_disable:
1145 kvm_arch_destroy_vm(kvm);
1146out_err_no_arch_destroy_vm:
1147 WARN_ON_ONCE(!refcount_dec_and_test(&kvm->users_count));
1148 for (i = 0; i < KVM_NR_BUSES; i++)
1149 kfree(kvm_get_bus(kvm, i));
1150 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
1151 kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
1152 cleanup_srcu_struct(&kvm->irq_srcu);
1153out_err_no_irq_srcu:
1154 cleanup_srcu_struct(&kvm->srcu);
1155out_err_no_srcu:
1156 kvm_arch_free_vm(kvm);
1157 mmdrop(current->mm);
1158 return ERR_PTR(r);
1159}

--- 48 unchanged lines hidden (view full) ---

1208 */
1209 WARN_ON(rcuwait_active(&kvm->mn_memslots_update_rcuwait));
1210 kvm->mn_active_invalidate_count = 0;
1211#else
1212 kvm_arch_flush_shadow_all(kvm);
1213#endif
1214 kvm_arch_destroy_vm(kvm);
1215 kvm_destroy_devices(kvm);
1150 cleanup_srcu_struct(&kvm->irq_srcu);
1151out_err_no_irq_srcu:
1152 cleanup_srcu_struct(&kvm->srcu);
1153out_err_no_srcu:
1154 kvm_arch_free_vm(kvm);
1155 mmdrop(current->mm);
1156 return ERR_PTR(r);
1157}

--- 48 unchanged lines hidden (view full) ---

1206 */
1207 WARN_ON(rcuwait_active(&kvm->mn_memslots_update_rcuwait));
1208 kvm->mn_active_invalidate_count = 0;
1209#else
1210 kvm_arch_flush_shadow_all(kvm);
1211#endif
1212 kvm_arch_destroy_vm(kvm);
1213 kvm_destroy_devices(kvm);
1216 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++)
1217 kvm_free_memslots(kvm, __kvm_memslots(kvm, i));
1214 for (i = 0; i < KVM_ADDRESS_SPACE_NUM; i++) {
1215 kvm_free_memslots(kvm, &kvm->__memslots[i][0]);
1216 kvm_free_memslots(kvm, &kvm->__memslots[i][1]);
1217 }
1218 cleanup_srcu_struct(&kvm->irq_srcu);
1219 cleanup_srcu_struct(&kvm->srcu);
1220 kvm_arch_free_vm(kvm);
1221 preempt_notifier_dec();
1222 hardware_disable_all();
1223 mmdrop(mm);
1224}
1225

--- 53 unchanged lines hidden (view full) ---

1279
1280 memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL_ACCOUNT);
1281 if (!memslot->dirty_bitmap)
1282 return -ENOMEM;
1283
1284 return 0;
1285}
1286
1218 cleanup_srcu_struct(&kvm->irq_srcu);
1219 cleanup_srcu_struct(&kvm->srcu);
1220 kvm_arch_free_vm(kvm);
1221 preempt_notifier_dec();
1222 hardware_disable_all();
1223 mmdrop(mm);
1224}
1225

--- 53 unchanged lines hidden (view full) ---

1279
1280 memslot->dirty_bitmap = kvzalloc(dirty_bytes, GFP_KERNEL_ACCOUNT);
1281 if (!memslot->dirty_bitmap)
1282 return -ENOMEM;
1283
1284 return 0;
1285}
1286
1287static void kvm_replace_memslot(struct kvm_memslots *slots,
1288 struct kvm_memory_slot *old,
1289 struct kvm_memory_slot *new)
1287static struct kvm_memslots *kvm_get_inactive_memslots(struct kvm *kvm, int as_id)
1290{
1288{
1291 /*
1292 * Remove the old memslot from the hash list and interval tree, copying
1293 * the node data would corrupt the structures.
1294 */
1295 if (old) {
1296 hash_del(&old->id_node);
1297 interval_tree_remove(&old->hva_node, &slots->hva_tree);
1289 struct kvm_memslots *active = __kvm_memslots(kvm, as_id);
1290 int node_idx_inactive = active->node_idx ^ 1;
1298
1291
1299 if (!new)
1300 return;
1301
1302 /* Copy the source *data*, not the pointer, to the destination. */
1303 *new = *old;
1304 } else {
1305 /* If @old is NULL, initialize @new's hva range. */
1306 new->hva_node.start = new->userspace_addr;
1307 new->hva_node.last = new->userspace_addr +
1308 (new->npages << PAGE_SHIFT) - 1;
1309 }
1310
1311 /* (Re)Add the new memslot. */
1312 hash_add(slots->id_hash, &new->id_node, new->id);
1313 interval_tree_insert(&new->hva_node, &slots->hva_tree);
1292 return &kvm->__memslots[as_id][node_idx_inactive];
1314}
1315
1293}
1294
1316static void kvm_shift_memslot(struct kvm_memslots *slots, int dst, int src)
1317{
1318 struct kvm_memory_slot *mslots = slots->memslots;
1319
1320 kvm_replace_memslot(slots, &mslots[src], &mslots[dst]);
1321}
1322
1323/*
1295/*
1324 * Delete a memslot by decrementing the number of used slots and shifting all
1325 * other entries in the array forward one spot.
1326 * @memslot is a detached dummy struct with just .id and .as_id filled.
1296 * Helper to get the address space ID when one of memslot pointers may be NULL.
1297 * This also serves as a sanity that at least one of the pointers is non-NULL,
1298 * and that their address space IDs don't diverge.
1327 */
1299 */
1328static inline void kvm_memslot_delete(struct kvm_memslots *slots,
1329 struct kvm_memory_slot *memslot)
1300static int kvm_memslots_get_as_id(struct kvm_memory_slot *a,
1301 struct kvm_memory_slot *b)
1330{
1302{
1331 struct kvm_memory_slot *mslots = slots->memslots;
1332 struct kvm_memory_slot *oldslot = id_to_memslot(slots, memslot->id);
1333 int i;
1303 if (WARN_ON_ONCE(!a && !b))
1304 return 0;
1334
1305
1335 if (WARN_ON(!oldslot))
1336 return;
1306 if (!a)
1307 return b->as_id;
1308 if (!b)
1309 return a->as_id;
1337
1310
1338 slots->used_slots--;
1339
1340 if (atomic_read(&slots->last_used_slot) >= slots->used_slots)
1341 atomic_set(&slots->last_used_slot, 0);
1342
1343 /*
1344 * Remove the to-be-deleted memslot from the list/tree _before_ shifting
1345 * the trailing memslots forward, its data will be overwritten.
1346 * Defer the (somewhat pointless) copying of the memslot until after
1347 * the last slot has been shifted to avoid overwriting said last slot.
1348 */
1349 kvm_replace_memslot(slots, oldslot, NULL);
1350
1351 for (i = oldslot - mslots; i < slots->used_slots; i++)
1352 kvm_shift_memslot(slots, i, i + 1);
1353 mslots[i] = *memslot;
1311 WARN_ON_ONCE(a->as_id != b->as_id);
1312 return a->as_id;
1354}
1355
1313}
1314
1356/*
1357 * "Insert" a new memslot by incrementing the number of used slots. Returns
1358 * the new slot's initial index into the memslots array.
1359 */
1360static inline int kvm_memslot_insert_back(struct kvm_memslots *slots)
1315static void kvm_insert_gfn_node(struct kvm_memslots *slots,
1316 struct kvm_memory_slot *slot)
1361{
1317{
1362 return slots->used_slots++;
1363}
1318 struct rb_root *gfn_tree = &slots->gfn_tree;
1319 struct rb_node **node, *parent;
1320 int idx = slots->node_idx;
1364
1321
1365/*
1366 * Move a changed memslot backwards in the array by shifting existing slots
1367 * with a higher GFN toward the front of the array. Note, the changed memslot
1368 * itself is not preserved in the array, i.e. not swapped at this time, only
1369 * its new index into the array is tracked. Returns the changed memslot's
1370 * current index into the memslots array.
1371 * The memslot at the returned index will not be in @slots->hva_tree or
1372 * @slots->id_hash by then.
1373 * @memslot is a detached struct with desired final data of the changed slot.
1374 */
1375static inline int kvm_memslot_move_backward(struct kvm_memslots *slots,
1376 struct kvm_memory_slot *memslot)
1377{
1378 struct kvm_memory_slot *mslots = slots->memslots;
1379 struct kvm_memory_slot *oldslot = id_to_memslot(slots, memslot->id);
1380 int i;
1322 parent = NULL;
1323 for (node = &gfn_tree->rb_node; *node; ) {
1324 struct kvm_memory_slot *tmp;
1381
1325
1382 if (!oldslot || !slots->used_slots)
1383 return -1;
1326 tmp = container_of(*node, struct kvm_memory_slot, gfn_node[idx]);
1327 parent = *node;
1328 if (slot->base_gfn < tmp->base_gfn)
1329 node = &(*node)->rb_left;
1330 else if (slot->base_gfn > tmp->base_gfn)
1331 node = &(*node)->rb_right;
1332 else
1333 BUG();
1334 }
1384
1335
1385 /*
1386 * Delete the slot from the hash table and interval tree before sorting
1387 * the remaining slots, the slot's data may be overwritten when copying
1388 * slots as part of the sorting proccess. update_memslots() will
1389 * unconditionally rewrite and re-add the entire slot.
1390 */
1391 kvm_replace_memslot(slots, oldslot, NULL);
1336 rb_link_node(&slot->gfn_node[idx], parent, node);
1337 rb_insert_color(&slot->gfn_node[idx], gfn_tree);
1338}
1392
1339
1393 /*
1394 * Move the target memslot backward in the array by shifting existing
1395 * memslots with a higher GFN (than the target memslot) towards the
1396 * front of the array.
1397 */
1398 for (i = oldslot - mslots; i < slots->used_slots - 1; i++) {
1399 if (memslot->base_gfn > mslots[i + 1].base_gfn)
1400 break;
1401
1402 WARN_ON_ONCE(memslot->base_gfn == mslots[i + 1].base_gfn);
1403
1404 kvm_shift_memslot(slots, i, i + 1);
1405 }
1406 return i;
1340static void kvm_erase_gfn_node(struct kvm_memslots *slots,
1341 struct kvm_memory_slot *slot)
1342{
1343 rb_erase(&slot->gfn_node[slots->node_idx], &slots->gfn_tree);
1407}
1408
1344}
1345
1409/*
1410 * Move a changed memslot forwards in the array by shifting existing slots with
1411 * a lower GFN toward the back of the array. Note, the changed memslot itself
1412 * is not preserved in the array, i.e. not swapped at this time, only its new
1413 * index into the array is tracked. Returns the changed memslot's final index
1414 * into the memslots array.
1415 * The memslot at the returned index will not be in @slots->hva_tree or
1416 * @slots->id_hash by then.
1417 * @memslot is a detached struct with desired final data of the new or
1418 * changed slot.
1419 * Assumes that the memslot at @start index is not in @slots->hva_tree or
1420 * @slots->id_hash.
1421 */
1422static inline int kvm_memslot_move_forward(struct kvm_memslots *slots,
1423 struct kvm_memory_slot *memslot,
1424 int start)
1346static void kvm_replace_gfn_node(struct kvm_memslots *slots,
1347 struct kvm_memory_slot *old,
1348 struct kvm_memory_slot *new)
1425{
1349{
1426 struct kvm_memory_slot *mslots = slots->memslots;
1427 int i;
1350 int idx = slots->node_idx;
1428
1351
1429 for (i = start; i > 0; i--) {
1430 if (memslot->base_gfn < mslots[i - 1].base_gfn)
1431 break;
1352 WARN_ON_ONCE(old->base_gfn != new->base_gfn);
1432
1353
1433 WARN_ON_ONCE(memslot->base_gfn == mslots[i - 1].base_gfn);
1434
1435 kvm_shift_memslot(slots, i, i - 1);
1436 }
1437 return i;
1354 rb_replace_node(&old->gfn_node[idx], &new->gfn_node[idx],
1355 &slots->gfn_tree);
1438}
1439
1440/*
1356}
1357
1358/*
1441 * Re-sort memslots based on their GFN to account for an added, deleted, or
1442 * moved memslot. Sorting memslots by GFN allows using a binary search during
1443 * memslot lookup.
1359 * Replace @old with @new in the inactive memslots.
1444 *
1360 *
1445 * IMPORTANT: Slots are sorted from highest GFN to lowest GFN! I.e. the entry
1446 * at memslots[0] has the highest GFN.
1361 * With NULL @old this simply adds @new.
1362 * With NULL @new this simply removes @old.
1447 *
1363 *
1448 * The sorting algorithm takes advantage of having initially sorted memslots
1449 * and knowing the position of the changed memslot. Sorting is also optimized
1450 * by not swapping the updated memslot and instead only shifting other memslots
1451 * and tracking the new index for the update memslot. Only once its final
1452 * index is known is the updated memslot copied into its position in the array.
1453 *
1454 * - When deleting a memslot, the deleted memslot simply needs to be moved to
1455 * the end of the array.
1456 *
1457 * - When creating a memslot, the algorithm "inserts" the new memslot at the
1458 * end of the array and then it forward to its correct location.
1459 *
1460 * - When moving a memslot, the algorithm first moves the updated memslot
1461 * backward to handle the scenario where the memslot's GFN was changed to a
1462 * lower value. update_memslots() then falls through and runs the same flow
1463 * as creating a memslot to move the memslot forward to handle the scenario
1464 * where its GFN was changed to a higher value.
1465 *
1466 * Note, slots are sorted from highest->lowest instead of lowest->highest for
1467 * historical reasons. Originally, invalid memslots where denoted by having
1468 * GFN=0, thus sorting from highest->lowest naturally sorted invalid memslots
1469 * to the end of the array. The current algorithm uses dedicated logic to
1470 * delete a memslot and thus does not rely on invalid memslots having GFN=0.
1471 *
1472 * The other historical motiviation for highest->lowest was to improve the
1473 * performance of memslot lookup. KVM originally used a linear search starting
1474 * at memslots[0]. On x86, the largest memslot usually has one of the highest,
1475 * if not *the* highest, GFN, as the bulk of the guest's RAM is located in a
1476 * single memslot above the 4gb boundary. As the largest memslot is also the
1477 * most likely to be referenced, sorting it to the front of the array was
1478 * advantageous. The current binary search starts from the middle of the array
1479 * and uses an LRU pointer to improve performance for all memslots and GFNs.
1480 *
1481 * @memslot is a detached struct, not a part of the current or new memslot
1482 * array.
1364 * If @new is non-NULL its hva_node[slots_idx] range has to be set
1365 * appropriately.
1483 */
1366 */
1484static void update_memslots(struct kvm_memslots *slots,
1485 struct kvm_memory_slot *memslot,
1486 enum kvm_mr_change change)
1367static void kvm_replace_memslot(struct kvm *kvm,
1368 struct kvm_memory_slot *old,
1369 struct kvm_memory_slot *new)
1487{
1370{
1488 int i;
1371 int as_id = kvm_memslots_get_as_id(old, new);
1372 struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id);
1373 int idx = slots->node_idx;
1489
1374
1490 if (change == KVM_MR_DELETE) {
1491 kvm_memslot_delete(slots, memslot);
1492 } else {
1493 if (change == KVM_MR_CREATE)
1494 i = kvm_memslot_insert_back(slots);
1495 else
1496 i = kvm_memslot_move_backward(slots, memslot);
1497 i = kvm_memslot_move_forward(slots, memslot, i);
1375 if (old) {
1376 hash_del(&old->id_node[idx]);
1377 interval_tree_remove(&old->hva_node[idx], &slots->hva_tree);
1498
1378
1499 if (WARN_ON_ONCE(i < 0))
1379 if ((long)old == atomic_long_read(&slots->last_used_slot))
1380 atomic_long_set(&slots->last_used_slot, (long)new);
1381
1382 if (!new) {
1383 kvm_erase_gfn_node(slots, old);
1500 return;
1384 return;
1385 }
1386 }
1501
1387
1502 /*
1503 * Copy the memslot to its new position in memslots and update
1504 * its index accordingly.
1505 */
1506 slots->memslots[i] = *memslot;
1507 kvm_replace_memslot(slots, NULL, &slots->memslots[i]);
1388 /*
1389 * Initialize @new's hva range. Do this even when replacing an @old
1390 * slot, kvm_copy_memslot() deliberately does not touch node data.
1391 */
1392 new->hva_node[idx].start = new->userspace_addr;
1393 new->hva_node[idx].last = new->userspace_addr +
1394 (new->npages << PAGE_SHIFT) - 1;
1395
1396 /*
1397 * (Re)Add the new memslot. There is no O(1) interval_tree_replace(),
1398 * hva_node needs to be swapped with remove+insert even though hva can't
1399 * change when replacing an existing slot.
1400 */
1401 hash_add(slots->id_hash, &new->id_node[idx], new->id);
1402 interval_tree_insert(&new->hva_node[idx], &slots->hva_tree);
1403
1404 /*
1405 * If the memslot gfn is unchanged, rb_replace_node() can be used to
1406 * switch the node in the gfn tree instead of removing the old and
1407 * inserting the new as two separate operations. Replacement is a
1408 * single O(1) operation versus two O(log(n)) operations for
1409 * remove+insert.
1410 */
1411 if (old && old->base_gfn == new->base_gfn) {
1412 kvm_replace_gfn_node(slots, old, new);
1413 } else {
1414 if (old)
1415 kvm_erase_gfn_node(slots, old);
1416 kvm_insert_gfn_node(slots, new);
1508 }
1509}
1510
1511static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
1512{
1513 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
1514
1515#ifdef __KVM_HAVE_READONLY_MEM
1516 valid_flags |= KVM_MEM_READONLY;
1517#endif
1518
1519 if (mem->flags & ~valid_flags)
1520 return -EINVAL;
1521
1522 return 0;
1523}
1524
1417 }
1418}
1419
1420static int check_memory_region_flags(const struct kvm_userspace_memory_region *mem)
1421{
1422 u32 valid_flags = KVM_MEM_LOG_DIRTY_PAGES;
1423
1424#ifdef __KVM_HAVE_READONLY_MEM
1425 valid_flags |= KVM_MEM_READONLY;
1426#endif
1427
1428 if (mem->flags & ~valid_flags)
1429 return -EINVAL;
1430
1431 return 0;
1432}
1433
1525static struct kvm_memslots *install_new_memslots(struct kvm *kvm,
1526 int as_id, struct kvm_memslots *slots)
1434static void kvm_swap_active_memslots(struct kvm *kvm, int as_id)
1527{
1435{
1528 struct kvm_memslots *old_memslots = __kvm_memslots(kvm, as_id);
1529 u64 gen = old_memslots->generation;
1436 struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, as_id);
1530
1437
1438 /* Grab the generation from the activate memslots. */
1439 u64 gen = __kvm_memslots(kvm, as_id)->generation;
1440
1531 WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
1532 slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
1533
1534 /*
1535 * Do not store the new memslots while there are invalidations in
1536 * progress, otherwise the locking in invalidate_range_start and
1537 * invalidate_range_end will be unbalanced.
1538 */

--- 33 unchanged lines hidden (view full) ---

1572 * space 0 will use generations 0, 2, 4, ... while address space 1 will
1573 * use generations 1, 3, 5, ...
1574 */
1575 gen += KVM_ADDRESS_SPACE_NUM;
1576
1577 kvm_arch_memslots_updated(kvm, gen);
1578
1579 slots->generation = gen;
1441 WARN_ON(gen & KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS);
1442 slots->generation = gen | KVM_MEMSLOT_GEN_UPDATE_IN_PROGRESS;
1443
1444 /*
1445 * Do not store the new memslots while there are invalidations in
1446 * progress, otherwise the locking in invalidate_range_start and
1447 * invalidate_range_end will be unbalanced.
1448 */

--- 33 unchanged lines hidden (view full) ---

1482 * space 0 will use generations 0, 2, 4, ... while address space 1 will
1483 * use generations 1, 3, 5, ...
1484 */
1485 gen += KVM_ADDRESS_SPACE_NUM;
1486
1487 kvm_arch_memslots_updated(kvm, gen);
1488
1489 slots->generation = gen;
1580
1581 return old_memslots;
1582}
1583
1490}
1491
1584static size_t kvm_memslots_size(int slots)
1585{
1586 return sizeof(struct kvm_memslots) +
1587 (sizeof(struct kvm_memory_slot) * slots);
1588}
1589
1590/*
1591 * Note, at a minimum, the current number of used slots must be allocated, even
1592 * when deleting a memslot, as we need a complete duplicate of the memslots for
1593 * use when invalidating a memslot prior to deleting/moving the memslot.
1594 */
1595static struct kvm_memslots *kvm_dup_memslots(struct kvm_memslots *old,
1596 enum kvm_mr_change change)
1597{
1598 struct kvm_memslots *slots;
1599 size_t new_size;
1600 struct kvm_memory_slot *memslot;
1601
1602 if (change == KVM_MR_CREATE)
1603 new_size = kvm_memslots_size(old->used_slots + 1);
1604 else
1605 new_size = kvm_memslots_size(old->used_slots);
1606
1607 slots = kvzalloc(new_size, GFP_KERNEL_ACCOUNT);
1608 if (unlikely(!slots))
1609 return NULL;
1610
1611 memcpy(slots, old, kvm_memslots_size(old->used_slots));
1612
1613 slots->hva_tree = RB_ROOT_CACHED;
1614 hash_init(slots->id_hash);
1615 kvm_for_each_memslot(memslot, slots) {
1616 interval_tree_insert(&memslot->hva_node, &slots->hva_tree);
1617 hash_add(slots->id_hash, &memslot->id_node, memslot->id);
1618 }
1619
1620 return slots;
1621}
1622
1623static void kvm_copy_memslots_arch(struct kvm_memslots *to,
1624 struct kvm_memslots *from)
1625{
1626 int i;
1627
1628 WARN_ON_ONCE(to->used_slots != from->used_slots);
1629
1630 for (i = 0; i < from->used_slots; i++)
1631 to->memslots[i].arch = from->memslots[i].arch;
1632}
1633
1634static int kvm_prepare_memory_region(struct kvm *kvm,
1635 const struct kvm_memory_slot *old,
1636 struct kvm_memory_slot *new,
1637 enum kvm_mr_change change)
1638{
1639 int r;
1640
1641 /*

--- 36 unchanged lines hidden (view full) ---

1678 */
1679 if (change == KVM_MR_DELETE)
1680 kvm->nr_memslot_pages -= old->npages;
1681 else if (change == KVM_MR_CREATE)
1682 kvm->nr_memslot_pages += new->npages;
1683
1684 kvm_arch_commit_memory_region(kvm, old, new, change);
1685
1492static int kvm_prepare_memory_region(struct kvm *kvm,
1493 const struct kvm_memory_slot *old,
1494 struct kvm_memory_slot *new,
1495 enum kvm_mr_change change)
1496{
1497 int r;
1498
1499 /*

--- 36 unchanged lines hidden (view full) ---

1536 */
1537 if (change == KVM_MR_DELETE)
1538 kvm->nr_memslot_pages -= old->npages;
1539 else if (change == KVM_MR_CREATE)
1540 kvm->nr_memslot_pages += new->npages;
1541
1542 kvm_arch_commit_memory_region(kvm, old, new, change);
1543
1544 switch (change) {
1545 case KVM_MR_CREATE:
1546 /* Nothing more to do. */
1547 break;
1548 case KVM_MR_DELETE:
1549 /* Free the old memslot and all its metadata. */
1550 kvm_free_memslot(kvm, old);
1551 break;
1552 case KVM_MR_MOVE:
1553 case KVM_MR_FLAGS_ONLY:
1554 /*
1555 * Free the dirty bitmap as needed; the below check encompasses
1556 * both the flags and whether a ring buffer is being used)
1557 */
1558 if (old->dirty_bitmap && !new->dirty_bitmap)
1559 kvm_destroy_dirty_bitmap(old);
1560
1561 /*
1562 * The final quirk. Free the detached, old slot, but only its
1563 * memory, not any metadata. Metadata, including arch specific
1564 * data, may be reused by @new.
1565 */
1566 kfree(old);
1567 break;
1568 default:
1569 BUG();
1570 }
1571}
1572
1573/*
1574 * Activate @new, which must be installed in the inactive slots by the caller,
1575 * by swapping the active slots and then propagating @new to @old once @old is
1576 * unreachable and can be safely modified.
1577 *
1578 * With NULL @old this simply adds @new to @active (while swapping the sets).
1579 * With NULL @new this simply removes @old from @active and frees it
1580 * (while also swapping the sets).
1581 */
1582static void kvm_activate_memslot(struct kvm *kvm,
1583 struct kvm_memory_slot *old,
1584 struct kvm_memory_slot *new)
1585{
1586 int as_id = kvm_memslots_get_as_id(old, new);
1587
1588 kvm_swap_active_memslots(kvm, as_id);
1589
1590 /* Propagate the new memslot to the now inactive memslots. */
1591 kvm_replace_memslot(kvm, old, new);
1592}
1593
1594static void kvm_copy_memslot(struct kvm_memory_slot *dest,
1595 const struct kvm_memory_slot *src)
1596{
1597 dest->base_gfn = src->base_gfn;
1598 dest->npages = src->npages;
1599 dest->dirty_bitmap = src->dirty_bitmap;
1600 dest->arch = src->arch;
1601 dest->userspace_addr = src->userspace_addr;
1602 dest->flags = src->flags;
1603 dest->id = src->id;
1604 dest->as_id = src->as_id;
1605}
1606
1607static void kvm_invalidate_memslot(struct kvm *kvm,
1608 struct kvm_memory_slot *old,
1609 struct kvm_memory_slot *working_slot)
1610{
1686 /*
1611 /*
1687 * Free the old memslot's metadata. On DELETE, free the whole thing,
1688 * otherwise free the dirty bitmap as needed (the below effectively
1689 * checks both the flags and whether a ring buffer is being used).
1612 * Mark the current slot INVALID. As with all memslot modifications,
1613 * this must be done on an unreachable slot to avoid modifying the
1614 * current slot in the active tree.
1690 */
1615 */
1691 if (change == KVM_MR_DELETE)
1692 kvm_free_memslot(kvm, old);
1693 else if (old->dirty_bitmap && !new->dirty_bitmap)
1694 kvm_destroy_dirty_bitmap(old);
1616 kvm_copy_memslot(working_slot, old);
1617 working_slot->flags |= KVM_MEMSLOT_INVALID;
1618 kvm_replace_memslot(kvm, old, working_slot);
1619
1620 /*
1621 * Activate the slot that is now marked INVALID, but don't propagate
1622 * the slot to the now inactive slots. The slot is either going to be
1623 * deleted or recreated as a new slot.
1624 */
1625 kvm_swap_active_memslots(kvm, old->as_id);
1626
1627 /*
1628 * From this point no new shadow pages pointing to a deleted, or moved,
1629 * memslot will be created. Validation of sp->gfn happens in:
1630 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
1631 * - kvm_is_visible_gfn (mmu_check_root)
1632 */
1633 kvm_arch_flush_shadow_memslot(kvm, working_slot);
1634
1635 /* Was released by kvm_swap_active_memslots, reacquire. */
1636 mutex_lock(&kvm->slots_arch_lock);
1637
1638 /*
1639 * Copy the arch-specific field of the newly-installed slot back to the
1640 * old slot as the arch data could have changed between releasing
1641 * slots_arch_lock in install_new_memslots() and re-acquiring the lock
1642 * above. Writers are required to retrieve memslots *after* acquiring
1643 * slots_arch_lock, thus the active slot's data is guaranteed to be fresh.
1644 */
1645 old->arch = working_slot->arch;
1695}
1696
1646}
1647
1648static void kvm_create_memslot(struct kvm *kvm,
1649 const struct kvm_memory_slot *new,
1650 struct kvm_memory_slot *working)
1651{
1652 /*
1653 * Add the new memslot to the inactive set as a copy of the
1654 * new memslot data provided by userspace.
1655 */
1656 kvm_copy_memslot(working, new);
1657 kvm_replace_memslot(kvm, NULL, working);
1658 kvm_activate_memslot(kvm, NULL, working);
1659}
1660
1661static void kvm_delete_memslot(struct kvm *kvm,
1662 struct kvm_memory_slot *old,
1663 struct kvm_memory_slot *invalid_slot)
1664{
1665 /*
1666 * Remove the old memslot (in the inactive memslots) by passing NULL as
1667 * the "new" slot.
1668 */
1669 kvm_replace_memslot(kvm, old, NULL);
1670
1671 /* And do the same for the invalid version in the active slot. */
1672 kvm_activate_memslot(kvm, invalid_slot, NULL);
1673
1674 /* Free the invalid slot, the caller will clean up the old slot. */
1675 kfree(invalid_slot);
1676}
1677
1678static struct kvm_memory_slot *kvm_move_memslot(struct kvm *kvm,
1679 struct kvm_memory_slot *old,
1680 const struct kvm_memory_slot *new,
1681 struct kvm_memory_slot *invalid_slot)
1682{
1683 struct kvm_memslots *slots = kvm_get_inactive_memslots(kvm, old->as_id);
1684
1685 /*
1686 * The memslot's gfn is changing, remove it from the inactive tree, it
1687 * will be re-added with its updated gfn. Because its range is
1688 * changing, an in-place replace is not possible.
1689 */
1690 kvm_erase_gfn_node(slots, old);
1691
1692 /*
1693 * The old slot is now fully disconnected, reuse its memory for the
1694 * persistent copy of "new".
1695 */
1696 kvm_copy_memslot(old, new);
1697
1698 /* Re-add to the gfn tree with the updated gfn */
1699 kvm_insert_gfn_node(slots, old);
1700
1701 /* Replace the current INVALID slot with the updated memslot. */
1702 kvm_activate_memslot(kvm, invalid_slot, old);
1703
1704 /*
1705 * Clear the INVALID flag so that the invalid_slot is now a perfect
1706 * copy of the old slot. Return it for cleanup in the caller.
1707 */
1708 WARN_ON_ONCE(!(invalid_slot->flags & KVM_MEMSLOT_INVALID));
1709 invalid_slot->flags &= ~KVM_MEMSLOT_INVALID;
1710 return invalid_slot;
1711}
1712
1713static void kvm_update_flags_memslot(struct kvm *kvm,
1714 struct kvm_memory_slot *old,
1715 const struct kvm_memory_slot *new,
1716 struct kvm_memory_slot *working_slot)
1717{
1718 /*
1719 * Similar to the MOVE case, but the slot doesn't need to be zapped as
1720 * an intermediate step. Instead, the old memslot is simply replaced
1721 * with a new, updated copy in both memslot sets.
1722 */
1723 kvm_copy_memslot(working_slot, new);
1724 kvm_replace_memslot(kvm, old, working_slot);
1725 kvm_activate_memslot(kvm, old, working_slot);
1726}
1727
1697static int kvm_set_memslot(struct kvm *kvm,
1728static int kvm_set_memslot(struct kvm *kvm,
1729 struct kvm_memory_slot *old,
1698 struct kvm_memory_slot *new,
1699 enum kvm_mr_change change)
1700{
1730 struct kvm_memory_slot *new,
1731 enum kvm_mr_change change)
1732{
1701 struct kvm_memory_slot *slot, old;
1702 struct kvm_memslots *slots;
1733 struct kvm_memory_slot *working;
1703 int r;
1704
1705 /*
1734 int r;
1735
1736 /*
1706 * Released in install_new_memslots.
1737 * Modifications are done on an unreachable slot. Any changes are then
1738 * (eventually) propagated to both the active and inactive slots. This
1739 * allocation would ideally be on-demand (in helpers), but is done here
1740 * to avoid having to handle failure after kvm_prepare_memory_region().
1741 */
1742 working = kzalloc(sizeof(*working), GFP_KERNEL_ACCOUNT);
1743 if (!working)
1744 return -ENOMEM;
1745
1746 /*
1747 * Released in kvm_swap_active_memslots.
1707 *
1708 * Must be held from before the current memslots are copied until
1709 * after the new memslots are installed with rcu_assign_pointer,
1748 *
1749 * Must be held from before the current memslots are copied until
1750 * after the new memslots are installed with rcu_assign_pointer,
1710 * then released before the synchronize srcu in install_new_memslots.
1751 * then released before the synchronize srcu in kvm_swap_active_memslots.
1711 *
1712 * When modifying memslots outside of the slots_lock, must be held
1713 * before reading the pointer to the current memslots until after all
1714 * changes to those memslots are complete.
1715 *
1716 * These rules ensure that installing new memslots does not lose
1717 * changes made to the previous memslots.
1718 */
1719 mutex_lock(&kvm->slots_arch_lock);
1720
1752 *
1753 * When modifying memslots outside of the slots_lock, must be held
1754 * before reading the pointer to the current memslots until after all
1755 * changes to those memslots are complete.
1756 *
1757 * These rules ensure that installing new memslots does not lose
1758 * changes made to the previous memslots.
1759 */
1760 mutex_lock(&kvm->slots_arch_lock);
1761
1721 slots = kvm_dup_memslots(__kvm_memslots(kvm, new->as_id), change);
1722 if (!slots) {
1723 mutex_unlock(&kvm->slots_arch_lock);
1724 return -ENOMEM;
1725 }
1762 /*
1763 * Invalidate the old slot if it's being deleted or moved. This is
1764 * done prior to actually deleting/moving the memslot to allow vCPUs to
1765 * continue running by ensuring there are no mappings or shadow pages
1766 * for the memslot when it is deleted/moved. Without pre-invalidation
1767 * (and without a lock), a window would exist between effecting the
1768 * delete/move and committing the changes in arch code where KVM or a
1769 * guest could access a non-existent memslot.
1770 */
1771 if (change == KVM_MR_DELETE || change == KVM_MR_MOVE)
1772 kvm_invalidate_memslot(kvm, old, working);
1726
1773
1727 if (change == KVM_MR_DELETE || change == KVM_MR_MOVE) {
1774 r = kvm_prepare_memory_region(kvm, old, new, change);
1775 if (r) {
1728 /*
1776 /*
1729 * Note, the INVALID flag needs to be in the appropriate entry
1730 * in the freshly allocated memslots, not in @old or @new.
1777 * For DELETE/MOVE, revert the above INVALID change. No
1778 * modifications required since the original slot was preserved
1779 * in the inactive slots. Changing the active memslots also
1780 * release slots_arch_lock.
1731 */
1781 */
1732 slot = id_to_memslot(slots, new->id);
1733 slot->flags |= KVM_MEMSLOT_INVALID;
1734
1735 /*
1736 * We can re-use the old memslots, the only difference from the
1737 * newly installed memslots is the invalid flag, which will get
1738 * dropped by update_memslots anyway. We'll also revert to the
1739 * old memslots if preparing the new memory region fails.
1740 */
1741 slots = install_new_memslots(kvm, new->as_id, slots);
1742
1743 /* From this point no new shadow pages pointing to a deleted,
1744 * or moved, memslot will be created.
1745 *
1746 * validation of sp->gfn happens in:
1747 * - gfn_to_hva (kvm_read_guest, gfn_to_pfn)
1748 * - kvm_is_visible_gfn (mmu_check_root)
1749 */
1750 kvm_arch_flush_shadow_memslot(kvm, slot);
1751
1752 /* Released in install_new_memslots. */
1753 mutex_lock(&kvm->slots_arch_lock);
1754
1755 /*
1756 * The arch-specific fields of the now-active memslots could
1757 * have been modified between releasing slots_arch_lock in
1758 * install_new_memslots and re-acquiring slots_arch_lock above.
1759 * Copy them to the inactive memslots. Arch code is required
1760 * to retrieve memslots *after* acquiring slots_arch_lock, thus
1761 * the active memslots are guaranteed to be fresh.
1762 */
1763 kvm_copy_memslots_arch(slots, __kvm_memslots(kvm, new->as_id));
1782 if (change == KVM_MR_DELETE || change == KVM_MR_MOVE)
1783 kvm_activate_memslot(kvm, working, old);
1784 else
1785 mutex_unlock(&kvm->slots_arch_lock);
1786 kfree(working);
1787 return r;
1764 }
1765
1766 /*
1788 }
1789
1790 /*
1767 * Make a full copy of the old memslot, the pointer will become stale
1768 * when the memslots are re-sorted by update_memslots(), and the old
1769 * memslot needs to be referenced after calling update_memslots(), e.g.
1770 * to free its resources and for arch specific behavior. This needs to
1771 * happen *after* (re)acquiring slots_arch_lock.
1791 * For DELETE and MOVE, the working slot is now active as the INVALID
1792 * version of the old slot. MOVE is particularly special as it reuses
1793 * the old slot and returns a copy of the old slot (in working_slot).
1794 * For CREATE, there is no old slot. For DELETE and FLAGS_ONLY, the
1795 * old slot is detached but otherwise preserved.
1772 */
1796 */
1773 slot = id_to_memslot(slots, new->id);
1774 if (slot) {
1775 old = *slot;
1776 } else {
1777 WARN_ON_ONCE(change != KVM_MR_CREATE);
1778 memset(&old, 0, sizeof(old));
1779 old.id = new->id;
1780 old.as_id = new->as_id;
1781 }
1797 if (change == KVM_MR_CREATE)
1798 kvm_create_memslot(kvm, new, working);
1799 else if (change == KVM_MR_DELETE)
1800 kvm_delete_memslot(kvm, old, working);
1801 else if (change == KVM_MR_MOVE)
1802 old = kvm_move_memslot(kvm, old, new, working);
1803 else if (change == KVM_MR_FLAGS_ONLY)
1804 kvm_update_flags_memslot(kvm, old, new, working);
1805 else
1806 BUG();
1782
1807
1783 r = kvm_prepare_memory_region(kvm, &old, new, change);
1784 if (r)
1785 goto out_slots;
1808 /*
1809 * No need to refresh new->arch, changes after dropping slots_arch_lock
1810 * will directly hit the final, active memsot. Architectures are
1811 * responsible for knowing that new->arch may be stale.
1812 */
1813 kvm_commit_memory_region(kvm, old, new, change);
1786
1814
1787 update_memslots(slots, new, change);
1788 slots = install_new_memslots(kvm, new->as_id, slots);
1789
1790 kvm_commit_memory_region(kvm, &old, new, change);
1791
1792 kvfree(slots);
1793 return 0;
1815 return 0;
1794
1795out_slots:
1796 if (change == KVM_MR_DELETE || change == KVM_MR_MOVE)
1797 slots = install_new_memslots(kvm, new->as_id, slots);
1798 else
1799 mutex_unlock(&kvm->slots_arch_lock);
1800 kvfree(slots);
1801 return r;
1802}
1803
1804/*
1805 * Allocate some memory and give it an address in the guest physical address
1806 * space.
1807 *
1808 * Discontiguous memory is allowed, mostly for framebuffers.
1809 *

--- 44 unchanged lines hidden (view full) ---

1854
1855 if (WARN_ON_ONCE(kvm->nr_memslot_pages < old->npages))
1856 return -EIO;
1857
1858 memset(&new, 0, sizeof(new));
1859 new.id = id;
1860 new.as_id = as_id;
1861
1816}
1817
1818/*
1819 * Allocate some memory and give it an address in the guest physical address
1820 * space.
1821 *
1822 * Discontiguous memory is allowed, mostly for framebuffers.
1823 *

--- 44 unchanged lines hidden (view full) ---

1868
1869 if (WARN_ON_ONCE(kvm->nr_memslot_pages < old->npages))
1870 return -EIO;
1871
1872 memset(&new, 0, sizeof(new));
1873 new.id = id;
1874 new.as_id = as_id;
1875
1862 return kvm_set_memslot(kvm, &new, KVM_MR_DELETE);
1876 return kvm_set_memslot(kvm, old, &new, KVM_MR_DELETE);
1863 }
1864
1865 new.as_id = as_id;
1866 new.id = id;
1867 new.base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
1868 new.npages = mem->memory_size >> PAGE_SHIFT;
1869 new.flags = mem->flags;
1870 new.userspace_addr = mem->userspace_addr;

--- 20 unchanged lines hidden (view full) ---

1891 change = KVM_MR_MOVE;
1892 else if (new.flags != old->flags)
1893 change = KVM_MR_FLAGS_ONLY;
1894 else /* Nothing to change. */
1895 return 0;
1896 }
1897
1898 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
1877 }
1878
1879 new.as_id = as_id;
1880 new.id = id;
1881 new.base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
1882 new.npages = mem->memory_size >> PAGE_SHIFT;
1883 new.flags = mem->flags;
1884 new.userspace_addr = mem->userspace_addr;

--- 20 unchanged lines hidden (view full) ---

1905 change = KVM_MR_MOVE;
1906 else if (new.flags != old->flags)
1907 change = KVM_MR_FLAGS_ONLY;
1908 else /* Nothing to change. */
1909 return 0;
1910 }
1911
1912 if ((change == KVM_MR_CREATE) || (change == KVM_MR_MOVE)) {
1913 int bkt;
1914
1899 /* Check for overlaps */
1915 /* Check for overlaps */
1900 kvm_for_each_memslot(tmp, __kvm_memslots(kvm, as_id)) {
1916 kvm_for_each_memslot(tmp, bkt, __kvm_memslots(kvm, as_id)) {
1901 if (tmp->id == id)
1902 continue;
1903 if (!((new.base_gfn + new.npages <= tmp->base_gfn) ||
1904 (new.base_gfn >= tmp->base_gfn + tmp->npages)))
1905 return -EEXIST;
1906 }
1907 }
1908
1917 if (tmp->id == id)
1918 continue;
1919 if (!((new.base_gfn + new.npages <= tmp->base_gfn) ||
1920 (new.base_gfn >= tmp->base_gfn + tmp->npages)))
1921 return -EEXIST;
1922 }
1923 }
1924
1909 return kvm_set_memslot(kvm, &new, change);
1925 return kvm_set_memslot(kvm, old, &new, change);
1910}
1911EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
1912
1913int kvm_set_memory_region(struct kvm *kvm,
1914 const struct kvm_userspace_memory_region *mem)
1915{
1916 int r;
1917

--- 288 unchanged lines hidden (view full) ---

2206{
2207 return __gfn_to_memslot(kvm_memslots(kvm), gfn);
2208}
2209EXPORT_SYMBOL_GPL(gfn_to_memslot);
2210
2211struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
2212{
2213 struct kvm_memslots *slots = kvm_vcpu_memslots(vcpu);
1926}
1927EXPORT_SYMBOL_GPL(__kvm_set_memory_region);
1928
1929int kvm_set_memory_region(struct kvm *kvm,
1930 const struct kvm_userspace_memory_region *mem)
1931{
1932 int r;
1933

--- 288 unchanged lines hidden (view full) ---

2222{
2223 return __gfn_to_memslot(kvm_memslots(kvm), gfn);
2224}
2225EXPORT_SYMBOL_GPL(gfn_to_memslot);
2226
2227struct kvm_memory_slot *kvm_vcpu_gfn_to_memslot(struct kvm_vcpu *vcpu, gfn_t gfn)
2228{
2229 struct kvm_memslots *slots = kvm_vcpu_memslots(vcpu);
2230 u64 gen = slots->generation;
2214 struct kvm_memory_slot *slot;
2231 struct kvm_memory_slot *slot;
2215 int slot_index;
2216
2232
2217 slot = try_get_memslot(slots, vcpu->last_used_slot, gfn);
2233 /*
2234 * This also protects against using a memslot from a different address space,
2235 * since different address spaces have different generation numbers.
2236 */
2237 if (unlikely(gen != vcpu->last_used_slot_gen)) {
2238 vcpu->last_used_slot = NULL;
2239 vcpu->last_used_slot_gen = gen;
2240 }
2241
2242 slot = try_get_memslot(vcpu->last_used_slot, gfn);
2218 if (slot)
2219 return slot;
2220
2221 /*
2222 * Fall back to searching all memslots. We purposely use
2223 * search_memslots() instead of __gfn_to_memslot() to avoid
2243 if (slot)
2244 return slot;
2245
2246 /*
2247 * Fall back to searching all memslots. We purposely use
2248 * search_memslots() instead of __gfn_to_memslot() to avoid
2224 * thrashing the VM-wide last_used_index in kvm_memslots.
2249 * thrashing the VM-wide last_used_slot in kvm_memslots.
2225 */
2250 */
2226 slot = search_memslots(slots, gfn, &slot_index, false);
2251 slot = search_memslots(slots, gfn, false);
2227 if (slot) {
2252 if (slot) {
2228 vcpu->last_used_slot = slot_index;
2253 vcpu->last_used_slot = slot;
2229 return slot;
2230 }
2231
2232 return NULL;
2233}
2234EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_memslot);
2235
2236bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)

--- 3539 unchanged lines hidden --- 		2254 return slot;
2255 }
2256
2257 return NULL;
2258}
2259EXPORT_SYMBOL_GPL(kvm_vcpu_gfn_to_memslot);
2260
2261bool kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)

--- 3539 unchanged lines hidden ---