1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0 24ccefbe5SStefano Stabellini /* 34ccefbe5SStefano Stabellini * Xen stolen ticks accounting. 44ccefbe5SStefano Stabellini */ 54ccefbe5SStefano Stabellini #include <linux/kernel.h> 64ccefbe5SStefano Stabellini #include <linux/kernel_stat.h> 74ccefbe5SStefano Stabellini #include <linux/math64.h> 84ccefbe5SStefano Stabellini #include <linux/gfp.h> 95e25f5dbSDongli Zhang #include <linux/slab.h> 104ccefbe5SStefano Stabellini 11ecb23dc6SJuergen Gross #include <asm/paravirt.h> 124ccefbe5SStefano Stabellini #include <asm/xen/hypervisor.h> 134ccefbe5SStefano Stabellini #include <asm/xen/hypercall.h> 144ccefbe5SStefano Stabellini 154ccefbe5SStefano Stabellini #include <xen/events.h> 164ccefbe5SStefano Stabellini #include <xen/features.h> 174ccefbe5SStefano Stabellini #include <xen/interface/xen.h> 184ccefbe5SStefano Stabellini #include <xen/interface/vcpu.h> 194ccefbe5SStefano Stabellini #include <xen/xen-ops.h> 204ccefbe5SStefano Stabellini 214ccefbe5SStefano Stabellini /* runstate info updated by Xen */ 224ccefbe5SStefano Stabellini static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate); 234ccefbe5SStefano Stabellini 245e25f5dbSDongli Zhang static DEFINE_PER_CPU(u64[4], old_runstate_time); 255e25f5dbSDongli Zhang 264ccefbe5SStefano Stabellini /* return an consistent snapshot of 64-bit time/counter value */ 274ccefbe5SStefano Stabellini static u64 get64(const u64 *p) 284ccefbe5SStefano Stabellini { 294ccefbe5SStefano Stabellini u64 ret; 304ccefbe5SStefano Stabellini 314ccefbe5SStefano Stabellini if (BITS_PER_LONG < 64) { 324ccefbe5SStefano Stabellini u32 *p32 = (u32 *)p; 332dd887e3SStefano Stabellini u32 h, l, h2; 344ccefbe5SStefano Stabellini 354ccefbe5SStefano Stabellini /* 364ccefbe5SStefano Stabellini * Read high then low, and then make sure high is 374ccefbe5SStefano Stabellini * still the same; this will only loop if low wraps 384ccefbe5SStefano Stabellini * and carries into high. 394ccefbe5SStefano Stabellini * XXX some clean way to make this endian-proof? 404ccefbe5SStefano Stabellini */ 414ccefbe5SStefano Stabellini do { 422dd887e3SStefano Stabellini h = READ_ONCE(p32[1]); 432dd887e3SStefano Stabellini l = READ_ONCE(p32[0]); 442dd887e3SStefano Stabellini h2 = READ_ONCE(p32[1]); 452dd887e3SStefano Stabellini } while(h2 != h); 464ccefbe5SStefano Stabellini 474ccefbe5SStefano Stabellini ret = (((u64)h) << 32) | l; 484ccefbe5SStefano Stabellini } else 492dd887e3SStefano Stabellini ret = READ_ONCE(*p); 504ccefbe5SStefano Stabellini 514ccefbe5SStefano Stabellini return ret; 524ccefbe5SStefano Stabellini } 534ccefbe5SStefano Stabellini 545e25f5dbSDongli Zhang static void xen_get_runstate_snapshot_cpu_delta( 555e25f5dbSDongli Zhang struct vcpu_runstate_info *res, unsigned int cpu) 564ccefbe5SStefano Stabellini { 574ccefbe5SStefano Stabellini u64 state_time; 584ccefbe5SStefano Stabellini struct vcpu_runstate_info *state; 594ccefbe5SStefano Stabellini 604ccefbe5SStefano Stabellini BUG_ON(preemptible()); 614ccefbe5SStefano Stabellini 626ba286adSJuergen Gross state = per_cpu_ptr(&xen_runstate, cpu); 634ccefbe5SStefano Stabellini 644ccefbe5SStefano Stabellini do { 654ccefbe5SStefano Stabellini state_time = get64(&state->state_entry_time); 666ba286adSJuergen Gross rmb(); /* Hypervisor might update data. */ 679e343b46SWill Deacon *res = __READ_ONCE(*state); 686ba286adSJuergen Gross rmb(); /* Hypervisor might update data. */ 696ba286adSJuergen Gross } while (get64(&state->state_entry_time) != state_time || 706ba286adSJuergen Gross (state_time & XEN_RUNSTATE_UPDATE)); 716ba286adSJuergen Gross } 726ba286adSJuergen Gross 735e25f5dbSDongli Zhang static void xen_get_runstate_snapshot_cpu(struct vcpu_runstate_info *res, 745e25f5dbSDongli Zhang unsigned int cpu) 755e25f5dbSDongli Zhang { 765e25f5dbSDongli Zhang int i; 775e25f5dbSDongli Zhang 785e25f5dbSDongli Zhang xen_get_runstate_snapshot_cpu_delta(res, cpu); 795e25f5dbSDongli Zhang 805e25f5dbSDongli Zhang for (i = 0; i < 4; i++) 815e25f5dbSDongli Zhang res->time[i] += per_cpu(old_runstate_time, cpu)[i]; 825e25f5dbSDongli Zhang } 835e25f5dbSDongli Zhang 845e25f5dbSDongli Zhang void xen_manage_runstate_time(int action) 855e25f5dbSDongli Zhang { 865e25f5dbSDongli Zhang static struct vcpu_runstate_info *runstate_delta; 875e25f5dbSDongli Zhang struct vcpu_runstate_info state; 885e25f5dbSDongli Zhang int cpu, i; 895e25f5dbSDongli Zhang 905e25f5dbSDongli Zhang switch (action) { 915e25f5dbSDongli Zhang case -1: /* backup runstate time before suspend */ 925e25f5dbSDongli Zhang if (unlikely(runstate_delta)) 935e25f5dbSDongli Zhang pr_warn_once("%s: memory leak as runstate_delta is not NULL\n", 945e25f5dbSDongli Zhang __func__); 955e25f5dbSDongli Zhang 965e25f5dbSDongli Zhang runstate_delta = kmalloc_array(num_possible_cpus(), 975e25f5dbSDongli Zhang sizeof(*runstate_delta), 985e25f5dbSDongli Zhang GFP_ATOMIC); 995e25f5dbSDongli Zhang if (unlikely(!runstate_delta)) { 1005e25f5dbSDongli Zhang pr_warn("%s: failed to allocate runstate_delta\n", 1015e25f5dbSDongli Zhang __func__); 1025e25f5dbSDongli Zhang return; 1035e25f5dbSDongli Zhang } 1045e25f5dbSDongli Zhang 1055e25f5dbSDongli Zhang for_each_possible_cpu(cpu) { 1065e25f5dbSDongli Zhang xen_get_runstate_snapshot_cpu_delta(&state, cpu); 1075e25f5dbSDongli Zhang memcpy(runstate_delta[cpu].time, state.time, 1085e25f5dbSDongli Zhang sizeof(runstate_delta[cpu].time)); 1095e25f5dbSDongli Zhang } 1105e25f5dbSDongli Zhang 1115e25f5dbSDongli Zhang break; 1125e25f5dbSDongli Zhang 1135e25f5dbSDongli Zhang case 0: /* backup runstate time after resume */ 1145e25f5dbSDongli Zhang if (unlikely(!runstate_delta)) { 1155e25f5dbSDongli Zhang pr_warn("%s: cannot accumulate runstate time as runstate_delta is NULL\n", 1165e25f5dbSDongli Zhang __func__); 1175e25f5dbSDongli Zhang return; 1185e25f5dbSDongli Zhang } 1195e25f5dbSDongli Zhang 1205e25f5dbSDongli Zhang for_each_possible_cpu(cpu) { 1215e25f5dbSDongli Zhang for (i = 0; i < 4; i++) 1225e25f5dbSDongli Zhang per_cpu(old_runstate_time, cpu)[i] += 1235e25f5dbSDongli Zhang runstate_delta[cpu].time[i]; 1245e25f5dbSDongli Zhang } 1255e25f5dbSDongli Zhang 1265e25f5dbSDongli Zhang break; 1275e25f5dbSDongli Zhang 1285e25f5dbSDongli Zhang default: /* do not accumulate runstate time for checkpointing */ 1295e25f5dbSDongli Zhang break; 1305e25f5dbSDongli Zhang } 1315e25f5dbSDongli Zhang 1325e25f5dbSDongli Zhang if (action != -1 && runstate_delta) { 1335e25f5dbSDongli Zhang kfree(runstate_delta); 1345e25f5dbSDongli Zhang runstate_delta = NULL; 1355e25f5dbSDongli Zhang } 1365e25f5dbSDongli Zhang } 1375e25f5dbSDongli Zhang 1386ba286adSJuergen Gross /* 1396ba286adSJuergen Gross * Runstate accounting 1406ba286adSJuergen Gross */ 1416ba286adSJuergen Gross void xen_get_runstate_snapshot(struct vcpu_runstate_info *res) 1426ba286adSJuergen Gross { 1436ba286adSJuergen Gross xen_get_runstate_snapshot_cpu(res, smp_processor_id()); 1444ccefbe5SStefano Stabellini } 1454ccefbe5SStefano Stabellini 1464ccefbe5SStefano Stabellini /* return true when a vcpu could run but has no real cpu to run on */ 1474ccefbe5SStefano Stabellini bool xen_vcpu_stolen(int vcpu) 1484ccefbe5SStefano Stabellini { 1494ccefbe5SStefano Stabellini return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable; 1504ccefbe5SStefano Stabellini } 1514ccefbe5SStefano Stabellini 152d34c30ccSJuergen Gross u64 xen_steal_clock(int cpu) 153ecb23dc6SJuergen Gross { 154ecb23dc6SJuergen Gross struct vcpu_runstate_info state; 155ecb23dc6SJuergen Gross 1566ba286adSJuergen Gross xen_get_runstate_snapshot_cpu(&state, cpu); 157ecb23dc6SJuergen Gross return state.time[RUNSTATE_runnable] + state.time[RUNSTATE_offline]; 158ecb23dc6SJuergen Gross } 159ecb23dc6SJuergen Gross 1604ccefbe5SStefano Stabellini void xen_setup_runstate_info(int cpu) 1614ccefbe5SStefano Stabellini { 1624ccefbe5SStefano Stabellini struct vcpu_register_runstate_memory_area area; 1634ccefbe5SStefano Stabellini 1644ccefbe5SStefano Stabellini area.addr.v = &per_cpu(xen_runstate, cpu); 1654ccefbe5SStefano Stabellini 1664ccefbe5SStefano Stabellini if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area, 167ad5475f9SVitaly Kuznetsov xen_vcpu_nr(cpu), &area)) 1684ccefbe5SStefano Stabellini BUG(); 1694ccefbe5SStefano Stabellini } 1704ccefbe5SStefano Stabellini 171ecb23dc6SJuergen Gross void __init xen_time_setup_guest(void) 172ecb23dc6SJuergen Gross { 1736ba286adSJuergen Gross bool xen_runstate_remote; 1746ba286adSJuergen Gross 1756ba286adSJuergen Gross xen_runstate_remote = !HYPERVISOR_vm_assist(VMASST_CMD_enable, 1766ba286adSJuergen Gross VMASST_TYPE_runstate_update_flag); 1776ba286adSJuergen Gross 1785c83511bSJuergen Gross pv_ops.time.steal_clock = xen_steal_clock; 179ecb23dc6SJuergen Gross 180ecb23dc6SJuergen Gross static_key_slow_inc(¶virt_steal_enabled); 1816ba286adSJuergen Gross if (xen_runstate_remote) 1826ba286adSJuergen Gross static_key_slow_inc(¶virt_steal_rq_enabled); 183ecb23dc6SJuergen Gross } 184