1b2441318SGreg Kroah-Hartman // SPDX-License-Identifier: GPL-2.0
24ccefbe5SStefano Stabellini /*
34ccefbe5SStefano Stabellini * Xen stolen ticks accounting.
44ccefbe5SStefano Stabellini */
54ccefbe5SStefano Stabellini #include <linux/kernel.h>
64ccefbe5SStefano Stabellini #include <linux/kernel_stat.h>
74ccefbe5SStefano Stabellini #include <linux/math64.h>
84ccefbe5SStefano Stabellini #include <linux/gfp.h>
95e25f5dbSDongli Zhang #include <linux/slab.h>
10*a0e2bf7cSJuergen Gross #include <linux/static_call.h>
114ccefbe5SStefano Stabellini
12ecb23dc6SJuergen Gross #include <asm/paravirt.h>
134ccefbe5SStefano Stabellini #include <asm/xen/hypervisor.h>
144ccefbe5SStefano Stabellini #include <asm/xen/hypercall.h>
154ccefbe5SStefano Stabellini
164ccefbe5SStefano Stabellini #include <xen/events.h>
174ccefbe5SStefano Stabellini #include <xen/features.h>
184ccefbe5SStefano Stabellini #include <xen/interface/xen.h>
194ccefbe5SStefano Stabellini #include <xen/interface/vcpu.h>
204ccefbe5SStefano Stabellini #include <xen/xen-ops.h>
214ccefbe5SStefano Stabellini
224ccefbe5SStefano Stabellini /* runstate info updated by Xen */
234ccefbe5SStefano Stabellini static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate);
244ccefbe5SStefano Stabellini
255e25f5dbSDongli Zhang static DEFINE_PER_CPU(u64[4], old_runstate_time);
265e25f5dbSDongli Zhang
274ccefbe5SStefano Stabellini /* return an consistent snapshot of 64-bit time/counter value */
get64(const u64 * p)284ccefbe5SStefano Stabellini static u64 get64(const u64 *p)
294ccefbe5SStefano Stabellini {
304ccefbe5SStefano Stabellini u64 ret;
314ccefbe5SStefano Stabellini
324ccefbe5SStefano Stabellini if (BITS_PER_LONG < 64) {
334ccefbe5SStefano Stabellini u32 *p32 = (u32 *)p;
342dd887e3SStefano Stabellini u32 h, l, h2;
354ccefbe5SStefano Stabellini
364ccefbe5SStefano Stabellini /*
374ccefbe5SStefano Stabellini * Read high then low, and then make sure high is
384ccefbe5SStefano Stabellini * still the same; this will only loop if low wraps
394ccefbe5SStefano Stabellini * and carries into high.
404ccefbe5SStefano Stabellini * XXX some clean way to make this endian-proof?
414ccefbe5SStefano Stabellini */
424ccefbe5SStefano Stabellini do {
432dd887e3SStefano Stabellini h = READ_ONCE(p32[1]);
442dd887e3SStefano Stabellini l = READ_ONCE(p32[0]);
452dd887e3SStefano Stabellini h2 = READ_ONCE(p32[1]);
462dd887e3SStefano Stabellini } while(h2 != h);
474ccefbe5SStefano Stabellini
484ccefbe5SStefano Stabellini ret = (((u64)h) << 32) | l;
494ccefbe5SStefano Stabellini } else
502dd887e3SStefano Stabellini ret = READ_ONCE(*p);
514ccefbe5SStefano Stabellini
524ccefbe5SStefano Stabellini return ret;
534ccefbe5SStefano Stabellini }
544ccefbe5SStefano Stabellini
xen_get_runstate_snapshot_cpu_delta(struct vcpu_runstate_info * res,unsigned int cpu)555e25f5dbSDongli Zhang static void xen_get_runstate_snapshot_cpu_delta(
565e25f5dbSDongli Zhang struct vcpu_runstate_info *res, unsigned int cpu)
574ccefbe5SStefano Stabellini {
584ccefbe5SStefano Stabellini u64 state_time;
594ccefbe5SStefano Stabellini struct vcpu_runstate_info *state;
604ccefbe5SStefano Stabellini
614ccefbe5SStefano Stabellini BUG_ON(preemptible());
624ccefbe5SStefano Stabellini
636ba286adSJuergen Gross state = per_cpu_ptr(&xen_runstate, cpu);
644ccefbe5SStefano Stabellini
654ccefbe5SStefano Stabellini do {
664ccefbe5SStefano Stabellini state_time = get64(&state->state_entry_time);
676ba286adSJuergen Gross rmb(); /* Hypervisor might update data. */
689e343b46SWill Deacon *res = __READ_ONCE(*state);
696ba286adSJuergen Gross rmb(); /* Hypervisor might update data. */
706ba286adSJuergen Gross } while (get64(&state->state_entry_time) != state_time ||
716ba286adSJuergen Gross (state_time & XEN_RUNSTATE_UPDATE));
726ba286adSJuergen Gross }
736ba286adSJuergen Gross
xen_get_runstate_snapshot_cpu(struct vcpu_runstate_info * res,unsigned int cpu)745e25f5dbSDongli Zhang static void xen_get_runstate_snapshot_cpu(struct vcpu_runstate_info *res,
755e25f5dbSDongli Zhang unsigned int cpu)
765e25f5dbSDongli Zhang {
775e25f5dbSDongli Zhang int i;
785e25f5dbSDongli Zhang
795e25f5dbSDongli Zhang xen_get_runstate_snapshot_cpu_delta(res, cpu);
805e25f5dbSDongli Zhang
815e25f5dbSDongli Zhang for (i = 0; i < 4; i++)
825e25f5dbSDongli Zhang res->time[i] += per_cpu(old_runstate_time, cpu)[i];
835e25f5dbSDongli Zhang }
845e25f5dbSDongli Zhang
xen_manage_runstate_time(int action)855e25f5dbSDongli Zhang void xen_manage_runstate_time(int action)
865e25f5dbSDongli Zhang {
875e25f5dbSDongli Zhang static struct vcpu_runstate_info *runstate_delta;
885e25f5dbSDongli Zhang struct vcpu_runstate_info state;
895e25f5dbSDongli Zhang int cpu, i;
905e25f5dbSDongli Zhang
915e25f5dbSDongli Zhang switch (action) {
925e25f5dbSDongli Zhang case -1: /* backup runstate time before suspend */
935e25f5dbSDongli Zhang if (unlikely(runstate_delta))
945e25f5dbSDongli Zhang pr_warn_once("%s: memory leak as runstate_delta is not NULL\n",
955e25f5dbSDongli Zhang __func__);
965e25f5dbSDongli Zhang
975e25f5dbSDongli Zhang runstate_delta = kmalloc_array(num_possible_cpus(),
985e25f5dbSDongli Zhang sizeof(*runstate_delta),
995e25f5dbSDongli Zhang GFP_ATOMIC);
1005e25f5dbSDongli Zhang if (unlikely(!runstate_delta)) {
1015e25f5dbSDongli Zhang pr_warn("%s: failed to allocate runstate_delta\n",
1025e25f5dbSDongli Zhang __func__);
1035e25f5dbSDongli Zhang return;
1045e25f5dbSDongli Zhang }
1055e25f5dbSDongli Zhang
1065e25f5dbSDongli Zhang for_each_possible_cpu(cpu) {
1075e25f5dbSDongli Zhang xen_get_runstate_snapshot_cpu_delta(&state, cpu);
1085e25f5dbSDongli Zhang memcpy(runstate_delta[cpu].time, state.time,
1095e25f5dbSDongli Zhang sizeof(runstate_delta[cpu].time));
1105e25f5dbSDongli Zhang }
1115e25f5dbSDongli Zhang
1125e25f5dbSDongli Zhang break;
1135e25f5dbSDongli Zhang
1145e25f5dbSDongli Zhang case 0: /* backup runstate time after resume */
1155e25f5dbSDongli Zhang if (unlikely(!runstate_delta)) {
1165e25f5dbSDongli Zhang pr_warn("%s: cannot accumulate runstate time as runstate_delta is NULL\n",
1175e25f5dbSDongli Zhang __func__);
1185e25f5dbSDongli Zhang return;
1195e25f5dbSDongli Zhang }
1205e25f5dbSDongli Zhang
1215e25f5dbSDongli Zhang for_each_possible_cpu(cpu) {
1225e25f5dbSDongli Zhang for (i = 0; i < 4; i++)
1235e25f5dbSDongli Zhang per_cpu(old_runstate_time, cpu)[i] +=
1245e25f5dbSDongli Zhang runstate_delta[cpu].time[i];
1255e25f5dbSDongli Zhang }
1265e25f5dbSDongli Zhang
1275e25f5dbSDongli Zhang break;
1285e25f5dbSDongli Zhang
1295e25f5dbSDongli Zhang default: /* do not accumulate runstate time for checkpointing */
1305e25f5dbSDongli Zhang break;
1315e25f5dbSDongli Zhang }
1325e25f5dbSDongli Zhang
1335e25f5dbSDongli Zhang if (action != -1 && runstate_delta) {
1345e25f5dbSDongli Zhang kfree(runstate_delta);
1355e25f5dbSDongli Zhang runstate_delta = NULL;
1365e25f5dbSDongli Zhang }
1375e25f5dbSDongli Zhang }
1385e25f5dbSDongli Zhang
1396ba286adSJuergen Gross /*
1406ba286adSJuergen Gross * Runstate accounting
1416ba286adSJuergen Gross */
xen_get_runstate_snapshot(struct vcpu_runstate_info * res)1426ba286adSJuergen Gross void xen_get_runstate_snapshot(struct vcpu_runstate_info *res)
1436ba286adSJuergen Gross {
1446ba286adSJuergen Gross xen_get_runstate_snapshot_cpu(res, smp_processor_id());
1454ccefbe5SStefano Stabellini }
1464ccefbe5SStefano Stabellini
1474ccefbe5SStefano Stabellini /* return true when a vcpu could run but has no real cpu to run on */
xen_vcpu_stolen(int vcpu)1484ccefbe5SStefano Stabellini bool xen_vcpu_stolen(int vcpu)
1494ccefbe5SStefano Stabellini {
1504ccefbe5SStefano Stabellini return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable;
1514ccefbe5SStefano Stabellini }
1524ccefbe5SStefano Stabellini
xen_steal_clock(int cpu)153d34c30ccSJuergen Gross u64 xen_steal_clock(int cpu)
154ecb23dc6SJuergen Gross {
155ecb23dc6SJuergen Gross struct vcpu_runstate_info state;
156ecb23dc6SJuergen Gross
1576ba286adSJuergen Gross xen_get_runstate_snapshot_cpu(&state, cpu);
158ecb23dc6SJuergen Gross return state.time[RUNSTATE_runnable] + state.time[RUNSTATE_offline];
159ecb23dc6SJuergen Gross }
160ecb23dc6SJuergen Gross
xen_setup_runstate_info(int cpu)1614ccefbe5SStefano Stabellini void xen_setup_runstate_info(int cpu)
1624ccefbe5SStefano Stabellini {
1634ccefbe5SStefano Stabellini struct vcpu_register_runstate_memory_area area;
1644ccefbe5SStefano Stabellini
1654ccefbe5SStefano Stabellini area.addr.v = &per_cpu(xen_runstate, cpu);
1664ccefbe5SStefano Stabellini
1674ccefbe5SStefano Stabellini if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area,
168ad5475f9SVitaly Kuznetsov xen_vcpu_nr(cpu), &area))
1694ccefbe5SStefano Stabellini BUG();
1704ccefbe5SStefano Stabellini }
1714ccefbe5SStefano Stabellini
xen_time_setup_guest(void)172ecb23dc6SJuergen Gross void __init xen_time_setup_guest(void)
173ecb23dc6SJuergen Gross {
1746ba286adSJuergen Gross bool xen_runstate_remote;
1756ba286adSJuergen Gross
1766ba286adSJuergen Gross xen_runstate_remote = !HYPERVISOR_vm_assist(VMASST_CMD_enable,
1776ba286adSJuergen Gross VMASST_TYPE_runstate_update_flag);
1786ba286adSJuergen Gross
179*a0e2bf7cSJuergen Gross static_call_update(pv_steal_clock, xen_steal_clock);
180ecb23dc6SJuergen Gross
181ecb23dc6SJuergen Gross static_key_slow_inc(¶virt_steal_enabled);
1826ba286adSJuergen Gross if (xen_runstate_remote)
1836ba286adSJuergen Gross static_key_slow_inc(¶virt_steal_rq_enabled);
184ecb23dc6SJuergen Gross }
185