14ccefbe5SStefano Stabellini /* 24ccefbe5SStefano Stabellini * Xen stolen ticks accounting. 34ccefbe5SStefano Stabellini */ 44ccefbe5SStefano Stabellini #include <linux/kernel.h> 54ccefbe5SStefano Stabellini #include <linux/kernel_stat.h> 64ccefbe5SStefano Stabellini #include <linux/math64.h> 74ccefbe5SStefano Stabellini #include <linux/gfp.h> 85e25f5dbSDongli Zhang #include <linux/slab.h> 94ccefbe5SStefano Stabellini 10ecb23dc6SJuergen Gross #include <asm/paravirt.h> 114ccefbe5SStefano Stabellini #include <asm/xen/hypervisor.h> 124ccefbe5SStefano Stabellini #include <asm/xen/hypercall.h> 134ccefbe5SStefano Stabellini 144ccefbe5SStefano Stabellini #include <xen/events.h> 154ccefbe5SStefano Stabellini #include <xen/features.h> 164ccefbe5SStefano Stabellini #include <xen/interface/xen.h> 174ccefbe5SStefano Stabellini #include <xen/interface/vcpu.h> 184ccefbe5SStefano Stabellini #include <xen/xen-ops.h> 194ccefbe5SStefano Stabellini 204ccefbe5SStefano Stabellini /* runstate info updated by Xen */ 214ccefbe5SStefano Stabellini static DEFINE_PER_CPU(struct vcpu_runstate_info, xen_runstate); 224ccefbe5SStefano Stabellini 235e25f5dbSDongli Zhang static DEFINE_PER_CPU(u64[4], old_runstate_time); 245e25f5dbSDongli Zhang 254ccefbe5SStefano Stabellini /* return an consistent snapshot of 64-bit time/counter value */ 264ccefbe5SStefano Stabellini static u64 get64(const u64 *p) 274ccefbe5SStefano Stabellini { 284ccefbe5SStefano Stabellini u64 ret; 294ccefbe5SStefano Stabellini 304ccefbe5SStefano Stabellini if (BITS_PER_LONG < 64) { 314ccefbe5SStefano Stabellini u32 *p32 = (u32 *)p; 322dd887e3SStefano Stabellini u32 h, l, h2; 334ccefbe5SStefano Stabellini 344ccefbe5SStefano Stabellini /* 354ccefbe5SStefano Stabellini * Read high then low, and then make sure high is 364ccefbe5SStefano Stabellini * still the same; this will only loop if low wraps 374ccefbe5SStefano Stabellini * and carries into high. 384ccefbe5SStefano Stabellini * XXX some clean way to make this endian-proof? 394ccefbe5SStefano Stabellini */ 404ccefbe5SStefano Stabellini do { 412dd887e3SStefano Stabellini h = READ_ONCE(p32[1]); 422dd887e3SStefano Stabellini l = READ_ONCE(p32[0]); 432dd887e3SStefano Stabellini h2 = READ_ONCE(p32[1]); 442dd887e3SStefano Stabellini } while(h2 != h); 454ccefbe5SStefano Stabellini 464ccefbe5SStefano Stabellini ret = (((u64)h) << 32) | l; 474ccefbe5SStefano Stabellini } else 482dd887e3SStefano Stabellini ret = READ_ONCE(*p); 494ccefbe5SStefano Stabellini 504ccefbe5SStefano Stabellini return ret; 514ccefbe5SStefano Stabellini } 524ccefbe5SStefano Stabellini 535e25f5dbSDongli Zhang static void xen_get_runstate_snapshot_cpu_delta( 545e25f5dbSDongli Zhang struct vcpu_runstate_info *res, unsigned int cpu) 554ccefbe5SStefano Stabellini { 564ccefbe5SStefano Stabellini u64 state_time; 574ccefbe5SStefano Stabellini struct vcpu_runstate_info *state; 584ccefbe5SStefano Stabellini 594ccefbe5SStefano Stabellini BUG_ON(preemptible()); 604ccefbe5SStefano Stabellini 616ba286adSJuergen Gross state = per_cpu_ptr(&xen_runstate, cpu); 624ccefbe5SStefano Stabellini 634ccefbe5SStefano Stabellini do { 644ccefbe5SStefano Stabellini state_time = get64(&state->state_entry_time); 656ba286adSJuergen Gross rmb(); /* Hypervisor might update data. */ 662dd887e3SStefano Stabellini *res = READ_ONCE(*state); 676ba286adSJuergen Gross rmb(); /* Hypervisor might update data. */ 686ba286adSJuergen Gross } while (get64(&state->state_entry_time) != state_time || 696ba286adSJuergen Gross (state_time & XEN_RUNSTATE_UPDATE)); 706ba286adSJuergen Gross } 716ba286adSJuergen Gross 725e25f5dbSDongli Zhang static void xen_get_runstate_snapshot_cpu(struct vcpu_runstate_info *res, 735e25f5dbSDongli Zhang unsigned int cpu) 745e25f5dbSDongli Zhang { 755e25f5dbSDongli Zhang int i; 765e25f5dbSDongli Zhang 775e25f5dbSDongli Zhang xen_get_runstate_snapshot_cpu_delta(res, cpu); 785e25f5dbSDongli Zhang 795e25f5dbSDongli Zhang for (i = 0; i < 4; i++) 805e25f5dbSDongli Zhang res->time[i] += per_cpu(old_runstate_time, cpu)[i]; 815e25f5dbSDongli Zhang } 825e25f5dbSDongli Zhang 835e25f5dbSDongli Zhang void xen_manage_runstate_time(int action) 845e25f5dbSDongli Zhang { 855e25f5dbSDongli Zhang static struct vcpu_runstate_info *runstate_delta; 865e25f5dbSDongli Zhang struct vcpu_runstate_info state; 875e25f5dbSDongli Zhang int cpu, i; 885e25f5dbSDongli Zhang 895e25f5dbSDongli Zhang switch (action) { 905e25f5dbSDongli Zhang case -1: /* backup runstate time before suspend */ 915e25f5dbSDongli Zhang if (unlikely(runstate_delta)) 925e25f5dbSDongli Zhang pr_warn_once("%s: memory leak as runstate_delta is not NULL\n", 935e25f5dbSDongli Zhang __func__); 945e25f5dbSDongli Zhang 955e25f5dbSDongli Zhang runstate_delta = kmalloc_array(num_possible_cpus(), 965e25f5dbSDongli Zhang sizeof(*runstate_delta), 975e25f5dbSDongli Zhang GFP_ATOMIC); 985e25f5dbSDongli Zhang if (unlikely(!runstate_delta)) { 995e25f5dbSDongli Zhang pr_warn("%s: failed to allocate runstate_delta\n", 1005e25f5dbSDongli Zhang __func__); 1015e25f5dbSDongli Zhang return; 1025e25f5dbSDongli Zhang } 1035e25f5dbSDongli Zhang 1045e25f5dbSDongli Zhang for_each_possible_cpu(cpu) { 1055e25f5dbSDongli Zhang xen_get_runstate_snapshot_cpu_delta(&state, cpu); 1065e25f5dbSDongli Zhang memcpy(runstate_delta[cpu].time, state.time, 1075e25f5dbSDongli Zhang sizeof(runstate_delta[cpu].time)); 1085e25f5dbSDongli Zhang } 1095e25f5dbSDongli Zhang 1105e25f5dbSDongli Zhang break; 1115e25f5dbSDongli Zhang 1125e25f5dbSDongli Zhang case 0: /* backup runstate time after resume */ 1135e25f5dbSDongli Zhang if (unlikely(!runstate_delta)) { 1145e25f5dbSDongli Zhang pr_warn("%s: cannot accumulate runstate time as runstate_delta is NULL\n", 1155e25f5dbSDongli Zhang __func__); 1165e25f5dbSDongli Zhang return; 1175e25f5dbSDongli Zhang } 1185e25f5dbSDongli Zhang 1195e25f5dbSDongli Zhang for_each_possible_cpu(cpu) { 1205e25f5dbSDongli Zhang for (i = 0; i < 4; i++) 1215e25f5dbSDongli Zhang per_cpu(old_runstate_time, cpu)[i] += 1225e25f5dbSDongli Zhang runstate_delta[cpu].time[i]; 1235e25f5dbSDongli Zhang } 1245e25f5dbSDongli Zhang 1255e25f5dbSDongli Zhang break; 1265e25f5dbSDongli Zhang 1275e25f5dbSDongli Zhang default: /* do not accumulate runstate time for checkpointing */ 1285e25f5dbSDongli Zhang break; 1295e25f5dbSDongli Zhang } 1305e25f5dbSDongli Zhang 1315e25f5dbSDongli Zhang if (action != -1 && runstate_delta) { 1325e25f5dbSDongli Zhang kfree(runstate_delta); 1335e25f5dbSDongli Zhang runstate_delta = NULL; 1345e25f5dbSDongli Zhang } 1355e25f5dbSDongli Zhang } 1365e25f5dbSDongli Zhang 1376ba286adSJuergen Gross /* 1386ba286adSJuergen Gross * Runstate accounting 1396ba286adSJuergen Gross */ 1406ba286adSJuergen Gross void xen_get_runstate_snapshot(struct vcpu_runstate_info *res) 1416ba286adSJuergen Gross { 1426ba286adSJuergen Gross xen_get_runstate_snapshot_cpu(res, smp_processor_id()); 1434ccefbe5SStefano Stabellini } 1444ccefbe5SStefano Stabellini 1454ccefbe5SStefano Stabellini /* return true when a vcpu could run but has no real cpu to run on */ 1464ccefbe5SStefano Stabellini bool xen_vcpu_stolen(int vcpu) 1474ccefbe5SStefano Stabellini { 1484ccefbe5SStefano Stabellini return per_cpu(xen_runstate, vcpu).state == RUNSTATE_runnable; 1494ccefbe5SStefano Stabellini } 1504ccefbe5SStefano Stabellini 151d34c30ccSJuergen Gross u64 xen_steal_clock(int cpu) 152ecb23dc6SJuergen Gross { 153ecb23dc6SJuergen Gross struct vcpu_runstate_info state; 154ecb23dc6SJuergen Gross 1556ba286adSJuergen Gross xen_get_runstate_snapshot_cpu(&state, cpu); 156ecb23dc6SJuergen Gross return state.time[RUNSTATE_runnable] + state.time[RUNSTATE_offline]; 157ecb23dc6SJuergen Gross } 158ecb23dc6SJuergen Gross 1594ccefbe5SStefano Stabellini void xen_setup_runstate_info(int cpu) 1604ccefbe5SStefano Stabellini { 1614ccefbe5SStefano Stabellini struct vcpu_register_runstate_memory_area area; 1624ccefbe5SStefano Stabellini 1634ccefbe5SStefano Stabellini area.addr.v = &per_cpu(xen_runstate, cpu); 1644ccefbe5SStefano Stabellini 1654ccefbe5SStefano Stabellini if (HYPERVISOR_vcpu_op(VCPUOP_register_runstate_memory_area, 166ad5475f9SVitaly Kuznetsov xen_vcpu_nr(cpu), &area)) 1674ccefbe5SStefano Stabellini BUG(); 1684ccefbe5SStefano Stabellini } 1694ccefbe5SStefano Stabellini 170ecb23dc6SJuergen Gross void __init xen_time_setup_guest(void) 171ecb23dc6SJuergen Gross { 1726ba286adSJuergen Gross bool xen_runstate_remote; 1736ba286adSJuergen Gross 1746ba286adSJuergen Gross xen_runstate_remote = !HYPERVISOR_vm_assist(VMASST_CMD_enable, 1756ba286adSJuergen Gross VMASST_TYPE_runstate_update_flag); 1766ba286adSJuergen Gross 177ecb23dc6SJuergen Gross pv_time_ops.steal_clock = xen_steal_clock; 178ecb23dc6SJuergen Gross 179ecb23dc6SJuergen Gross static_key_slow_inc(¶virt_steal_enabled); 1806ba286adSJuergen Gross if (xen_runstate_remote) 1816ba286adSJuergen Gross static_key_slow_inc(¶virt_steal_rq_enabled); 182ecb23dc6SJuergen Gross } 183