1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * 4 * Copyright (C) 2013 Citrix Systems 5 * 6 * Author: Stefano Stabellini <stefano.stabellini@eu.citrix.com> 7 */ 8 9 #define pr_fmt(fmt) "arm-pv: " fmt 10 11 #include <linux/arm-smccc.h> 12 #include <linux/cpuhotplug.h> 13 #include <linux/export.h> 14 #include <linux/io.h> 15 #include <linux/jump_label.h> 16 #include <linux/printk.h> 17 #include <linux/psci.h> 18 #include <linux/reboot.h> 19 #include <linux/slab.h> 20 #include <linux/types.h> 21 #include <linux/static_call.h> 22 23 #include <asm/paravirt.h> 24 #include <asm/pvclock-abi.h> 25 #include <asm/smp_plat.h> 26 27 struct static_key paravirt_steal_enabled; 28 struct static_key paravirt_steal_rq_enabled; 29 30 static u64 native_steal_clock(int cpu) 31 { 32 return 0; 33 } 34 35 DEFINE_STATIC_CALL(pv_steal_clock, native_steal_clock); 36 37 struct pv_time_stolen_time_region { 38 struct pvclock_vcpu_stolen_time __rcu *kaddr; 39 }; 40 41 static DEFINE_PER_CPU(struct pv_time_stolen_time_region, stolen_time_region); 42 43 static bool steal_acc = true; 44 static int __init parse_no_stealacc(char *arg) 45 { 46 steal_acc = false; 47 return 0; 48 } 49 50 early_param("no-steal-acc", parse_no_stealacc); 51 52 /* return stolen time in ns by asking the hypervisor */ 53 static u64 para_steal_clock(int cpu) 54 { 55 struct pvclock_vcpu_stolen_time *kaddr = NULL; 56 struct pv_time_stolen_time_region *reg; 57 u64 ret = 0; 58 59 reg = per_cpu_ptr(&stolen_time_region, cpu); 60 61 /* 62 * paravirt_steal_clock() may be called before the CPU 63 * online notification callback runs. Until the callback 64 * has run we just return zero. 65 */ 66 rcu_read_lock(); 67 kaddr = rcu_dereference(reg->kaddr); 68 if (!kaddr) { 69 rcu_read_unlock(); 70 return 0; 71 } 72 73 ret = le64_to_cpu(READ_ONCE(kaddr->stolen_time)); 74 rcu_read_unlock(); 75 return ret; 76 } 77 78 static int stolen_time_cpu_down_prepare(unsigned int cpu) 79 { 80 struct pvclock_vcpu_stolen_time *kaddr = NULL; 81 struct pv_time_stolen_time_region *reg; 82 83 reg = this_cpu_ptr(&stolen_time_region); 84 if (!reg->kaddr) 85 return 0; 86 87 kaddr = rcu_replace_pointer(reg->kaddr, NULL, true); 88 synchronize_rcu(); 89 memunmap(kaddr); 90 91 return 0; 92 } 93 94 static int stolen_time_cpu_online(unsigned int cpu) 95 { 96 struct pvclock_vcpu_stolen_time *kaddr = NULL; 97 struct pv_time_stolen_time_region *reg; 98 struct arm_smccc_res res; 99 100 reg = this_cpu_ptr(&stolen_time_region); 101 102 arm_smccc_1_1_invoke(ARM_SMCCC_HV_PV_TIME_ST, &res); 103 104 if (res.a0 == SMCCC_RET_NOT_SUPPORTED) 105 return -EINVAL; 106 107 kaddr = memremap(res.a0, 108 sizeof(struct pvclock_vcpu_stolen_time), 109 MEMREMAP_WB); 110 111 rcu_assign_pointer(reg->kaddr, kaddr); 112 113 if (!reg->kaddr) { 114 pr_warn("Failed to map stolen time data structure\n"); 115 return -ENOMEM; 116 } 117 118 if (le32_to_cpu(kaddr->revision) != 0 || 119 le32_to_cpu(kaddr->attributes) != 0) { 120 pr_warn_once("Unexpected revision or attributes in stolen time data\n"); 121 return -ENXIO; 122 } 123 124 return 0; 125 } 126 127 static int __init pv_time_init_stolen_time(void) 128 { 129 int ret; 130 131 ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, 132 "hypervisor/arm/pvtime:online", 133 stolen_time_cpu_online, 134 stolen_time_cpu_down_prepare); 135 if (ret < 0) 136 return ret; 137 return 0; 138 } 139 140 static bool __init has_pv_steal_clock(void) 141 { 142 struct arm_smccc_res res; 143 144 arm_smccc_1_1_invoke(ARM_SMCCC_ARCH_FEATURES_FUNC_ID, 145 ARM_SMCCC_HV_PV_TIME_FEATURES, &res); 146 147 if (res.a0 != SMCCC_RET_SUCCESS) 148 return false; 149 150 arm_smccc_1_1_invoke(ARM_SMCCC_HV_PV_TIME_FEATURES, 151 ARM_SMCCC_HV_PV_TIME_ST, &res); 152 153 return (res.a0 == SMCCC_RET_SUCCESS); 154 } 155 156 int __init pv_time_init(void) 157 { 158 int ret; 159 160 if (!has_pv_steal_clock()) 161 return 0; 162 163 ret = pv_time_init_stolen_time(); 164 if (ret) 165 return ret; 166 167 static_call_update(pv_steal_clock, para_steal_clock); 168 169 static_key_slow_inc(¶virt_steal_enabled); 170 if (steal_acc) 171 static_key_slow_inc(¶virt_steal_rq_enabled); 172 173 pr_info("using stolen time PV\n"); 174 175 return 0; 176 } 177