1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * arch/arm64/include/asm/arch_timer.h 4 * 5 * Copyright (C) 2012 ARM Ltd. 6 * Author: Marc Zyngier <marc.zyngier@arm.com> 7 */ 8 #ifndef __ASM_ARCH_TIMER_H 9 #define __ASM_ARCH_TIMER_H 10 11 #include <asm/barrier.h> 12 #include <asm/hwcap.h> 13 #include <asm/sysreg.h> 14 15 #include <linux/bug.h> 16 #include <linux/init.h> 17 #include <linux/jump_label.h> 18 #include <linux/smp.h> 19 #include <linux/types.h> 20 21 #include <clocksource/arm_arch_timer.h> 22 23 #if IS_ENABLED(CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND) 24 #define has_erratum_handler(h) \ 25 ({ \ 26 const struct arch_timer_erratum_workaround *__wa; \ 27 __wa = __this_cpu_read(timer_unstable_counter_workaround); \ 28 (__wa && __wa->h); \ 29 }) 30 31 #define erratum_handler(h) \ 32 ({ \ 33 const struct arch_timer_erratum_workaround *__wa; \ 34 __wa = __this_cpu_read(timer_unstable_counter_workaround); \ 35 (__wa && __wa->h) ? __wa->h : arch_timer_##h; \ 36 }) 37 38 #else 39 #define has_erratum_handler(h) false 40 #define erratum_handler(h) (arch_timer_##h) 41 #endif 42 43 enum arch_timer_erratum_match_type { 44 ate_match_dt, 45 ate_match_local_cap_id, 46 ate_match_acpi_oem_info, 47 }; 48 49 struct clock_event_device; 50 51 struct arch_timer_erratum_workaround { 52 enum arch_timer_erratum_match_type match_type; 53 const void *id; 54 const char *desc; 55 u32 (*read_cntp_tval_el0)(void); 56 u32 (*read_cntv_tval_el0)(void); 57 u64 (*read_cntpct_el0)(void); 58 u64 (*read_cntvct_el0)(void); 59 int (*set_next_event_phys)(unsigned long, struct clock_event_device *); 60 int (*set_next_event_virt)(unsigned long, struct clock_event_device *); 61 bool disable_compat_vdso; 62 }; 63 64 DECLARE_PER_CPU(const struct arch_timer_erratum_workaround *, 65 timer_unstable_counter_workaround); 66 67 /* inline sysreg accessors that make erratum_handler() work */ 68 static inline notrace u32 arch_timer_read_cntp_tval_el0(void) 69 { 70 return read_sysreg(cntp_tval_el0); 71 } 72 73 static inline notrace u32 arch_timer_read_cntv_tval_el0(void) 74 { 75 return read_sysreg(cntv_tval_el0); 76 } 77 78 static inline notrace u64 arch_timer_read_cntpct_el0(void) 79 { 80 return read_sysreg(cntpct_el0); 81 } 82 83 static inline notrace u64 arch_timer_read_cntvct_el0(void) 84 { 85 return read_sysreg(cntvct_el0); 86 } 87 88 #define arch_timer_reg_read_stable(reg) \ 89 ({ \ 90 u64 _val; \ 91 \ 92 preempt_disable_notrace(); \ 93 _val = erratum_handler(read_ ## reg)(); \ 94 preempt_enable_notrace(); \ 95 \ 96 _val; \ 97 }) 98 99 /* 100 * These register accessors are marked inline so the compiler can 101 * nicely work out which register we want, and chuck away the rest of 102 * the code. 103 */ 104 static __always_inline 105 void arch_timer_reg_write_cp15(int access, enum arch_timer_reg reg, u32 val) 106 { 107 if (access == ARCH_TIMER_PHYS_ACCESS) { 108 switch (reg) { 109 case ARCH_TIMER_REG_CTRL: 110 write_sysreg(val, cntp_ctl_el0); 111 break; 112 case ARCH_TIMER_REG_TVAL: 113 write_sysreg(val, cntp_tval_el0); 114 break; 115 } 116 } else if (access == ARCH_TIMER_VIRT_ACCESS) { 117 switch (reg) { 118 case ARCH_TIMER_REG_CTRL: 119 write_sysreg(val, cntv_ctl_el0); 120 break; 121 case ARCH_TIMER_REG_TVAL: 122 write_sysreg(val, cntv_tval_el0); 123 break; 124 } 125 } 126 127 isb(); 128 } 129 130 static __always_inline 131 u32 arch_timer_reg_read_cp15(int access, enum arch_timer_reg reg) 132 { 133 if (access == ARCH_TIMER_PHYS_ACCESS) { 134 switch (reg) { 135 case ARCH_TIMER_REG_CTRL: 136 return read_sysreg(cntp_ctl_el0); 137 case ARCH_TIMER_REG_TVAL: 138 return arch_timer_reg_read_stable(cntp_tval_el0); 139 } 140 } else if (access == ARCH_TIMER_VIRT_ACCESS) { 141 switch (reg) { 142 case ARCH_TIMER_REG_CTRL: 143 return read_sysreg(cntv_ctl_el0); 144 case ARCH_TIMER_REG_TVAL: 145 return arch_timer_reg_read_stable(cntv_tval_el0); 146 } 147 } 148 149 BUG(); 150 } 151 152 static inline u32 arch_timer_get_cntfrq(void) 153 { 154 return read_sysreg(cntfrq_el0); 155 } 156 157 static inline u32 arch_timer_get_cntkctl(void) 158 { 159 return read_sysreg(cntkctl_el1); 160 } 161 162 static inline void arch_timer_set_cntkctl(u32 cntkctl) 163 { 164 write_sysreg(cntkctl, cntkctl_el1); 165 isb(); 166 } 167 168 /* 169 * Ensure that reads of the counter are treated the same as memory reads 170 * for the purposes of ordering by subsequent memory barriers. 171 * 172 * This insanity brought to you by speculative system register reads, 173 * out-of-order memory accesses, sequence locks and Thomas Gleixner. 174 * 175 * http://lists.infradead.org/pipermail/linux-arm-kernel/2019-February/631195.html 176 */ 177 #define arch_counter_enforce_ordering(val) do { \ 178 u64 tmp, _val = (val); \ 179 \ 180 asm volatile( \ 181 " eor %0, %1, %1\n" \ 182 " add %0, sp, %0\n" \ 183 " ldr xzr, [%0]" \ 184 : "=r" (tmp) : "r" (_val)); \ 185 } while (0) 186 187 static __always_inline u64 __arch_counter_get_cntpct_stable(void) 188 { 189 u64 cnt; 190 191 isb(); 192 cnt = arch_timer_reg_read_stable(cntpct_el0); 193 arch_counter_enforce_ordering(cnt); 194 return cnt; 195 } 196 197 static __always_inline u64 __arch_counter_get_cntpct(void) 198 { 199 u64 cnt; 200 201 isb(); 202 cnt = read_sysreg(cntpct_el0); 203 arch_counter_enforce_ordering(cnt); 204 return cnt; 205 } 206 207 static __always_inline u64 __arch_counter_get_cntvct_stable(void) 208 { 209 u64 cnt; 210 211 isb(); 212 cnt = arch_timer_reg_read_stable(cntvct_el0); 213 arch_counter_enforce_ordering(cnt); 214 return cnt; 215 } 216 217 static __always_inline u64 __arch_counter_get_cntvct(void) 218 { 219 u64 cnt; 220 221 isb(); 222 cnt = read_sysreg(cntvct_el0); 223 arch_counter_enforce_ordering(cnt); 224 return cnt; 225 } 226 227 #undef arch_counter_enforce_ordering 228 229 static inline int arch_timer_arch_init(void) 230 { 231 return 0; 232 } 233 234 static inline void arch_timer_set_evtstrm_feature(void) 235 { 236 cpu_set_named_feature(EVTSTRM); 237 #ifdef CONFIG_COMPAT 238 compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM; 239 #endif 240 } 241 242 static inline bool arch_timer_have_evtstrm_feature(void) 243 { 244 return cpu_have_named_feature(EVTSTRM); 245 } 246 #endif 247