1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * arch/arm64/include/asm/arch_timer.h
4  *
5  * Copyright (C) 2012 ARM Ltd.
6  * Author: Marc Zyngier <marc.zyngier@arm.com>
7  */
8 #ifndef __ASM_ARCH_TIMER_H
9 #define __ASM_ARCH_TIMER_H
10 
11 #include <asm/barrier.h>
12 #include <asm/hwcap.h>
13 #include <asm/sysreg.h>
14 
15 #include <linux/bug.h>
16 #include <linux/init.h>
17 #include <linux/jump_label.h>
18 #include <linux/smp.h>
19 #include <linux/types.h>
20 
21 #include <clocksource/arm_arch_timer.h>
22 
23 #if IS_ENABLED(CONFIG_ARM_ARCH_TIMER_OOL_WORKAROUND)
24 #define has_erratum_handler(h)						\
25 	({								\
26 		const struct arch_timer_erratum_workaround *__wa;	\
27 		__wa = __this_cpu_read(timer_unstable_counter_workaround); \
28 		(__wa && __wa->h);					\
29 	})
30 
31 #define erratum_handler(h)						\
32 	({								\
33 		const struct arch_timer_erratum_workaround *__wa;	\
34 		__wa = __this_cpu_read(timer_unstable_counter_workaround); \
35 		(__wa && __wa->h) ? ({ isb(); __wa->h;}) : arch_timer_##h; \
36 	})
37 
38 #else
39 #define has_erratum_handler(h)			   false
40 #define erratum_handler(h)			   (arch_timer_##h)
41 #endif
42 
43 enum arch_timer_erratum_match_type {
44 	ate_match_dt,
45 	ate_match_local_cap_id,
46 	ate_match_acpi_oem_info,
47 };
48 
49 struct clock_event_device;
50 
51 struct arch_timer_erratum_workaround {
52 	enum arch_timer_erratum_match_type match_type;
53 	const void *id;
54 	const char *desc;
55 	u64 (*read_cntpct_el0)(void);
56 	u64 (*read_cntvct_el0)(void);
57 	int (*set_next_event_phys)(unsigned long, struct clock_event_device *);
58 	int (*set_next_event_virt)(unsigned long, struct clock_event_device *);
59 	bool disable_compat_vdso;
60 };
61 
62 DECLARE_PER_CPU(const struct arch_timer_erratum_workaround *,
63 		timer_unstable_counter_workaround);
64 
arch_timer_read_cntpct_el0(void)65 static inline notrace u64 arch_timer_read_cntpct_el0(void)
66 {
67 	u64 cnt;
68 
69 	asm volatile(ALTERNATIVE("isb\n mrs %0, cntpct_el0",
70 				 "nop\n" __mrs_s("%0", SYS_CNTPCTSS_EL0),
71 				 ARM64_HAS_ECV)
72 		     : "=r" (cnt));
73 
74 	return cnt;
75 }
76 
arch_timer_read_cntvct_el0(void)77 static inline notrace u64 arch_timer_read_cntvct_el0(void)
78 {
79 	u64 cnt;
80 
81 	asm volatile(ALTERNATIVE("isb\n mrs %0, cntvct_el0",
82 				 "nop\n" __mrs_s("%0", SYS_CNTVCTSS_EL0),
83 				 ARM64_HAS_ECV)
84 		     : "=r" (cnt));
85 
86 	return cnt;
87 }
88 
89 #define arch_timer_reg_read_stable(reg)					\
90 	({								\
91 		erratum_handler(read_ ## reg)();			\
92 	})
93 
94 /*
95  * These register accessors are marked inline so the compiler can
96  * nicely work out which register we want, and chuck away the rest of
97  * the code.
98  */
99 static __always_inline
arch_timer_reg_write_cp15(int access,enum arch_timer_reg reg,u64 val)100 void arch_timer_reg_write_cp15(int access, enum arch_timer_reg reg, u64 val)
101 {
102 	if (access == ARCH_TIMER_PHYS_ACCESS) {
103 		switch (reg) {
104 		case ARCH_TIMER_REG_CTRL:
105 			write_sysreg(val, cntp_ctl_el0);
106 			isb();
107 			break;
108 		case ARCH_TIMER_REG_CVAL:
109 			write_sysreg(val, cntp_cval_el0);
110 			break;
111 		default:
112 			BUILD_BUG();
113 		}
114 	} else if (access == ARCH_TIMER_VIRT_ACCESS) {
115 		switch (reg) {
116 		case ARCH_TIMER_REG_CTRL:
117 			write_sysreg(val, cntv_ctl_el0);
118 			isb();
119 			break;
120 		case ARCH_TIMER_REG_CVAL:
121 			write_sysreg(val, cntv_cval_el0);
122 			break;
123 		default:
124 			BUILD_BUG();
125 		}
126 	} else {
127 		BUILD_BUG();
128 	}
129 }
130 
131 static __always_inline
arch_timer_reg_read_cp15(int access,enum arch_timer_reg reg)132 u64 arch_timer_reg_read_cp15(int access, enum arch_timer_reg reg)
133 {
134 	if (access == ARCH_TIMER_PHYS_ACCESS) {
135 		switch (reg) {
136 		case ARCH_TIMER_REG_CTRL:
137 			return read_sysreg(cntp_ctl_el0);
138 		default:
139 			BUILD_BUG();
140 		}
141 	} else if (access == ARCH_TIMER_VIRT_ACCESS) {
142 		switch (reg) {
143 		case ARCH_TIMER_REG_CTRL:
144 			return read_sysreg(cntv_ctl_el0);
145 		default:
146 			BUILD_BUG();
147 		}
148 	}
149 
150 	BUILD_BUG();
151 	unreachable();
152 }
153 
arch_timer_get_cntfrq(void)154 static inline u32 arch_timer_get_cntfrq(void)
155 {
156 	return read_sysreg(cntfrq_el0);
157 }
158 
arch_timer_get_cntkctl(void)159 static inline u32 arch_timer_get_cntkctl(void)
160 {
161 	return read_sysreg(cntkctl_el1);
162 }
163 
arch_timer_set_cntkctl(u32 cntkctl)164 static inline void arch_timer_set_cntkctl(u32 cntkctl)
165 {
166 	write_sysreg(cntkctl, cntkctl_el1);
167 	isb();
168 }
169 
__arch_counter_get_cntpct_stable(void)170 static __always_inline u64 __arch_counter_get_cntpct_stable(void)
171 {
172 	u64 cnt;
173 
174 	cnt = arch_timer_reg_read_stable(cntpct_el0);
175 	arch_counter_enforce_ordering(cnt);
176 	return cnt;
177 }
178 
__arch_counter_get_cntpct(void)179 static __always_inline u64 __arch_counter_get_cntpct(void)
180 {
181 	u64 cnt;
182 
183 	asm volatile(ALTERNATIVE("isb\n mrs %0, cntpct_el0",
184 				 "nop\n" __mrs_s("%0", SYS_CNTPCTSS_EL0),
185 				 ARM64_HAS_ECV)
186 		     : "=r" (cnt));
187 	arch_counter_enforce_ordering(cnt);
188 	return cnt;
189 }
190 
__arch_counter_get_cntvct_stable(void)191 static __always_inline u64 __arch_counter_get_cntvct_stable(void)
192 {
193 	u64 cnt;
194 
195 	cnt = arch_timer_reg_read_stable(cntvct_el0);
196 	arch_counter_enforce_ordering(cnt);
197 	return cnt;
198 }
199 
__arch_counter_get_cntvct(void)200 static __always_inline u64 __arch_counter_get_cntvct(void)
201 {
202 	u64 cnt;
203 
204 	asm volatile(ALTERNATIVE("isb\n mrs %0, cntvct_el0",
205 				 "nop\n" __mrs_s("%0", SYS_CNTVCTSS_EL0),
206 				 ARM64_HAS_ECV)
207 		     : "=r" (cnt));
208 	arch_counter_enforce_ordering(cnt);
209 	return cnt;
210 }
211 
arch_timer_arch_init(void)212 static inline int arch_timer_arch_init(void)
213 {
214 	return 0;
215 }
216 
arch_timer_set_evtstrm_feature(void)217 static inline void arch_timer_set_evtstrm_feature(void)
218 {
219 	cpu_set_named_feature(EVTSTRM);
220 #ifdef CONFIG_COMPAT
221 	compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM;
222 #endif
223 }
224 
arch_timer_have_evtstrm_feature(void)225 static inline bool arch_timer_have_evtstrm_feature(void)
226 {
227 	return cpu_have_named_feature(EVTSTRM);
228 }
229 #endif
230