1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright 2015 Mentor Graphics Corporation. 4 */ 5 6 #include <linux/compiler.h> 7 #include <linux/hrtimer.h> 8 #include <linux/time.h> 9 #include <asm/barrier.h> 10 #include <asm/bug.h> 11 #include <asm/cp15.h> 12 #include <asm/page.h> 13 #include <asm/unistd.h> 14 #include <asm/vdso_datapage.h> 15 16 #ifndef CONFIG_AEABI 17 #error This code depends on AEABI system call conventions 18 #endif 19 20 extern struct vdso_data *__get_datapage(void); 21 22 static notrace u32 __vdso_read_begin(const struct vdso_data *vdata) 23 { 24 u32 seq; 25 repeat: 26 seq = READ_ONCE(vdata->seq_count); 27 if (seq & 1) { 28 cpu_relax(); 29 goto repeat; 30 } 31 return seq; 32 } 33 34 static notrace u32 vdso_read_begin(const struct vdso_data *vdata) 35 { 36 u32 seq; 37 38 seq = __vdso_read_begin(vdata); 39 40 smp_rmb(); /* Pairs with smp_wmb in vdso_write_end */ 41 return seq; 42 } 43 44 static notrace int vdso_read_retry(const struct vdso_data *vdata, u32 start) 45 { 46 smp_rmb(); /* Pairs with smp_wmb in vdso_write_begin */ 47 return vdata->seq_count != start; 48 } 49 50 static notrace long clock_gettime_fallback(clockid_t _clkid, 51 struct timespec *_ts) 52 { 53 register struct timespec *ts asm("r1") = _ts; 54 register clockid_t clkid asm("r0") = _clkid; 55 register long ret asm ("r0"); 56 register long nr asm("r7") = __NR_clock_gettime; 57 58 asm volatile( 59 " swi #0\n" 60 : "=r" (ret) 61 : "r" (clkid), "r" (ts), "r" (nr) 62 : "memory"); 63 64 return ret; 65 } 66 67 static notrace int do_realtime_coarse(struct timespec *ts, 68 struct vdso_data *vdata) 69 { 70 u32 seq; 71 72 do { 73 seq = vdso_read_begin(vdata); 74 75 ts->tv_sec = vdata->xtime_coarse_sec; 76 ts->tv_nsec = vdata->xtime_coarse_nsec; 77 78 } while (vdso_read_retry(vdata, seq)); 79 80 return 0; 81 } 82 83 static notrace int do_monotonic_coarse(struct timespec *ts, 84 struct vdso_data *vdata) 85 { 86 struct timespec tomono; 87 u32 seq; 88 89 do { 90 seq = vdso_read_begin(vdata); 91 92 ts->tv_sec = vdata->xtime_coarse_sec; 93 ts->tv_nsec = vdata->xtime_coarse_nsec; 94 95 tomono.tv_sec = vdata->wtm_clock_sec; 96 tomono.tv_nsec = vdata->wtm_clock_nsec; 97 98 } while (vdso_read_retry(vdata, seq)); 99 100 ts->tv_sec += tomono.tv_sec; 101 timespec_add_ns(ts, tomono.tv_nsec); 102 103 return 0; 104 } 105 106 #ifdef CONFIG_ARM_ARCH_TIMER 107 108 static notrace u64 get_ns(struct vdso_data *vdata) 109 { 110 u64 cycle_delta; 111 u64 cycle_now; 112 u64 nsec; 113 114 isb(); 115 cycle_now = read_sysreg(CNTVCT); 116 117 cycle_delta = (cycle_now - vdata->cs_cycle_last) & vdata->cs_mask; 118 119 nsec = (cycle_delta * vdata->cs_mult) + vdata->xtime_clock_snsec; 120 nsec >>= vdata->cs_shift; 121 122 return nsec; 123 } 124 125 static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata) 126 { 127 u64 nsecs; 128 u32 seq; 129 130 do { 131 seq = vdso_read_begin(vdata); 132 133 if (!vdata->tk_is_cntvct) 134 return -1; 135 136 ts->tv_sec = vdata->xtime_clock_sec; 137 nsecs = get_ns(vdata); 138 139 } while (vdso_read_retry(vdata, seq)); 140 141 ts->tv_nsec = 0; 142 timespec_add_ns(ts, nsecs); 143 144 return 0; 145 } 146 147 static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata) 148 { 149 struct timespec tomono; 150 u64 nsecs; 151 u32 seq; 152 153 do { 154 seq = vdso_read_begin(vdata); 155 156 if (!vdata->tk_is_cntvct) 157 return -1; 158 159 ts->tv_sec = vdata->xtime_clock_sec; 160 nsecs = get_ns(vdata); 161 162 tomono.tv_sec = vdata->wtm_clock_sec; 163 tomono.tv_nsec = vdata->wtm_clock_nsec; 164 165 } while (vdso_read_retry(vdata, seq)); 166 167 ts->tv_sec += tomono.tv_sec; 168 ts->tv_nsec = 0; 169 timespec_add_ns(ts, nsecs + tomono.tv_nsec); 170 171 return 0; 172 } 173 174 #else /* CONFIG_ARM_ARCH_TIMER */ 175 176 static notrace int do_realtime(struct timespec *ts, struct vdso_data *vdata) 177 { 178 return -1; 179 } 180 181 static notrace int do_monotonic(struct timespec *ts, struct vdso_data *vdata) 182 { 183 return -1; 184 } 185 186 #endif /* CONFIG_ARM_ARCH_TIMER */ 187 188 notrace int __vdso_clock_gettime(clockid_t clkid, struct timespec *ts) 189 { 190 struct vdso_data *vdata; 191 int ret = -1; 192 193 vdata = __get_datapage(); 194 195 switch (clkid) { 196 case CLOCK_REALTIME_COARSE: 197 ret = do_realtime_coarse(ts, vdata); 198 break; 199 case CLOCK_MONOTONIC_COARSE: 200 ret = do_monotonic_coarse(ts, vdata); 201 break; 202 case CLOCK_REALTIME: 203 ret = do_realtime(ts, vdata); 204 break; 205 case CLOCK_MONOTONIC: 206 ret = do_monotonic(ts, vdata); 207 break; 208 default: 209 break; 210 } 211 212 if (ret) 213 ret = clock_gettime_fallback(clkid, ts); 214 215 return ret; 216 } 217 218 static notrace long gettimeofday_fallback(struct timeval *_tv, 219 struct timezone *_tz) 220 { 221 register struct timezone *tz asm("r1") = _tz; 222 register struct timeval *tv asm("r0") = _tv; 223 register long ret asm ("r0"); 224 register long nr asm("r7") = __NR_gettimeofday; 225 226 asm volatile( 227 " swi #0\n" 228 : "=r" (ret) 229 : "r" (tv), "r" (tz), "r" (nr) 230 : "memory"); 231 232 return ret; 233 } 234 235 notrace int __vdso_gettimeofday(struct timeval *tv, struct timezone *tz) 236 { 237 struct timespec ts; 238 struct vdso_data *vdata; 239 int ret; 240 241 vdata = __get_datapage(); 242 243 ret = do_realtime(&ts, vdata); 244 if (ret) 245 return gettimeofday_fallback(tv, tz); 246 247 if (tv) { 248 tv->tv_sec = ts.tv_sec; 249 tv->tv_usec = ts.tv_nsec / 1000; 250 } 251 if (tz) { 252 tz->tz_minuteswest = vdata->tz_minuteswest; 253 tz->tz_dsttime = vdata->tz_dsttime; 254 } 255 256 return ret; 257 } 258 259 /* Avoid unresolved references emitted by GCC */ 260 261 void __aeabi_unwind_cpp_pr0(void) 262 { 263 } 264 265 void __aeabi_unwind_cpp_pr1(void) 266 { 267 } 268 269 void __aeabi_unwind_cpp_pr2(void) 270 { 271 } 272