1b2441318SGreg Kroah-Hartman /* SPDX-License-Identifier: GPL-2.0 */ 21965aae3SH. Peter Anvin #ifndef _ASM_X86_VGTOD_H 31965aae3SH. Peter Anvin #define _ASM_X86_VGTOD_H 4bb898558SAl Viro 57c03156fSStefani Seibold #include <linux/compiler.h> 6bb898558SAl Viro #include <linux/clocksource.h> 7bb898558SAl Viro 87c03156fSStefani Seibold #ifdef BUILD_VDSO32_64 97c03156fSStefani Seibold typedef u64 gtod_long_t; 107c03156fSStefani Seibold #else 117c03156fSStefani Seibold typedef unsigned long gtod_long_t; 127c03156fSStefani Seibold #endif 137c03156fSStefani Seibold /* 147c03156fSStefani Seibold * vsyscall_gtod_data will be accessed by 32 and 64 bit code at the same time 157c03156fSStefani Seibold * so be carefull by modifying this structure. 167c03156fSStefani Seibold */ 17bb898558SAl Viro struct vsyscall_gtod_data { 187c03156fSStefani Seibold unsigned seq; 19bb898558SAl Viro 2098d0ac38SAndy Lutomirski int vclock_mode; 21a5a1d1c2SThomas Gleixner u64 cycle_last; 22a5a1d1c2SThomas Gleixner u64 mask; 23bb898558SAl Viro u32 mult; 24bb898558SAl Viro u32 shift; 2591ec87d5SAndy Lutomirski 2691ec87d5SAndy Lutomirski /* open coded 'struct timespec' */ 27650ea024SJohn Stultz u64 wall_time_snsec; 287c03156fSStefani Seibold gtod_long_t wall_time_sec; 297c03156fSStefani Seibold gtod_long_t monotonic_time_sec; 30650ea024SJohn Stultz u64 monotonic_time_snsec; 317c03156fSStefani Seibold gtod_long_t wall_time_coarse_sec; 327c03156fSStefani Seibold gtod_long_t wall_time_coarse_nsec; 337c03156fSStefani Seibold gtod_long_t monotonic_time_coarse_sec; 347c03156fSStefani Seibold gtod_long_t monotonic_time_coarse_nsec; 3591ec87d5SAndy Lutomirski 367c03156fSStefani Seibold int tz_minuteswest; 377c03156fSStefani Seibold int tz_dsttime; 38bb898558SAl Viro }; 39bb898558SAl Viro extern struct vsyscall_gtod_data vsyscall_gtod_data; 40bb898558SAl Viro 41bd902c53SAndy Lutomirski extern int vclocks_used; 42bd902c53SAndy Lutomirski static inline bool vclock_was_used(int vclock) 43bd902c53SAndy Lutomirski { 44bd902c53SAndy Lutomirski return READ_ONCE(vclocks_used) & (1 << vclock); 45bd902c53SAndy Lutomirski } 46bd902c53SAndy Lutomirski 477c03156fSStefani Seibold static inline unsigned gtod_read_begin(const struct vsyscall_gtod_data *s) 487c03156fSStefani Seibold { 497c03156fSStefani Seibold unsigned ret; 507c03156fSStefani Seibold 517c03156fSStefani Seibold repeat: 527c03156fSStefani Seibold ret = ACCESS_ONCE(s->seq); 537c03156fSStefani Seibold if (unlikely(ret & 1)) { 547c03156fSStefani Seibold cpu_relax(); 557c03156fSStefani Seibold goto repeat; 567c03156fSStefani Seibold } 577c03156fSStefani Seibold smp_rmb(); 587c03156fSStefani Seibold return ret; 597c03156fSStefani Seibold } 607c03156fSStefani Seibold 617c03156fSStefani Seibold static inline int gtod_read_retry(const struct vsyscall_gtod_data *s, 627c03156fSStefani Seibold unsigned start) 637c03156fSStefani Seibold { 647c03156fSStefani Seibold smp_rmb(); 657c03156fSStefani Seibold return unlikely(s->seq != start); 667c03156fSStefani Seibold } 677c03156fSStefani Seibold 687c03156fSStefani Seibold static inline void gtod_write_begin(struct vsyscall_gtod_data *s) 697c03156fSStefani Seibold { 707c03156fSStefani Seibold ++s->seq; 717c03156fSStefani Seibold smp_wmb(); 727c03156fSStefani Seibold } 737c03156fSStefani Seibold 747c03156fSStefani Seibold static inline void gtod_write_end(struct vsyscall_gtod_data *s) 757c03156fSStefani Seibold { 767c03156fSStefani Seibold smp_wmb(); 777c03156fSStefani Seibold ++s->seq; 787c03156fSStefani Seibold } 797c03156fSStefani Seibold 80e76b027eSAndy Lutomirski #ifdef CONFIG_X86_64 81e76b027eSAndy Lutomirski 82e76b027eSAndy Lutomirski #define VGETCPU_CPU_MASK 0xfff 83e76b027eSAndy Lutomirski 84e76b027eSAndy Lutomirski static inline unsigned int __getcpu(void) 85e76b027eSAndy Lutomirski { 86e76b027eSAndy Lutomirski unsigned int p; 87e76b027eSAndy Lutomirski 88e76b027eSAndy Lutomirski /* 89e76b027eSAndy Lutomirski * Load per CPU data from GDT. LSL is faster than RDTSCP and 901ddf0b1bSAndy Lutomirski * works on all CPUs. This is volatile so that it orders 911ddf0b1bSAndy Lutomirski * correctly wrt barrier() and to keep gcc from cleverly 921ddf0b1bSAndy Lutomirski * hoisting it out of the calling function. 93a582c540SAndy Lutomirski * 94a582c540SAndy Lutomirski * If RDPID is available, use it. 95e76b027eSAndy Lutomirski */ 96a582c540SAndy Lutomirski alternative_io ("lsl %[p],%[seg]", 97a582c540SAndy Lutomirski ".byte 0xf3,0x0f,0xc7,0xf8", /* RDPID %eax/rax */ 98a582c540SAndy Lutomirski X86_FEATURE_RDPID, 99a582c540SAndy Lutomirski [p] "=a" (p), [seg] "r" (__PER_CPU_SEG)); 100e76b027eSAndy Lutomirski 101e76b027eSAndy Lutomirski return p; 102e76b027eSAndy Lutomirski } 103e76b027eSAndy Lutomirski 104e76b027eSAndy Lutomirski #endif /* CONFIG_X86_64 */ 105e76b027eSAndy Lutomirski 1061965aae3SH. Peter Anvin #endif /* _ASM_X86_VGTOD_H */ 107