11965aae3SH. Peter Anvin #ifndef _ASM_X86_VGTOD_H 21965aae3SH. Peter Anvin #define _ASM_X86_VGTOD_H 3bb898558SAl Viro 47c03156fSStefani Seibold #include <linux/compiler.h> 5bb898558SAl Viro #include <linux/clocksource.h> 6bb898558SAl Viro 77c03156fSStefani Seibold #ifdef BUILD_VDSO32_64 87c03156fSStefani Seibold typedef u64 gtod_long_t; 97c03156fSStefani Seibold #else 107c03156fSStefani Seibold typedef unsigned long gtod_long_t; 117c03156fSStefani Seibold #endif 127c03156fSStefani Seibold /* 137c03156fSStefani Seibold * vsyscall_gtod_data will be accessed by 32 and 64 bit code at the same time 147c03156fSStefani Seibold * so be carefull by modifying this structure. 157c03156fSStefani Seibold */ 16bb898558SAl Viro struct vsyscall_gtod_data { 177c03156fSStefani Seibold unsigned seq; 18bb898558SAl Viro 1998d0ac38SAndy Lutomirski int vclock_mode; 20bb898558SAl Viro cycle_t cycle_last; 21bb898558SAl Viro cycle_t mask; 22bb898558SAl Viro u32 mult; 23bb898558SAl Viro u32 shift; 2491ec87d5SAndy Lutomirski 2591ec87d5SAndy Lutomirski /* open coded 'struct timespec' */ 26650ea024SJohn Stultz u64 wall_time_snsec; 277c03156fSStefani Seibold gtod_long_t wall_time_sec; 287c03156fSStefani Seibold gtod_long_t monotonic_time_sec; 29650ea024SJohn Stultz u64 monotonic_time_snsec; 307c03156fSStefani Seibold gtod_long_t wall_time_coarse_sec; 317c03156fSStefani Seibold gtod_long_t wall_time_coarse_nsec; 327c03156fSStefani Seibold gtod_long_t monotonic_time_coarse_sec; 337c03156fSStefani Seibold gtod_long_t monotonic_time_coarse_nsec; 3491ec87d5SAndy Lutomirski 357c03156fSStefani Seibold int tz_minuteswest; 367c03156fSStefani Seibold int tz_dsttime; 37bb898558SAl Viro }; 38bb898558SAl Viro extern struct vsyscall_gtod_data vsyscall_gtod_data; 39bb898558SAl Viro 40bd902c53SAndy Lutomirski extern int vclocks_used; 41bd902c53SAndy Lutomirski static inline bool vclock_was_used(int vclock) 42bd902c53SAndy Lutomirski { 43bd902c53SAndy Lutomirski return READ_ONCE(vclocks_used) & (1 << vclock); 44bd902c53SAndy Lutomirski } 45bd902c53SAndy Lutomirski 467c03156fSStefani Seibold static inline unsigned gtod_read_begin(const struct vsyscall_gtod_data *s) 477c03156fSStefani Seibold { 487c03156fSStefani Seibold unsigned ret; 497c03156fSStefani Seibold 507c03156fSStefani Seibold repeat: 517c03156fSStefani Seibold ret = ACCESS_ONCE(s->seq); 527c03156fSStefani Seibold if (unlikely(ret & 1)) { 537c03156fSStefani Seibold cpu_relax(); 547c03156fSStefani Seibold goto repeat; 557c03156fSStefani Seibold } 567c03156fSStefani Seibold smp_rmb(); 577c03156fSStefani Seibold return ret; 587c03156fSStefani Seibold } 597c03156fSStefani Seibold 607c03156fSStefani Seibold static inline int gtod_read_retry(const struct vsyscall_gtod_data *s, 617c03156fSStefani Seibold unsigned start) 627c03156fSStefani Seibold { 637c03156fSStefani Seibold smp_rmb(); 647c03156fSStefani Seibold return unlikely(s->seq != start); 657c03156fSStefani Seibold } 667c03156fSStefani Seibold 677c03156fSStefani Seibold static inline void gtod_write_begin(struct vsyscall_gtod_data *s) 687c03156fSStefani Seibold { 697c03156fSStefani Seibold ++s->seq; 707c03156fSStefani Seibold smp_wmb(); 717c03156fSStefani Seibold } 727c03156fSStefani Seibold 737c03156fSStefani Seibold static inline void gtod_write_end(struct vsyscall_gtod_data *s) 747c03156fSStefani Seibold { 757c03156fSStefani Seibold smp_wmb(); 767c03156fSStefani Seibold ++s->seq; 777c03156fSStefani Seibold } 787c03156fSStefani Seibold 79e76b027eSAndy Lutomirski #ifdef CONFIG_X86_64 80e76b027eSAndy Lutomirski 81e76b027eSAndy Lutomirski #define VGETCPU_CPU_MASK 0xfff 82e76b027eSAndy Lutomirski 83e76b027eSAndy Lutomirski static inline unsigned int __getcpu(void) 84e76b027eSAndy Lutomirski { 85e76b027eSAndy Lutomirski unsigned int p; 86e76b027eSAndy Lutomirski 87e76b027eSAndy Lutomirski /* 88e76b027eSAndy Lutomirski * Load per CPU data from GDT. LSL is faster than RDTSCP and 891ddf0b1bSAndy Lutomirski * works on all CPUs. This is volatile so that it orders 901ddf0b1bSAndy Lutomirski * correctly wrt barrier() and to keep gcc from cleverly 911ddf0b1bSAndy Lutomirski * hoisting it out of the calling function. 92e76b027eSAndy Lutomirski */ 931ddf0b1bSAndy Lutomirski asm volatile ("lsl %1,%0" : "=r" (p) : "r" (__PER_CPU_SEG)); 94e76b027eSAndy Lutomirski 95e76b027eSAndy Lutomirski return p; 96e76b027eSAndy Lutomirski } 97e76b027eSAndy Lutomirski 98e76b027eSAndy Lutomirski #endif /* CONFIG_X86_64 */ 99e76b027eSAndy Lutomirski 1001965aae3SH. Peter Anvin #endif /* _ASM_X86_VGTOD_H */ 101