1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __ASM_PREEMPT_H 3 #define __ASM_PREEMPT_H 4 5 #include <linux/jump_label.h> 6 #include <linux/thread_info.h> 7 8 #define PREEMPT_NEED_RESCHED BIT(32) 9 #define PREEMPT_ENABLED (PREEMPT_NEED_RESCHED) 10 11 static inline int preempt_count(void) 12 { 13 return READ_ONCE(current_thread_info()->preempt.count); 14 } 15 16 static inline void preempt_count_set(u64 pc) 17 { 18 /* Preserve existing value of PREEMPT_NEED_RESCHED */ 19 WRITE_ONCE(current_thread_info()->preempt.count, pc); 20 } 21 22 #define init_task_preempt_count(p) do { \ 23 task_thread_info(p)->preempt_count = FORK_PREEMPT_COUNT; \ 24 } while (0) 25 26 #define init_idle_preempt_count(p, cpu) do { \ 27 task_thread_info(p)->preempt_count = PREEMPT_DISABLED; \ 28 } while (0) 29 30 static inline void set_preempt_need_resched(void) 31 { 32 current_thread_info()->preempt.need_resched = 0; 33 } 34 35 static inline void clear_preempt_need_resched(void) 36 { 37 current_thread_info()->preempt.need_resched = 1; 38 } 39 40 static inline bool test_preempt_need_resched(void) 41 { 42 return !current_thread_info()->preempt.need_resched; 43 } 44 45 static inline void __preempt_count_add(int val) 46 { 47 u32 pc = READ_ONCE(current_thread_info()->preempt.count); 48 pc += val; 49 WRITE_ONCE(current_thread_info()->preempt.count, pc); 50 } 51 52 static inline void __preempt_count_sub(int val) 53 { 54 u32 pc = READ_ONCE(current_thread_info()->preempt.count); 55 pc -= val; 56 WRITE_ONCE(current_thread_info()->preempt.count, pc); 57 } 58 59 static inline bool __preempt_count_dec_and_test(void) 60 { 61 struct thread_info *ti = current_thread_info(); 62 u64 pc = READ_ONCE(ti->preempt_count); 63 64 /* Update only the count field, leaving need_resched unchanged */ 65 WRITE_ONCE(ti->preempt.count, --pc); 66 67 /* 68 * If we wrote back all zeroes, then we're preemptible and in 69 * need of a reschedule. Otherwise, we need to reload the 70 * preempt_count in case the need_resched flag was cleared by an 71 * interrupt occurring between the non-atomic READ_ONCE/WRITE_ONCE 72 * pair. 73 */ 74 return !pc || !READ_ONCE(ti->preempt_count); 75 } 76 77 static inline bool should_resched(int preempt_offset) 78 { 79 u64 pc = READ_ONCE(current_thread_info()->preempt_count); 80 return pc == preempt_offset; 81 } 82 83 #ifdef CONFIG_PREEMPTION 84 85 void preempt_schedule(void); 86 void preempt_schedule_notrace(void); 87 88 #ifdef CONFIG_PREEMPT_DYNAMIC 89 90 DECLARE_STATIC_KEY_TRUE(sk_dynamic_irqentry_exit_cond_resched); 91 void dynamic_preempt_schedule(void); 92 #define __preempt_schedule() dynamic_preempt_schedule() 93 void dynamic_preempt_schedule_notrace(void); 94 #define __preempt_schedule_notrace() dynamic_preempt_schedule_notrace() 95 96 #else /* CONFIG_PREEMPT_DYNAMIC */ 97 98 #define __preempt_schedule() preempt_schedule() 99 #define __preempt_schedule_notrace() preempt_schedule_notrace() 100 101 #endif /* CONFIG_PREEMPT_DYNAMIC */ 102 #endif /* CONFIG_PREEMPTION */ 103 104 #endif /* __ASM_PREEMPT_H */ 105