72 	jea->key	= jeb->key - delta;  in jump_label_swap()
76 	jeb->key	= tmp.key + delta;  in jump_label_swap()
93 static void jump_label_update(struct static_key *key);
104 int static_key_count(struct static_key *key)  in static_key_count()  argument
110 	int n = atomic_read(&key->enabled);  in static_key_count()
127 bool static_key_fast_inc_not_disabled(struct static_key *key)  in static_key_fast_inc_not_disabled()  argument
131 	STATIC_KEY_CHECK_USE(key);  in static_key_fast_inc_not_disabled()
138 	v = atomic_read(&key->enabled);  in static_key_fast_inc_not_disabled()
142 	} while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1)));  in static_key_fast_inc_not_disabled()
148 bool static_key_slow_inc_cpuslocked(struct static_key *key)  in static_key_slow_inc_cpuslocked()  argument
159 	if (static_key_fast_inc_not_disabled(key))  in static_key_slow_inc_cpuslocked()
163 	if (atomic_read(&key->enabled) == 0) {  in static_key_slow_inc_cpuslocked()
164 		atomic_set(&key->enabled, -1);  in static_key_slow_inc_cpuslocked()
165 		jump_label_update(key);  in static_key_slow_inc_cpuslocked()
170 		atomic_set_release(&key->enabled, 1);  in static_key_slow_inc_cpuslocked()
172 		if (WARN_ON_ONCE(!static_key_fast_inc_not_disabled(key))) {  in static_key_slow_inc_cpuslocked()
181 bool static_key_slow_inc(struct static_key *key)  in static_key_slow_inc()  argument
186 	ret = static_key_slow_inc_cpuslocked(key);  in static_key_slow_inc()
192 void static_key_enable_cpuslocked(struct static_key *key)  in static_key_enable_cpuslocked()  argument
194 	STATIC_KEY_CHECK_USE(key);  in static_key_enable_cpuslocked()
197 	if (atomic_read(&key->enabled) > 0) {  in static_key_enable_cpuslocked()
198 		WARN_ON_ONCE(atomic_read(&key->enabled) != 1);  in static_key_enable_cpuslocked()
203 	if (atomic_read(&key->enabled) == 0) {  in static_key_enable_cpuslocked()
204 		atomic_set(&key->enabled, -1);  in static_key_enable_cpuslocked()
205 		jump_label_update(key);  in static_key_enable_cpuslocked()
209 		atomic_set_release(&key->enabled, 1);  in static_key_enable_cpuslocked()
215 void static_key_enable(struct static_key *key)  in static_key_enable()  argument
218 	static_key_enable_cpuslocked(key);  in static_key_enable()
223 void static_key_disable_cpuslocked(struct static_key *key)  in static_key_disable_cpuslocked()  argument
225 	STATIC_KEY_CHECK_USE(key);  in static_key_disable_cpuslocked()
228 	if (atomic_read(&key->enabled) != 1) {  in static_key_disable_cpuslocked()
229 		WARN_ON_ONCE(atomic_read(&key->enabled) != 0);  in static_key_disable_cpuslocked()
234 	if (atomic_cmpxchg(&key->enabled, 1, 0) == 1)  in static_key_disable_cpuslocked()
235 		jump_label_update(key);  in static_key_disable_cpuslocked()
240 void static_key_disable(struct static_key *key)  in static_key_disable()  argument
243 	static_key_disable_cpuslocked(key);  in static_key_disable()
248 static bool static_key_slow_try_dec(struct static_key *key)  in static_key_slow_try_dec()  argument
263 	v = atomic_read(&key->enabled);  in static_key_slow_try_dec()
274 	} while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v - 1)));  in static_key_slow_try_dec()
279 static void __static_key_slow_dec_cpuslocked(struct static_key *key)  in __static_key_slow_dec_cpuslocked()  argument
283 	if (static_key_slow_try_dec(key))  in __static_key_slow_dec_cpuslocked()
287 	if (atomic_cmpxchg(&key->enabled, 1, 0) == 1)  in __static_key_slow_dec_cpuslocked()
288 		jump_label_update(key);  in __static_key_slow_dec_cpuslocked()
290 		WARN_ON_ONCE(!static_key_slow_try_dec(key));  in __static_key_slow_dec_cpuslocked()
293 static void __static_key_slow_dec(struct static_key *key)  in __static_key_slow_dec()  argument
296 	__static_key_slow_dec_cpuslocked(key);  in __static_key_slow_dec()
302 	struct static_key_deferred *key =  in jump_label_update_timeout()  local
304 	__static_key_slow_dec(&key->key);  in jump_label_update_timeout()
308 void static_key_slow_dec(struct static_key *key)  in static_key_slow_dec()  argument
310 	STATIC_KEY_CHECK_USE(key);  in static_key_slow_dec()
311 	__static_key_slow_dec(key);  in static_key_slow_dec()
315 void static_key_slow_dec_cpuslocked(struct static_key *key)  in static_key_slow_dec_cpuslocked()  argument
317 	STATIC_KEY_CHECK_USE(key);  in static_key_slow_dec_cpuslocked()
318 	__static_key_slow_dec_cpuslocked(key);  in static_key_slow_dec_cpuslocked()
321 void __static_key_slow_dec_deferred(struct static_key *key,  in __static_key_slow_dec_deferred()  argument
325 	STATIC_KEY_CHECK_USE(key);  in __static_key_slow_dec_deferred()
327 	if (static_key_slow_try_dec(key))  in __static_key_slow_dec_deferred()
334 void __static_key_deferred_flush(void *key, struct delayed_work *work)  in __static_key_deferred_flush()  argument
336 	STATIC_KEY_CHECK_USE(key);  in __static_key_deferred_flush()
341 void jump_label_rate_limit(struct static_key_deferred *key,  in jump_label_rate_limit()  argument
344 	STATIC_KEY_CHECK_USE(key);  in jump_label_rate_limit()
345 	key->timeout = rl;  in jump_label_rate_limit()
346 	INIT_DELAYED_WORK(&key->work, jump_label_update_timeout);  in jump_label_rate_limit()
384 static inline struct jump_entry *static_key_entries(struct static_key *key)  in static_key_entries()  argument
386 	WARN_ON_ONCE(key->type & JUMP_TYPE_LINKED);  in static_key_entries()
387 	return (struct jump_entry *)(key->type & ~JUMP_TYPE_MASK);  in static_key_entries()
390 static inline bool static_key_type(struct static_key *key)  in static_key_type()  argument
392 	return key->type & JUMP_TYPE_TRUE;  in static_key_type()
395 static inline bool static_key_linked(struct static_key *key)  in static_key_linked()  argument
397 	return key->type & JUMP_TYPE_LINKED;  in static_key_linked()
400 static inline void static_key_clear_linked(struct static_key *key)  in static_key_clear_linked()  argument
402 	key->type &= ~JUMP_TYPE_LINKED;  in static_key_clear_linked()
405 static inline void static_key_set_linked(struct static_key *key)  in static_key_set_linked()  argument
407 	key->type |= JUMP_TYPE_LINKED;  in static_key_set_linked()
419 static void static_key_set_entries(struct static_key *key,  in static_key_set_entries()  argument
425 	type = key->type & JUMP_TYPE_MASK;  in static_key_set_entries()
426 	key->entries = entries;  in static_key_set_entries()
427 	key->type |= type;  in static_key_set_entries()
432 	struct static_key *key = jump_entry_key(entry);  in jump_label_type()  local
433 	bool enabled = static_key_enabled(key);  in jump_label_type()
467 static void __jump_label_update(struct static_key *key,  in __jump_label_update()  argument
472 	for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {  in __jump_label_update()
478 static void __jump_label_update(struct static_key *key,  in __jump_label_update()  argument
483 	for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) {  in __jump_label_update()
504 	struct static_key *key = NULL;  in jump_label_init()  local
535 		if (iterk == key)  in jump_label_init()
538 		key = iterk;  in jump_label_init()
539 		static_key_set_entries(key, iter);  in jump_label_init()
550 	struct static_key *key = jump_entry_key(entry);  in jump_label_init_type()  local
551 	bool type = static_key_type(key);  in jump_label_init_type()
564 static inline struct static_key_mod *static_key_mod(struct static_key *key)  in static_key_mod()  argument
566 	WARN_ON_ONCE(!static_key_linked(key));  in static_key_mod()
567 	return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK);  in static_key_mod()
576 static void static_key_set_mod(struct static_key *key,  in static_key_set_mod()  argument
582 	type = key->type & JUMP_TYPE_MASK;  in static_key_set_mod()
583 	key->next = mod;  in static_key_set_mod()
584 	key->type |= type;  in static_key_set_mod()
611 static void __jump_label_mod_update(struct static_key *key)  in __jump_label_mod_update()  argument
615 	for (mod = static_key_mod(key); mod; mod = mod->next) {  in __jump_label_mod_update()
631 		__jump_label_update(key, mod->entries, stop,  in __jump_label_mod_update()
641 	struct static_key *key = NULL;  in jump_label_add_module()  local
658 		if (iterk == key)  in jump_label_add_module()
661 		key = iterk;  in jump_label_add_module()
662 		if (within_module((unsigned long)key, mod)) {  in jump_label_add_module()
663 			static_key_set_entries(key, iter);  in jump_label_add_module()
669 		if (!static_key_linked(key)) {  in jump_label_add_module()
677 			jlm2->mod = __module_address((unsigned long)key);  in jump_label_add_module()
679 			jlm2->entries = static_key_entries(key);  in jump_label_add_module()
681 			static_key_set_mod(key, jlm2);  in jump_label_add_module()
682 			static_key_set_linked(key);  in jump_label_add_module()
686 		jlm->next = static_key_mod(key);  in jump_label_add_module()
687 		static_key_set_mod(key, jlm);  in jump_label_add_module()
688 		static_key_set_linked(key);  in jump_label_add_module()
692 			__jump_label_update(key, iter, iter_stop, true);  in jump_label_add_module()
703 	struct static_key *key = NULL;  in jump_label_del_module()  local
707 		if (jump_entry_key(iter) == key)  in jump_label_del_module()
710 		key = jump_entry_key(iter);  in jump_label_del_module()
712 		if (within_module((unsigned long)key, mod))  in jump_label_del_module()
716 		if (WARN_ON(!static_key_linked(key)))  in jump_label_del_module()
719 		prev = &key->next;  in jump_label_del_module()
720 		jlm = static_key_mod(key);  in jump_label_del_module()
731 		if (prev == &key->next)  in jump_label_del_module()
732 			static_key_set_mod(key, jlm->next);  in jump_label_del_module()
738 		jlm = static_key_mod(key);  in jump_label_del_module()
741 			static_key_set_entries(key, jlm->entries);  in jump_label_del_module()
742 			static_key_clear_linked(key);  in jump_label_del_module()
818 static void jump_label_update(struct static_key *key)  in jump_label_update()  argument
826 	if (static_key_linked(key)) {  in jump_label_update()
827 		__jump_label_mod_update(key);  in jump_label_update()
832 	mod = __module_address((unsigned long)key);  in jump_label_update()
839 	entry = static_key_entries(key);  in jump_label_update()
842 		__jump_label_update(key, entry, stop, init);  in jump_label_update()
854 		WARN_ON(static_key_enabled(&sk_true.key) != true);  in jump_label_test()
855 		WARN_ON(static_key_enabled(&sk_false.key) != false);  in jump_label_test()
865 		WARN_ON(static_key_enabled(&sk_true.key) == true);  in jump_label_test()
866 		WARN_ON(static_key_enabled(&sk_false.key) == false);  in jump_label_test()