Lines Matching refs:key
72 jea->key = jeb->key - delta; in jump_label_swap()
76 jeb->key = tmp.key + delta; in jump_label_swap()
93 static void jump_label_update(struct static_key *key);
104 int static_key_count(struct static_key *key) in static_key_count() argument
110 int n = atomic_read(&key->enabled); in static_key_count()
127 bool static_key_fast_inc_not_disabled(struct static_key *key) in static_key_fast_inc_not_disabled() argument
131 STATIC_KEY_CHECK_USE(key); in static_key_fast_inc_not_disabled()
138 v = atomic_read(&key->enabled); in static_key_fast_inc_not_disabled()
142 } while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v + 1))); in static_key_fast_inc_not_disabled()
148 bool static_key_slow_inc_cpuslocked(struct static_key *key) in static_key_slow_inc_cpuslocked() argument
159 if (static_key_fast_inc_not_disabled(key)) in static_key_slow_inc_cpuslocked()
163 if (atomic_read(&key->enabled) == 0) { in static_key_slow_inc_cpuslocked()
164 atomic_set(&key->enabled, -1); in static_key_slow_inc_cpuslocked()
165 jump_label_update(key); in static_key_slow_inc_cpuslocked()
170 atomic_set_release(&key->enabled, 1); in static_key_slow_inc_cpuslocked()
172 if (WARN_ON_ONCE(!static_key_fast_inc_not_disabled(key))) { in static_key_slow_inc_cpuslocked()
181 bool static_key_slow_inc(struct static_key *key) in static_key_slow_inc() argument
186 ret = static_key_slow_inc_cpuslocked(key); in static_key_slow_inc()
192 void static_key_enable_cpuslocked(struct static_key *key) in static_key_enable_cpuslocked() argument
194 STATIC_KEY_CHECK_USE(key); in static_key_enable_cpuslocked()
197 if (atomic_read(&key->enabled) > 0) { in static_key_enable_cpuslocked()
198 WARN_ON_ONCE(atomic_read(&key->enabled) != 1); in static_key_enable_cpuslocked()
203 if (atomic_read(&key->enabled) == 0) { in static_key_enable_cpuslocked()
204 atomic_set(&key->enabled, -1); in static_key_enable_cpuslocked()
205 jump_label_update(key); in static_key_enable_cpuslocked()
209 atomic_set_release(&key->enabled, 1); in static_key_enable_cpuslocked()
215 void static_key_enable(struct static_key *key) in static_key_enable() argument
218 static_key_enable_cpuslocked(key); in static_key_enable()
223 void static_key_disable_cpuslocked(struct static_key *key) in static_key_disable_cpuslocked() argument
225 STATIC_KEY_CHECK_USE(key); in static_key_disable_cpuslocked()
228 if (atomic_read(&key->enabled) != 1) { in static_key_disable_cpuslocked()
229 WARN_ON_ONCE(atomic_read(&key->enabled) != 0); in static_key_disable_cpuslocked()
234 if (atomic_cmpxchg(&key->enabled, 1, 0) == 1) in static_key_disable_cpuslocked()
235 jump_label_update(key); in static_key_disable_cpuslocked()
240 void static_key_disable(struct static_key *key) in static_key_disable() argument
243 static_key_disable_cpuslocked(key); in static_key_disable()
248 static bool static_key_slow_try_dec(struct static_key *key) in static_key_slow_try_dec() argument
263 v = atomic_read(&key->enabled); in static_key_slow_try_dec()
274 } while (!likely(atomic_try_cmpxchg(&key->enabled, &v, v - 1))); in static_key_slow_try_dec()
279 static void __static_key_slow_dec_cpuslocked(struct static_key *key) in __static_key_slow_dec_cpuslocked() argument
283 if (static_key_slow_try_dec(key)) in __static_key_slow_dec_cpuslocked()
287 if (atomic_cmpxchg(&key->enabled, 1, 0) == 1) in __static_key_slow_dec_cpuslocked()
288 jump_label_update(key); in __static_key_slow_dec_cpuslocked()
290 WARN_ON_ONCE(!static_key_slow_try_dec(key)); in __static_key_slow_dec_cpuslocked()
293 static void __static_key_slow_dec(struct static_key *key) in __static_key_slow_dec() argument
296 __static_key_slow_dec_cpuslocked(key); in __static_key_slow_dec()
302 struct static_key_deferred *key = in jump_label_update_timeout() local
304 __static_key_slow_dec(&key->key); in jump_label_update_timeout()
308 void static_key_slow_dec(struct static_key *key) in static_key_slow_dec() argument
310 STATIC_KEY_CHECK_USE(key); in static_key_slow_dec()
311 __static_key_slow_dec(key); in static_key_slow_dec()
315 void static_key_slow_dec_cpuslocked(struct static_key *key) in static_key_slow_dec_cpuslocked() argument
317 STATIC_KEY_CHECK_USE(key); in static_key_slow_dec_cpuslocked()
318 __static_key_slow_dec_cpuslocked(key); in static_key_slow_dec_cpuslocked()
321 void __static_key_slow_dec_deferred(struct static_key *key, in __static_key_slow_dec_deferred() argument
325 STATIC_KEY_CHECK_USE(key); in __static_key_slow_dec_deferred()
327 if (static_key_slow_try_dec(key)) in __static_key_slow_dec_deferred()
334 void __static_key_deferred_flush(void *key, struct delayed_work *work) in __static_key_deferred_flush() argument
336 STATIC_KEY_CHECK_USE(key); in __static_key_deferred_flush()
341 void jump_label_rate_limit(struct static_key_deferred *key, in jump_label_rate_limit() argument
344 STATIC_KEY_CHECK_USE(key); in jump_label_rate_limit()
345 key->timeout = rl; in jump_label_rate_limit()
346 INIT_DELAYED_WORK(&key->work, jump_label_update_timeout); in jump_label_rate_limit()
384 static inline struct jump_entry *static_key_entries(struct static_key *key) in static_key_entries() argument
386 WARN_ON_ONCE(key->type & JUMP_TYPE_LINKED); in static_key_entries()
387 return (struct jump_entry *)(key->type & ~JUMP_TYPE_MASK); in static_key_entries()
390 static inline bool static_key_type(struct static_key *key) in static_key_type() argument
392 return key->type & JUMP_TYPE_TRUE; in static_key_type()
395 static inline bool static_key_linked(struct static_key *key) in static_key_linked() argument
397 return key->type & JUMP_TYPE_LINKED; in static_key_linked()
400 static inline void static_key_clear_linked(struct static_key *key) in static_key_clear_linked() argument
402 key->type &= ~JUMP_TYPE_LINKED; in static_key_clear_linked()
405 static inline void static_key_set_linked(struct static_key *key) in static_key_set_linked() argument
407 key->type |= JUMP_TYPE_LINKED; in static_key_set_linked()
419 static void static_key_set_entries(struct static_key *key, in static_key_set_entries() argument
425 type = key->type & JUMP_TYPE_MASK; in static_key_set_entries()
426 key->entries = entries; in static_key_set_entries()
427 key->type |= type; in static_key_set_entries()
432 struct static_key *key = jump_entry_key(entry); in jump_label_type() local
433 bool enabled = static_key_enabled(key); in jump_label_type()
467 static void __jump_label_update(struct static_key *key, in __jump_label_update() argument
472 for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) { in __jump_label_update()
478 static void __jump_label_update(struct static_key *key, in __jump_label_update() argument
483 for (; (entry < stop) && (jump_entry_key(entry) == key); entry++) { in __jump_label_update()
504 struct static_key *key = NULL; in jump_label_init() local
535 if (iterk == key) in jump_label_init()
538 key = iterk; in jump_label_init()
539 static_key_set_entries(key, iter); in jump_label_init()
550 struct static_key *key = jump_entry_key(entry); in jump_label_init_type() local
551 bool type = static_key_type(key); in jump_label_init_type()
564 static inline struct static_key_mod *static_key_mod(struct static_key *key) in static_key_mod() argument
566 WARN_ON_ONCE(!static_key_linked(key)); in static_key_mod()
567 return (struct static_key_mod *)(key->type & ~JUMP_TYPE_MASK); in static_key_mod()
576 static void static_key_set_mod(struct static_key *key, in static_key_set_mod() argument
582 type = key->type & JUMP_TYPE_MASK; in static_key_set_mod()
583 key->next = mod; in static_key_set_mod()
584 key->type |= type; in static_key_set_mod()
611 static void __jump_label_mod_update(struct static_key *key) in __jump_label_mod_update() argument
615 for (mod = static_key_mod(key); mod; mod = mod->next) { in __jump_label_mod_update()
631 __jump_label_update(key, mod->entries, stop, in __jump_label_mod_update()
641 struct static_key *key = NULL; in jump_label_add_module() local
658 if (iterk == key) in jump_label_add_module()
661 key = iterk; in jump_label_add_module()
662 if (within_module((unsigned long)key, mod)) { in jump_label_add_module()
663 static_key_set_entries(key, iter); in jump_label_add_module()
669 if (!static_key_linked(key)) { in jump_label_add_module()
677 jlm2->mod = __module_address((unsigned long)key); in jump_label_add_module()
679 jlm2->entries = static_key_entries(key); in jump_label_add_module()
681 static_key_set_mod(key, jlm2); in jump_label_add_module()
682 static_key_set_linked(key); in jump_label_add_module()
686 jlm->next = static_key_mod(key); in jump_label_add_module()
687 static_key_set_mod(key, jlm); in jump_label_add_module()
688 static_key_set_linked(key); in jump_label_add_module()
692 __jump_label_update(key, iter, iter_stop, true); in jump_label_add_module()
703 struct static_key *key = NULL; in jump_label_del_module() local
707 if (jump_entry_key(iter) == key) in jump_label_del_module()
710 key = jump_entry_key(iter); in jump_label_del_module()
712 if (within_module((unsigned long)key, mod)) in jump_label_del_module()
716 if (WARN_ON(!static_key_linked(key))) in jump_label_del_module()
719 prev = &key->next; in jump_label_del_module()
720 jlm = static_key_mod(key); in jump_label_del_module()
731 if (prev == &key->next) in jump_label_del_module()
732 static_key_set_mod(key, jlm->next); in jump_label_del_module()
738 jlm = static_key_mod(key); in jump_label_del_module()
741 static_key_set_entries(key, jlm->entries); in jump_label_del_module()
742 static_key_clear_linked(key); in jump_label_del_module()
818 static void jump_label_update(struct static_key *key) in jump_label_update() argument
826 if (static_key_linked(key)) { in jump_label_update()
827 __jump_label_mod_update(key); in jump_label_update()
832 mod = __module_address((unsigned long)key); in jump_label_update()
839 entry = static_key_entries(key); in jump_label_update()
842 __jump_label_update(key, entry, stop, init); in jump_label_update()
854 WARN_ON(static_key_enabled(&sk_true.key) != true); in jump_label_test()
855 WARN_ON(static_key_enabled(&sk_false.key) != false); in jump_label_test()
865 WARN_ON(static_key_enabled(&sk_true.key) == true); in jump_label_test()
866 WARN_ON(static_key_enabled(&sk_false.key) == false); in jump_label_test()