1*58d208deSAaron Tomlin // SPDX-License-Identifier: GPL-2.0-or-later 2*58d208deSAaron Tomlin /* 3*58d208deSAaron Tomlin * Modules tree lookup 4*58d208deSAaron Tomlin * 5*58d208deSAaron Tomlin * Copyright (C) 2015 Peter Zijlstra 6*58d208deSAaron Tomlin * Copyright (C) 2015 Rusty Russell 7*58d208deSAaron Tomlin */ 8*58d208deSAaron Tomlin 9*58d208deSAaron Tomlin #include <linux/module.h> 10*58d208deSAaron Tomlin #include <linux/rbtree_latch.h> 11*58d208deSAaron Tomlin #include "internal.h" 12*58d208deSAaron Tomlin 13*58d208deSAaron Tomlin /* 14*58d208deSAaron Tomlin * Use a latched RB-tree for __module_address(); this allows us to use 15*58d208deSAaron Tomlin * RCU-sched lookups of the address from any context. 16*58d208deSAaron Tomlin * 17*58d208deSAaron Tomlin * This is conditional on PERF_EVENTS || TRACING because those can really hit 18*58d208deSAaron Tomlin * __module_address() hard by doing a lot of stack unwinding; potentially from 19*58d208deSAaron Tomlin * NMI context. 20*58d208deSAaron Tomlin */ 21*58d208deSAaron Tomlin 22*58d208deSAaron Tomlin static __always_inline unsigned long __mod_tree_val(struct latch_tree_node *n) 23*58d208deSAaron Tomlin { 24*58d208deSAaron Tomlin struct module_layout *layout = container_of(n, struct module_layout, mtn.node); 25*58d208deSAaron Tomlin 26*58d208deSAaron Tomlin return (unsigned long)layout->base; 27*58d208deSAaron Tomlin } 28*58d208deSAaron Tomlin 29*58d208deSAaron Tomlin static __always_inline unsigned long __mod_tree_size(struct latch_tree_node *n) 30*58d208deSAaron Tomlin { 31*58d208deSAaron Tomlin struct module_layout *layout = container_of(n, struct module_layout, mtn.node); 32*58d208deSAaron Tomlin 33*58d208deSAaron Tomlin return (unsigned long)layout->size; 34*58d208deSAaron Tomlin } 35*58d208deSAaron Tomlin 36*58d208deSAaron Tomlin static __always_inline bool 37*58d208deSAaron Tomlin mod_tree_less(struct latch_tree_node *a, struct latch_tree_node *b) 38*58d208deSAaron Tomlin { 39*58d208deSAaron Tomlin return __mod_tree_val(a) < __mod_tree_val(b); 40*58d208deSAaron Tomlin } 41*58d208deSAaron Tomlin 42*58d208deSAaron Tomlin static __always_inline int 43*58d208deSAaron Tomlin mod_tree_comp(void *key, struct latch_tree_node *n) 44*58d208deSAaron Tomlin { 45*58d208deSAaron Tomlin unsigned long val = (unsigned long)key; 46*58d208deSAaron Tomlin unsigned long start, end; 47*58d208deSAaron Tomlin 48*58d208deSAaron Tomlin start = __mod_tree_val(n); 49*58d208deSAaron Tomlin if (val < start) 50*58d208deSAaron Tomlin return -1; 51*58d208deSAaron Tomlin 52*58d208deSAaron Tomlin end = start + __mod_tree_size(n); 53*58d208deSAaron Tomlin if (val >= end) 54*58d208deSAaron Tomlin return 1; 55*58d208deSAaron Tomlin 56*58d208deSAaron Tomlin return 0; 57*58d208deSAaron Tomlin } 58*58d208deSAaron Tomlin 59*58d208deSAaron Tomlin static const struct latch_tree_ops mod_tree_ops = { 60*58d208deSAaron Tomlin .less = mod_tree_less, 61*58d208deSAaron Tomlin .comp = mod_tree_comp, 62*58d208deSAaron Tomlin }; 63*58d208deSAaron Tomlin 64*58d208deSAaron Tomlin static noinline void __mod_tree_insert(struct mod_tree_node *node) 65*58d208deSAaron Tomlin { 66*58d208deSAaron Tomlin latch_tree_insert(&node->node, &mod_tree.root, &mod_tree_ops); 67*58d208deSAaron Tomlin } 68*58d208deSAaron Tomlin 69*58d208deSAaron Tomlin static void __mod_tree_remove(struct mod_tree_node *node) 70*58d208deSAaron Tomlin { 71*58d208deSAaron Tomlin latch_tree_erase(&node->node, &mod_tree.root, &mod_tree_ops); 72*58d208deSAaron Tomlin } 73*58d208deSAaron Tomlin 74*58d208deSAaron Tomlin /* 75*58d208deSAaron Tomlin * These modifications: insert, remove_init and remove; are serialized by the 76*58d208deSAaron Tomlin * module_mutex. 77*58d208deSAaron Tomlin */ 78*58d208deSAaron Tomlin void mod_tree_insert(struct module *mod) 79*58d208deSAaron Tomlin { 80*58d208deSAaron Tomlin mod->core_layout.mtn.mod = mod; 81*58d208deSAaron Tomlin mod->init_layout.mtn.mod = mod; 82*58d208deSAaron Tomlin 83*58d208deSAaron Tomlin __mod_tree_insert(&mod->core_layout.mtn); 84*58d208deSAaron Tomlin if (mod->init_layout.size) 85*58d208deSAaron Tomlin __mod_tree_insert(&mod->init_layout.mtn); 86*58d208deSAaron Tomlin } 87*58d208deSAaron Tomlin 88*58d208deSAaron Tomlin void mod_tree_remove_init(struct module *mod) 89*58d208deSAaron Tomlin { 90*58d208deSAaron Tomlin if (mod->init_layout.size) 91*58d208deSAaron Tomlin __mod_tree_remove(&mod->init_layout.mtn); 92*58d208deSAaron Tomlin } 93*58d208deSAaron Tomlin 94*58d208deSAaron Tomlin void mod_tree_remove(struct module *mod) 95*58d208deSAaron Tomlin { 96*58d208deSAaron Tomlin __mod_tree_remove(&mod->core_layout.mtn); 97*58d208deSAaron Tomlin mod_tree_remove_init(mod); 98*58d208deSAaron Tomlin } 99*58d208deSAaron Tomlin 100*58d208deSAaron Tomlin struct module *mod_find(unsigned long addr) 101*58d208deSAaron Tomlin { 102*58d208deSAaron Tomlin struct latch_tree_node *ltn; 103*58d208deSAaron Tomlin 104*58d208deSAaron Tomlin ltn = latch_tree_find((void *)addr, &mod_tree.root, &mod_tree_ops); 105*58d208deSAaron Tomlin if (!ltn) 106*58d208deSAaron Tomlin return NULL; 107*58d208deSAaron Tomlin 108*58d208deSAaron Tomlin return container_of(ltn, struct mod_tree_node, node)->mod; 109*58d208deSAaron Tomlin } 110