1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright(c) 2019 Intel Corporation. */ 3 4 #include <linux/hash.h> 5 #include <linux/bpf.h> 6 #include <linux/filter.h> 7 8 /* The BPF dispatcher is a multiway branch code generator. The 9 * dispatcher is a mechanism to avoid the performance penalty of an 10 * indirect call, which is expensive when retpolines are enabled. A 11 * dispatch client registers a BPF program into the dispatcher, and if 12 * there is available room in the dispatcher a direct call to the BPF 13 * program will be generated. All calls to the BPF programs called via 14 * the dispatcher will then be a direct call, instead of an 15 * indirect. The dispatcher hijacks a trampoline function it via the 16 * __fentry__ of the trampoline. The trampoline function has the 17 * following signature: 18 * 19 * unsigned int trampoline(const void *ctx, const struct bpf_insn *insnsi, 20 * unsigned int (*bpf_func)(const void *, 21 * const struct bpf_insn *)); 22 */ 23 24 static struct bpf_dispatcher_prog *bpf_dispatcher_find_prog( 25 struct bpf_dispatcher *d, struct bpf_prog *prog) 26 { 27 int i; 28 29 for (i = 0; i < BPF_DISPATCHER_MAX; i++) { 30 if (prog == d->progs[i].prog) 31 return &d->progs[i]; 32 } 33 return NULL; 34 } 35 36 static struct bpf_dispatcher_prog *bpf_dispatcher_find_free( 37 struct bpf_dispatcher *d) 38 { 39 return bpf_dispatcher_find_prog(d, NULL); 40 } 41 42 static bool bpf_dispatcher_add_prog(struct bpf_dispatcher *d, 43 struct bpf_prog *prog) 44 { 45 struct bpf_dispatcher_prog *entry; 46 47 if (!prog) 48 return false; 49 50 entry = bpf_dispatcher_find_prog(d, prog); 51 if (entry) { 52 refcount_inc(&entry->users); 53 return false; 54 } 55 56 entry = bpf_dispatcher_find_free(d); 57 if (!entry) 58 return false; 59 60 bpf_prog_inc(prog); 61 entry->prog = prog; 62 refcount_set(&entry->users, 1); 63 d->num_progs++; 64 return true; 65 } 66 67 static bool bpf_dispatcher_remove_prog(struct bpf_dispatcher *d, 68 struct bpf_prog *prog) 69 { 70 struct bpf_dispatcher_prog *entry; 71 72 if (!prog) 73 return false; 74 75 entry = bpf_dispatcher_find_prog(d, prog); 76 if (!entry) 77 return false; 78 79 if (refcount_dec_and_test(&entry->users)) { 80 entry->prog = NULL; 81 bpf_prog_put(prog); 82 d->num_progs--; 83 return true; 84 } 85 return false; 86 } 87 88 int __weak arch_prepare_bpf_dispatcher(void *image, void *buf, s64 *funcs, int num_funcs) 89 { 90 return -ENOTSUPP; 91 } 92 93 static int bpf_dispatcher_prepare(struct bpf_dispatcher *d, void *image, void *buf) 94 { 95 s64 ips[BPF_DISPATCHER_MAX] = {}, *ipsp = &ips[0]; 96 int i; 97 98 for (i = 0; i < BPF_DISPATCHER_MAX; i++) { 99 if (d->progs[i].prog) 100 *ipsp++ = (s64)(uintptr_t)d->progs[i].prog->bpf_func; 101 } 102 return arch_prepare_bpf_dispatcher(image, buf, &ips[0], d->num_progs); 103 } 104 105 static void bpf_dispatcher_update(struct bpf_dispatcher *d, int prev_num_progs) 106 { 107 void *old, *new, *tmp; 108 u32 noff; 109 int err; 110 111 if (!prev_num_progs) { 112 old = NULL; 113 noff = 0; 114 } else { 115 old = d->image + d->image_off; 116 noff = d->image_off ^ (PAGE_SIZE / 2); 117 } 118 119 new = d->num_progs ? d->image + noff : NULL; 120 tmp = d->num_progs ? d->rw_image + noff : NULL; 121 if (new) { 122 /* Prepare the dispatcher in d->rw_image. Then use 123 * bpf_arch_text_copy to update d->image, which is RO+X. 124 */ 125 if (bpf_dispatcher_prepare(d, new, tmp)) 126 return; 127 if (IS_ERR(bpf_arch_text_copy(new, tmp, PAGE_SIZE / 2))) 128 return; 129 } 130 131 err = bpf_arch_text_poke(d->func, BPF_MOD_JUMP, old, new); 132 if (err || !new) 133 return; 134 135 d->image_off = noff; 136 } 137 138 void bpf_dispatcher_change_prog(struct bpf_dispatcher *d, struct bpf_prog *from, 139 struct bpf_prog *to) 140 { 141 bool changed = false; 142 int prev_num_progs; 143 144 if (from == to) 145 return; 146 147 mutex_lock(&d->mutex); 148 if (!d->image) { 149 d->image = bpf_prog_pack_alloc(PAGE_SIZE, bpf_jit_fill_hole_with_zero); 150 if (!d->image) 151 goto out; 152 d->rw_image = bpf_jit_alloc_exec(PAGE_SIZE); 153 if (!d->rw_image) { 154 u32 size = PAGE_SIZE; 155 156 bpf_arch_text_copy(d->image, &size, sizeof(size)); 157 bpf_prog_pack_free((struct bpf_binary_header *)d->image); 158 d->image = NULL; 159 goto out; 160 } 161 bpf_image_ksym_add(d->image, &d->ksym); 162 } 163 164 prev_num_progs = d->num_progs; 165 changed |= bpf_dispatcher_remove_prog(d, from); 166 changed |= bpf_dispatcher_add_prog(d, to); 167 168 if (!changed) 169 goto out; 170 171 bpf_dispatcher_update(d, prev_num_progs); 172 out: 173 mutex_unlock(&d->mutex); 174 } 175