1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Xen hypercall batching. 4 * 5 * Xen allows multiple hypercalls to be issued at once, using the 6 * multicall interface. This allows the cost of trapping into the 7 * hypervisor to be amortized over several calls. 8 * 9 * This file implements a simple interface for multicalls. There's a 10 * per-cpu buffer of outstanding multicalls. When you want to queue a 11 * multicall for issuing, you can allocate a multicall slot for the 12 * call and its arguments, along with storage for space which is 13 * pointed to by the arguments (for passing pointers to structures, 14 * etc). When the multicall is actually issued, all the space for the 15 * commands and allocated memory is freed for reuse. 16 * 17 * Multicalls are flushed whenever any of the buffers get full, or 18 * when explicitly requested. There's no way to get per-multicall 19 * return results back. It will BUG if any of the multicalls fail. 20 * 21 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 22 */ 23 #include <linux/percpu.h> 24 #include <linux/hardirq.h> 25 #include <linux/debugfs.h> 26 27 #include <asm/xen/hypercall.h> 28 29 #include "multicalls.h" 30 #include "debugfs.h" 31 32 #define MC_BATCH 32 33 34 #define MC_DEBUG 0 35 36 #define MC_ARGS (MC_BATCH * 16) 37 38 39 struct mc_buffer { 40 unsigned mcidx, argidx, cbidx; 41 struct multicall_entry entries[MC_BATCH]; 42 #if MC_DEBUG 43 struct multicall_entry debug[MC_BATCH]; 44 void *caller[MC_BATCH]; 45 #endif 46 unsigned char args[MC_ARGS]; 47 struct callback { 48 void (*fn)(void *); 49 void *data; 50 } callbacks[MC_BATCH]; 51 }; 52 53 static DEFINE_PER_CPU(struct mc_buffer, mc_buffer); 54 DEFINE_PER_CPU(unsigned long, xen_mc_irq_flags); 55 56 void xen_mc_flush(void) 57 { 58 struct mc_buffer *b = this_cpu_ptr(&mc_buffer); 59 struct multicall_entry *mc; 60 int ret = 0; 61 unsigned long flags; 62 int i; 63 64 BUG_ON(preemptible()); 65 66 /* Disable interrupts in case someone comes in and queues 67 something in the middle */ 68 local_irq_save(flags); 69 70 trace_xen_mc_flush(b->mcidx, b->argidx, b->cbidx); 71 72 switch (b->mcidx) { 73 case 0: 74 /* no-op */ 75 BUG_ON(b->argidx != 0); 76 break; 77 78 case 1: 79 /* Singleton multicall - bypass multicall machinery 80 and just do the call directly. */ 81 mc = &b->entries[0]; 82 83 mc->result = xen_single_call(mc->op, mc->args[0], mc->args[1], 84 mc->args[2], mc->args[3], 85 mc->args[4]); 86 ret = mc->result < 0; 87 break; 88 89 default: 90 #if MC_DEBUG 91 memcpy(b->debug, b->entries, 92 b->mcidx * sizeof(struct multicall_entry)); 93 #endif 94 95 if (HYPERVISOR_multicall(b->entries, b->mcidx) != 0) 96 BUG(); 97 for (i = 0; i < b->mcidx; i++) 98 if (b->entries[i].result < 0) 99 ret++; 100 101 #if MC_DEBUG 102 if (ret) { 103 printk(KERN_ERR "%d multicall(s) failed: cpu %d\n", 104 ret, smp_processor_id()); 105 dump_stack(); 106 for (i = 0; i < b->mcidx; i++) { 107 printk(KERN_DEBUG " call %2d/%d: op=%lu arg=[%lx] result=%ld\t%pF\n", 108 i+1, b->mcidx, 109 b->debug[i].op, 110 b->debug[i].args[0], 111 b->entries[i].result, 112 b->caller[i]); 113 } 114 } 115 #endif 116 } 117 118 b->mcidx = 0; 119 b->argidx = 0; 120 121 for (i = 0; i < b->cbidx; i++) { 122 struct callback *cb = &b->callbacks[i]; 123 124 (*cb->fn)(cb->data); 125 } 126 b->cbidx = 0; 127 128 local_irq_restore(flags); 129 130 WARN_ON(ret); 131 } 132 133 struct multicall_space __xen_mc_entry(size_t args) 134 { 135 struct mc_buffer *b = this_cpu_ptr(&mc_buffer); 136 struct multicall_space ret; 137 unsigned argidx = roundup(b->argidx, sizeof(u64)); 138 139 trace_xen_mc_entry_alloc(args); 140 141 BUG_ON(preemptible()); 142 BUG_ON(b->argidx >= MC_ARGS); 143 144 if (unlikely(b->mcidx == MC_BATCH || 145 (argidx + args) >= MC_ARGS)) { 146 trace_xen_mc_flush_reason((b->mcidx == MC_BATCH) ? 147 XEN_MC_FL_BATCH : XEN_MC_FL_ARGS); 148 xen_mc_flush(); 149 argidx = roundup(b->argidx, sizeof(u64)); 150 } 151 152 ret.mc = &b->entries[b->mcidx]; 153 #if MC_DEBUG 154 b->caller[b->mcidx] = __builtin_return_address(0); 155 #endif 156 b->mcidx++; 157 ret.args = &b->args[argidx]; 158 b->argidx = argidx + args; 159 160 BUG_ON(b->argidx >= MC_ARGS); 161 return ret; 162 } 163 164 struct multicall_space xen_mc_extend_args(unsigned long op, size_t size) 165 { 166 struct mc_buffer *b = this_cpu_ptr(&mc_buffer); 167 struct multicall_space ret = { NULL, NULL }; 168 169 BUG_ON(preemptible()); 170 BUG_ON(b->argidx >= MC_ARGS); 171 172 if (unlikely(b->mcidx == 0 || 173 b->entries[b->mcidx - 1].op != op)) { 174 trace_xen_mc_extend_args(op, size, XEN_MC_XE_BAD_OP); 175 goto out; 176 } 177 178 if (unlikely((b->argidx + size) >= MC_ARGS)) { 179 trace_xen_mc_extend_args(op, size, XEN_MC_XE_NO_SPACE); 180 goto out; 181 } 182 183 ret.mc = &b->entries[b->mcidx - 1]; 184 ret.args = &b->args[b->argidx]; 185 b->argidx += size; 186 187 BUG_ON(b->argidx >= MC_ARGS); 188 189 trace_xen_mc_extend_args(op, size, XEN_MC_XE_OK); 190 out: 191 return ret; 192 } 193 194 void xen_mc_callback(void (*fn)(void *), void *data) 195 { 196 struct mc_buffer *b = this_cpu_ptr(&mc_buffer); 197 struct callback *cb; 198 199 if (b->cbidx == MC_BATCH) { 200 trace_xen_mc_flush_reason(XEN_MC_FL_CALLBACK); 201 xen_mc_flush(); 202 } 203 204 trace_xen_mc_callback(fn, data); 205 206 cb = &b->callbacks[b->cbidx++]; 207 cb->fn = fn; 208 cb->data = data; 209 } 210