1 /* 2 * Xen hypercall batching. 3 * 4 * Xen allows multiple hypercalls to be issued at once, using the 5 * multicall interface. This allows the cost of trapping into the 6 * hypervisor to be amortized over several calls. 7 * 8 * This file implements a simple interface for multicalls. There's a 9 * per-cpu buffer of outstanding multicalls. When you want to queue a 10 * multicall for issuing, you can allocate a multicall slot for the 11 * call and its arguments, along with storage for space which is 12 * pointed to by the arguments (for passing pointers to structures, 13 * etc). When the multicall is actually issued, all the space for the 14 * commands and allocated memory is freed for reuse. 15 * 16 * Multicalls are flushed whenever any of the buffers get full, or 17 * when explicitly requested. There's no way to get per-multicall 18 * return results back. It will BUG if any of the multicalls fail. 19 * 20 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 21 */ 22 #include <linux/percpu.h> 23 #include <linux/hardirq.h> 24 #include <linux/debugfs.h> 25 26 #include <asm/xen/hypercall.h> 27 28 #include "multicalls.h" 29 #include "debugfs.h" 30 31 #define MC_BATCH 32 32 33 #define MC_DEBUG 1 34 35 #define MC_ARGS (MC_BATCH * 16) 36 37 38 struct mc_buffer { 39 struct multicall_entry entries[MC_BATCH]; 40 #if MC_DEBUG 41 struct multicall_entry debug[MC_BATCH]; 42 #endif 43 unsigned char args[MC_ARGS]; 44 struct callback { 45 void (*fn)(void *); 46 void *data; 47 } callbacks[MC_BATCH]; 48 unsigned mcidx, argidx, cbidx; 49 }; 50 51 static DEFINE_PER_CPU(struct mc_buffer, mc_buffer); 52 DEFINE_PER_CPU(unsigned long, xen_mc_irq_flags); 53 54 /* flush reasons 0- slots, 1- args, 2- callbacks */ 55 enum flush_reasons 56 { 57 FL_SLOTS, 58 FL_ARGS, 59 FL_CALLBACKS, 60 61 FL_N_REASONS 62 }; 63 64 #ifdef CONFIG_XEN_DEBUG_FS 65 #define NHYPERCALLS 40 /* not really */ 66 67 static struct { 68 unsigned histo[MC_BATCH+1]; 69 70 unsigned issued; 71 unsigned arg_total; 72 unsigned hypercalls; 73 unsigned histo_hypercalls[NHYPERCALLS]; 74 75 unsigned flush[FL_N_REASONS]; 76 } mc_stats; 77 78 static u8 zero_stats; 79 80 static inline void check_zero(void) 81 { 82 if (unlikely(zero_stats)) { 83 memset(&mc_stats, 0, sizeof(mc_stats)); 84 zero_stats = 0; 85 } 86 } 87 88 static void mc_add_stats(const struct mc_buffer *mc) 89 { 90 int i; 91 92 check_zero(); 93 94 mc_stats.issued++; 95 mc_stats.hypercalls += mc->mcidx; 96 mc_stats.arg_total += mc->argidx; 97 98 mc_stats.histo[mc->mcidx]++; 99 for(i = 0; i < mc->mcidx; i++) { 100 unsigned op = mc->entries[i].op; 101 if (op < NHYPERCALLS) 102 mc_stats.histo_hypercalls[op]++; 103 } 104 } 105 106 static void mc_stats_flush(enum flush_reasons idx) 107 { 108 check_zero(); 109 110 mc_stats.flush[idx]++; 111 } 112 113 #else /* !CONFIG_XEN_DEBUG_FS */ 114 115 static inline void mc_add_stats(const struct mc_buffer *mc) 116 { 117 } 118 119 static inline void mc_stats_flush(enum flush_reasons idx) 120 { 121 } 122 #endif /* CONFIG_XEN_DEBUG_FS */ 123 124 void xen_mc_flush(void) 125 { 126 struct mc_buffer *b = &__get_cpu_var(mc_buffer); 127 int ret = 0; 128 unsigned long flags; 129 int i; 130 131 BUG_ON(preemptible()); 132 133 /* Disable interrupts in case someone comes in and queues 134 something in the middle */ 135 local_irq_save(flags); 136 137 mc_add_stats(b); 138 139 if (b->mcidx) { 140 #if MC_DEBUG 141 memcpy(b->debug, b->entries, 142 b->mcidx * sizeof(struct multicall_entry)); 143 #endif 144 145 if (HYPERVISOR_multicall(b->entries, b->mcidx) != 0) 146 BUG(); 147 for (i = 0; i < b->mcidx; i++) 148 if (b->entries[i].result < 0) 149 ret++; 150 151 #if MC_DEBUG 152 if (ret) { 153 printk(KERN_ERR "%d multicall(s) failed: cpu %d\n", 154 ret, smp_processor_id()); 155 dump_stack(); 156 for (i = 0; i < b->mcidx; i++) { 157 printk(" call %2d/%d: op=%lu arg=[%lx] result=%ld\n", 158 i+1, b->mcidx, 159 b->debug[i].op, 160 b->debug[i].args[0], 161 b->entries[i].result); 162 } 163 } 164 #endif 165 166 b->mcidx = 0; 167 b->argidx = 0; 168 } else 169 BUG_ON(b->argidx != 0); 170 171 local_irq_restore(flags); 172 173 for (i = 0; i < b->cbidx; i++) { 174 struct callback *cb = &b->callbacks[i]; 175 176 (*cb->fn)(cb->data); 177 } 178 b->cbidx = 0; 179 180 BUG_ON(ret); 181 } 182 183 struct multicall_space __xen_mc_entry(size_t args) 184 { 185 struct mc_buffer *b = &__get_cpu_var(mc_buffer); 186 struct multicall_space ret; 187 unsigned argidx = roundup(b->argidx, sizeof(u64)); 188 189 BUG_ON(preemptible()); 190 BUG_ON(b->argidx > MC_ARGS); 191 192 if (b->mcidx == MC_BATCH || 193 (argidx + args) > MC_ARGS) { 194 mc_stats_flush(b->mcidx == MC_BATCH ? FL_SLOTS : FL_ARGS); 195 xen_mc_flush(); 196 argidx = roundup(b->argidx, sizeof(u64)); 197 } 198 199 ret.mc = &b->entries[b->mcidx]; 200 b->mcidx++; 201 ret.args = &b->args[argidx]; 202 b->argidx = argidx + args; 203 204 BUG_ON(b->argidx > MC_ARGS); 205 return ret; 206 } 207 208 struct multicall_space xen_mc_extend_args(unsigned long op, size_t size) 209 { 210 struct mc_buffer *b = &__get_cpu_var(mc_buffer); 211 struct multicall_space ret = { NULL, NULL }; 212 213 BUG_ON(preemptible()); 214 BUG_ON(b->argidx > MC_ARGS); 215 216 if (b->mcidx == 0) 217 return ret; 218 219 if (b->entries[b->mcidx - 1].op != op) 220 return ret; 221 222 if ((b->argidx + size) > MC_ARGS) 223 return ret; 224 225 ret.mc = &b->entries[b->mcidx - 1]; 226 ret.args = &b->args[b->argidx]; 227 b->argidx += size; 228 229 BUG_ON(b->argidx > MC_ARGS); 230 return ret; 231 } 232 233 void xen_mc_callback(void (*fn)(void *), void *data) 234 { 235 struct mc_buffer *b = &__get_cpu_var(mc_buffer); 236 struct callback *cb; 237 238 if (b->cbidx == MC_BATCH) { 239 mc_stats_flush(FL_CALLBACKS); 240 xen_mc_flush(); 241 } 242 243 cb = &b->callbacks[b->cbidx++]; 244 cb->fn = fn; 245 cb->data = data; 246 } 247 248 #ifdef CONFIG_XEN_DEBUG_FS 249 250 static struct dentry *d_mc_debug; 251 252 static int __init xen_mc_debugfs(void) 253 { 254 struct dentry *d_xen = xen_init_debugfs(); 255 256 if (d_xen == NULL) 257 return -ENOMEM; 258 259 d_mc_debug = debugfs_create_dir("multicalls", d_xen); 260 261 debugfs_create_u8("zero_stats", 0644, d_mc_debug, &zero_stats); 262 263 debugfs_create_u32("batches", 0444, d_mc_debug, &mc_stats.issued); 264 debugfs_create_u32("hypercalls", 0444, d_mc_debug, &mc_stats.hypercalls); 265 debugfs_create_u32("arg_total", 0444, d_mc_debug, &mc_stats.arg_total); 266 267 xen_debugfs_create_u32_array("batch_histo", 0444, d_mc_debug, 268 mc_stats.histo, MC_BATCH); 269 xen_debugfs_create_u32_array("hypercall_histo", 0444, d_mc_debug, 270 mc_stats.histo_hypercalls, NHYPERCALLS); 271 xen_debugfs_create_u32_array("flush_reasons", 0444, d_mc_debug, 272 mc_stats.flush, FL_N_REASONS); 273 274 return 0; 275 } 276 fs_initcall(xen_mc_debugfs); 277 278 #endif /* CONFIG_XEN_DEBUG_FS */ 279