multicalls.c revision 84cdee76b15f3669f012d5916287d124c805ef2f
1/* 2 * Xen hypercall batching. 3 * 4 * Xen allows multiple hypercalls to be issued at once, using the 5 * multicall interface. This allows the cost of trapping into the 6 * hypervisor to be amortized over several calls. 7 * 8 * This file implements a simple interface for multicalls. There's a 9 * per-cpu buffer of outstanding multicalls. When you want to queue a 10 * multicall for issuing, you can allocate a multicall slot for the 11 * call and its arguments, along with storage for space which is 12 * pointed to by the arguments (for passing pointers to structures, 13 * etc). When the multicall is actually issued, all the space for the 14 * commands and allocated memory is freed for reuse. 15 * 16 * Multicalls are flushed whenever any of the buffers get full, or 17 * when explicitly requested. There's no way to get per-multicall 18 * return results back. It will BUG if any of the multicalls fail. 19 * 20 * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007 21 */ 22#include <linux/percpu.h> 23#include <linux/hardirq.h> 24#include <linux/debugfs.h> 25 26#include <asm/xen/hypercall.h> 27 28#include "multicalls.h" 29#include "debugfs.h" 30 31#define MC_BATCH 32 32 33#define MC_DEBUG 1 34 35#define MC_ARGS (MC_BATCH * 16) 36 37 38struct mc_buffer { 39 struct multicall_entry entries[MC_BATCH]; 40#if MC_DEBUG 41 struct multicall_entry debug[MC_BATCH]; 42 void *caller[MC_BATCH]; 43#endif 44 unsigned char args[MC_ARGS]; 45 struct callback { 46 void (*fn)(void *); 47 void *data; 48 } callbacks[MC_BATCH]; 49 unsigned mcidx, argidx, cbidx; 50}; 51 52static DEFINE_PER_CPU(struct mc_buffer, mc_buffer); 53DEFINE_PER_CPU(unsigned long, xen_mc_irq_flags); 54 55void xen_mc_flush(void) 56{ 57 struct mc_buffer *b = &__get_cpu_var(mc_buffer); 58 int ret = 0; 59 unsigned long flags; 60 int i; 61 62 BUG_ON(preemptible()); 63 64 /* Disable interrupts in case someone comes in and queues 65 something in the middle */ 66 local_irq_save(flags); 67 68 if (b->mcidx) { 69#if MC_DEBUG 70 memcpy(b->debug, b->entries, 71 b->mcidx * sizeof(struct multicall_entry)); 72#endif 73 74 if (HYPERVISOR_multicall(b->entries, b->mcidx) != 0) 75 BUG(); 76 for (i = 0; i < b->mcidx; i++) 77 if (b->entries[i].result < 0) 78 ret++; 79 80#if MC_DEBUG 81 if (ret) { 82 printk(KERN_ERR "%d multicall(s) failed: cpu %d\n", 83 ret, smp_processor_id()); 84 dump_stack(); 85 for (i = 0; i < b->mcidx; i++) { 86 printk(KERN_DEBUG " call %2d/%d: op=%lu arg=[%lx] result=%ld\t%pF\n", 87 i+1, b->mcidx, 88 b->debug[i].op, 89 b->debug[i].args[0], 90 b->entries[i].result, 91 b->caller[i]); 92 } 93 } 94#endif 95 96 b->mcidx = 0; 97 b->argidx = 0; 98 } else 99 BUG_ON(b->argidx != 0); 100 101 for (i = 0; i < b->cbidx; i++) { 102 struct callback *cb = &b->callbacks[i]; 103 104 (*cb->fn)(cb->data); 105 } 106 b->cbidx = 0; 107 108 local_irq_restore(flags); 109 110 WARN_ON(ret); 111} 112 113struct multicall_space __xen_mc_entry(size_t args) 114{ 115 struct mc_buffer *b = &__get_cpu_var(mc_buffer); 116 struct multicall_space ret; 117 unsigned argidx = roundup(b->argidx, sizeof(u64)); 118 119 BUG_ON(preemptible()); 120 BUG_ON(b->argidx >= MC_ARGS); 121 122 if (b->mcidx == MC_BATCH || 123 (argidx + args) >= MC_ARGS) { 124 xen_mc_flush(); 125 argidx = roundup(b->argidx, sizeof(u64)); 126 } 127 128 ret.mc = &b->entries[b->mcidx]; 129#ifdef MC_DEBUG 130 b->caller[b->mcidx] = __builtin_return_address(0); 131#endif 132 b->mcidx++; 133 ret.args = &b->args[argidx]; 134 b->argidx = argidx + args; 135 136 BUG_ON(b->argidx >= MC_ARGS); 137 return ret; 138} 139 140struct multicall_space xen_mc_extend_args(unsigned long op, size_t size) 141{ 142 struct mc_buffer *b = &__get_cpu_var(mc_buffer); 143 struct multicall_space ret = { NULL, NULL }; 144 145 BUG_ON(preemptible()); 146 BUG_ON(b->argidx >= MC_ARGS); 147 148 if (b->mcidx == 0) 149 return ret; 150 151 if (b->entries[b->mcidx - 1].op != op) 152 return ret; 153 154 if ((b->argidx + size) >= MC_ARGS) 155 return ret; 156 157 ret.mc = &b->entries[b->mcidx - 1]; 158 ret.args = &b->args[b->argidx]; 159 b->argidx += size; 160 161 BUG_ON(b->argidx >= MC_ARGS); 162 return ret; 163} 164 165void xen_mc_callback(void (*fn)(void *), void *data) 166{ 167 struct mc_buffer *b = &__get_cpu_var(mc_buffer); 168 struct callback *cb; 169 170 if (b->cbidx == MC_BATCH) 171 xen_mc_flush(); 172 173 cb = &b->callbacks[b->cbidx++]; 174 cb->fn = fn; 175 cb->data = data; 176} 177