1 /* 2 * Copyright (C) 2016 Red Hat, Inc. 3 * Author: Michael S. Tsirkin <mst@redhat.com> 4 * This work is licensed under the terms of the GNU GPL, version 2. 5 * 6 * Partial implementation of virtio 0.9. event index is used for signalling, 7 * unconditionally. Design roughly follows linux kernel implementation in order 8 * to be able to judge its performance. 9 */ 10 #define _GNU_SOURCE 11 #include "main.h" 12 #include <stdlib.h> 13 #include <stdio.h> 14 #include <assert.h> 15 #include <string.h> 16 #include <linux/virtio_ring.h> 17 18 struct data { 19 void *data; 20 } *data; 21 22 struct vring ring; 23 24 /* enabling the below activates experimental ring polling code 25 * (which skips index reads on consumer in favor of looking at 26 * high bits of ring id ^ 0x8000). 27 */ 28 /* #ifdef RING_POLL */ 29 /* enabling the below activates experimental in-order code 30 * (which skips ring updates and reads and writes len in descriptor). 31 */ 32 /* #ifdef INORDER */ 33 34 #if defined(RING_POLL) && defined(INORDER) 35 #error "RING_POLL and INORDER are mutually exclusive" 36 #endif 37 38 /* how much padding is needed to avoid false cache sharing */ 39 #define HOST_GUEST_PADDING 0x80 40 41 struct guest { 42 unsigned short avail_idx; 43 unsigned short last_used_idx; 44 unsigned short num_free; 45 unsigned short kicked_avail_idx; 46 #ifndef INORDER 47 unsigned short free_head; 48 #else 49 unsigned short reserved_free_head; 50 #endif 51 unsigned char reserved[HOST_GUEST_PADDING - 10]; 52 } guest; 53 54 struct host { 55 /* we do not need to track last avail index 56 * unless we have more than one in flight. 57 */ 58 unsigned short used_idx; 59 unsigned short called_used_idx; 60 unsigned char reserved[HOST_GUEST_PADDING - 4]; 61 } host; 62 63 /* implemented by ring */ 64 void alloc_ring(void) 65 { 66 int ret; 67 int i; 68 void *p; 69 70 ret = posix_memalign(&p, 0x1000, vring_size(ring_size, 0x1000)); 71 if (ret) { 72 perror("Unable to allocate ring buffer.\n"); 73 exit(3); 74 } 75 memset(p, 0, vring_size(ring_size, 0x1000)); 76 vring_init(&ring, ring_size, p, 0x1000); 77 78 guest.avail_idx = 0; 79 guest.kicked_avail_idx = -1; 80 guest.last_used_idx = 0; 81 #ifndef INORDER 82 /* Put everything in free lists. */ 83 guest.free_head = 0; 84 #endif 85 for (i = 0; i < ring_size - 1; i++) 86 ring.desc[i].next = i + 1; 87 host.used_idx = 0; 88 host.called_used_idx = -1; 89 guest.num_free = ring_size; 90 data = malloc(ring_size * sizeof *data); 91 if (!data) { 92 perror("Unable to allocate data buffer.\n"); 93 exit(3); 94 } 95 memset(data, 0, ring_size * sizeof *data); 96 } 97 98 /* guest side */ 99 int add_inbuf(unsigned len, void *buf, void *datap) 100 { 101 unsigned head; 102 #ifndef INORDER 103 unsigned avail; 104 #endif 105 struct vring_desc *desc; 106 107 if (!guest.num_free) 108 return -1; 109 110 #ifdef INORDER 111 head = (ring_size - 1) & (guest.avail_idx++); 112 #else 113 head = guest.free_head; 114 #endif 115 guest.num_free--; 116 117 desc = ring.desc; 118 desc[head].flags = VRING_DESC_F_NEXT; 119 desc[head].addr = (unsigned long)(void *)buf; 120 desc[head].len = len; 121 /* We do it like this to simulate the way 122 * we'd have to flip it if we had multiple 123 * descriptors. 124 */ 125 desc[head].flags &= ~VRING_DESC_F_NEXT; 126 #ifndef INORDER 127 guest.free_head = desc[head].next; 128 #endif 129 130 data[head].data = datap; 131 132 #ifdef RING_POLL 133 /* Barrier A (for pairing) */ 134 smp_release(); 135 avail = guest.avail_idx++; 136 ring.avail->ring[avail & (ring_size - 1)] = 137 (head | (avail & ~(ring_size - 1))) ^ 0x8000; 138 #else 139 #ifndef INORDER 140 /* Barrier A (for pairing) */ 141 smp_release(); 142 avail = (ring_size - 1) & (guest.avail_idx++); 143 ring.avail->ring[avail] = head; 144 #endif 145 /* Barrier A (for pairing) */ 146 smp_release(); 147 #endif 148 ring.avail->idx = guest.avail_idx; 149 return 0; 150 } 151 152 void *get_buf(unsigned *lenp, void **bufp) 153 { 154 unsigned head; 155 unsigned index; 156 void *datap; 157 158 #ifdef RING_POLL 159 head = (ring_size - 1) & guest.last_used_idx; 160 index = ring.used->ring[head].id; 161 if ((index ^ guest.last_used_idx ^ 0x8000) & ~(ring_size - 1)) 162 return NULL; 163 /* Barrier B (for pairing) */ 164 smp_acquire(); 165 index &= ring_size - 1; 166 #else 167 if (ring.used->idx == guest.last_used_idx) 168 return NULL; 169 /* Barrier B (for pairing) */ 170 smp_acquire(); 171 #ifdef INORDER 172 head = (ring_size - 1) & guest.last_used_idx; 173 index = head; 174 #else 175 head = (ring_size - 1) & guest.last_used_idx; 176 index = ring.used->ring[head].id; 177 #endif 178 179 #endif 180 #ifdef INORDER 181 *lenp = ring.desc[index].len; 182 #else 183 *lenp = ring.used->ring[head].len; 184 #endif 185 datap = data[index].data; 186 *bufp = (void*)(unsigned long)ring.desc[index].addr; 187 data[index].data = NULL; 188 #ifndef INORDER 189 ring.desc[index].next = guest.free_head; 190 guest.free_head = index; 191 #endif 192 guest.num_free++; 193 guest.last_used_idx++; 194 return datap; 195 } 196 197 void poll_used(void) 198 { 199 #ifdef RING_POLL 200 unsigned head = (ring_size - 1) & guest.last_used_idx; 201 202 for (;;) { 203 unsigned index = ring.used->ring[head].id; 204 205 if ((index ^ guest.last_used_idx ^ 0x8000) & ~(ring_size - 1)) 206 busy_wait(); 207 else 208 break; 209 } 210 #else 211 unsigned head = guest.last_used_idx; 212 213 while (ring.used->idx == head) 214 busy_wait(); 215 #endif 216 } 217 218 void disable_call() 219 { 220 /* Doing nothing to disable calls might cause 221 * extra interrupts, but reduces the number of cache misses. 222 */ 223 } 224 225 bool enable_call() 226 { 227 unsigned short last_used_idx; 228 229 vring_used_event(&ring) = (last_used_idx = guest.last_used_idx); 230 /* Flush call index write */ 231 /* Barrier D (for pairing) */ 232 smp_mb(); 233 #ifdef RING_POLL 234 { 235 unsigned short head = last_used_idx & (ring_size - 1); 236 unsigned index = ring.used->ring[head].id; 237 238 return (index ^ last_used_idx ^ 0x8000) & ~(ring_size - 1); 239 } 240 #else 241 return ring.used->idx == last_used_idx; 242 #endif 243 } 244 245 void kick_available(void) 246 { 247 /* Flush in previous flags write */ 248 /* Barrier C (for pairing) */ 249 smp_mb(); 250 if (!vring_need_event(vring_avail_event(&ring), 251 guest.avail_idx, 252 guest.kicked_avail_idx)) 253 return; 254 255 guest.kicked_avail_idx = guest.avail_idx; 256 kick(); 257 } 258 259 /* host side */ 260 void disable_kick() 261 { 262 /* Doing nothing to disable kicks might cause 263 * extra interrupts, but reduces the number of cache misses. 264 */ 265 } 266 267 bool enable_kick() 268 { 269 unsigned head = host.used_idx; 270 271 vring_avail_event(&ring) = head; 272 /* Barrier C (for pairing) */ 273 smp_mb(); 274 #ifdef RING_POLL 275 { 276 unsigned index = ring.avail->ring[head & (ring_size - 1)]; 277 278 return (index ^ head ^ 0x8000) & ~(ring_size - 1); 279 } 280 #else 281 return head == ring.avail->idx; 282 #endif 283 } 284 285 void poll_avail(void) 286 { 287 unsigned head = host.used_idx; 288 #ifdef RING_POLL 289 for (;;) { 290 unsigned index = ring.avail->ring[head & (ring_size - 1)]; 291 if ((index ^ head ^ 0x8000) & ~(ring_size - 1)) 292 busy_wait(); 293 else 294 break; 295 } 296 #else 297 while (ring.avail->idx == head) 298 busy_wait(); 299 #endif 300 } 301 302 bool use_buf(unsigned *lenp, void **bufp) 303 { 304 unsigned used_idx = host.used_idx; 305 struct vring_desc *desc; 306 unsigned head; 307 308 #ifdef RING_POLL 309 head = ring.avail->ring[used_idx & (ring_size - 1)]; 310 if ((used_idx ^ head ^ 0x8000) & ~(ring_size - 1)) 311 return false; 312 /* Barrier A (for pairing) */ 313 smp_acquire(); 314 315 used_idx &= ring_size - 1; 316 desc = &ring.desc[head & (ring_size - 1)]; 317 #else 318 if (used_idx == ring.avail->idx) 319 return false; 320 321 /* Barrier A (for pairing) */ 322 smp_acquire(); 323 324 used_idx &= ring_size - 1; 325 #ifdef INORDER 326 head = used_idx; 327 #else 328 head = ring.avail->ring[used_idx]; 329 #endif 330 desc = &ring.desc[head]; 331 #endif 332 333 *lenp = desc->len; 334 *bufp = (void *)(unsigned long)desc->addr; 335 336 #ifdef INORDER 337 desc->len = desc->len - 1; 338 #else 339 /* now update used ring */ 340 ring.used->ring[used_idx].id = head; 341 ring.used->ring[used_idx].len = desc->len - 1; 342 #endif 343 /* Barrier B (for pairing) */ 344 smp_release(); 345 host.used_idx++; 346 ring.used->idx = host.used_idx; 347 348 return true; 349 } 350 351 void call_used(void) 352 { 353 /* Flush in previous flags write */ 354 /* Barrier D (for pairing) */ 355 smp_mb(); 356 if (!vring_need_event(vring_used_event(&ring), 357 host.used_idx, 358 host.called_used_idx)) 359 return; 360 361 host.called_used_idx = host.used_idx; 362 call(); 363 } 364