1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* Virtio ring implementation. 3 * 4 * Copyright 2007 Rusty Russell IBM Corporation 5 */ 6 #include <linux/virtio.h> 7 #include <linux/virtio_ring.h> 8 #include <linux/virtio_config.h> 9 #include <linux/device.h> 10 #include <linux/slab.h> 11 #include <linux/module.h> 12 #include <linux/hrtimer.h> 13 #include <linux/dma-mapping.h> 14 #include <linux/kmsan.h> 15 #include <linux/spinlock.h> 16 #include <xen/xen.h> 17 18 #ifdef DEBUG 19 /* For development, we want to crash whenever the ring is screwed. */ 20 #define BAD_RING(_vq, fmt, args...) \ 21 do { \ 22 dev_err(&(_vq)->vq.vdev->dev, \ 23 "%s:"fmt, (_vq)->vq.name, ##args); \ 24 BUG(); \ 25 } while (0) 26 /* Caller is supposed to guarantee no reentry. */ 27 #define START_USE(_vq) \ 28 do { \ 29 if ((_vq)->in_use) \ 30 panic("%s:in_use = %i\n", \ 31 (_vq)->vq.name, (_vq)->in_use); \ 32 (_vq)->in_use = __LINE__; \ 33 } while (0) 34 #define END_USE(_vq) \ 35 do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0) 36 #define LAST_ADD_TIME_UPDATE(_vq) \ 37 do { \ 38 ktime_t now = ktime_get(); \ 39 \ 40 /* No kick or get, with .1 second between? Warn. */ \ 41 if ((_vq)->last_add_time_valid) \ 42 WARN_ON(ktime_to_ms(ktime_sub(now, \ 43 (_vq)->last_add_time)) > 100); \ 44 (_vq)->last_add_time = now; \ 45 (_vq)->last_add_time_valid = true; \ 46 } while (0) 47 #define LAST_ADD_TIME_CHECK(_vq) \ 48 do { \ 49 if ((_vq)->last_add_time_valid) { \ 50 WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), \ 51 (_vq)->last_add_time)) > 100); \ 52 } \ 53 } while (0) 54 #define LAST_ADD_TIME_INVALID(_vq) \ 55 ((_vq)->last_add_time_valid = false) 56 #else 57 #define BAD_RING(_vq, fmt, args...) \ 58 do { \ 59 dev_err(&_vq->vq.vdev->dev, \ 60 "%s:"fmt, (_vq)->vq.name, ##args); \ 61 (_vq)->broken = true; \ 62 } while (0) 63 #define START_USE(vq) 64 #define END_USE(vq) 65 #define LAST_ADD_TIME_UPDATE(vq) 66 #define LAST_ADD_TIME_CHECK(vq) 67 #define LAST_ADD_TIME_INVALID(vq) 68 #endif 69 70 struct vring_desc_state_split { 71 void *data; /* Data for callback. */ 72 struct vring_desc *indir_desc; /* Indirect descriptor, if any. */ 73 }; 74 75 struct vring_desc_state_packed { 76 void *data; /* Data for callback. */ 77 struct vring_packed_desc *indir_desc; /* Indirect descriptor, if any. */ 78 u16 num; /* Descriptor list length. */ 79 u16 last; /* The last desc state in a list. */ 80 }; 81 82 struct vring_desc_extra { 83 dma_addr_t addr; /* Descriptor DMA addr. */ 84 u32 len; /* Descriptor length. */ 85 u16 flags; /* Descriptor flags. */ 86 u16 next; /* The next desc state in a list. */ 87 }; 88 89 struct vring_virtqueue_split { 90 /* Actual memory layout for this queue. */ 91 struct vring vring; 92 93 /* Last written value to avail->flags */ 94 u16 avail_flags_shadow; 95 96 /* 97 * Last written value to avail->idx in 98 * guest byte order. 99 */ 100 u16 avail_idx_shadow; 101 102 /* Per-descriptor state. */ 103 struct vring_desc_state_split *desc_state; 104 struct vring_desc_extra *desc_extra; 105 106 /* DMA address and size information */ 107 dma_addr_t queue_dma_addr; 108 size_t queue_size_in_bytes; 109 110 /* 111 * The parameters for creating vrings are reserved for creating new 112 * vring. 113 */ 114 u32 vring_align; 115 bool may_reduce_num; 116 }; 117 118 struct vring_virtqueue_packed { 119 /* Actual memory layout for this queue. */ 120 struct { 121 unsigned int num; 122 struct vring_packed_desc *desc; 123 struct vring_packed_desc_event *driver; 124 struct vring_packed_desc_event *device; 125 } vring; 126 127 /* Driver ring wrap counter. */ 128 bool avail_wrap_counter; 129 130 /* Avail used flags. */ 131 u16 avail_used_flags; 132 133 /* Index of the next avail descriptor. */ 134 u16 next_avail_idx; 135 136 /* 137 * Last written value to driver->flags in 138 * guest byte order. 139 */ 140 u16 event_flags_shadow; 141 142 /* Per-descriptor state. */ 143 struct vring_desc_state_packed *desc_state; 144 struct vring_desc_extra *desc_extra; 145 146 /* DMA address and size information */ 147 dma_addr_t ring_dma_addr; 148 dma_addr_t driver_event_dma_addr; 149 dma_addr_t device_event_dma_addr; 150 size_t ring_size_in_bytes; 151 size_t event_size_in_bytes; 152 }; 153 154 struct vring_virtqueue { 155 struct virtqueue vq; 156 157 /* Is this a packed ring? */ 158 bool packed_ring; 159 160 /* Is DMA API used? */ 161 bool use_dma_api; 162 163 /* Can we use weak barriers? */ 164 bool weak_barriers; 165 166 /* Other side has made a mess, don't try any more. */ 167 bool broken; 168 169 /* Host supports indirect buffers */ 170 bool indirect; 171 172 /* Host publishes avail event idx */ 173 bool event; 174 175 /* Do DMA mapping by driver */ 176 bool premapped; 177 178 /* Do unmap or not for desc. Just when premapped is False and 179 * use_dma_api is true, this is true. 180 */ 181 bool do_unmap; 182 183 /* Head of free buffer list. */ 184 unsigned int free_head; 185 /* Number we've added since last sync. */ 186 unsigned int num_added; 187 188 /* Last used index we've seen. 189 * for split ring, it just contains last used index 190 * for packed ring: 191 * bits up to VRING_PACKED_EVENT_F_WRAP_CTR include the last used index. 192 * bits from VRING_PACKED_EVENT_F_WRAP_CTR include the used wrap counter. 193 */ 194 u16 last_used_idx; 195 196 /* Hint for event idx: already triggered no need to disable. */ 197 bool event_triggered; 198 199 union { 200 /* Available for split ring */ 201 struct vring_virtqueue_split split; 202 203 /* Available for packed ring */ 204 struct vring_virtqueue_packed packed; 205 }; 206 207 /* How to notify other side. FIXME: commonalize hcalls! */ 208 bool (*notify)(struct virtqueue *vq); 209 210 /* DMA, allocation, and size information */ 211 bool we_own_ring; 212 213 /* Device used for doing DMA */ 214 struct device *dma_dev; 215 216 #ifdef DEBUG 217 /* They're supposed to lock for us. */ 218 unsigned int in_use; 219 220 /* Figure out if their kicks are too delayed. */ 221 bool last_add_time_valid; 222 ktime_t last_add_time; 223 #endif 224 }; 225 226 static struct virtqueue *__vring_new_virtqueue(unsigned int index, 227 struct vring_virtqueue_split *vring_split, 228 struct virtio_device *vdev, 229 bool weak_barriers, 230 bool context, 231 bool (*notify)(struct virtqueue *), 232 void (*callback)(struct virtqueue *), 233 const char *name, 234 struct device *dma_dev); 235 static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num); 236 static void vring_free(struct virtqueue *_vq); 237 238 /* 239 * Helpers. 240 */ 241 242 #define to_vvq(_vq) container_of_const(_vq, struct vring_virtqueue, vq) 243 244 static bool virtqueue_use_indirect(const struct vring_virtqueue *vq, 245 unsigned int total_sg) 246 { 247 /* 248 * If the host supports indirect descriptor tables, and we have multiple 249 * buffers, then go indirect. FIXME: tune this threshold 250 */ 251 return (vq->indirect && total_sg > 1 && vq->vq.num_free); 252 } 253 254 /* 255 * Modern virtio devices have feature bits to specify whether they need a 256 * quirk and bypass the IOMMU. If not there, just use the DMA API. 257 * 258 * If there, the interaction between virtio and DMA API is messy. 259 * 260 * On most systems with virtio, physical addresses match bus addresses, 261 * and it doesn't particularly matter whether we use the DMA API. 262 * 263 * On some systems, including Xen and any system with a physical device 264 * that speaks virtio behind a physical IOMMU, we must use the DMA API 265 * for virtio DMA to work at all. 266 * 267 * On other systems, including SPARC and PPC64, virtio-pci devices are 268 * enumerated as though they are behind an IOMMU, but the virtio host 269 * ignores the IOMMU, so we must either pretend that the IOMMU isn't 270 * there or somehow map everything as the identity. 271 * 272 * For the time being, we preserve historic behavior and bypass the DMA 273 * API. 274 * 275 * TODO: install a per-device DMA ops structure that does the right thing 276 * taking into account all the above quirks, and use the DMA API 277 * unconditionally on data path. 278 */ 279 280 static bool vring_use_dma_api(const struct virtio_device *vdev) 281 { 282 if (!virtio_has_dma_quirk(vdev)) 283 return true; 284 285 /* Otherwise, we are left to guess. */ 286 /* 287 * In theory, it's possible to have a buggy QEMU-supposed 288 * emulated Q35 IOMMU and Xen enabled at the same time. On 289 * such a configuration, virtio has never worked and will 290 * not work without an even larger kludge. Instead, enable 291 * the DMA API if we're a Xen guest, which at least allows 292 * all of the sensible Xen configurations to work correctly. 293 */ 294 if (xen_domain()) 295 return true; 296 297 return false; 298 } 299 300 size_t virtio_max_dma_size(const struct virtio_device *vdev) 301 { 302 size_t max_segment_size = SIZE_MAX; 303 304 if (vring_use_dma_api(vdev)) 305 max_segment_size = dma_max_mapping_size(vdev->dev.parent); 306 307 return max_segment_size; 308 } 309 EXPORT_SYMBOL_GPL(virtio_max_dma_size); 310 311 static void *vring_alloc_queue(struct virtio_device *vdev, size_t size, 312 dma_addr_t *dma_handle, gfp_t flag, 313 struct device *dma_dev) 314 { 315 if (vring_use_dma_api(vdev)) { 316 return dma_alloc_coherent(dma_dev, size, 317 dma_handle, flag); 318 } else { 319 void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag); 320 321 if (queue) { 322 phys_addr_t phys_addr = virt_to_phys(queue); 323 *dma_handle = (dma_addr_t)phys_addr; 324 325 /* 326 * Sanity check: make sure we dind't truncate 327 * the address. The only arches I can find that 328 * have 64-bit phys_addr_t but 32-bit dma_addr_t 329 * are certain non-highmem MIPS and x86 330 * configurations, but these configurations 331 * should never allocate physical pages above 32 332 * bits, so this is fine. Just in case, throw a 333 * warning and abort if we end up with an 334 * unrepresentable address. 335 */ 336 if (WARN_ON_ONCE(*dma_handle != phys_addr)) { 337 free_pages_exact(queue, PAGE_ALIGN(size)); 338 return NULL; 339 } 340 } 341 return queue; 342 } 343 } 344 345 static void vring_free_queue(struct virtio_device *vdev, size_t size, 346 void *queue, dma_addr_t dma_handle, 347 struct device *dma_dev) 348 { 349 if (vring_use_dma_api(vdev)) 350 dma_free_coherent(dma_dev, size, queue, dma_handle); 351 else 352 free_pages_exact(queue, PAGE_ALIGN(size)); 353 } 354 355 /* 356 * The DMA ops on various arches are rather gnarly right now, and 357 * making all of the arch DMA ops work on the vring device itself 358 * is a mess. 359 */ 360 static struct device *vring_dma_dev(const struct vring_virtqueue *vq) 361 { 362 return vq->dma_dev; 363 } 364 365 /* Map one sg entry. */ 366 static int vring_map_one_sg(const struct vring_virtqueue *vq, struct scatterlist *sg, 367 enum dma_data_direction direction, dma_addr_t *addr) 368 { 369 if (vq->premapped) { 370 *addr = sg_dma_address(sg); 371 return 0; 372 } 373 374 if (!vq->use_dma_api) { 375 /* 376 * If DMA is not used, KMSAN doesn't know that the scatterlist 377 * is initialized by the hardware. Explicitly check/unpoison it 378 * depending on the direction. 379 */ 380 kmsan_handle_dma(sg_page(sg), sg->offset, sg->length, direction); 381 *addr = (dma_addr_t)sg_phys(sg); 382 return 0; 383 } 384 385 /* 386 * We can't use dma_map_sg, because we don't use scatterlists in 387 * the way it expects (we don't guarantee that the scatterlist 388 * will exist for the lifetime of the mapping). 389 */ 390 *addr = dma_map_page(vring_dma_dev(vq), 391 sg_page(sg), sg->offset, sg->length, 392 direction); 393 394 if (dma_mapping_error(vring_dma_dev(vq), *addr)) 395 return -ENOMEM; 396 397 return 0; 398 } 399 400 static dma_addr_t vring_map_single(const struct vring_virtqueue *vq, 401 void *cpu_addr, size_t size, 402 enum dma_data_direction direction) 403 { 404 if (!vq->use_dma_api) 405 return (dma_addr_t)virt_to_phys(cpu_addr); 406 407 return dma_map_single(vring_dma_dev(vq), 408 cpu_addr, size, direction); 409 } 410 411 static int vring_mapping_error(const struct vring_virtqueue *vq, 412 dma_addr_t addr) 413 { 414 if (!vq->use_dma_api) 415 return 0; 416 417 return dma_mapping_error(vring_dma_dev(vq), addr); 418 } 419 420 static void virtqueue_init(struct vring_virtqueue *vq, u32 num) 421 { 422 vq->vq.num_free = num; 423 424 if (vq->packed_ring) 425 vq->last_used_idx = 0 | (1 << VRING_PACKED_EVENT_F_WRAP_CTR); 426 else 427 vq->last_used_idx = 0; 428 429 vq->event_triggered = false; 430 vq->num_added = 0; 431 432 #ifdef DEBUG 433 vq->in_use = false; 434 vq->last_add_time_valid = false; 435 #endif 436 } 437 438 439 /* 440 * Split ring specific functions - *_split(). 441 */ 442 443 static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq, 444 const struct vring_desc *desc) 445 { 446 u16 flags; 447 448 if (!vq->do_unmap) 449 return; 450 451 flags = virtio16_to_cpu(vq->vq.vdev, desc->flags); 452 453 dma_unmap_page(vring_dma_dev(vq), 454 virtio64_to_cpu(vq->vq.vdev, desc->addr), 455 virtio32_to_cpu(vq->vq.vdev, desc->len), 456 (flags & VRING_DESC_F_WRITE) ? 457 DMA_FROM_DEVICE : DMA_TO_DEVICE); 458 } 459 460 static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq, 461 unsigned int i) 462 { 463 struct vring_desc_extra *extra = vq->split.desc_extra; 464 u16 flags; 465 466 flags = extra[i].flags; 467 468 if (flags & VRING_DESC_F_INDIRECT) { 469 if (!vq->use_dma_api) 470 goto out; 471 472 dma_unmap_single(vring_dma_dev(vq), 473 extra[i].addr, 474 extra[i].len, 475 (flags & VRING_DESC_F_WRITE) ? 476 DMA_FROM_DEVICE : DMA_TO_DEVICE); 477 } else { 478 if (!vq->do_unmap) 479 goto out; 480 481 dma_unmap_page(vring_dma_dev(vq), 482 extra[i].addr, 483 extra[i].len, 484 (flags & VRING_DESC_F_WRITE) ? 485 DMA_FROM_DEVICE : DMA_TO_DEVICE); 486 } 487 488 out: 489 return extra[i].next; 490 } 491 492 static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq, 493 unsigned int total_sg, 494 gfp_t gfp) 495 { 496 struct vring_desc *desc; 497 unsigned int i; 498 499 /* 500 * We require lowmem mappings for the descriptors because 501 * otherwise virt_to_phys will give us bogus addresses in the 502 * virtqueue. 503 */ 504 gfp &= ~__GFP_HIGHMEM; 505 506 desc = kmalloc_array(total_sg, sizeof(struct vring_desc), gfp); 507 if (!desc) 508 return NULL; 509 510 for (i = 0; i < total_sg; i++) 511 desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1); 512 return desc; 513 } 514 515 static inline unsigned int virtqueue_add_desc_split(struct virtqueue *vq, 516 struct vring_desc *desc, 517 unsigned int i, 518 dma_addr_t addr, 519 unsigned int len, 520 u16 flags, 521 bool indirect) 522 { 523 struct vring_virtqueue *vring = to_vvq(vq); 524 struct vring_desc_extra *extra = vring->split.desc_extra; 525 u16 next; 526 527 desc[i].flags = cpu_to_virtio16(vq->vdev, flags); 528 desc[i].addr = cpu_to_virtio64(vq->vdev, addr); 529 desc[i].len = cpu_to_virtio32(vq->vdev, len); 530 531 if (!indirect) { 532 next = extra[i].next; 533 desc[i].next = cpu_to_virtio16(vq->vdev, next); 534 535 extra[i].addr = addr; 536 extra[i].len = len; 537 extra[i].flags = flags; 538 } else 539 next = virtio16_to_cpu(vq->vdev, desc[i].next); 540 541 return next; 542 } 543 544 static inline int virtqueue_add_split(struct virtqueue *_vq, 545 struct scatterlist *sgs[], 546 unsigned int total_sg, 547 unsigned int out_sgs, 548 unsigned int in_sgs, 549 void *data, 550 void *ctx, 551 gfp_t gfp) 552 { 553 struct vring_virtqueue *vq = to_vvq(_vq); 554 struct scatterlist *sg; 555 struct vring_desc *desc; 556 unsigned int i, n, avail, descs_used, prev, err_idx; 557 int head; 558 bool indirect; 559 560 START_USE(vq); 561 562 BUG_ON(data == NULL); 563 BUG_ON(ctx && vq->indirect); 564 565 if (unlikely(vq->broken)) { 566 END_USE(vq); 567 return -EIO; 568 } 569 570 LAST_ADD_TIME_UPDATE(vq); 571 572 BUG_ON(total_sg == 0); 573 574 head = vq->free_head; 575 576 if (virtqueue_use_indirect(vq, total_sg)) 577 desc = alloc_indirect_split(_vq, total_sg, gfp); 578 else { 579 desc = NULL; 580 WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect); 581 } 582 583 if (desc) { 584 /* Use a single buffer which doesn't continue */ 585 indirect = true; 586 /* Set up rest to use this indirect table. */ 587 i = 0; 588 descs_used = 1; 589 } else { 590 indirect = false; 591 desc = vq->split.vring.desc; 592 i = head; 593 descs_used = total_sg; 594 } 595 596 if (unlikely(vq->vq.num_free < descs_used)) { 597 pr_debug("Can't add buf len %i - avail = %i\n", 598 descs_used, vq->vq.num_free); 599 /* FIXME: for historical reasons, we force a notify here if 600 * there are outgoing parts to the buffer. Presumably the 601 * host should service the ring ASAP. */ 602 if (out_sgs) 603 vq->notify(&vq->vq); 604 if (indirect) 605 kfree(desc); 606 END_USE(vq); 607 return -ENOSPC; 608 } 609 610 for (n = 0; n < out_sgs; n++) { 611 for (sg = sgs[n]; sg; sg = sg_next(sg)) { 612 dma_addr_t addr; 613 614 if (vring_map_one_sg(vq, sg, DMA_TO_DEVICE, &addr)) 615 goto unmap_release; 616 617 prev = i; 618 /* Note that we trust indirect descriptor 619 * table since it use stream DMA mapping. 620 */ 621 i = virtqueue_add_desc_split(_vq, desc, i, addr, sg->length, 622 VRING_DESC_F_NEXT, 623 indirect); 624 } 625 } 626 for (; n < (out_sgs + in_sgs); n++) { 627 for (sg = sgs[n]; sg; sg = sg_next(sg)) { 628 dma_addr_t addr; 629 630 if (vring_map_one_sg(vq, sg, DMA_FROM_DEVICE, &addr)) 631 goto unmap_release; 632 633 prev = i; 634 /* Note that we trust indirect descriptor 635 * table since it use stream DMA mapping. 636 */ 637 i = virtqueue_add_desc_split(_vq, desc, i, addr, 638 sg->length, 639 VRING_DESC_F_NEXT | 640 VRING_DESC_F_WRITE, 641 indirect); 642 } 643 } 644 /* Last one doesn't continue. */ 645 desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT); 646 if (!indirect && vq->do_unmap) 647 vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &= 648 ~VRING_DESC_F_NEXT; 649 650 if (indirect) { 651 /* Now that the indirect table is filled in, map it. */ 652 dma_addr_t addr = vring_map_single( 653 vq, desc, total_sg * sizeof(struct vring_desc), 654 DMA_TO_DEVICE); 655 if (vring_mapping_error(vq, addr)) { 656 if (vq->premapped) 657 goto free_indirect; 658 659 goto unmap_release; 660 } 661 662 virtqueue_add_desc_split(_vq, vq->split.vring.desc, 663 head, addr, 664 total_sg * sizeof(struct vring_desc), 665 VRING_DESC_F_INDIRECT, 666 false); 667 } 668 669 /* We're using some buffers from the free list. */ 670 vq->vq.num_free -= descs_used; 671 672 /* Update free pointer */ 673 if (indirect) 674 vq->free_head = vq->split.desc_extra[head].next; 675 else 676 vq->free_head = i; 677 678 /* Store token and indirect buffer state. */ 679 vq->split.desc_state[head].data = data; 680 if (indirect) 681 vq->split.desc_state[head].indir_desc = desc; 682 else 683 vq->split.desc_state[head].indir_desc = ctx; 684 685 /* Put entry in available array (but don't update avail->idx until they 686 * do sync). */ 687 avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1); 688 vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head); 689 690 /* Descriptors and available array need to be set before we expose the 691 * new available array entries. */ 692 virtio_wmb(vq->weak_barriers); 693 vq->split.avail_idx_shadow++; 694 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, 695 vq->split.avail_idx_shadow); 696 vq->num_added++; 697 698 pr_debug("Added buffer head %i to %p\n", head, vq); 699 END_USE(vq); 700 701 /* This is very unlikely, but theoretically possible. Kick 702 * just in case. */ 703 if (unlikely(vq->num_added == (1 << 16) - 1)) 704 virtqueue_kick(_vq); 705 706 return 0; 707 708 unmap_release: 709 err_idx = i; 710 711 if (indirect) 712 i = 0; 713 else 714 i = head; 715 716 for (n = 0; n < total_sg; n++) { 717 if (i == err_idx) 718 break; 719 if (indirect) { 720 vring_unmap_one_split_indirect(vq, &desc[i]); 721 i = virtio16_to_cpu(_vq->vdev, desc[i].next); 722 } else 723 i = vring_unmap_one_split(vq, i); 724 } 725 726 free_indirect: 727 if (indirect) 728 kfree(desc); 729 730 END_USE(vq); 731 return -ENOMEM; 732 } 733 734 static bool virtqueue_kick_prepare_split(struct virtqueue *_vq) 735 { 736 struct vring_virtqueue *vq = to_vvq(_vq); 737 u16 new, old; 738 bool needs_kick; 739 740 START_USE(vq); 741 /* We need to expose available array entries before checking avail 742 * event. */ 743 virtio_mb(vq->weak_barriers); 744 745 old = vq->split.avail_idx_shadow - vq->num_added; 746 new = vq->split.avail_idx_shadow; 747 vq->num_added = 0; 748 749 LAST_ADD_TIME_CHECK(vq); 750 LAST_ADD_TIME_INVALID(vq); 751 752 if (vq->event) { 753 needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev, 754 vring_avail_event(&vq->split.vring)), 755 new, old); 756 } else { 757 needs_kick = !(vq->split.vring.used->flags & 758 cpu_to_virtio16(_vq->vdev, 759 VRING_USED_F_NO_NOTIFY)); 760 } 761 END_USE(vq); 762 return needs_kick; 763 } 764 765 static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head, 766 void **ctx) 767 { 768 unsigned int i, j; 769 __virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT); 770 771 /* Clear data ptr. */ 772 vq->split.desc_state[head].data = NULL; 773 774 /* Put back on free list: unmap first-level descriptors and find end */ 775 i = head; 776 777 while (vq->split.vring.desc[i].flags & nextflag) { 778 vring_unmap_one_split(vq, i); 779 i = vq->split.desc_extra[i].next; 780 vq->vq.num_free++; 781 } 782 783 vring_unmap_one_split(vq, i); 784 vq->split.desc_extra[i].next = vq->free_head; 785 vq->free_head = head; 786 787 /* Plus final descriptor */ 788 vq->vq.num_free++; 789 790 if (vq->indirect) { 791 struct vring_desc *indir_desc = 792 vq->split.desc_state[head].indir_desc; 793 u32 len; 794 795 /* Free the indirect table, if any, now that it's unmapped. */ 796 if (!indir_desc) 797 return; 798 799 len = vq->split.desc_extra[head].len; 800 801 BUG_ON(!(vq->split.desc_extra[head].flags & 802 VRING_DESC_F_INDIRECT)); 803 BUG_ON(len == 0 || len % sizeof(struct vring_desc)); 804 805 if (vq->do_unmap) { 806 for (j = 0; j < len / sizeof(struct vring_desc); j++) 807 vring_unmap_one_split_indirect(vq, &indir_desc[j]); 808 } 809 810 kfree(indir_desc); 811 vq->split.desc_state[head].indir_desc = NULL; 812 } else if (ctx) { 813 *ctx = vq->split.desc_state[head].indir_desc; 814 } 815 } 816 817 static bool more_used_split(const struct vring_virtqueue *vq) 818 { 819 return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev, 820 vq->split.vring.used->idx); 821 } 822 823 static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq, 824 unsigned int *len, 825 void **ctx) 826 { 827 struct vring_virtqueue *vq = to_vvq(_vq); 828 void *ret; 829 unsigned int i; 830 u16 last_used; 831 832 START_USE(vq); 833 834 if (unlikely(vq->broken)) { 835 END_USE(vq); 836 return NULL; 837 } 838 839 if (!more_used_split(vq)) { 840 pr_debug("No more buffers in queue\n"); 841 END_USE(vq); 842 return NULL; 843 } 844 845 /* Only get used array entries after they have been exposed by host. */ 846 virtio_rmb(vq->weak_barriers); 847 848 last_used = (vq->last_used_idx & (vq->split.vring.num - 1)); 849 i = virtio32_to_cpu(_vq->vdev, 850 vq->split.vring.used->ring[last_used].id); 851 *len = virtio32_to_cpu(_vq->vdev, 852 vq->split.vring.used->ring[last_used].len); 853 854 if (unlikely(i >= vq->split.vring.num)) { 855 BAD_RING(vq, "id %u out of range\n", i); 856 return NULL; 857 } 858 if (unlikely(!vq->split.desc_state[i].data)) { 859 BAD_RING(vq, "id %u is not a head!\n", i); 860 return NULL; 861 } 862 863 /* detach_buf_split clears data, so grab it now. */ 864 ret = vq->split.desc_state[i].data; 865 detach_buf_split(vq, i, ctx); 866 vq->last_used_idx++; 867 /* If we expect an interrupt for the next entry, tell host 868 * by writing event index and flush out the write before 869 * the read in the next get_buf call. */ 870 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) 871 virtio_store_mb(vq->weak_barriers, 872 &vring_used_event(&vq->split.vring), 873 cpu_to_virtio16(_vq->vdev, vq->last_used_idx)); 874 875 LAST_ADD_TIME_INVALID(vq); 876 877 END_USE(vq); 878 return ret; 879 } 880 881 static void virtqueue_disable_cb_split(struct virtqueue *_vq) 882 { 883 struct vring_virtqueue *vq = to_vvq(_vq); 884 885 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) { 886 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; 887 888 /* 889 * If device triggered an event already it won't trigger one again: 890 * no need to disable. 891 */ 892 if (vq->event_triggered) 893 return; 894 895 if (vq->event) 896 /* TODO: this is a hack. Figure out a cleaner value to write. */ 897 vring_used_event(&vq->split.vring) = 0x0; 898 else 899 vq->split.vring.avail->flags = 900 cpu_to_virtio16(_vq->vdev, 901 vq->split.avail_flags_shadow); 902 } 903 } 904 905 static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq) 906 { 907 struct vring_virtqueue *vq = to_vvq(_vq); 908 u16 last_used_idx; 909 910 START_USE(vq); 911 912 /* We optimistically turn back on interrupts, then check if there was 913 * more to do. */ 914 /* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to 915 * either clear the flags bit or point the event index at the next 916 * entry. Always do both to keep code simple. */ 917 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { 918 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; 919 if (!vq->event) 920 vq->split.vring.avail->flags = 921 cpu_to_virtio16(_vq->vdev, 922 vq->split.avail_flags_shadow); 923 } 924 vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev, 925 last_used_idx = vq->last_used_idx); 926 END_USE(vq); 927 return last_used_idx; 928 } 929 930 static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned int last_used_idx) 931 { 932 struct vring_virtqueue *vq = to_vvq(_vq); 933 934 return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev, 935 vq->split.vring.used->idx); 936 } 937 938 static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq) 939 { 940 struct vring_virtqueue *vq = to_vvq(_vq); 941 u16 bufs; 942 943 START_USE(vq); 944 945 /* We optimistically turn back on interrupts, then check if there was 946 * more to do. */ 947 /* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to 948 * either clear the flags bit or point the event index at the next 949 * entry. Always update the event index to keep code simple. */ 950 if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) { 951 vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT; 952 if (!vq->event) 953 vq->split.vring.avail->flags = 954 cpu_to_virtio16(_vq->vdev, 955 vq->split.avail_flags_shadow); 956 } 957 /* TODO: tune this threshold */ 958 bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4; 959 960 virtio_store_mb(vq->weak_barriers, 961 &vring_used_event(&vq->split.vring), 962 cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs)); 963 964 if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx) 965 - vq->last_used_idx) > bufs)) { 966 END_USE(vq); 967 return false; 968 } 969 970 END_USE(vq); 971 return true; 972 } 973 974 static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq) 975 { 976 struct vring_virtqueue *vq = to_vvq(_vq); 977 unsigned int i; 978 void *buf; 979 980 START_USE(vq); 981 982 for (i = 0; i < vq->split.vring.num; i++) { 983 if (!vq->split.desc_state[i].data) 984 continue; 985 /* detach_buf_split clears data, so grab it now. */ 986 buf = vq->split.desc_state[i].data; 987 detach_buf_split(vq, i, NULL); 988 vq->split.avail_idx_shadow--; 989 vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev, 990 vq->split.avail_idx_shadow); 991 END_USE(vq); 992 return buf; 993 } 994 /* That should have freed everything. */ 995 BUG_ON(vq->vq.num_free != vq->split.vring.num); 996 997 END_USE(vq); 998 return NULL; 999 } 1000 1001 static void virtqueue_vring_init_split(struct vring_virtqueue_split *vring_split, 1002 struct vring_virtqueue *vq) 1003 { 1004 struct virtio_device *vdev; 1005 1006 vdev = vq->vq.vdev; 1007 1008 vring_split->avail_flags_shadow = 0; 1009 vring_split->avail_idx_shadow = 0; 1010 1011 /* No callback? Tell other side not to bother us. */ 1012 if (!vq->vq.callback) { 1013 vring_split->avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT; 1014 if (!vq->event) 1015 vring_split->vring.avail->flags = cpu_to_virtio16(vdev, 1016 vring_split->avail_flags_shadow); 1017 } 1018 } 1019 1020 static void virtqueue_reinit_split(struct vring_virtqueue *vq) 1021 { 1022 int num; 1023 1024 num = vq->split.vring.num; 1025 1026 vq->split.vring.avail->flags = 0; 1027 vq->split.vring.avail->idx = 0; 1028 1029 /* reset avail event */ 1030 vq->split.vring.avail->ring[num] = 0; 1031 1032 vq->split.vring.used->flags = 0; 1033 vq->split.vring.used->idx = 0; 1034 1035 /* reset used event */ 1036 *(__virtio16 *)&(vq->split.vring.used->ring[num]) = 0; 1037 1038 virtqueue_init(vq, num); 1039 1040 virtqueue_vring_init_split(&vq->split, vq); 1041 } 1042 1043 static void virtqueue_vring_attach_split(struct vring_virtqueue *vq, 1044 struct vring_virtqueue_split *vring_split) 1045 { 1046 vq->split = *vring_split; 1047 1048 /* Put everything in free lists. */ 1049 vq->free_head = 0; 1050 } 1051 1052 static int vring_alloc_state_extra_split(struct vring_virtqueue_split *vring_split) 1053 { 1054 struct vring_desc_state_split *state; 1055 struct vring_desc_extra *extra; 1056 u32 num = vring_split->vring.num; 1057 1058 state = kmalloc_array(num, sizeof(struct vring_desc_state_split), GFP_KERNEL); 1059 if (!state) 1060 goto err_state; 1061 1062 extra = vring_alloc_desc_extra(num); 1063 if (!extra) 1064 goto err_extra; 1065 1066 memset(state, 0, num * sizeof(struct vring_desc_state_split)); 1067 1068 vring_split->desc_state = state; 1069 vring_split->desc_extra = extra; 1070 return 0; 1071 1072 err_extra: 1073 kfree(state); 1074 err_state: 1075 return -ENOMEM; 1076 } 1077 1078 static void vring_free_split(struct vring_virtqueue_split *vring_split, 1079 struct virtio_device *vdev, struct device *dma_dev) 1080 { 1081 vring_free_queue(vdev, vring_split->queue_size_in_bytes, 1082 vring_split->vring.desc, 1083 vring_split->queue_dma_addr, 1084 dma_dev); 1085 1086 kfree(vring_split->desc_state); 1087 kfree(vring_split->desc_extra); 1088 } 1089 1090 static int vring_alloc_queue_split(struct vring_virtqueue_split *vring_split, 1091 struct virtio_device *vdev, 1092 u32 num, 1093 unsigned int vring_align, 1094 bool may_reduce_num, 1095 struct device *dma_dev) 1096 { 1097 void *queue = NULL; 1098 dma_addr_t dma_addr; 1099 1100 /* We assume num is a power of 2. */ 1101 if (!is_power_of_2(num)) { 1102 dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num); 1103 return -EINVAL; 1104 } 1105 1106 /* TODO: allocate each queue chunk individually */ 1107 for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) { 1108 queue = vring_alloc_queue(vdev, vring_size(num, vring_align), 1109 &dma_addr, 1110 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 1111 dma_dev); 1112 if (queue) 1113 break; 1114 if (!may_reduce_num) 1115 return -ENOMEM; 1116 } 1117 1118 if (!num) 1119 return -ENOMEM; 1120 1121 if (!queue) { 1122 /* Try to get a single page. You are my only hope! */ 1123 queue = vring_alloc_queue(vdev, vring_size(num, vring_align), 1124 &dma_addr, GFP_KERNEL | __GFP_ZERO, 1125 dma_dev); 1126 } 1127 if (!queue) 1128 return -ENOMEM; 1129 1130 vring_init(&vring_split->vring, num, queue, vring_align); 1131 1132 vring_split->queue_dma_addr = dma_addr; 1133 vring_split->queue_size_in_bytes = vring_size(num, vring_align); 1134 1135 vring_split->vring_align = vring_align; 1136 vring_split->may_reduce_num = may_reduce_num; 1137 1138 return 0; 1139 } 1140 1141 static struct virtqueue *vring_create_virtqueue_split( 1142 unsigned int index, 1143 unsigned int num, 1144 unsigned int vring_align, 1145 struct virtio_device *vdev, 1146 bool weak_barriers, 1147 bool may_reduce_num, 1148 bool context, 1149 bool (*notify)(struct virtqueue *), 1150 void (*callback)(struct virtqueue *), 1151 const char *name, 1152 struct device *dma_dev) 1153 { 1154 struct vring_virtqueue_split vring_split = {}; 1155 struct virtqueue *vq; 1156 int err; 1157 1158 err = vring_alloc_queue_split(&vring_split, vdev, num, vring_align, 1159 may_reduce_num, dma_dev); 1160 if (err) 1161 return NULL; 1162 1163 vq = __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers, 1164 context, notify, callback, name, dma_dev); 1165 if (!vq) { 1166 vring_free_split(&vring_split, vdev, dma_dev); 1167 return NULL; 1168 } 1169 1170 to_vvq(vq)->we_own_ring = true; 1171 1172 return vq; 1173 } 1174 1175 static int virtqueue_resize_split(struct virtqueue *_vq, u32 num) 1176 { 1177 struct vring_virtqueue_split vring_split = {}; 1178 struct vring_virtqueue *vq = to_vvq(_vq); 1179 struct virtio_device *vdev = _vq->vdev; 1180 int err; 1181 1182 err = vring_alloc_queue_split(&vring_split, vdev, num, 1183 vq->split.vring_align, 1184 vq->split.may_reduce_num, 1185 vring_dma_dev(vq)); 1186 if (err) 1187 goto err; 1188 1189 err = vring_alloc_state_extra_split(&vring_split); 1190 if (err) 1191 goto err_state_extra; 1192 1193 vring_free(&vq->vq); 1194 1195 virtqueue_vring_init_split(&vring_split, vq); 1196 1197 virtqueue_init(vq, vring_split.vring.num); 1198 virtqueue_vring_attach_split(vq, &vring_split); 1199 1200 return 0; 1201 1202 err_state_extra: 1203 vring_free_split(&vring_split, vdev, vring_dma_dev(vq)); 1204 err: 1205 virtqueue_reinit_split(vq); 1206 return -ENOMEM; 1207 } 1208 1209 1210 /* 1211 * Packed ring specific functions - *_packed(). 1212 */ 1213 static bool packed_used_wrap_counter(u16 last_used_idx) 1214 { 1215 return !!(last_used_idx & (1 << VRING_PACKED_EVENT_F_WRAP_CTR)); 1216 } 1217 1218 static u16 packed_last_used(u16 last_used_idx) 1219 { 1220 return last_used_idx & ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR)); 1221 } 1222 1223 static void vring_unmap_extra_packed(const struct vring_virtqueue *vq, 1224 const struct vring_desc_extra *extra) 1225 { 1226 u16 flags; 1227 1228 flags = extra->flags; 1229 1230 if (flags & VRING_DESC_F_INDIRECT) { 1231 if (!vq->use_dma_api) 1232 return; 1233 1234 dma_unmap_single(vring_dma_dev(vq), 1235 extra->addr, extra->len, 1236 (flags & VRING_DESC_F_WRITE) ? 1237 DMA_FROM_DEVICE : DMA_TO_DEVICE); 1238 } else { 1239 if (!vq->do_unmap) 1240 return; 1241 1242 dma_unmap_page(vring_dma_dev(vq), 1243 extra->addr, extra->len, 1244 (flags & VRING_DESC_F_WRITE) ? 1245 DMA_FROM_DEVICE : DMA_TO_DEVICE); 1246 } 1247 } 1248 1249 static void vring_unmap_desc_packed(const struct vring_virtqueue *vq, 1250 const struct vring_packed_desc *desc) 1251 { 1252 u16 flags; 1253 1254 if (!vq->do_unmap) 1255 return; 1256 1257 flags = le16_to_cpu(desc->flags); 1258 1259 dma_unmap_page(vring_dma_dev(vq), 1260 le64_to_cpu(desc->addr), 1261 le32_to_cpu(desc->len), 1262 (flags & VRING_DESC_F_WRITE) ? 1263 DMA_FROM_DEVICE : DMA_TO_DEVICE); 1264 } 1265 1266 static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg, 1267 gfp_t gfp) 1268 { 1269 struct vring_packed_desc *desc; 1270 1271 /* 1272 * We require lowmem mappings for the descriptors because 1273 * otherwise virt_to_phys will give us bogus addresses in the 1274 * virtqueue. 1275 */ 1276 gfp &= ~__GFP_HIGHMEM; 1277 1278 desc = kmalloc_array(total_sg, sizeof(struct vring_packed_desc), gfp); 1279 1280 return desc; 1281 } 1282 1283 static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq, 1284 struct scatterlist *sgs[], 1285 unsigned int total_sg, 1286 unsigned int out_sgs, 1287 unsigned int in_sgs, 1288 void *data, 1289 gfp_t gfp) 1290 { 1291 struct vring_packed_desc *desc; 1292 struct scatterlist *sg; 1293 unsigned int i, n, err_idx; 1294 u16 head, id; 1295 dma_addr_t addr; 1296 1297 head = vq->packed.next_avail_idx; 1298 desc = alloc_indirect_packed(total_sg, gfp); 1299 if (!desc) 1300 return -ENOMEM; 1301 1302 if (unlikely(vq->vq.num_free < 1)) { 1303 pr_debug("Can't add buf len 1 - avail = 0\n"); 1304 kfree(desc); 1305 END_USE(vq); 1306 return -ENOSPC; 1307 } 1308 1309 i = 0; 1310 id = vq->free_head; 1311 BUG_ON(id == vq->packed.vring.num); 1312 1313 for (n = 0; n < out_sgs + in_sgs; n++) { 1314 for (sg = sgs[n]; sg; sg = sg_next(sg)) { 1315 if (vring_map_one_sg(vq, sg, n < out_sgs ? 1316 DMA_TO_DEVICE : DMA_FROM_DEVICE, &addr)) 1317 goto unmap_release; 1318 1319 desc[i].flags = cpu_to_le16(n < out_sgs ? 1320 0 : VRING_DESC_F_WRITE); 1321 desc[i].addr = cpu_to_le64(addr); 1322 desc[i].len = cpu_to_le32(sg->length); 1323 i++; 1324 } 1325 } 1326 1327 /* Now that the indirect table is filled in, map it. */ 1328 addr = vring_map_single(vq, desc, 1329 total_sg * sizeof(struct vring_packed_desc), 1330 DMA_TO_DEVICE); 1331 if (vring_mapping_error(vq, addr)) { 1332 if (vq->premapped) 1333 goto free_desc; 1334 1335 goto unmap_release; 1336 } 1337 1338 vq->packed.vring.desc[head].addr = cpu_to_le64(addr); 1339 vq->packed.vring.desc[head].len = cpu_to_le32(total_sg * 1340 sizeof(struct vring_packed_desc)); 1341 vq->packed.vring.desc[head].id = cpu_to_le16(id); 1342 1343 if (vq->do_unmap) { 1344 vq->packed.desc_extra[id].addr = addr; 1345 vq->packed.desc_extra[id].len = total_sg * 1346 sizeof(struct vring_packed_desc); 1347 vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT | 1348 vq->packed.avail_used_flags; 1349 } 1350 1351 /* 1352 * A driver MUST NOT make the first descriptor in the list 1353 * available before all subsequent descriptors comprising 1354 * the list are made available. 1355 */ 1356 virtio_wmb(vq->weak_barriers); 1357 vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT | 1358 vq->packed.avail_used_flags); 1359 1360 /* We're using some buffers from the free list. */ 1361 vq->vq.num_free -= 1; 1362 1363 /* Update free pointer */ 1364 n = head + 1; 1365 if (n >= vq->packed.vring.num) { 1366 n = 0; 1367 vq->packed.avail_wrap_counter ^= 1; 1368 vq->packed.avail_used_flags ^= 1369 1 << VRING_PACKED_DESC_F_AVAIL | 1370 1 << VRING_PACKED_DESC_F_USED; 1371 } 1372 vq->packed.next_avail_idx = n; 1373 vq->free_head = vq->packed.desc_extra[id].next; 1374 1375 /* Store token and indirect buffer state. */ 1376 vq->packed.desc_state[id].num = 1; 1377 vq->packed.desc_state[id].data = data; 1378 vq->packed.desc_state[id].indir_desc = desc; 1379 vq->packed.desc_state[id].last = id; 1380 1381 vq->num_added += 1; 1382 1383 pr_debug("Added buffer head %i to %p\n", head, vq); 1384 END_USE(vq); 1385 1386 return 0; 1387 1388 unmap_release: 1389 err_idx = i; 1390 1391 for (i = 0; i < err_idx; i++) 1392 vring_unmap_desc_packed(vq, &desc[i]); 1393 1394 free_desc: 1395 kfree(desc); 1396 1397 END_USE(vq); 1398 return -ENOMEM; 1399 } 1400 1401 static inline int virtqueue_add_packed(struct virtqueue *_vq, 1402 struct scatterlist *sgs[], 1403 unsigned int total_sg, 1404 unsigned int out_sgs, 1405 unsigned int in_sgs, 1406 void *data, 1407 void *ctx, 1408 gfp_t gfp) 1409 { 1410 struct vring_virtqueue *vq = to_vvq(_vq); 1411 struct vring_packed_desc *desc; 1412 struct scatterlist *sg; 1413 unsigned int i, n, c, descs_used, err_idx; 1414 __le16 head_flags, flags; 1415 u16 head, id, prev, curr, avail_used_flags; 1416 int err; 1417 1418 START_USE(vq); 1419 1420 BUG_ON(data == NULL); 1421 BUG_ON(ctx && vq->indirect); 1422 1423 if (unlikely(vq->broken)) { 1424 END_USE(vq); 1425 return -EIO; 1426 } 1427 1428 LAST_ADD_TIME_UPDATE(vq); 1429 1430 BUG_ON(total_sg == 0); 1431 1432 if (virtqueue_use_indirect(vq, total_sg)) { 1433 err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs, 1434 in_sgs, data, gfp); 1435 if (err != -ENOMEM) { 1436 END_USE(vq); 1437 return err; 1438 } 1439 1440 /* fall back on direct */ 1441 } 1442 1443 head = vq->packed.next_avail_idx; 1444 avail_used_flags = vq->packed.avail_used_flags; 1445 1446 WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect); 1447 1448 desc = vq->packed.vring.desc; 1449 i = head; 1450 descs_used = total_sg; 1451 1452 if (unlikely(vq->vq.num_free < descs_used)) { 1453 pr_debug("Can't add buf len %i - avail = %i\n", 1454 descs_used, vq->vq.num_free); 1455 END_USE(vq); 1456 return -ENOSPC; 1457 } 1458 1459 id = vq->free_head; 1460 BUG_ON(id == vq->packed.vring.num); 1461 1462 curr = id; 1463 c = 0; 1464 for (n = 0; n < out_sgs + in_sgs; n++) { 1465 for (sg = sgs[n]; sg; sg = sg_next(sg)) { 1466 dma_addr_t addr; 1467 1468 if (vring_map_one_sg(vq, sg, n < out_sgs ? 1469 DMA_TO_DEVICE : DMA_FROM_DEVICE, &addr)) 1470 goto unmap_release; 1471 1472 flags = cpu_to_le16(vq->packed.avail_used_flags | 1473 (++c == total_sg ? 0 : VRING_DESC_F_NEXT) | 1474 (n < out_sgs ? 0 : VRING_DESC_F_WRITE)); 1475 if (i == head) 1476 head_flags = flags; 1477 else 1478 desc[i].flags = flags; 1479 1480 desc[i].addr = cpu_to_le64(addr); 1481 desc[i].len = cpu_to_le32(sg->length); 1482 desc[i].id = cpu_to_le16(id); 1483 1484 if (unlikely(vq->do_unmap)) { 1485 vq->packed.desc_extra[curr].addr = addr; 1486 vq->packed.desc_extra[curr].len = sg->length; 1487 vq->packed.desc_extra[curr].flags = 1488 le16_to_cpu(flags); 1489 } 1490 prev = curr; 1491 curr = vq->packed.desc_extra[curr].next; 1492 1493 if ((unlikely(++i >= vq->packed.vring.num))) { 1494 i = 0; 1495 vq->packed.avail_used_flags ^= 1496 1 << VRING_PACKED_DESC_F_AVAIL | 1497 1 << VRING_PACKED_DESC_F_USED; 1498 } 1499 } 1500 } 1501 1502 if (i <= head) 1503 vq->packed.avail_wrap_counter ^= 1; 1504 1505 /* We're using some buffers from the free list. */ 1506 vq->vq.num_free -= descs_used; 1507 1508 /* Update free pointer */ 1509 vq->packed.next_avail_idx = i; 1510 vq->free_head = curr; 1511 1512 /* Store token. */ 1513 vq->packed.desc_state[id].num = descs_used; 1514 vq->packed.desc_state[id].data = data; 1515 vq->packed.desc_state[id].indir_desc = ctx; 1516 vq->packed.desc_state[id].last = prev; 1517 1518 /* 1519 * A driver MUST NOT make the first descriptor in the list 1520 * available before all subsequent descriptors comprising 1521 * the list are made available. 1522 */ 1523 virtio_wmb(vq->weak_barriers); 1524 vq->packed.vring.desc[head].flags = head_flags; 1525 vq->num_added += descs_used; 1526 1527 pr_debug("Added buffer head %i to %p\n", head, vq); 1528 END_USE(vq); 1529 1530 return 0; 1531 1532 unmap_release: 1533 err_idx = i; 1534 i = head; 1535 curr = vq->free_head; 1536 1537 vq->packed.avail_used_flags = avail_used_flags; 1538 1539 for (n = 0; n < total_sg; n++) { 1540 if (i == err_idx) 1541 break; 1542 vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]); 1543 curr = vq->packed.desc_extra[curr].next; 1544 i++; 1545 if (i >= vq->packed.vring.num) 1546 i = 0; 1547 } 1548 1549 END_USE(vq); 1550 return -EIO; 1551 } 1552 1553 static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq) 1554 { 1555 struct vring_virtqueue *vq = to_vvq(_vq); 1556 u16 new, old, off_wrap, flags, wrap_counter, event_idx; 1557 bool needs_kick; 1558 union { 1559 struct { 1560 __le16 off_wrap; 1561 __le16 flags; 1562 }; 1563 u32 u32; 1564 } snapshot; 1565 1566 START_USE(vq); 1567 1568 /* 1569 * We need to expose the new flags value before checking notification 1570 * suppressions. 1571 */ 1572 virtio_mb(vq->weak_barriers); 1573 1574 old = vq->packed.next_avail_idx - vq->num_added; 1575 new = vq->packed.next_avail_idx; 1576 vq->num_added = 0; 1577 1578 snapshot.u32 = *(u32 *)vq->packed.vring.device; 1579 flags = le16_to_cpu(snapshot.flags); 1580 1581 LAST_ADD_TIME_CHECK(vq); 1582 LAST_ADD_TIME_INVALID(vq); 1583 1584 if (flags != VRING_PACKED_EVENT_FLAG_DESC) { 1585 needs_kick = (flags != VRING_PACKED_EVENT_FLAG_DISABLE); 1586 goto out; 1587 } 1588 1589 off_wrap = le16_to_cpu(snapshot.off_wrap); 1590 1591 wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR; 1592 event_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR); 1593 if (wrap_counter != vq->packed.avail_wrap_counter) 1594 event_idx -= vq->packed.vring.num; 1595 1596 needs_kick = vring_need_event(event_idx, new, old); 1597 out: 1598 END_USE(vq); 1599 return needs_kick; 1600 } 1601 1602 static void detach_buf_packed(struct vring_virtqueue *vq, 1603 unsigned int id, void **ctx) 1604 { 1605 struct vring_desc_state_packed *state = NULL; 1606 struct vring_packed_desc *desc; 1607 unsigned int i, curr; 1608 1609 state = &vq->packed.desc_state[id]; 1610 1611 /* Clear data ptr. */ 1612 state->data = NULL; 1613 1614 vq->packed.desc_extra[state->last].next = vq->free_head; 1615 vq->free_head = id; 1616 vq->vq.num_free += state->num; 1617 1618 if (unlikely(vq->do_unmap)) { 1619 curr = id; 1620 for (i = 0; i < state->num; i++) { 1621 vring_unmap_extra_packed(vq, 1622 &vq->packed.desc_extra[curr]); 1623 curr = vq->packed.desc_extra[curr].next; 1624 } 1625 } 1626 1627 if (vq->indirect) { 1628 u32 len; 1629 1630 /* Free the indirect table, if any, now that it's unmapped. */ 1631 desc = state->indir_desc; 1632 if (!desc) 1633 return; 1634 1635 if (vq->do_unmap) { 1636 len = vq->packed.desc_extra[id].len; 1637 for (i = 0; i < len / sizeof(struct vring_packed_desc); 1638 i++) 1639 vring_unmap_desc_packed(vq, &desc[i]); 1640 } 1641 kfree(desc); 1642 state->indir_desc = NULL; 1643 } else if (ctx) { 1644 *ctx = state->indir_desc; 1645 } 1646 } 1647 1648 static inline bool is_used_desc_packed(const struct vring_virtqueue *vq, 1649 u16 idx, bool used_wrap_counter) 1650 { 1651 bool avail, used; 1652 u16 flags; 1653 1654 flags = le16_to_cpu(vq->packed.vring.desc[idx].flags); 1655 avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL)); 1656 used = !!(flags & (1 << VRING_PACKED_DESC_F_USED)); 1657 1658 return avail == used && used == used_wrap_counter; 1659 } 1660 1661 static bool more_used_packed(const struct vring_virtqueue *vq) 1662 { 1663 u16 last_used; 1664 u16 last_used_idx; 1665 bool used_wrap_counter; 1666 1667 last_used_idx = READ_ONCE(vq->last_used_idx); 1668 last_used = packed_last_used(last_used_idx); 1669 used_wrap_counter = packed_used_wrap_counter(last_used_idx); 1670 return is_used_desc_packed(vq, last_used, used_wrap_counter); 1671 } 1672 1673 static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq, 1674 unsigned int *len, 1675 void **ctx) 1676 { 1677 struct vring_virtqueue *vq = to_vvq(_vq); 1678 u16 last_used, id, last_used_idx; 1679 bool used_wrap_counter; 1680 void *ret; 1681 1682 START_USE(vq); 1683 1684 if (unlikely(vq->broken)) { 1685 END_USE(vq); 1686 return NULL; 1687 } 1688 1689 if (!more_used_packed(vq)) { 1690 pr_debug("No more buffers in queue\n"); 1691 END_USE(vq); 1692 return NULL; 1693 } 1694 1695 /* Only get used elements after they have been exposed by host. */ 1696 virtio_rmb(vq->weak_barriers); 1697 1698 last_used_idx = READ_ONCE(vq->last_used_idx); 1699 used_wrap_counter = packed_used_wrap_counter(last_used_idx); 1700 last_used = packed_last_used(last_used_idx); 1701 id = le16_to_cpu(vq->packed.vring.desc[last_used].id); 1702 *len = le32_to_cpu(vq->packed.vring.desc[last_used].len); 1703 1704 if (unlikely(id >= vq->packed.vring.num)) { 1705 BAD_RING(vq, "id %u out of range\n", id); 1706 return NULL; 1707 } 1708 if (unlikely(!vq->packed.desc_state[id].data)) { 1709 BAD_RING(vq, "id %u is not a head!\n", id); 1710 return NULL; 1711 } 1712 1713 /* detach_buf_packed clears data, so grab it now. */ 1714 ret = vq->packed.desc_state[id].data; 1715 detach_buf_packed(vq, id, ctx); 1716 1717 last_used += vq->packed.desc_state[id].num; 1718 if (unlikely(last_used >= vq->packed.vring.num)) { 1719 last_used -= vq->packed.vring.num; 1720 used_wrap_counter ^= 1; 1721 } 1722 1723 last_used = (last_used | (used_wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR)); 1724 WRITE_ONCE(vq->last_used_idx, last_used); 1725 1726 /* 1727 * If we expect an interrupt for the next entry, tell host 1728 * by writing event index and flush out the write before 1729 * the read in the next get_buf call. 1730 */ 1731 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC) 1732 virtio_store_mb(vq->weak_barriers, 1733 &vq->packed.vring.driver->off_wrap, 1734 cpu_to_le16(vq->last_used_idx)); 1735 1736 LAST_ADD_TIME_INVALID(vq); 1737 1738 END_USE(vq); 1739 return ret; 1740 } 1741 1742 static void virtqueue_disable_cb_packed(struct virtqueue *_vq) 1743 { 1744 struct vring_virtqueue *vq = to_vvq(_vq); 1745 1746 if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) { 1747 vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE; 1748 1749 /* 1750 * If device triggered an event already it won't trigger one again: 1751 * no need to disable. 1752 */ 1753 if (vq->event_triggered) 1754 return; 1755 1756 vq->packed.vring.driver->flags = 1757 cpu_to_le16(vq->packed.event_flags_shadow); 1758 } 1759 } 1760 1761 static unsigned int virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq) 1762 { 1763 struct vring_virtqueue *vq = to_vvq(_vq); 1764 1765 START_USE(vq); 1766 1767 /* 1768 * We optimistically turn back on interrupts, then check if there was 1769 * more to do. 1770 */ 1771 1772 if (vq->event) { 1773 vq->packed.vring.driver->off_wrap = 1774 cpu_to_le16(vq->last_used_idx); 1775 /* 1776 * We need to update event offset and event wrap 1777 * counter first before updating event flags. 1778 */ 1779 virtio_wmb(vq->weak_barriers); 1780 } 1781 1782 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) { 1783 vq->packed.event_flags_shadow = vq->event ? 1784 VRING_PACKED_EVENT_FLAG_DESC : 1785 VRING_PACKED_EVENT_FLAG_ENABLE; 1786 vq->packed.vring.driver->flags = 1787 cpu_to_le16(vq->packed.event_flags_shadow); 1788 } 1789 1790 END_USE(vq); 1791 return vq->last_used_idx; 1792 } 1793 1794 static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap) 1795 { 1796 struct vring_virtqueue *vq = to_vvq(_vq); 1797 bool wrap_counter; 1798 u16 used_idx; 1799 1800 wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR; 1801 used_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR); 1802 1803 return is_used_desc_packed(vq, used_idx, wrap_counter); 1804 } 1805 1806 static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq) 1807 { 1808 struct vring_virtqueue *vq = to_vvq(_vq); 1809 u16 used_idx, wrap_counter, last_used_idx; 1810 u16 bufs; 1811 1812 START_USE(vq); 1813 1814 /* 1815 * We optimistically turn back on interrupts, then check if there was 1816 * more to do. 1817 */ 1818 1819 if (vq->event) { 1820 /* TODO: tune this threshold */ 1821 bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4; 1822 last_used_idx = READ_ONCE(vq->last_used_idx); 1823 wrap_counter = packed_used_wrap_counter(last_used_idx); 1824 1825 used_idx = packed_last_used(last_used_idx) + bufs; 1826 if (used_idx >= vq->packed.vring.num) { 1827 used_idx -= vq->packed.vring.num; 1828 wrap_counter ^= 1; 1829 } 1830 1831 vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx | 1832 (wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR)); 1833 1834 /* 1835 * We need to update event offset and event wrap 1836 * counter first before updating event flags. 1837 */ 1838 virtio_wmb(vq->weak_barriers); 1839 } 1840 1841 if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) { 1842 vq->packed.event_flags_shadow = vq->event ? 1843 VRING_PACKED_EVENT_FLAG_DESC : 1844 VRING_PACKED_EVENT_FLAG_ENABLE; 1845 vq->packed.vring.driver->flags = 1846 cpu_to_le16(vq->packed.event_flags_shadow); 1847 } 1848 1849 /* 1850 * We need to update event suppression structure first 1851 * before re-checking for more used buffers. 1852 */ 1853 virtio_mb(vq->weak_barriers); 1854 1855 last_used_idx = READ_ONCE(vq->last_used_idx); 1856 wrap_counter = packed_used_wrap_counter(last_used_idx); 1857 used_idx = packed_last_used(last_used_idx); 1858 if (is_used_desc_packed(vq, used_idx, wrap_counter)) { 1859 END_USE(vq); 1860 return false; 1861 } 1862 1863 END_USE(vq); 1864 return true; 1865 } 1866 1867 static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq) 1868 { 1869 struct vring_virtqueue *vq = to_vvq(_vq); 1870 unsigned int i; 1871 void *buf; 1872 1873 START_USE(vq); 1874 1875 for (i = 0; i < vq->packed.vring.num; i++) { 1876 if (!vq->packed.desc_state[i].data) 1877 continue; 1878 /* detach_buf clears data, so grab it now. */ 1879 buf = vq->packed.desc_state[i].data; 1880 detach_buf_packed(vq, i, NULL); 1881 END_USE(vq); 1882 return buf; 1883 } 1884 /* That should have freed everything. */ 1885 BUG_ON(vq->vq.num_free != vq->packed.vring.num); 1886 1887 END_USE(vq); 1888 return NULL; 1889 } 1890 1891 static struct vring_desc_extra *vring_alloc_desc_extra(unsigned int num) 1892 { 1893 struct vring_desc_extra *desc_extra; 1894 unsigned int i; 1895 1896 desc_extra = kmalloc_array(num, sizeof(struct vring_desc_extra), 1897 GFP_KERNEL); 1898 if (!desc_extra) 1899 return NULL; 1900 1901 memset(desc_extra, 0, num * sizeof(struct vring_desc_extra)); 1902 1903 for (i = 0; i < num - 1; i++) 1904 desc_extra[i].next = i + 1; 1905 1906 return desc_extra; 1907 } 1908 1909 static void vring_free_packed(struct vring_virtqueue_packed *vring_packed, 1910 struct virtio_device *vdev, 1911 struct device *dma_dev) 1912 { 1913 if (vring_packed->vring.desc) 1914 vring_free_queue(vdev, vring_packed->ring_size_in_bytes, 1915 vring_packed->vring.desc, 1916 vring_packed->ring_dma_addr, 1917 dma_dev); 1918 1919 if (vring_packed->vring.driver) 1920 vring_free_queue(vdev, vring_packed->event_size_in_bytes, 1921 vring_packed->vring.driver, 1922 vring_packed->driver_event_dma_addr, 1923 dma_dev); 1924 1925 if (vring_packed->vring.device) 1926 vring_free_queue(vdev, vring_packed->event_size_in_bytes, 1927 vring_packed->vring.device, 1928 vring_packed->device_event_dma_addr, 1929 dma_dev); 1930 1931 kfree(vring_packed->desc_state); 1932 kfree(vring_packed->desc_extra); 1933 } 1934 1935 static int vring_alloc_queue_packed(struct vring_virtqueue_packed *vring_packed, 1936 struct virtio_device *vdev, 1937 u32 num, struct device *dma_dev) 1938 { 1939 struct vring_packed_desc *ring; 1940 struct vring_packed_desc_event *driver, *device; 1941 dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr; 1942 size_t ring_size_in_bytes, event_size_in_bytes; 1943 1944 ring_size_in_bytes = num * sizeof(struct vring_packed_desc); 1945 1946 ring = vring_alloc_queue(vdev, ring_size_in_bytes, 1947 &ring_dma_addr, 1948 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 1949 dma_dev); 1950 if (!ring) 1951 goto err; 1952 1953 vring_packed->vring.desc = ring; 1954 vring_packed->ring_dma_addr = ring_dma_addr; 1955 vring_packed->ring_size_in_bytes = ring_size_in_bytes; 1956 1957 event_size_in_bytes = sizeof(struct vring_packed_desc_event); 1958 1959 driver = vring_alloc_queue(vdev, event_size_in_bytes, 1960 &driver_event_dma_addr, 1961 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 1962 dma_dev); 1963 if (!driver) 1964 goto err; 1965 1966 vring_packed->vring.driver = driver; 1967 vring_packed->event_size_in_bytes = event_size_in_bytes; 1968 vring_packed->driver_event_dma_addr = driver_event_dma_addr; 1969 1970 device = vring_alloc_queue(vdev, event_size_in_bytes, 1971 &device_event_dma_addr, 1972 GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, 1973 dma_dev); 1974 if (!device) 1975 goto err; 1976 1977 vring_packed->vring.device = device; 1978 vring_packed->device_event_dma_addr = device_event_dma_addr; 1979 1980 vring_packed->vring.num = num; 1981 1982 return 0; 1983 1984 err: 1985 vring_free_packed(vring_packed, vdev, dma_dev); 1986 return -ENOMEM; 1987 } 1988 1989 static int vring_alloc_state_extra_packed(struct vring_virtqueue_packed *vring_packed) 1990 { 1991 struct vring_desc_state_packed *state; 1992 struct vring_desc_extra *extra; 1993 u32 num = vring_packed->vring.num; 1994 1995 state = kmalloc_array(num, sizeof(struct vring_desc_state_packed), GFP_KERNEL); 1996 if (!state) 1997 goto err_desc_state; 1998 1999 memset(state, 0, num * sizeof(struct vring_desc_state_packed)); 2000 2001 extra = vring_alloc_desc_extra(num); 2002 if (!extra) 2003 goto err_desc_extra; 2004 2005 vring_packed->desc_state = state; 2006 vring_packed->desc_extra = extra; 2007 2008 return 0; 2009 2010 err_desc_extra: 2011 kfree(state); 2012 err_desc_state: 2013 return -ENOMEM; 2014 } 2015 2016 static void virtqueue_vring_init_packed(struct vring_virtqueue_packed *vring_packed, 2017 bool callback) 2018 { 2019 vring_packed->next_avail_idx = 0; 2020 vring_packed->avail_wrap_counter = 1; 2021 vring_packed->event_flags_shadow = 0; 2022 vring_packed->avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL; 2023 2024 /* No callback? Tell other side not to bother us. */ 2025 if (!callback) { 2026 vring_packed->event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE; 2027 vring_packed->vring.driver->flags = 2028 cpu_to_le16(vring_packed->event_flags_shadow); 2029 } 2030 } 2031 2032 static void virtqueue_vring_attach_packed(struct vring_virtqueue *vq, 2033 struct vring_virtqueue_packed *vring_packed) 2034 { 2035 vq->packed = *vring_packed; 2036 2037 /* Put everything in free lists. */ 2038 vq->free_head = 0; 2039 } 2040 2041 static void virtqueue_reinit_packed(struct vring_virtqueue *vq) 2042 { 2043 memset(vq->packed.vring.device, 0, vq->packed.event_size_in_bytes); 2044 memset(vq->packed.vring.driver, 0, vq->packed.event_size_in_bytes); 2045 2046 /* we need to reset the desc.flags. For more, see is_used_desc_packed() */ 2047 memset(vq->packed.vring.desc, 0, vq->packed.ring_size_in_bytes); 2048 2049 virtqueue_init(vq, vq->packed.vring.num); 2050 virtqueue_vring_init_packed(&vq->packed, !!vq->vq.callback); 2051 } 2052 2053 static struct virtqueue *vring_create_virtqueue_packed( 2054 unsigned int index, 2055 unsigned int num, 2056 unsigned int vring_align, 2057 struct virtio_device *vdev, 2058 bool weak_barriers, 2059 bool may_reduce_num, 2060 bool context, 2061 bool (*notify)(struct virtqueue *), 2062 void (*callback)(struct virtqueue *), 2063 const char *name, 2064 struct device *dma_dev) 2065 { 2066 struct vring_virtqueue_packed vring_packed = {}; 2067 struct vring_virtqueue *vq; 2068 int err; 2069 2070 if (vring_alloc_queue_packed(&vring_packed, vdev, num, dma_dev)) 2071 goto err_ring; 2072 2073 vq = kmalloc(sizeof(*vq), GFP_KERNEL); 2074 if (!vq) 2075 goto err_vq; 2076 2077 vq->vq.callback = callback; 2078 vq->vq.vdev = vdev; 2079 vq->vq.name = name; 2080 vq->vq.index = index; 2081 vq->vq.reset = false; 2082 vq->we_own_ring = true; 2083 vq->notify = notify; 2084 vq->weak_barriers = weak_barriers; 2085 #ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION 2086 vq->broken = true; 2087 #else 2088 vq->broken = false; 2089 #endif 2090 vq->packed_ring = true; 2091 vq->dma_dev = dma_dev; 2092 vq->use_dma_api = vring_use_dma_api(vdev); 2093 vq->premapped = false; 2094 vq->do_unmap = vq->use_dma_api; 2095 2096 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) && 2097 !context; 2098 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); 2099 2100 if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM)) 2101 vq->weak_barriers = false; 2102 2103 err = vring_alloc_state_extra_packed(&vring_packed); 2104 if (err) 2105 goto err_state_extra; 2106 2107 virtqueue_vring_init_packed(&vring_packed, !!callback); 2108 2109 virtqueue_init(vq, num); 2110 virtqueue_vring_attach_packed(vq, &vring_packed); 2111 2112 spin_lock(&vdev->vqs_list_lock); 2113 list_add_tail(&vq->vq.list, &vdev->vqs); 2114 spin_unlock(&vdev->vqs_list_lock); 2115 return &vq->vq; 2116 2117 err_state_extra: 2118 kfree(vq); 2119 err_vq: 2120 vring_free_packed(&vring_packed, vdev, dma_dev); 2121 err_ring: 2122 return NULL; 2123 } 2124 2125 static int virtqueue_resize_packed(struct virtqueue *_vq, u32 num) 2126 { 2127 struct vring_virtqueue_packed vring_packed = {}; 2128 struct vring_virtqueue *vq = to_vvq(_vq); 2129 struct virtio_device *vdev = _vq->vdev; 2130 int err; 2131 2132 if (vring_alloc_queue_packed(&vring_packed, vdev, num, vring_dma_dev(vq))) 2133 goto err_ring; 2134 2135 err = vring_alloc_state_extra_packed(&vring_packed); 2136 if (err) 2137 goto err_state_extra; 2138 2139 vring_free(&vq->vq); 2140 2141 virtqueue_vring_init_packed(&vring_packed, !!vq->vq.callback); 2142 2143 virtqueue_init(vq, vring_packed.vring.num); 2144 virtqueue_vring_attach_packed(vq, &vring_packed); 2145 2146 return 0; 2147 2148 err_state_extra: 2149 vring_free_packed(&vring_packed, vdev, vring_dma_dev(vq)); 2150 err_ring: 2151 virtqueue_reinit_packed(vq); 2152 return -ENOMEM; 2153 } 2154 2155 static int virtqueue_disable_and_recycle(struct virtqueue *_vq, 2156 void (*recycle)(struct virtqueue *vq, void *buf)) 2157 { 2158 struct vring_virtqueue *vq = to_vvq(_vq); 2159 struct virtio_device *vdev = vq->vq.vdev; 2160 void *buf; 2161 int err; 2162 2163 if (!vq->we_own_ring) 2164 return -EPERM; 2165 2166 if (!vdev->config->disable_vq_and_reset) 2167 return -ENOENT; 2168 2169 if (!vdev->config->enable_vq_after_reset) 2170 return -ENOENT; 2171 2172 err = vdev->config->disable_vq_and_reset(_vq); 2173 if (err) 2174 return err; 2175 2176 while ((buf = virtqueue_detach_unused_buf(_vq)) != NULL) 2177 recycle(_vq, buf); 2178 2179 return 0; 2180 } 2181 2182 static int virtqueue_enable_after_reset(struct virtqueue *_vq) 2183 { 2184 struct vring_virtqueue *vq = to_vvq(_vq); 2185 struct virtio_device *vdev = vq->vq.vdev; 2186 2187 if (vdev->config->enable_vq_after_reset(_vq)) 2188 return -EBUSY; 2189 2190 return 0; 2191 } 2192 2193 /* 2194 * Generic functions and exported symbols. 2195 */ 2196 2197 static inline int virtqueue_add(struct virtqueue *_vq, 2198 struct scatterlist *sgs[], 2199 unsigned int total_sg, 2200 unsigned int out_sgs, 2201 unsigned int in_sgs, 2202 void *data, 2203 void *ctx, 2204 gfp_t gfp) 2205 { 2206 struct vring_virtqueue *vq = to_vvq(_vq); 2207 2208 return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg, 2209 out_sgs, in_sgs, data, ctx, gfp) : 2210 virtqueue_add_split(_vq, sgs, total_sg, 2211 out_sgs, in_sgs, data, ctx, gfp); 2212 } 2213 2214 /** 2215 * virtqueue_add_sgs - expose buffers to other end 2216 * @_vq: the struct virtqueue we're talking about. 2217 * @sgs: array of terminated scatterlists. 2218 * @out_sgs: the number of scatterlists readable by other side 2219 * @in_sgs: the number of scatterlists which are writable (after readable ones) 2220 * @data: the token identifying the buffer. 2221 * @gfp: how to do memory allocations (if necessary). 2222 * 2223 * Caller must ensure we don't call this with other virtqueue operations 2224 * at the same time (except where noted). 2225 * 2226 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO). 2227 */ 2228 int virtqueue_add_sgs(struct virtqueue *_vq, 2229 struct scatterlist *sgs[], 2230 unsigned int out_sgs, 2231 unsigned int in_sgs, 2232 void *data, 2233 gfp_t gfp) 2234 { 2235 unsigned int i, total_sg = 0; 2236 2237 /* Count them first. */ 2238 for (i = 0; i < out_sgs + in_sgs; i++) { 2239 struct scatterlist *sg; 2240 2241 for (sg = sgs[i]; sg; sg = sg_next(sg)) 2242 total_sg++; 2243 } 2244 return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs, 2245 data, NULL, gfp); 2246 } 2247 EXPORT_SYMBOL_GPL(virtqueue_add_sgs); 2248 2249 /** 2250 * virtqueue_add_outbuf - expose output buffers to other end 2251 * @vq: the struct virtqueue we're talking about. 2252 * @sg: scatterlist (must be well-formed and terminated!) 2253 * @num: the number of entries in @sg readable by other side 2254 * @data: the token identifying the buffer. 2255 * @gfp: how to do memory allocations (if necessary). 2256 * 2257 * Caller must ensure we don't call this with other virtqueue operations 2258 * at the same time (except where noted). 2259 * 2260 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO). 2261 */ 2262 int virtqueue_add_outbuf(struct virtqueue *vq, 2263 struct scatterlist *sg, unsigned int num, 2264 void *data, 2265 gfp_t gfp) 2266 { 2267 return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, gfp); 2268 } 2269 EXPORT_SYMBOL_GPL(virtqueue_add_outbuf); 2270 2271 /** 2272 * virtqueue_add_inbuf - expose input buffers to other end 2273 * @vq: the struct virtqueue we're talking about. 2274 * @sg: scatterlist (must be well-formed and terminated!) 2275 * @num: the number of entries in @sg writable by other side 2276 * @data: the token identifying the buffer. 2277 * @gfp: how to do memory allocations (if necessary). 2278 * 2279 * Caller must ensure we don't call this with other virtqueue operations 2280 * at the same time (except where noted). 2281 * 2282 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO). 2283 */ 2284 int virtqueue_add_inbuf(struct virtqueue *vq, 2285 struct scatterlist *sg, unsigned int num, 2286 void *data, 2287 gfp_t gfp) 2288 { 2289 return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, gfp); 2290 } 2291 EXPORT_SYMBOL_GPL(virtqueue_add_inbuf); 2292 2293 /** 2294 * virtqueue_add_inbuf_ctx - expose input buffers to other end 2295 * @vq: the struct virtqueue we're talking about. 2296 * @sg: scatterlist (must be well-formed and terminated!) 2297 * @num: the number of entries in @sg writable by other side 2298 * @data: the token identifying the buffer. 2299 * @ctx: extra context for the token 2300 * @gfp: how to do memory allocations (if necessary). 2301 * 2302 * Caller must ensure we don't call this with other virtqueue operations 2303 * at the same time (except where noted). 2304 * 2305 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO). 2306 */ 2307 int virtqueue_add_inbuf_ctx(struct virtqueue *vq, 2308 struct scatterlist *sg, unsigned int num, 2309 void *data, 2310 void *ctx, 2311 gfp_t gfp) 2312 { 2313 return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, gfp); 2314 } 2315 EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_ctx); 2316 2317 /** 2318 * virtqueue_dma_dev - get the dma dev 2319 * @_vq: the struct virtqueue we're talking about. 2320 * 2321 * Returns the dma dev. That can been used for dma api. 2322 */ 2323 struct device *virtqueue_dma_dev(struct virtqueue *_vq) 2324 { 2325 struct vring_virtqueue *vq = to_vvq(_vq); 2326 2327 if (vq->use_dma_api) 2328 return vring_dma_dev(vq); 2329 else 2330 return NULL; 2331 } 2332 EXPORT_SYMBOL_GPL(virtqueue_dma_dev); 2333 2334 /** 2335 * virtqueue_kick_prepare - first half of split virtqueue_kick call. 2336 * @_vq: the struct virtqueue 2337 * 2338 * Instead of virtqueue_kick(), you can do: 2339 * if (virtqueue_kick_prepare(vq)) 2340 * virtqueue_notify(vq); 2341 * 2342 * This is sometimes useful because the virtqueue_kick_prepare() needs 2343 * to be serialized, but the actual virtqueue_notify() call does not. 2344 */ 2345 bool virtqueue_kick_prepare(struct virtqueue *_vq) 2346 { 2347 struct vring_virtqueue *vq = to_vvq(_vq); 2348 2349 return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) : 2350 virtqueue_kick_prepare_split(_vq); 2351 } 2352 EXPORT_SYMBOL_GPL(virtqueue_kick_prepare); 2353 2354 /** 2355 * virtqueue_notify - second half of split virtqueue_kick call. 2356 * @_vq: the struct virtqueue 2357 * 2358 * This does not need to be serialized. 2359 * 2360 * Returns false if host notify failed or queue is broken, otherwise true. 2361 */ 2362 bool virtqueue_notify(struct virtqueue *_vq) 2363 { 2364 struct vring_virtqueue *vq = to_vvq(_vq); 2365 2366 if (unlikely(vq->broken)) 2367 return false; 2368 2369 /* Prod other side to tell it about changes. */ 2370 if (!vq->notify(_vq)) { 2371 vq->broken = true; 2372 return false; 2373 } 2374 return true; 2375 } 2376 EXPORT_SYMBOL_GPL(virtqueue_notify); 2377 2378 /** 2379 * virtqueue_kick - update after add_buf 2380 * @vq: the struct virtqueue 2381 * 2382 * After one or more virtqueue_add_* calls, invoke this to kick 2383 * the other side. 2384 * 2385 * Caller must ensure we don't call this with other virtqueue 2386 * operations at the same time (except where noted). 2387 * 2388 * Returns false if kick failed, otherwise true. 2389 */ 2390 bool virtqueue_kick(struct virtqueue *vq) 2391 { 2392 if (virtqueue_kick_prepare(vq)) 2393 return virtqueue_notify(vq); 2394 return true; 2395 } 2396 EXPORT_SYMBOL_GPL(virtqueue_kick); 2397 2398 /** 2399 * virtqueue_get_buf_ctx - get the next used buffer 2400 * @_vq: the struct virtqueue we're talking about. 2401 * @len: the length written into the buffer 2402 * @ctx: extra context for the token 2403 * 2404 * If the device wrote data into the buffer, @len will be set to the 2405 * amount written. This means you don't need to clear the buffer 2406 * beforehand to ensure there's no data leakage in the case of short 2407 * writes. 2408 * 2409 * Caller must ensure we don't call this with other virtqueue 2410 * operations at the same time (except where noted). 2411 * 2412 * Returns NULL if there are no used buffers, or the "data" token 2413 * handed to virtqueue_add_*(). 2414 */ 2415 void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len, 2416 void **ctx) 2417 { 2418 struct vring_virtqueue *vq = to_vvq(_vq); 2419 2420 return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) : 2421 virtqueue_get_buf_ctx_split(_vq, len, ctx); 2422 } 2423 EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx); 2424 2425 void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len) 2426 { 2427 return virtqueue_get_buf_ctx(_vq, len, NULL); 2428 } 2429 EXPORT_SYMBOL_GPL(virtqueue_get_buf); 2430 /** 2431 * virtqueue_disable_cb - disable callbacks 2432 * @_vq: the struct virtqueue we're talking about. 2433 * 2434 * Note that this is not necessarily synchronous, hence unreliable and only 2435 * useful as an optimization. 2436 * 2437 * Unlike other operations, this need not be serialized. 2438 */ 2439 void virtqueue_disable_cb(struct virtqueue *_vq) 2440 { 2441 struct vring_virtqueue *vq = to_vvq(_vq); 2442 2443 if (vq->packed_ring) 2444 virtqueue_disable_cb_packed(_vq); 2445 else 2446 virtqueue_disable_cb_split(_vq); 2447 } 2448 EXPORT_SYMBOL_GPL(virtqueue_disable_cb); 2449 2450 /** 2451 * virtqueue_enable_cb_prepare - restart callbacks after disable_cb 2452 * @_vq: the struct virtqueue we're talking about. 2453 * 2454 * This re-enables callbacks; it returns current queue state 2455 * in an opaque unsigned value. This value should be later tested by 2456 * virtqueue_poll, to detect a possible race between the driver checking for 2457 * more work, and enabling callbacks. 2458 * 2459 * Caller must ensure we don't call this with other virtqueue 2460 * operations at the same time (except where noted). 2461 */ 2462 unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq) 2463 { 2464 struct vring_virtqueue *vq = to_vvq(_vq); 2465 2466 if (vq->event_triggered) 2467 vq->event_triggered = false; 2468 2469 return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) : 2470 virtqueue_enable_cb_prepare_split(_vq); 2471 } 2472 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare); 2473 2474 /** 2475 * virtqueue_poll - query pending used buffers 2476 * @_vq: the struct virtqueue we're talking about. 2477 * @last_used_idx: virtqueue state (from call to virtqueue_enable_cb_prepare). 2478 * 2479 * Returns "true" if there are pending used buffers in the queue. 2480 * 2481 * This does not need to be serialized. 2482 */ 2483 bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx) 2484 { 2485 struct vring_virtqueue *vq = to_vvq(_vq); 2486 2487 if (unlikely(vq->broken)) 2488 return false; 2489 2490 virtio_mb(vq->weak_barriers); 2491 return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) : 2492 virtqueue_poll_split(_vq, last_used_idx); 2493 } 2494 EXPORT_SYMBOL_GPL(virtqueue_poll); 2495 2496 /** 2497 * virtqueue_enable_cb - restart callbacks after disable_cb. 2498 * @_vq: the struct virtqueue we're talking about. 2499 * 2500 * This re-enables callbacks; it returns "false" if there are pending 2501 * buffers in the queue, to detect a possible race between the driver 2502 * checking for more work, and enabling callbacks. 2503 * 2504 * Caller must ensure we don't call this with other virtqueue 2505 * operations at the same time (except where noted). 2506 */ 2507 bool virtqueue_enable_cb(struct virtqueue *_vq) 2508 { 2509 unsigned int last_used_idx = virtqueue_enable_cb_prepare(_vq); 2510 2511 return !virtqueue_poll(_vq, last_used_idx); 2512 } 2513 EXPORT_SYMBOL_GPL(virtqueue_enable_cb); 2514 2515 /** 2516 * virtqueue_enable_cb_delayed - restart callbacks after disable_cb. 2517 * @_vq: the struct virtqueue we're talking about. 2518 * 2519 * This re-enables callbacks but hints to the other side to delay 2520 * interrupts until most of the available buffers have been processed; 2521 * it returns "false" if there are many pending buffers in the queue, 2522 * to detect a possible race between the driver checking for more work, 2523 * and enabling callbacks. 2524 * 2525 * Caller must ensure we don't call this with other virtqueue 2526 * operations at the same time (except where noted). 2527 */ 2528 bool virtqueue_enable_cb_delayed(struct virtqueue *_vq) 2529 { 2530 struct vring_virtqueue *vq = to_vvq(_vq); 2531 2532 if (vq->event_triggered) 2533 vq->event_triggered = false; 2534 2535 return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) : 2536 virtqueue_enable_cb_delayed_split(_vq); 2537 } 2538 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed); 2539 2540 /** 2541 * virtqueue_detach_unused_buf - detach first unused buffer 2542 * @_vq: the struct virtqueue we're talking about. 2543 * 2544 * Returns NULL or the "data" token handed to virtqueue_add_*(). 2545 * This is not valid on an active queue; it is useful for device 2546 * shutdown or the reset queue. 2547 */ 2548 void *virtqueue_detach_unused_buf(struct virtqueue *_vq) 2549 { 2550 struct vring_virtqueue *vq = to_vvq(_vq); 2551 2552 return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) : 2553 virtqueue_detach_unused_buf_split(_vq); 2554 } 2555 EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf); 2556 2557 static inline bool more_used(const struct vring_virtqueue *vq) 2558 { 2559 return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq); 2560 } 2561 2562 /** 2563 * vring_interrupt - notify a virtqueue on an interrupt 2564 * @irq: the IRQ number (ignored) 2565 * @_vq: the struct virtqueue to notify 2566 * 2567 * Calls the callback function of @_vq to process the virtqueue 2568 * notification. 2569 */ 2570 irqreturn_t vring_interrupt(int irq, void *_vq) 2571 { 2572 struct vring_virtqueue *vq = to_vvq(_vq); 2573 2574 if (!more_used(vq)) { 2575 pr_debug("virtqueue interrupt with no work for %p\n", vq); 2576 return IRQ_NONE; 2577 } 2578 2579 if (unlikely(vq->broken)) { 2580 #ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION 2581 dev_warn_once(&vq->vq.vdev->dev, 2582 "virtio vring IRQ raised before DRIVER_OK"); 2583 return IRQ_NONE; 2584 #else 2585 return IRQ_HANDLED; 2586 #endif 2587 } 2588 2589 /* Just a hint for performance: so it's ok that this can be racy! */ 2590 if (vq->event) 2591 vq->event_triggered = true; 2592 2593 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback); 2594 if (vq->vq.callback) 2595 vq->vq.callback(&vq->vq); 2596 2597 return IRQ_HANDLED; 2598 } 2599 EXPORT_SYMBOL_GPL(vring_interrupt); 2600 2601 /* Only available for split ring */ 2602 static struct virtqueue *__vring_new_virtqueue(unsigned int index, 2603 struct vring_virtqueue_split *vring_split, 2604 struct virtio_device *vdev, 2605 bool weak_barriers, 2606 bool context, 2607 bool (*notify)(struct virtqueue *), 2608 void (*callback)(struct virtqueue *), 2609 const char *name, 2610 struct device *dma_dev) 2611 { 2612 struct vring_virtqueue *vq; 2613 int err; 2614 2615 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) 2616 return NULL; 2617 2618 vq = kmalloc(sizeof(*vq), GFP_KERNEL); 2619 if (!vq) 2620 return NULL; 2621 2622 vq->packed_ring = false; 2623 vq->vq.callback = callback; 2624 vq->vq.vdev = vdev; 2625 vq->vq.name = name; 2626 vq->vq.index = index; 2627 vq->vq.reset = false; 2628 vq->we_own_ring = false; 2629 vq->notify = notify; 2630 vq->weak_barriers = weak_barriers; 2631 #ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION 2632 vq->broken = true; 2633 #else 2634 vq->broken = false; 2635 #endif 2636 vq->dma_dev = dma_dev; 2637 vq->use_dma_api = vring_use_dma_api(vdev); 2638 vq->premapped = false; 2639 vq->do_unmap = vq->use_dma_api; 2640 2641 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) && 2642 !context; 2643 vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX); 2644 2645 if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM)) 2646 vq->weak_barriers = false; 2647 2648 err = vring_alloc_state_extra_split(vring_split); 2649 if (err) { 2650 kfree(vq); 2651 return NULL; 2652 } 2653 2654 virtqueue_vring_init_split(vring_split, vq); 2655 2656 virtqueue_init(vq, vring_split->vring.num); 2657 virtqueue_vring_attach_split(vq, vring_split); 2658 2659 spin_lock(&vdev->vqs_list_lock); 2660 list_add_tail(&vq->vq.list, &vdev->vqs); 2661 spin_unlock(&vdev->vqs_list_lock); 2662 return &vq->vq; 2663 } 2664 2665 struct virtqueue *vring_create_virtqueue( 2666 unsigned int index, 2667 unsigned int num, 2668 unsigned int vring_align, 2669 struct virtio_device *vdev, 2670 bool weak_barriers, 2671 bool may_reduce_num, 2672 bool context, 2673 bool (*notify)(struct virtqueue *), 2674 void (*callback)(struct virtqueue *), 2675 const char *name) 2676 { 2677 2678 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) 2679 return vring_create_virtqueue_packed(index, num, vring_align, 2680 vdev, weak_barriers, may_reduce_num, 2681 context, notify, callback, name, vdev->dev.parent); 2682 2683 return vring_create_virtqueue_split(index, num, vring_align, 2684 vdev, weak_barriers, may_reduce_num, 2685 context, notify, callback, name, vdev->dev.parent); 2686 } 2687 EXPORT_SYMBOL_GPL(vring_create_virtqueue); 2688 2689 struct virtqueue *vring_create_virtqueue_dma( 2690 unsigned int index, 2691 unsigned int num, 2692 unsigned int vring_align, 2693 struct virtio_device *vdev, 2694 bool weak_barriers, 2695 bool may_reduce_num, 2696 bool context, 2697 bool (*notify)(struct virtqueue *), 2698 void (*callback)(struct virtqueue *), 2699 const char *name, 2700 struct device *dma_dev) 2701 { 2702 2703 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) 2704 return vring_create_virtqueue_packed(index, num, vring_align, 2705 vdev, weak_barriers, may_reduce_num, 2706 context, notify, callback, name, dma_dev); 2707 2708 return vring_create_virtqueue_split(index, num, vring_align, 2709 vdev, weak_barriers, may_reduce_num, 2710 context, notify, callback, name, dma_dev); 2711 } 2712 EXPORT_SYMBOL_GPL(vring_create_virtqueue_dma); 2713 2714 /** 2715 * virtqueue_resize - resize the vring of vq 2716 * @_vq: the struct virtqueue we're talking about. 2717 * @num: new ring num 2718 * @recycle: callback to recycle unused buffers 2719 * 2720 * When it is really necessary to create a new vring, it will set the current vq 2721 * into the reset state. Then call the passed callback to recycle the buffer 2722 * that is no longer used. Only after the new vring is successfully created, the 2723 * old vring will be released. 2724 * 2725 * Caller must ensure we don't call this with other virtqueue operations 2726 * at the same time (except where noted). 2727 * 2728 * Returns zero or a negative error. 2729 * 0: success. 2730 * -ENOMEM: Failed to allocate a new ring, fall back to the original ring size. 2731 * vq can still work normally 2732 * -EBUSY: Failed to sync with device, vq may not work properly 2733 * -ENOENT: Transport or device not supported 2734 * -E2BIG/-EINVAL: num error 2735 * -EPERM: Operation not permitted 2736 * 2737 */ 2738 int virtqueue_resize(struct virtqueue *_vq, u32 num, 2739 void (*recycle)(struct virtqueue *vq, void *buf)) 2740 { 2741 struct vring_virtqueue *vq = to_vvq(_vq); 2742 int err; 2743 2744 if (num > vq->vq.num_max) 2745 return -E2BIG; 2746 2747 if (!num) 2748 return -EINVAL; 2749 2750 if ((vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num) == num) 2751 return 0; 2752 2753 err = virtqueue_disable_and_recycle(_vq, recycle); 2754 if (err) 2755 return err; 2756 2757 if (vq->packed_ring) 2758 err = virtqueue_resize_packed(_vq, num); 2759 else 2760 err = virtqueue_resize_split(_vq, num); 2761 2762 return virtqueue_enable_after_reset(_vq); 2763 } 2764 EXPORT_SYMBOL_GPL(virtqueue_resize); 2765 2766 /** 2767 * virtqueue_set_dma_premapped - set the vring premapped mode 2768 * @_vq: the struct virtqueue we're talking about. 2769 * 2770 * Enable the premapped mode of the vq. 2771 * 2772 * The vring in premapped mode does not do dma internally, so the driver must 2773 * do dma mapping in advance. The driver must pass the dma_address through 2774 * dma_address of scatterlist. When the driver got a used buffer from 2775 * the vring, it has to unmap the dma address. 2776 * 2777 * This function must be called immediately after creating the vq, or after vq 2778 * reset, and before adding any buffers to it. 2779 * 2780 * Caller must ensure we don't call this with other virtqueue operations 2781 * at the same time (except where noted). 2782 * 2783 * Returns zero or a negative error. 2784 * 0: success. 2785 * -EINVAL: vring does not use the dma api, so we can not enable premapped mode. 2786 */ 2787 int virtqueue_set_dma_premapped(struct virtqueue *_vq) 2788 { 2789 struct vring_virtqueue *vq = to_vvq(_vq); 2790 u32 num; 2791 2792 START_USE(vq); 2793 2794 num = vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num; 2795 2796 if (num != vq->vq.num_free) { 2797 END_USE(vq); 2798 return -EINVAL; 2799 } 2800 2801 if (!vq->use_dma_api) { 2802 END_USE(vq); 2803 return -EINVAL; 2804 } 2805 2806 vq->premapped = true; 2807 vq->do_unmap = false; 2808 2809 END_USE(vq); 2810 2811 return 0; 2812 } 2813 EXPORT_SYMBOL_GPL(virtqueue_set_dma_premapped); 2814 2815 /** 2816 * virtqueue_reset - detach and recycle all unused buffers 2817 * @_vq: the struct virtqueue we're talking about. 2818 * @recycle: callback to recycle unused buffers 2819 * 2820 * Caller must ensure we don't call this with other virtqueue operations 2821 * at the same time (except where noted). 2822 * 2823 * Returns zero or a negative error. 2824 * 0: success. 2825 * -EBUSY: Failed to sync with device, vq may not work properly 2826 * -ENOENT: Transport or device not supported 2827 * -EPERM: Operation not permitted 2828 */ 2829 int virtqueue_reset(struct virtqueue *_vq, 2830 void (*recycle)(struct virtqueue *vq, void *buf)) 2831 { 2832 struct vring_virtqueue *vq = to_vvq(_vq); 2833 int err; 2834 2835 err = virtqueue_disable_and_recycle(_vq, recycle); 2836 if (err) 2837 return err; 2838 2839 if (vq->packed_ring) 2840 virtqueue_reinit_packed(vq); 2841 else 2842 virtqueue_reinit_split(vq); 2843 2844 return virtqueue_enable_after_reset(_vq); 2845 } 2846 EXPORT_SYMBOL_GPL(virtqueue_reset); 2847 2848 /* Only available for split ring */ 2849 struct virtqueue *vring_new_virtqueue(unsigned int index, 2850 unsigned int num, 2851 unsigned int vring_align, 2852 struct virtio_device *vdev, 2853 bool weak_barriers, 2854 bool context, 2855 void *pages, 2856 bool (*notify)(struct virtqueue *vq), 2857 void (*callback)(struct virtqueue *vq), 2858 const char *name) 2859 { 2860 struct vring_virtqueue_split vring_split = {}; 2861 2862 if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED)) 2863 return NULL; 2864 2865 vring_init(&vring_split.vring, num, pages, vring_align); 2866 return __vring_new_virtqueue(index, &vring_split, vdev, weak_barriers, 2867 context, notify, callback, name, 2868 vdev->dev.parent); 2869 } 2870 EXPORT_SYMBOL_GPL(vring_new_virtqueue); 2871 2872 static void vring_free(struct virtqueue *_vq) 2873 { 2874 struct vring_virtqueue *vq = to_vvq(_vq); 2875 2876 if (vq->we_own_ring) { 2877 if (vq->packed_ring) { 2878 vring_free_queue(vq->vq.vdev, 2879 vq->packed.ring_size_in_bytes, 2880 vq->packed.vring.desc, 2881 vq->packed.ring_dma_addr, 2882 vring_dma_dev(vq)); 2883 2884 vring_free_queue(vq->vq.vdev, 2885 vq->packed.event_size_in_bytes, 2886 vq->packed.vring.driver, 2887 vq->packed.driver_event_dma_addr, 2888 vring_dma_dev(vq)); 2889 2890 vring_free_queue(vq->vq.vdev, 2891 vq->packed.event_size_in_bytes, 2892 vq->packed.vring.device, 2893 vq->packed.device_event_dma_addr, 2894 vring_dma_dev(vq)); 2895 2896 kfree(vq->packed.desc_state); 2897 kfree(vq->packed.desc_extra); 2898 } else { 2899 vring_free_queue(vq->vq.vdev, 2900 vq->split.queue_size_in_bytes, 2901 vq->split.vring.desc, 2902 vq->split.queue_dma_addr, 2903 vring_dma_dev(vq)); 2904 } 2905 } 2906 if (!vq->packed_ring) { 2907 kfree(vq->split.desc_state); 2908 kfree(vq->split.desc_extra); 2909 } 2910 } 2911 2912 void vring_del_virtqueue(struct virtqueue *_vq) 2913 { 2914 struct vring_virtqueue *vq = to_vvq(_vq); 2915 2916 spin_lock(&vq->vq.vdev->vqs_list_lock); 2917 list_del(&_vq->list); 2918 spin_unlock(&vq->vq.vdev->vqs_list_lock); 2919 2920 vring_free(_vq); 2921 2922 kfree(vq); 2923 } 2924 EXPORT_SYMBOL_GPL(vring_del_virtqueue); 2925 2926 u32 vring_notification_data(struct virtqueue *_vq) 2927 { 2928 struct vring_virtqueue *vq = to_vvq(_vq); 2929 u16 next; 2930 2931 if (vq->packed_ring) 2932 next = (vq->packed.next_avail_idx & 2933 ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR))) | 2934 vq->packed.avail_wrap_counter << 2935 VRING_PACKED_EVENT_F_WRAP_CTR; 2936 else 2937 next = vq->split.avail_idx_shadow; 2938 2939 return next << 16 | _vq->index; 2940 } 2941 EXPORT_SYMBOL_GPL(vring_notification_data); 2942 2943 /* Manipulates transport-specific feature bits. */ 2944 void vring_transport_features(struct virtio_device *vdev) 2945 { 2946 unsigned int i; 2947 2948 for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) { 2949 switch (i) { 2950 case VIRTIO_RING_F_INDIRECT_DESC: 2951 break; 2952 case VIRTIO_RING_F_EVENT_IDX: 2953 break; 2954 case VIRTIO_F_VERSION_1: 2955 break; 2956 case VIRTIO_F_ACCESS_PLATFORM: 2957 break; 2958 case VIRTIO_F_RING_PACKED: 2959 break; 2960 case VIRTIO_F_ORDER_PLATFORM: 2961 break; 2962 case VIRTIO_F_NOTIFICATION_DATA: 2963 break; 2964 default: 2965 /* We don't understand this bit. */ 2966 __virtio_clear_bit(vdev, i); 2967 } 2968 } 2969 } 2970 EXPORT_SYMBOL_GPL(vring_transport_features); 2971 2972 /** 2973 * virtqueue_get_vring_size - return the size of the virtqueue's vring 2974 * @_vq: the struct virtqueue containing the vring of interest. 2975 * 2976 * Returns the size of the vring. This is mainly used for boasting to 2977 * userspace. Unlike other operations, this need not be serialized. 2978 */ 2979 unsigned int virtqueue_get_vring_size(const struct virtqueue *_vq) 2980 { 2981 2982 const struct vring_virtqueue *vq = to_vvq(_vq); 2983 2984 return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num; 2985 } 2986 EXPORT_SYMBOL_GPL(virtqueue_get_vring_size); 2987 2988 /* 2989 * This function should only be called by the core, not directly by the driver. 2990 */ 2991 void __virtqueue_break(struct virtqueue *_vq) 2992 { 2993 struct vring_virtqueue *vq = to_vvq(_vq); 2994 2995 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */ 2996 WRITE_ONCE(vq->broken, true); 2997 } 2998 EXPORT_SYMBOL_GPL(__virtqueue_break); 2999 3000 /* 3001 * This function should only be called by the core, not directly by the driver. 3002 */ 3003 void __virtqueue_unbreak(struct virtqueue *_vq) 3004 { 3005 struct vring_virtqueue *vq = to_vvq(_vq); 3006 3007 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */ 3008 WRITE_ONCE(vq->broken, false); 3009 } 3010 EXPORT_SYMBOL_GPL(__virtqueue_unbreak); 3011 3012 bool virtqueue_is_broken(const struct virtqueue *_vq) 3013 { 3014 const struct vring_virtqueue *vq = to_vvq(_vq); 3015 3016 return READ_ONCE(vq->broken); 3017 } 3018 EXPORT_SYMBOL_GPL(virtqueue_is_broken); 3019 3020 /* 3021 * This should prevent the device from being used, allowing drivers to 3022 * recover. You may need to grab appropriate locks to flush. 3023 */ 3024 void virtio_break_device(struct virtio_device *dev) 3025 { 3026 struct virtqueue *_vq; 3027 3028 spin_lock(&dev->vqs_list_lock); 3029 list_for_each_entry(_vq, &dev->vqs, list) { 3030 struct vring_virtqueue *vq = to_vvq(_vq); 3031 3032 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */ 3033 WRITE_ONCE(vq->broken, true); 3034 } 3035 spin_unlock(&dev->vqs_list_lock); 3036 } 3037 EXPORT_SYMBOL_GPL(virtio_break_device); 3038 3039 /* 3040 * This should allow the device to be used by the driver. You may 3041 * need to grab appropriate locks to flush the write to 3042 * vq->broken. This should only be used in some specific case e.g 3043 * (probing and restoring). This function should only be called by the 3044 * core, not directly by the driver. 3045 */ 3046 void __virtio_unbreak_device(struct virtio_device *dev) 3047 { 3048 struct virtqueue *_vq; 3049 3050 spin_lock(&dev->vqs_list_lock); 3051 list_for_each_entry(_vq, &dev->vqs, list) { 3052 struct vring_virtqueue *vq = to_vvq(_vq); 3053 3054 /* Pairs with READ_ONCE() in virtqueue_is_broken(). */ 3055 WRITE_ONCE(vq->broken, false); 3056 } 3057 spin_unlock(&dev->vqs_list_lock); 3058 } 3059 EXPORT_SYMBOL_GPL(__virtio_unbreak_device); 3060 3061 dma_addr_t virtqueue_get_desc_addr(const struct virtqueue *_vq) 3062 { 3063 const struct vring_virtqueue *vq = to_vvq(_vq); 3064 3065 BUG_ON(!vq->we_own_ring); 3066 3067 if (vq->packed_ring) 3068 return vq->packed.ring_dma_addr; 3069 3070 return vq->split.queue_dma_addr; 3071 } 3072 EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr); 3073 3074 dma_addr_t virtqueue_get_avail_addr(const struct virtqueue *_vq) 3075 { 3076 const struct vring_virtqueue *vq = to_vvq(_vq); 3077 3078 BUG_ON(!vq->we_own_ring); 3079 3080 if (vq->packed_ring) 3081 return vq->packed.driver_event_dma_addr; 3082 3083 return vq->split.queue_dma_addr + 3084 ((char *)vq->split.vring.avail - (char *)vq->split.vring.desc); 3085 } 3086 EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr); 3087 3088 dma_addr_t virtqueue_get_used_addr(const struct virtqueue *_vq) 3089 { 3090 const struct vring_virtqueue *vq = to_vvq(_vq); 3091 3092 BUG_ON(!vq->we_own_ring); 3093 3094 if (vq->packed_ring) 3095 return vq->packed.device_event_dma_addr; 3096 3097 return vq->split.queue_dma_addr + 3098 ((char *)vq->split.vring.used - (char *)vq->split.vring.desc); 3099 } 3100 EXPORT_SYMBOL_GPL(virtqueue_get_used_addr); 3101 3102 /* Only available for split ring */ 3103 const struct vring *virtqueue_get_vring(const struct virtqueue *vq) 3104 { 3105 return &to_vvq(vq)->split.vring; 3106 } 3107 EXPORT_SYMBOL_GPL(virtqueue_get_vring); 3108 3109 /** 3110 * virtqueue_dma_map_single_attrs - map DMA for _vq 3111 * @_vq: the struct virtqueue we're talking about. 3112 * @ptr: the pointer of the buffer to do dma 3113 * @size: the size of the buffer to do dma 3114 * @dir: DMA direction 3115 * @attrs: DMA Attrs 3116 * 3117 * The caller calls this to do dma mapping in advance. The DMA address can be 3118 * passed to this _vq when it is in pre-mapped mode. 3119 * 3120 * return DMA address. Caller should check that by virtqueue_dma_mapping_error(). 3121 */ 3122 dma_addr_t virtqueue_dma_map_single_attrs(struct virtqueue *_vq, void *ptr, 3123 size_t size, 3124 enum dma_data_direction dir, 3125 unsigned long attrs) 3126 { 3127 struct vring_virtqueue *vq = to_vvq(_vq); 3128 3129 if (!vq->use_dma_api) 3130 return (dma_addr_t)virt_to_phys(ptr); 3131 3132 return dma_map_single_attrs(vring_dma_dev(vq), ptr, size, dir, attrs); 3133 } 3134 EXPORT_SYMBOL_GPL(virtqueue_dma_map_single_attrs); 3135 3136 /** 3137 * virtqueue_dma_unmap_single_attrs - unmap DMA for _vq 3138 * @_vq: the struct virtqueue we're talking about. 3139 * @addr: the dma address to unmap 3140 * @size: the size of the buffer 3141 * @dir: DMA direction 3142 * @attrs: DMA Attrs 3143 * 3144 * Unmap the address that is mapped by the virtqueue_dma_map_* APIs. 3145 * 3146 */ 3147 void virtqueue_dma_unmap_single_attrs(struct virtqueue *_vq, dma_addr_t addr, 3148 size_t size, enum dma_data_direction dir, 3149 unsigned long attrs) 3150 { 3151 struct vring_virtqueue *vq = to_vvq(_vq); 3152 3153 if (!vq->use_dma_api) 3154 return; 3155 3156 dma_unmap_single_attrs(vring_dma_dev(vq), addr, size, dir, attrs); 3157 } 3158 EXPORT_SYMBOL_GPL(virtqueue_dma_unmap_single_attrs); 3159 3160 /** 3161 * virtqueue_dma_mapping_error - check dma address 3162 * @_vq: the struct virtqueue we're talking about. 3163 * @addr: DMA address 3164 * 3165 * Returns 0 means dma valid. Other means invalid dma address. 3166 */ 3167 int virtqueue_dma_mapping_error(struct virtqueue *_vq, dma_addr_t addr) 3168 { 3169 struct vring_virtqueue *vq = to_vvq(_vq); 3170 3171 if (!vq->use_dma_api) 3172 return 0; 3173 3174 return dma_mapping_error(vring_dma_dev(vq), addr); 3175 } 3176 EXPORT_SYMBOL_GPL(virtqueue_dma_mapping_error); 3177 3178 /** 3179 * virtqueue_dma_need_sync - check a dma address needs sync 3180 * @_vq: the struct virtqueue we're talking about. 3181 * @addr: DMA address 3182 * 3183 * Check if the dma address mapped by the virtqueue_dma_map_* APIs needs to be 3184 * synchronized 3185 * 3186 * return bool 3187 */ 3188 bool virtqueue_dma_need_sync(struct virtqueue *_vq, dma_addr_t addr) 3189 { 3190 struct vring_virtqueue *vq = to_vvq(_vq); 3191 3192 if (!vq->use_dma_api) 3193 return false; 3194 3195 return dma_need_sync(vring_dma_dev(vq), addr); 3196 } 3197 EXPORT_SYMBOL_GPL(virtqueue_dma_need_sync); 3198 3199 /** 3200 * virtqueue_dma_sync_single_range_for_cpu - dma sync for cpu 3201 * @_vq: the struct virtqueue we're talking about. 3202 * @addr: DMA address 3203 * @offset: DMA address offset 3204 * @size: buf size for sync 3205 * @dir: DMA direction 3206 * 3207 * Before calling this function, use virtqueue_dma_need_sync() to confirm that 3208 * the DMA address really needs to be synchronized 3209 * 3210 */ 3211 void virtqueue_dma_sync_single_range_for_cpu(struct virtqueue *_vq, 3212 dma_addr_t addr, 3213 unsigned long offset, size_t size, 3214 enum dma_data_direction dir) 3215 { 3216 struct vring_virtqueue *vq = to_vvq(_vq); 3217 struct device *dev = vring_dma_dev(vq); 3218 3219 if (!vq->use_dma_api) 3220 return; 3221 3222 dma_sync_single_range_for_cpu(dev, addr, offset, size, dir); 3223 } 3224 EXPORT_SYMBOL_GPL(virtqueue_dma_sync_single_range_for_cpu); 3225 3226 /** 3227 * virtqueue_dma_sync_single_range_for_device - dma sync for device 3228 * @_vq: the struct virtqueue we're talking about. 3229 * @addr: DMA address 3230 * @offset: DMA address offset 3231 * @size: buf size for sync 3232 * @dir: DMA direction 3233 * 3234 * Before calling this function, use virtqueue_dma_need_sync() to confirm that 3235 * the DMA address really needs to be synchronized 3236 */ 3237 void virtqueue_dma_sync_single_range_for_device(struct virtqueue *_vq, 3238 dma_addr_t addr, 3239 unsigned long offset, size_t size, 3240 enum dma_data_direction dir) 3241 { 3242 struct vring_virtqueue *vq = to_vvq(_vq); 3243 struct device *dev = vring_dma_dev(vq); 3244 3245 if (!vq->use_dma_api) 3246 return; 3247 3248 dma_sync_single_range_for_device(dev, addr, offset, size, dir); 3249 } 3250 EXPORT_SYMBOL_GPL(virtqueue_dma_sync_single_range_for_device); 3251 3252 MODULE_LICENSE("GPL"); 3253