xref: /openbmc/linux/drivers/virtio/virtio_ring.c (revision aac28965)
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* Virtio ring implementation.
3  *
4  *  Copyright 2007 Rusty Russell IBM Corporation
5  */
6 #include <linux/virtio.h>
7 #include <linux/virtio_ring.h>
8 #include <linux/virtio_config.h>
9 #include <linux/device.h>
10 #include <linux/slab.h>
11 #include <linux/module.h>
12 #include <linux/hrtimer.h>
13 #include <linux/dma-mapping.h>
14 #include <linux/spinlock.h>
15 #include <xen/xen.h>
16 
17 #ifdef DEBUG
18 /* For development, we want to crash whenever the ring is screwed. */
19 #define BAD_RING(_vq, fmt, args...)				\
20 	do {							\
21 		dev_err(&(_vq)->vq.vdev->dev,			\
22 			"%s:"fmt, (_vq)->vq.name, ##args);	\
23 		BUG();						\
24 	} while (0)
25 /* Caller is supposed to guarantee no reentry. */
26 #define START_USE(_vq)						\
27 	do {							\
28 		if ((_vq)->in_use)				\
29 			panic("%s:in_use = %i\n",		\
30 			      (_vq)->vq.name, (_vq)->in_use);	\
31 		(_vq)->in_use = __LINE__;			\
32 	} while (0)
33 #define END_USE(_vq) \
34 	do { BUG_ON(!(_vq)->in_use); (_vq)->in_use = 0; } while(0)
35 #define LAST_ADD_TIME_UPDATE(_vq)				\
36 	do {							\
37 		ktime_t now = ktime_get();			\
38 								\
39 		/* No kick or get, with .1 second between?  Warn. */ \
40 		if ((_vq)->last_add_time_valid)			\
41 			WARN_ON(ktime_to_ms(ktime_sub(now,	\
42 				(_vq)->last_add_time)) > 100);	\
43 		(_vq)->last_add_time = now;			\
44 		(_vq)->last_add_time_valid = true;		\
45 	} while (0)
46 #define LAST_ADD_TIME_CHECK(_vq)				\
47 	do {							\
48 		if ((_vq)->last_add_time_valid) {		\
49 			WARN_ON(ktime_to_ms(ktime_sub(ktime_get(), \
50 				      (_vq)->last_add_time)) > 100); \
51 		}						\
52 	} while (0)
53 #define LAST_ADD_TIME_INVALID(_vq)				\
54 	((_vq)->last_add_time_valid = false)
55 #else
56 #define BAD_RING(_vq, fmt, args...)				\
57 	do {							\
58 		dev_err(&_vq->vq.vdev->dev,			\
59 			"%s:"fmt, (_vq)->vq.name, ##args);	\
60 		(_vq)->broken = true;				\
61 	} while (0)
62 #define START_USE(vq)
63 #define END_USE(vq)
64 #define LAST_ADD_TIME_UPDATE(vq)
65 #define LAST_ADD_TIME_CHECK(vq)
66 #define LAST_ADD_TIME_INVALID(vq)
67 #endif
68 
69 struct vring_desc_state_split {
70 	void *data;			/* Data for callback. */
71 	struct vring_desc *indir_desc;	/* Indirect descriptor, if any. */
72 };
73 
74 struct vring_desc_state_packed {
75 	void *data;			/* Data for callback. */
76 	struct vring_packed_desc *indir_desc; /* Indirect descriptor, if any. */
77 	u16 num;			/* Descriptor list length. */
78 	u16 last;			/* The last desc state in a list. */
79 };
80 
81 struct vring_desc_extra {
82 	dma_addr_t addr;		/* Descriptor DMA addr. */
83 	u32 len;			/* Descriptor length. */
84 	u16 flags;			/* Descriptor flags. */
85 	u16 next;			/* The next desc state in a list. */
86 };
87 
88 struct vring_virtqueue {
89 	struct virtqueue vq;
90 
91 	/* Is this a packed ring? */
92 	bool packed_ring;
93 
94 	/* Is DMA API used? */
95 	bool use_dma_api;
96 
97 	/* Can we use weak barriers? */
98 	bool weak_barriers;
99 
100 	/* Other side has made a mess, don't try any more. */
101 	bool broken;
102 
103 	/* Host supports indirect buffers */
104 	bool indirect;
105 
106 	/* Host publishes avail event idx */
107 	bool event;
108 
109 	/* Head of free buffer list. */
110 	unsigned int free_head;
111 	/* Number we've added since last sync. */
112 	unsigned int num_added;
113 
114 	/* Last used index  we've seen.
115 	 * for split ring, it just contains last used index
116 	 * for packed ring:
117 	 * bits up to VRING_PACKED_EVENT_F_WRAP_CTR include the last used index.
118 	 * bits from VRING_PACKED_EVENT_F_WRAP_CTR include the used wrap counter.
119 	 */
120 	u16 last_used_idx;
121 
122 	/* Hint for event idx: already triggered no need to disable. */
123 	bool event_triggered;
124 
125 	union {
126 		/* Available for split ring */
127 		struct {
128 			/* Actual memory layout for this queue. */
129 			struct vring vring;
130 
131 			/* Last written value to avail->flags */
132 			u16 avail_flags_shadow;
133 
134 			/*
135 			 * Last written value to avail->idx in
136 			 * guest byte order.
137 			 */
138 			u16 avail_idx_shadow;
139 
140 			/* Per-descriptor state. */
141 			struct vring_desc_state_split *desc_state;
142 			struct vring_desc_extra *desc_extra;
143 
144 			/* DMA address and size information */
145 			dma_addr_t queue_dma_addr;
146 			size_t queue_size_in_bytes;
147 		} split;
148 
149 		/* Available for packed ring */
150 		struct {
151 			/* Actual memory layout for this queue. */
152 			struct {
153 				unsigned int num;
154 				struct vring_packed_desc *desc;
155 				struct vring_packed_desc_event *driver;
156 				struct vring_packed_desc_event *device;
157 			} vring;
158 
159 			/* Driver ring wrap counter. */
160 			bool avail_wrap_counter;
161 
162 			/* Avail used flags. */
163 			u16 avail_used_flags;
164 
165 			/* Index of the next avail descriptor. */
166 			u16 next_avail_idx;
167 
168 			/*
169 			 * Last written value to driver->flags in
170 			 * guest byte order.
171 			 */
172 			u16 event_flags_shadow;
173 
174 			/* Per-descriptor state. */
175 			struct vring_desc_state_packed *desc_state;
176 			struct vring_desc_extra *desc_extra;
177 
178 			/* DMA address and size information */
179 			dma_addr_t ring_dma_addr;
180 			dma_addr_t driver_event_dma_addr;
181 			dma_addr_t device_event_dma_addr;
182 			size_t ring_size_in_bytes;
183 			size_t event_size_in_bytes;
184 		} packed;
185 	};
186 
187 	/* How to notify other side. FIXME: commonalize hcalls! */
188 	bool (*notify)(struct virtqueue *vq);
189 
190 	/* DMA, allocation, and size information */
191 	bool we_own_ring;
192 
193 #ifdef DEBUG
194 	/* They're supposed to lock for us. */
195 	unsigned int in_use;
196 
197 	/* Figure out if their kicks are too delayed. */
198 	bool last_add_time_valid;
199 	ktime_t last_add_time;
200 #endif
201 };
202 
203 
204 /*
205  * Helpers.
206  */
207 
208 #define to_vvq(_vq) container_of(_vq, struct vring_virtqueue, vq)
209 
210 static inline bool virtqueue_use_indirect(struct vring_virtqueue *vq,
211 					  unsigned int total_sg)
212 {
213 	/*
214 	 * If the host supports indirect descriptor tables, and we have multiple
215 	 * buffers, then go indirect. FIXME: tune this threshold
216 	 */
217 	return (vq->indirect && total_sg > 1 && vq->vq.num_free);
218 }
219 
220 /*
221  * Modern virtio devices have feature bits to specify whether they need a
222  * quirk and bypass the IOMMU. If not there, just use the DMA API.
223  *
224  * If there, the interaction between virtio and DMA API is messy.
225  *
226  * On most systems with virtio, physical addresses match bus addresses,
227  * and it doesn't particularly matter whether we use the DMA API.
228  *
229  * On some systems, including Xen and any system with a physical device
230  * that speaks virtio behind a physical IOMMU, we must use the DMA API
231  * for virtio DMA to work at all.
232  *
233  * On other systems, including SPARC and PPC64, virtio-pci devices are
234  * enumerated as though they are behind an IOMMU, but the virtio host
235  * ignores the IOMMU, so we must either pretend that the IOMMU isn't
236  * there or somehow map everything as the identity.
237  *
238  * For the time being, we preserve historic behavior and bypass the DMA
239  * API.
240  *
241  * TODO: install a per-device DMA ops structure that does the right thing
242  * taking into account all the above quirks, and use the DMA API
243  * unconditionally on data path.
244  */
245 
246 static bool vring_use_dma_api(struct virtio_device *vdev)
247 {
248 	if (!virtio_has_dma_quirk(vdev))
249 		return true;
250 
251 	/* Otherwise, we are left to guess. */
252 	/*
253 	 * In theory, it's possible to have a buggy QEMU-supposed
254 	 * emulated Q35 IOMMU and Xen enabled at the same time.  On
255 	 * such a configuration, virtio has never worked and will
256 	 * not work without an even larger kludge.  Instead, enable
257 	 * the DMA API if we're a Xen guest, which at least allows
258 	 * all of the sensible Xen configurations to work correctly.
259 	 */
260 	if (xen_domain())
261 		return true;
262 
263 	return false;
264 }
265 
266 size_t virtio_max_dma_size(struct virtio_device *vdev)
267 {
268 	size_t max_segment_size = SIZE_MAX;
269 
270 	if (vring_use_dma_api(vdev))
271 		max_segment_size = dma_max_mapping_size(vdev->dev.parent);
272 
273 	return max_segment_size;
274 }
275 EXPORT_SYMBOL_GPL(virtio_max_dma_size);
276 
277 static void *vring_alloc_queue(struct virtio_device *vdev, size_t size,
278 			      dma_addr_t *dma_handle, gfp_t flag)
279 {
280 	if (vring_use_dma_api(vdev)) {
281 		return dma_alloc_coherent(vdev->dev.parent, size,
282 					  dma_handle, flag);
283 	} else {
284 		void *queue = alloc_pages_exact(PAGE_ALIGN(size), flag);
285 
286 		if (queue) {
287 			phys_addr_t phys_addr = virt_to_phys(queue);
288 			*dma_handle = (dma_addr_t)phys_addr;
289 
290 			/*
291 			 * Sanity check: make sure we dind't truncate
292 			 * the address.  The only arches I can find that
293 			 * have 64-bit phys_addr_t but 32-bit dma_addr_t
294 			 * are certain non-highmem MIPS and x86
295 			 * configurations, but these configurations
296 			 * should never allocate physical pages above 32
297 			 * bits, so this is fine.  Just in case, throw a
298 			 * warning and abort if we end up with an
299 			 * unrepresentable address.
300 			 */
301 			if (WARN_ON_ONCE(*dma_handle != phys_addr)) {
302 				free_pages_exact(queue, PAGE_ALIGN(size));
303 				return NULL;
304 			}
305 		}
306 		return queue;
307 	}
308 }
309 
310 static void vring_free_queue(struct virtio_device *vdev, size_t size,
311 			     void *queue, dma_addr_t dma_handle)
312 {
313 	if (vring_use_dma_api(vdev))
314 		dma_free_coherent(vdev->dev.parent, size, queue, dma_handle);
315 	else
316 		free_pages_exact(queue, PAGE_ALIGN(size));
317 }
318 
319 /*
320  * The DMA ops on various arches are rather gnarly right now, and
321  * making all of the arch DMA ops work on the vring device itself
322  * is a mess.  For now, we use the parent device for DMA ops.
323  */
324 static inline struct device *vring_dma_dev(const struct vring_virtqueue *vq)
325 {
326 	return vq->vq.vdev->dev.parent;
327 }
328 
329 /* Map one sg entry. */
330 static dma_addr_t vring_map_one_sg(const struct vring_virtqueue *vq,
331 				   struct scatterlist *sg,
332 				   enum dma_data_direction direction)
333 {
334 	if (!vq->use_dma_api)
335 		return (dma_addr_t)sg_phys(sg);
336 
337 	/*
338 	 * We can't use dma_map_sg, because we don't use scatterlists in
339 	 * the way it expects (we don't guarantee that the scatterlist
340 	 * will exist for the lifetime of the mapping).
341 	 */
342 	return dma_map_page(vring_dma_dev(vq),
343 			    sg_page(sg), sg->offset, sg->length,
344 			    direction);
345 }
346 
347 static dma_addr_t vring_map_single(const struct vring_virtqueue *vq,
348 				   void *cpu_addr, size_t size,
349 				   enum dma_data_direction direction)
350 {
351 	if (!vq->use_dma_api)
352 		return (dma_addr_t)virt_to_phys(cpu_addr);
353 
354 	return dma_map_single(vring_dma_dev(vq),
355 			      cpu_addr, size, direction);
356 }
357 
358 static int vring_mapping_error(const struct vring_virtqueue *vq,
359 			       dma_addr_t addr)
360 {
361 	if (!vq->use_dma_api)
362 		return 0;
363 
364 	return dma_mapping_error(vring_dma_dev(vq), addr);
365 }
366 
367 
368 /*
369  * Split ring specific functions - *_split().
370  */
371 
372 static void vring_unmap_one_split_indirect(const struct vring_virtqueue *vq,
373 					   struct vring_desc *desc)
374 {
375 	u16 flags;
376 
377 	if (!vq->use_dma_api)
378 		return;
379 
380 	flags = virtio16_to_cpu(vq->vq.vdev, desc->flags);
381 
382 	dma_unmap_page(vring_dma_dev(vq),
383 		       virtio64_to_cpu(vq->vq.vdev, desc->addr),
384 		       virtio32_to_cpu(vq->vq.vdev, desc->len),
385 		       (flags & VRING_DESC_F_WRITE) ?
386 		       DMA_FROM_DEVICE : DMA_TO_DEVICE);
387 }
388 
389 static unsigned int vring_unmap_one_split(const struct vring_virtqueue *vq,
390 					  unsigned int i)
391 {
392 	struct vring_desc_extra *extra = vq->split.desc_extra;
393 	u16 flags;
394 
395 	if (!vq->use_dma_api)
396 		goto out;
397 
398 	flags = extra[i].flags;
399 
400 	if (flags & VRING_DESC_F_INDIRECT) {
401 		dma_unmap_single(vring_dma_dev(vq),
402 				 extra[i].addr,
403 				 extra[i].len,
404 				 (flags & VRING_DESC_F_WRITE) ?
405 				 DMA_FROM_DEVICE : DMA_TO_DEVICE);
406 	} else {
407 		dma_unmap_page(vring_dma_dev(vq),
408 			       extra[i].addr,
409 			       extra[i].len,
410 			       (flags & VRING_DESC_F_WRITE) ?
411 			       DMA_FROM_DEVICE : DMA_TO_DEVICE);
412 	}
413 
414 out:
415 	return extra[i].next;
416 }
417 
418 static struct vring_desc *alloc_indirect_split(struct virtqueue *_vq,
419 					       unsigned int total_sg,
420 					       gfp_t gfp)
421 {
422 	struct vring_desc *desc;
423 	unsigned int i;
424 
425 	/*
426 	 * We require lowmem mappings for the descriptors because
427 	 * otherwise virt_to_phys will give us bogus addresses in the
428 	 * virtqueue.
429 	 */
430 	gfp &= ~__GFP_HIGHMEM;
431 
432 	desc = kmalloc_array(total_sg, sizeof(struct vring_desc), gfp);
433 	if (!desc)
434 		return NULL;
435 
436 	for (i = 0; i < total_sg; i++)
437 		desc[i].next = cpu_to_virtio16(_vq->vdev, i + 1);
438 	return desc;
439 }
440 
441 static inline unsigned int virtqueue_add_desc_split(struct virtqueue *vq,
442 						    struct vring_desc *desc,
443 						    unsigned int i,
444 						    dma_addr_t addr,
445 						    unsigned int len,
446 						    u16 flags,
447 						    bool indirect)
448 {
449 	struct vring_virtqueue *vring = to_vvq(vq);
450 	struct vring_desc_extra *extra = vring->split.desc_extra;
451 	u16 next;
452 
453 	desc[i].flags = cpu_to_virtio16(vq->vdev, flags);
454 	desc[i].addr = cpu_to_virtio64(vq->vdev, addr);
455 	desc[i].len = cpu_to_virtio32(vq->vdev, len);
456 
457 	if (!indirect) {
458 		next = extra[i].next;
459 		desc[i].next = cpu_to_virtio16(vq->vdev, next);
460 
461 		extra[i].addr = addr;
462 		extra[i].len = len;
463 		extra[i].flags = flags;
464 	} else
465 		next = virtio16_to_cpu(vq->vdev, desc[i].next);
466 
467 	return next;
468 }
469 
470 static inline int virtqueue_add_split(struct virtqueue *_vq,
471 				      struct scatterlist *sgs[],
472 				      unsigned int total_sg,
473 				      unsigned int out_sgs,
474 				      unsigned int in_sgs,
475 				      void *data,
476 				      void *ctx,
477 				      gfp_t gfp)
478 {
479 	struct vring_virtqueue *vq = to_vvq(_vq);
480 	struct scatterlist *sg;
481 	struct vring_desc *desc;
482 	unsigned int i, n, avail, descs_used, prev, err_idx;
483 	int head;
484 	bool indirect;
485 
486 	START_USE(vq);
487 
488 	BUG_ON(data == NULL);
489 	BUG_ON(ctx && vq->indirect);
490 
491 	if (unlikely(vq->broken)) {
492 		END_USE(vq);
493 		return -EIO;
494 	}
495 
496 	LAST_ADD_TIME_UPDATE(vq);
497 
498 	BUG_ON(total_sg == 0);
499 
500 	head = vq->free_head;
501 
502 	if (virtqueue_use_indirect(vq, total_sg))
503 		desc = alloc_indirect_split(_vq, total_sg, gfp);
504 	else {
505 		desc = NULL;
506 		WARN_ON_ONCE(total_sg > vq->split.vring.num && !vq->indirect);
507 	}
508 
509 	if (desc) {
510 		/* Use a single buffer which doesn't continue */
511 		indirect = true;
512 		/* Set up rest to use this indirect table. */
513 		i = 0;
514 		descs_used = 1;
515 	} else {
516 		indirect = false;
517 		desc = vq->split.vring.desc;
518 		i = head;
519 		descs_used = total_sg;
520 	}
521 
522 	if (unlikely(vq->vq.num_free < descs_used)) {
523 		pr_debug("Can't add buf len %i - avail = %i\n",
524 			 descs_used, vq->vq.num_free);
525 		/* FIXME: for historical reasons, we force a notify here if
526 		 * there are outgoing parts to the buffer.  Presumably the
527 		 * host should service the ring ASAP. */
528 		if (out_sgs)
529 			vq->notify(&vq->vq);
530 		if (indirect)
531 			kfree(desc);
532 		END_USE(vq);
533 		return -ENOSPC;
534 	}
535 
536 	for (n = 0; n < out_sgs; n++) {
537 		for (sg = sgs[n]; sg; sg = sg_next(sg)) {
538 			dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_TO_DEVICE);
539 			if (vring_mapping_error(vq, addr))
540 				goto unmap_release;
541 
542 			prev = i;
543 			/* Note that we trust indirect descriptor
544 			 * table since it use stream DMA mapping.
545 			 */
546 			i = virtqueue_add_desc_split(_vq, desc, i, addr, sg->length,
547 						     VRING_DESC_F_NEXT,
548 						     indirect);
549 		}
550 	}
551 	for (; n < (out_sgs + in_sgs); n++) {
552 		for (sg = sgs[n]; sg; sg = sg_next(sg)) {
553 			dma_addr_t addr = vring_map_one_sg(vq, sg, DMA_FROM_DEVICE);
554 			if (vring_mapping_error(vq, addr))
555 				goto unmap_release;
556 
557 			prev = i;
558 			/* Note that we trust indirect descriptor
559 			 * table since it use stream DMA mapping.
560 			 */
561 			i = virtqueue_add_desc_split(_vq, desc, i, addr,
562 						     sg->length,
563 						     VRING_DESC_F_NEXT |
564 						     VRING_DESC_F_WRITE,
565 						     indirect);
566 		}
567 	}
568 	/* Last one doesn't continue. */
569 	desc[prev].flags &= cpu_to_virtio16(_vq->vdev, ~VRING_DESC_F_NEXT);
570 	if (!indirect && vq->use_dma_api)
571 		vq->split.desc_extra[prev & (vq->split.vring.num - 1)].flags &=
572 			~VRING_DESC_F_NEXT;
573 
574 	if (indirect) {
575 		/* Now that the indirect table is filled in, map it. */
576 		dma_addr_t addr = vring_map_single(
577 			vq, desc, total_sg * sizeof(struct vring_desc),
578 			DMA_TO_DEVICE);
579 		if (vring_mapping_error(vq, addr))
580 			goto unmap_release;
581 
582 		virtqueue_add_desc_split(_vq, vq->split.vring.desc,
583 					 head, addr,
584 					 total_sg * sizeof(struct vring_desc),
585 					 VRING_DESC_F_INDIRECT,
586 					 false);
587 	}
588 
589 	/* We're using some buffers from the free list. */
590 	vq->vq.num_free -= descs_used;
591 
592 	/* Update free pointer */
593 	if (indirect)
594 		vq->free_head = vq->split.desc_extra[head].next;
595 	else
596 		vq->free_head = i;
597 
598 	/* Store token and indirect buffer state. */
599 	vq->split.desc_state[head].data = data;
600 	if (indirect)
601 		vq->split.desc_state[head].indir_desc = desc;
602 	else
603 		vq->split.desc_state[head].indir_desc = ctx;
604 
605 	/* Put entry in available array (but don't update avail->idx until they
606 	 * do sync). */
607 	avail = vq->split.avail_idx_shadow & (vq->split.vring.num - 1);
608 	vq->split.vring.avail->ring[avail] = cpu_to_virtio16(_vq->vdev, head);
609 
610 	/* Descriptors and available array need to be set before we expose the
611 	 * new available array entries. */
612 	virtio_wmb(vq->weak_barriers);
613 	vq->split.avail_idx_shadow++;
614 	vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
615 						vq->split.avail_idx_shadow);
616 	vq->num_added++;
617 
618 	pr_debug("Added buffer head %i to %p\n", head, vq);
619 	END_USE(vq);
620 
621 	/* This is very unlikely, but theoretically possible.  Kick
622 	 * just in case. */
623 	if (unlikely(vq->num_added == (1 << 16) - 1))
624 		virtqueue_kick(_vq);
625 
626 	return 0;
627 
628 unmap_release:
629 	err_idx = i;
630 
631 	if (indirect)
632 		i = 0;
633 	else
634 		i = head;
635 
636 	for (n = 0; n < total_sg; n++) {
637 		if (i == err_idx)
638 			break;
639 		if (indirect) {
640 			vring_unmap_one_split_indirect(vq, &desc[i]);
641 			i = virtio16_to_cpu(_vq->vdev, desc[i].next);
642 		} else
643 			i = vring_unmap_one_split(vq, i);
644 	}
645 
646 	if (indirect)
647 		kfree(desc);
648 
649 	END_USE(vq);
650 	return -ENOMEM;
651 }
652 
653 static bool virtqueue_kick_prepare_split(struct virtqueue *_vq)
654 {
655 	struct vring_virtqueue *vq = to_vvq(_vq);
656 	u16 new, old;
657 	bool needs_kick;
658 
659 	START_USE(vq);
660 	/* We need to expose available array entries before checking avail
661 	 * event. */
662 	virtio_mb(vq->weak_barriers);
663 
664 	old = vq->split.avail_idx_shadow - vq->num_added;
665 	new = vq->split.avail_idx_shadow;
666 	vq->num_added = 0;
667 
668 	LAST_ADD_TIME_CHECK(vq);
669 	LAST_ADD_TIME_INVALID(vq);
670 
671 	if (vq->event) {
672 		needs_kick = vring_need_event(virtio16_to_cpu(_vq->vdev,
673 					vring_avail_event(&vq->split.vring)),
674 					      new, old);
675 	} else {
676 		needs_kick = !(vq->split.vring.used->flags &
677 					cpu_to_virtio16(_vq->vdev,
678 						VRING_USED_F_NO_NOTIFY));
679 	}
680 	END_USE(vq);
681 	return needs_kick;
682 }
683 
684 static void detach_buf_split(struct vring_virtqueue *vq, unsigned int head,
685 			     void **ctx)
686 {
687 	unsigned int i, j;
688 	__virtio16 nextflag = cpu_to_virtio16(vq->vq.vdev, VRING_DESC_F_NEXT);
689 
690 	/* Clear data ptr. */
691 	vq->split.desc_state[head].data = NULL;
692 
693 	/* Put back on free list: unmap first-level descriptors and find end */
694 	i = head;
695 
696 	while (vq->split.vring.desc[i].flags & nextflag) {
697 		vring_unmap_one_split(vq, i);
698 		i = vq->split.desc_extra[i].next;
699 		vq->vq.num_free++;
700 	}
701 
702 	vring_unmap_one_split(vq, i);
703 	vq->split.desc_extra[i].next = vq->free_head;
704 	vq->free_head = head;
705 
706 	/* Plus final descriptor */
707 	vq->vq.num_free++;
708 
709 	if (vq->indirect) {
710 		struct vring_desc *indir_desc =
711 				vq->split.desc_state[head].indir_desc;
712 		u32 len;
713 
714 		/* Free the indirect table, if any, now that it's unmapped. */
715 		if (!indir_desc)
716 			return;
717 
718 		len = vq->split.desc_extra[head].len;
719 
720 		BUG_ON(!(vq->split.desc_extra[head].flags &
721 				VRING_DESC_F_INDIRECT));
722 		BUG_ON(len == 0 || len % sizeof(struct vring_desc));
723 
724 		for (j = 0; j < len / sizeof(struct vring_desc); j++)
725 			vring_unmap_one_split_indirect(vq, &indir_desc[j]);
726 
727 		kfree(indir_desc);
728 		vq->split.desc_state[head].indir_desc = NULL;
729 	} else if (ctx) {
730 		*ctx = vq->split.desc_state[head].indir_desc;
731 	}
732 }
733 
734 static inline bool more_used_split(const struct vring_virtqueue *vq)
735 {
736 	return vq->last_used_idx != virtio16_to_cpu(vq->vq.vdev,
737 			vq->split.vring.used->idx);
738 }
739 
740 static void *virtqueue_get_buf_ctx_split(struct virtqueue *_vq,
741 					 unsigned int *len,
742 					 void **ctx)
743 {
744 	struct vring_virtqueue *vq = to_vvq(_vq);
745 	void *ret;
746 	unsigned int i;
747 	u16 last_used;
748 
749 	START_USE(vq);
750 
751 	if (unlikely(vq->broken)) {
752 		END_USE(vq);
753 		return NULL;
754 	}
755 
756 	if (!more_used_split(vq)) {
757 		pr_debug("No more buffers in queue\n");
758 		END_USE(vq);
759 		return NULL;
760 	}
761 
762 	/* Only get used array entries after they have been exposed by host. */
763 	virtio_rmb(vq->weak_barriers);
764 
765 	last_used = (vq->last_used_idx & (vq->split.vring.num - 1));
766 	i = virtio32_to_cpu(_vq->vdev,
767 			vq->split.vring.used->ring[last_used].id);
768 	*len = virtio32_to_cpu(_vq->vdev,
769 			vq->split.vring.used->ring[last_used].len);
770 
771 	if (unlikely(i >= vq->split.vring.num)) {
772 		BAD_RING(vq, "id %u out of range\n", i);
773 		return NULL;
774 	}
775 	if (unlikely(!vq->split.desc_state[i].data)) {
776 		BAD_RING(vq, "id %u is not a head!\n", i);
777 		return NULL;
778 	}
779 
780 	/* detach_buf_split clears data, so grab it now. */
781 	ret = vq->split.desc_state[i].data;
782 	detach_buf_split(vq, i, ctx);
783 	vq->last_used_idx++;
784 	/* If we expect an interrupt for the next entry, tell host
785 	 * by writing event index and flush out the write before
786 	 * the read in the next get_buf call. */
787 	if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT))
788 		virtio_store_mb(vq->weak_barriers,
789 				&vring_used_event(&vq->split.vring),
790 				cpu_to_virtio16(_vq->vdev, vq->last_used_idx));
791 
792 	LAST_ADD_TIME_INVALID(vq);
793 
794 	END_USE(vq);
795 	return ret;
796 }
797 
798 static void virtqueue_disable_cb_split(struct virtqueue *_vq)
799 {
800 	struct vring_virtqueue *vq = to_vvq(_vq);
801 
802 	if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
803 		vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
804 		if (vq->event)
805 			/* TODO: this is a hack. Figure out a cleaner value to write. */
806 			vring_used_event(&vq->split.vring) = 0x0;
807 		else
808 			vq->split.vring.avail->flags =
809 				cpu_to_virtio16(_vq->vdev,
810 						vq->split.avail_flags_shadow);
811 	}
812 }
813 
814 static unsigned int virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
815 {
816 	struct vring_virtqueue *vq = to_vvq(_vq);
817 	u16 last_used_idx;
818 
819 	START_USE(vq);
820 
821 	/* We optimistically turn back on interrupts, then check if there was
822 	 * more to do. */
823 	/* Depending on the VIRTIO_RING_F_EVENT_IDX feature, we need to
824 	 * either clear the flags bit or point the event index at the next
825 	 * entry. Always do both to keep code simple. */
826 	if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
827 		vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
828 		if (!vq->event)
829 			vq->split.vring.avail->flags =
830 				cpu_to_virtio16(_vq->vdev,
831 						vq->split.avail_flags_shadow);
832 	}
833 	vring_used_event(&vq->split.vring) = cpu_to_virtio16(_vq->vdev,
834 			last_used_idx = vq->last_used_idx);
835 	END_USE(vq);
836 	return last_used_idx;
837 }
838 
839 static bool virtqueue_poll_split(struct virtqueue *_vq, unsigned int last_used_idx)
840 {
841 	struct vring_virtqueue *vq = to_vvq(_vq);
842 
843 	return (u16)last_used_idx != virtio16_to_cpu(_vq->vdev,
844 			vq->split.vring.used->idx);
845 }
846 
847 static bool virtqueue_enable_cb_delayed_split(struct virtqueue *_vq)
848 {
849 	struct vring_virtqueue *vq = to_vvq(_vq);
850 	u16 bufs;
851 
852 	START_USE(vq);
853 
854 	/* We optimistically turn back on interrupts, then check if there was
855 	 * more to do. */
856 	/* Depending on the VIRTIO_RING_F_USED_EVENT_IDX feature, we need to
857 	 * either clear the flags bit or point the event index at the next
858 	 * entry. Always update the event index to keep code simple. */
859 	if (vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT) {
860 		vq->split.avail_flags_shadow &= ~VRING_AVAIL_F_NO_INTERRUPT;
861 		if (!vq->event)
862 			vq->split.vring.avail->flags =
863 				cpu_to_virtio16(_vq->vdev,
864 						vq->split.avail_flags_shadow);
865 	}
866 	/* TODO: tune this threshold */
867 	bufs = (u16)(vq->split.avail_idx_shadow - vq->last_used_idx) * 3 / 4;
868 
869 	virtio_store_mb(vq->weak_barriers,
870 			&vring_used_event(&vq->split.vring),
871 			cpu_to_virtio16(_vq->vdev, vq->last_used_idx + bufs));
872 
873 	if (unlikely((u16)(virtio16_to_cpu(_vq->vdev, vq->split.vring.used->idx)
874 					- vq->last_used_idx) > bufs)) {
875 		END_USE(vq);
876 		return false;
877 	}
878 
879 	END_USE(vq);
880 	return true;
881 }
882 
883 static void *virtqueue_detach_unused_buf_split(struct virtqueue *_vq)
884 {
885 	struct vring_virtqueue *vq = to_vvq(_vq);
886 	unsigned int i;
887 	void *buf;
888 
889 	START_USE(vq);
890 
891 	for (i = 0; i < vq->split.vring.num; i++) {
892 		if (!vq->split.desc_state[i].data)
893 			continue;
894 		/* detach_buf_split clears data, so grab it now. */
895 		buf = vq->split.desc_state[i].data;
896 		detach_buf_split(vq, i, NULL);
897 		vq->split.avail_idx_shadow--;
898 		vq->split.vring.avail->idx = cpu_to_virtio16(_vq->vdev,
899 				vq->split.avail_idx_shadow);
900 		END_USE(vq);
901 		return buf;
902 	}
903 	/* That should have freed everything. */
904 	BUG_ON(vq->vq.num_free != vq->split.vring.num);
905 
906 	END_USE(vq);
907 	return NULL;
908 }
909 
910 static struct virtqueue *vring_create_virtqueue_split(
911 	unsigned int index,
912 	unsigned int num,
913 	unsigned int vring_align,
914 	struct virtio_device *vdev,
915 	bool weak_barriers,
916 	bool may_reduce_num,
917 	bool context,
918 	bool (*notify)(struct virtqueue *),
919 	void (*callback)(struct virtqueue *),
920 	const char *name)
921 {
922 	struct virtqueue *vq;
923 	void *queue = NULL;
924 	dma_addr_t dma_addr;
925 	size_t queue_size_in_bytes;
926 	struct vring vring;
927 
928 	/* We assume num is a power of 2. */
929 	if (num & (num - 1)) {
930 		dev_warn(&vdev->dev, "Bad virtqueue length %u\n", num);
931 		return NULL;
932 	}
933 
934 	/* TODO: allocate each queue chunk individually */
935 	for (; num && vring_size(num, vring_align) > PAGE_SIZE; num /= 2) {
936 		queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
937 					  &dma_addr,
938 					  GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO);
939 		if (queue)
940 			break;
941 		if (!may_reduce_num)
942 			return NULL;
943 	}
944 
945 	if (!num)
946 		return NULL;
947 
948 	if (!queue) {
949 		/* Try to get a single page. You are my only hope! */
950 		queue = vring_alloc_queue(vdev, vring_size(num, vring_align),
951 					  &dma_addr, GFP_KERNEL|__GFP_ZERO);
952 	}
953 	if (!queue)
954 		return NULL;
955 
956 	queue_size_in_bytes = vring_size(num, vring_align);
957 	vring_init(&vring, num, queue, vring_align);
958 
959 	vq = __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
960 				   notify, callback, name);
961 	if (!vq) {
962 		vring_free_queue(vdev, queue_size_in_bytes, queue,
963 				 dma_addr);
964 		return NULL;
965 	}
966 
967 	to_vvq(vq)->split.queue_dma_addr = dma_addr;
968 	to_vvq(vq)->split.queue_size_in_bytes = queue_size_in_bytes;
969 	to_vvq(vq)->we_own_ring = true;
970 
971 	return vq;
972 }
973 
974 
975 /*
976  * Packed ring specific functions - *_packed().
977  */
978 static inline bool packed_used_wrap_counter(u16 last_used_idx)
979 {
980 	return !!(last_used_idx & (1 << VRING_PACKED_EVENT_F_WRAP_CTR));
981 }
982 
983 static inline u16 packed_last_used(u16 last_used_idx)
984 {
985 	return last_used_idx & ~(-(1 << VRING_PACKED_EVENT_F_WRAP_CTR));
986 }
987 
988 static void vring_unmap_extra_packed(const struct vring_virtqueue *vq,
989 				     struct vring_desc_extra *extra)
990 {
991 	u16 flags;
992 
993 	if (!vq->use_dma_api)
994 		return;
995 
996 	flags = extra->flags;
997 
998 	if (flags & VRING_DESC_F_INDIRECT) {
999 		dma_unmap_single(vring_dma_dev(vq),
1000 				 extra->addr, extra->len,
1001 				 (flags & VRING_DESC_F_WRITE) ?
1002 				 DMA_FROM_DEVICE : DMA_TO_DEVICE);
1003 	} else {
1004 		dma_unmap_page(vring_dma_dev(vq),
1005 			       extra->addr, extra->len,
1006 			       (flags & VRING_DESC_F_WRITE) ?
1007 			       DMA_FROM_DEVICE : DMA_TO_DEVICE);
1008 	}
1009 }
1010 
1011 static void vring_unmap_desc_packed(const struct vring_virtqueue *vq,
1012 				   struct vring_packed_desc *desc)
1013 {
1014 	u16 flags;
1015 
1016 	if (!vq->use_dma_api)
1017 		return;
1018 
1019 	flags = le16_to_cpu(desc->flags);
1020 
1021 	dma_unmap_page(vring_dma_dev(vq),
1022 		       le64_to_cpu(desc->addr),
1023 		       le32_to_cpu(desc->len),
1024 		       (flags & VRING_DESC_F_WRITE) ?
1025 		       DMA_FROM_DEVICE : DMA_TO_DEVICE);
1026 }
1027 
1028 static struct vring_packed_desc *alloc_indirect_packed(unsigned int total_sg,
1029 						       gfp_t gfp)
1030 {
1031 	struct vring_packed_desc *desc;
1032 
1033 	/*
1034 	 * We require lowmem mappings for the descriptors because
1035 	 * otherwise virt_to_phys will give us bogus addresses in the
1036 	 * virtqueue.
1037 	 */
1038 	gfp &= ~__GFP_HIGHMEM;
1039 
1040 	desc = kmalloc_array(total_sg, sizeof(struct vring_packed_desc), gfp);
1041 
1042 	return desc;
1043 }
1044 
1045 static int virtqueue_add_indirect_packed(struct vring_virtqueue *vq,
1046 					 struct scatterlist *sgs[],
1047 					 unsigned int total_sg,
1048 					 unsigned int out_sgs,
1049 					 unsigned int in_sgs,
1050 					 void *data,
1051 					 gfp_t gfp)
1052 {
1053 	struct vring_packed_desc *desc;
1054 	struct scatterlist *sg;
1055 	unsigned int i, n, err_idx;
1056 	u16 head, id;
1057 	dma_addr_t addr;
1058 
1059 	head = vq->packed.next_avail_idx;
1060 	desc = alloc_indirect_packed(total_sg, gfp);
1061 	if (!desc)
1062 		return -ENOMEM;
1063 
1064 	if (unlikely(vq->vq.num_free < 1)) {
1065 		pr_debug("Can't add buf len 1 - avail = 0\n");
1066 		kfree(desc);
1067 		END_USE(vq);
1068 		return -ENOSPC;
1069 	}
1070 
1071 	i = 0;
1072 	id = vq->free_head;
1073 	BUG_ON(id == vq->packed.vring.num);
1074 
1075 	for (n = 0; n < out_sgs + in_sgs; n++) {
1076 		for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1077 			addr = vring_map_one_sg(vq, sg, n < out_sgs ?
1078 					DMA_TO_DEVICE : DMA_FROM_DEVICE);
1079 			if (vring_mapping_error(vq, addr))
1080 				goto unmap_release;
1081 
1082 			desc[i].flags = cpu_to_le16(n < out_sgs ?
1083 						0 : VRING_DESC_F_WRITE);
1084 			desc[i].addr = cpu_to_le64(addr);
1085 			desc[i].len = cpu_to_le32(sg->length);
1086 			i++;
1087 		}
1088 	}
1089 
1090 	/* Now that the indirect table is filled in, map it. */
1091 	addr = vring_map_single(vq, desc,
1092 			total_sg * sizeof(struct vring_packed_desc),
1093 			DMA_TO_DEVICE);
1094 	if (vring_mapping_error(vq, addr))
1095 		goto unmap_release;
1096 
1097 	vq->packed.vring.desc[head].addr = cpu_to_le64(addr);
1098 	vq->packed.vring.desc[head].len = cpu_to_le32(total_sg *
1099 				sizeof(struct vring_packed_desc));
1100 	vq->packed.vring.desc[head].id = cpu_to_le16(id);
1101 
1102 	if (vq->use_dma_api) {
1103 		vq->packed.desc_extra[id].addr = addr;
1104 		vq->packed.desc_extra[id].len = total_sg *
1105 				sizeof(struct vring_packed_desc);
1106 		vq->packed.desc_extra[id].flags = VRING_DESC_F_INDIRECT |
1107 						  vq->packed.avail_used_flags;
1108 	}
1109 
1110 	/*
1111 	 * A driver MUST NOT make the first descriptor in the list
1112 	 * available before all subsequent descriptors comprising
1113 	 * the list are made available.
1114 	 */
1115 	virtio_wmb(vq->weak_barriers);
1116 	vq->packed.vring.desc[head].flags = cpu_to_le16(VRING_DESC_F_INDIRECT |
1117 						vq->packed.avail_used_flags);
1118 
1119 	/* We're using some buffers from the free list. */
1120 	vq->vq.num_free -= 1;
1121 
1122 	/* Update free pointer */
1123 	n = head + 1;
1124 	if (n >= vq->packed.vring.num) {
1125 		n = 0;
1126 		vq->packed.avail_wrap_counter ^= 1;
1127 		vq->packed.avail_used_flags ^=
1128 				1 << VRING_PACKED_DESC_F_AVAIL |
1129 				1 << VRING_PACKED_DESC_F_USED;
1130 	}
1131 	vq->packed.next_avail_idx = n;
1132 	vq->free_head = vq->packed.desc_extra[id].next;
1133 
1134 	/* Store token and indirect buffer state. */
1135 	vq->packed.desc_state[id].num = 1;
1136 	vq->packed.desc_state[id].data = data;
1137 	vq->packed.desc_state[id].indir_desc = desc;
1138 	vq->packed.desc_state[id].last = id;
1139 
1140 	vq->num_added += 1;
1141 
1142 	pr_debug("Added buffer head %i to %p\n", head, vq);
1143 	END_USE(vq);
1144 
1145 	return 0;
1146 
1147 unmap_release:
1148 	err_idx = i;
1149 
1150 	for (i = 0; i < err_idx; i++)
1151 		vring_unmap_desc_packed(vq, &desc[i]);
1152 
1153 	kfree(desc);
1154 
1155 	END_USE(vq);
1156 	return -ENOMEM;
1157 }
1158 
1159 static inline int virtqueue_add_packed(struct virtqueue *_vq,
1160 				       struct scatterlist *sgs[],
1161 				       unsigned int total_sg,
1162 				       unsigned int out_sgs,
1163 				       unsigned int in_sgs,
1164 				       void *data,
1165 				       void *ctx,
1166 				       gfp_t gfp)
1167 {
1168 	struct vring_virtqueue *vq = to_vvq(_vq);
1169 	struct vring_packed_desc *desc;
1170 	struct scatterlist *sg;
1171 	unsigned int i, n, c, descs_used, err_idx;
1172 	__le16 head_flags, flags;
1173 	u16 head, id, prev, curr, avail_used_flags;
1174 	int err;
1175 
1176 	START_USE(vq);
1177 
1178 	BUG_ON(data == NULL);
1179 	BUG_ON(ctx && vq->indirect);
1180 
1181 	if (unlikely(vq->broken)) {
1182 		END_USE(vq);
1183 		return -EIO;
1184 	}
1185 
1186 	LAST_ADD_TIME_UPDATE(vq);
1187 
1188 	BUG_ON(total_sg == 0);
1189 
1190 	if (virtqueue_use_indirect(vq, total_sg)) {
1191 		err = virtqueue_add_indirect_packed(vq, sgs, total_sg, out_sgs,
1192 						    in_sgs, data, gfp);
1193 		if (err != -ENOMEM) {
1194 			END_USE(vq);
1195 			return err;
1196 		}
1197 
1198 		/* fall back on direct */
1199 	}
1200 
1201 	head = vq->packed.next_avail_idx;
1202 	avail_used_flags = vq->packed.avail_used_flags;
1203 
1204 	WARN_ON_ONCE(total_sg > vq->packed.vring.num && !vq->indirect);
1205 
1206 	desc = vq->packed.vring.desc;
1207 	i = head;
1208 	descs_used = total_sg;
1209 
1210 	if (unlikely(vq->vq.num_free < descs_used)) {
1211 		pr_debug("Can't add buf len %i - avail = %i\n",
1212 			 descs_used, vq->vq.num_free);
1213 		END_USE(vq);
1214 		return -ENOSPC;
1215 	}
1216 
1217 	id = vq->free_head;
1218 	BUG_ON(id == vq->packed.vring.num);
1219 
1220 	curr = id;
1221 	c = 0;
1222 	for (n = 0; n < out_sgs + in_sgs; n++) {
1223 		for (sg = sgs[n]; sg; sg = sg_next(sg)) {
1224 			dma_addr_t addr = vring_map_one_sg(vq, sg, n < out_sgs ?
1225 					DMA_TO_DEVICE : DMA_FROM_DEVICE);
1226 			if (vring_mapping_error(vq, addr))
1227 				goto unmap_release;
1228 
1229 			flags = cpu_to_le16(vq->packed.avail_used_flags |
1230 				    (++c == total_sg ? 0 : VRING_DESC_F_NEXT) |
1231 				    (n < out_sgs ? 0 : VRING_DESC_F_WRITE));
1232 			if (i == head)
1233 				head_flags = flags;
1234 			else
1235 				desc[i].flags = flags;
1236 
1237 			desc[i].addr = cpu_to_le64(addr);
1238 			desc[i].len = cpu_to_le32(sg->length);
1239 			desc[i].id = cpu_to_le16(id);
1240 
1241 			if (unlikely(vq->use_dma_api)) {
1242 				vq->packed.desc_extra[curr].addr = addr;
1243 				vq->packed.desc_extra[curr].len = sg->length;
1244 				vq->packed.desc_extra[curr].flags =
1245 					le16_to_cpu(flags);
1246 			}
1247 			prev = curr;
1248 			curr = vq->packed.desc_extra[curr].next;
1249 
1250 			if ((unlikely(++i >= vq->packed.vring.num))) {
1251 				i = 0;
1252 				vq->packed.avail_used_flags ^=
1253 					1 << VRING_PACKED_DESC_F_AVAIL |
1254 					1 << VRING_PACKED_DESC_F_USED;
1255 			}
1256 		}
1257 	}
1258 
1259 	if (i < head)
1260 		vq->packed.avail_wrap_counter ^= 1;
1261 
1262 	/* We're using some buffers from the free list. */
1263 	vq->vq.num_free -= descs_used;
1264 
1265 	/* Update free pointer */
1266 	vq->packed.next_avail_idx = i;
1267 	vq->free_head = curr;
1268 
1269 	/* Store token. */
1270 	vq->packed.desc_state[id].num = descs_used;
1271 	vq->packed.desc_state[id].data = data;
1272 	vq->packed.desc_state[id].indir_desc = ctx;
1273 	vq->packed.desc_state[id].last = prev;
1274 
1275 	/*
1276 	 * A driver MUST NOT make the first descriptor in the list
1277 	 * available before all subsequent descriptors comprising
1278 	 * the list are made available.
1279 	 */
1280 	virtio_wmb(vq->weak_barriers);
1281 	vq->packed.vring.desc[head].flags = head_flags;
1282 	vq->num_added += descs_used;
1283 
1284 	pr_debug("Added buffer head %i to %p\n", head, vq);
1285 	END_USE(vq);
1286 
1287 	return 0;
1288 
1289 unmap_release:
1290 	err_idx = i;
1291 	i = head;
1292 	curr = vq->free_head;
1293 
1294 	vq->packed.avail_used_flags = avail_used_flags;
1295 
1296 	for (n = 0; n < total_sg; n++) {
1297 		if (i == err_idx)
1298 			break;
1299 		vring_unmap_extra_packed(vq, &vq->packed.desc_extra[curr]);
1300 		curr = vq->packed.desc_extra[curr].next;
1301 		i++;
1302 		if (i >= vq->packed.vring.num)
1303 			i = 0;
1304 	}
1305 
1306 	END_USE(vq);
1307 	return -EIO;
1308 }
1309 
1310 static bool virtqueue_kick_prepare_packed(struct virtqueue *_vq)
1311 {
1312 	struct vring_virtqueue *vq = to_vvq(_vq);
1313 	u16 new, old, off_wrap, flags, wrap_counter, event_idx;
1314 	bool needs_kick;
1315 	union {
1316 		struct {
1317 			__le16 off_wrap;
1318 			__le16 flags;
1319 		};
1320 		u32 u32;
1321 	} snapshot;
1322 
1323 	START_USE(vq);
1324 
1325 	/*
1326 	 * We need to expose the new flags value before checking notification
1327 	 * suppressions.
1328 	 */
1329 	virtio_mb(vq->weak_barriers);
1330 
1331 	old = vq->packed.next_avail_idx - vq->num_added;
1332 	new = vq->packed.next_avail_idx;
1333 	vq->num_added = 0;
1334 
1335 	snapshot.u32 = *(u32 *)vq->packed.vring.device;
1336 	flags = le16_to_cpu(snapshot.flags);
1337 
1338 	LAST_ADD_TIME_CHECK(vq);
1339 	LAST_ADD_TIME_INVALID(vq);
1340 
1341 	if (flags != VRING_PACKED_EVENT_FLAG_DESC) {
1342 		needs_kick = (flags != VRING_PACKED_EVENT_FLAG_DISABLE);
1343 		goto out;
1344 	}
1345 
1346 	off_wrap = le16_to_cpu(snapshot.off_wrap);
1347 
1348 	wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1349 	event_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1350 	if (wrap_counter != vq->packed.avail_wrap_counter)
1351 		event_idx -= vq->packed.vring.num;
1352 
1353 	needs_kick = vring_need_event(event_idx, new, old);
1354 out:
1355 	END_USE(vq);
1356 	return needs_kick;
1357 }
1358 
1359 static void detach_buf_packed(struct vring_virtqueue *vq,
1360 			      unsigned int id, void **ctx)
1361 {
1362 	struct vring_desc_state_packed *state = NULL;
1363 	struct vring_packed_desc *desc;
1364 	unsigned int i, curr;
1365 
1366 	state = &vq->packed.desc_state[id];
1367 
1368 	/* Clear data ptr. */
1369 	state->data = NULL;
1370 
1371 	vq->packed.desc_extra[state->last].next = vq->free_head;
1372 	vq->free_head = id;
1373 	vq->vq.num_free += state->num;
1374 
1375 	if (unlikely(vq->use_dma_api)) {
1376 		curr = id;
1377 		for (i = 0; i < state->num; i++) {
1378 			vring_unmap_extra_packed(vq,
1379 						 &vq->packed.desc_extra[curr]);
1380 			curr = vq->packed.desc_extra[curr].next;
1381 		}
1382 	}
1383 
1384 	if (vq->indirect) {
1385 		u32 len;
1386 
1387 		/* Free the indirect table, if any, now that it's unmapped. */
1388 		desc = state->indir_desc;
1389 		if (!desc)
1390 			return;
1391 
1392 		if (vq->use_dma_api) {
1393 			len = vq->packed.desc_extra[id].len;
1394 			for (i = 0; i < len / sizeof(struct vring_packed_desc);
1395 					i++)
1396 				vring_unmap_desc_packed(vq, &desc[i]);
1397 		}
1398 		kfree(desc);
1399 		state->indir_desc = NULL;
1400 	} else if (ctx) {
1401 		*ctx = state->indir_desc;
1402 	}
1403 }
1404 
1405 static inline bool is_used_desc_packed(const struct vring_virtqueue *vq,
1406 				       u16 idx, bool used_wrap_counter)
1407 {
1408 	bool avail, used;
1409 	u16 flags;
1410 
1411 	flags = le16_to_cpu(vq->packed.vring.desc[idx].flags);
1412 	avail = !!(flags & (1 << VRING_PACKED_DESC_F_AVAIL));
1413 	used = !!(flags & (1 << VRING_PACKED_DESC_F_USED));
1414 
1415 	return avail == used && used == used_wrap_counter;
1416 }
1417 
1418 static inline bool more_used_packed(const struct vring_virtqueue *vq)
1419 {
1420 	u16 last_used;
1421 	u16 last_used_idx;
1422 	bool used_wrap_counter;
1423 
1424 	last_used_idx = READ_ONCE(vq->last_used_idx);
1425 	last_used = packed_last_used(last_used_idx);
1426 	used_wrap_counter = packed_used_wrap_counter(last_used_idx);
1427 	return is_used_desc_packed(vq, last_used, used_wrap_counter);
1428 }
1429 
1430 static void *virtqueue_get_buf_ctx_packed(struct virtqueue *_vq,
1431 					  unsigned int *len,
1432 					  void **ctx)
1433 {
1434 	struct vring_virtqueue *vq = to_vvq(_vq);
1435 	u16 last_used, id, last_used_idx;
1436 	bool used_wrap_counter;
1437 	void *ret;
1438 
1439 	START_USE(vq);
1440 
1441 	if (unlikely(vq->broken)) {
1442 		END_USE(vq);
1443 		return NULL;
1444 	}
1445 
1446 	if (!more_used_packed(vq)) {
1447 		pr_debug("No more buffers in queue\n");
1448 		END_USE(vq);
1449 		return NULL;
1450 	}
1451 
1452 	/* Only get used elements after they have been exposed by host. */
1453 	virtio_rmb(vq->weak_barriers);
1454 
1455 	last_used_idx = READ_ONCE(vq->last_used_idx);
1456 	used_wrap_counter = packed_used_wrap_counter(last_used_idx);
1457 	last_used = packed_last_used(last_used_idx);
1458 	id = le16_to_cpu(vq->packed.vring.desc[last_used].id);
1459 	*len = le32_to_cpu(vq->packed.vring.desc[last_used].len);
1460 
1461 	if (unlikely(id >= vq->packed.vring.num)) {
1462 		BAD_RING(vq, "id %u out of range\n", id);
1463 		return NULL;
1464 	}
1465 	if (unlikely(!vq->packed.desc_state[id].data)) {
1466 		BAD_RING(vq, "id %u is not a head!\n", id);
1467 		return NULL;
1468 	}
1469 
1470 	/* detach_buf_packed clears data, so grab it now. */
1471 	ret = vq->packed.desc_state[id].data;
1472 	detach_buf_packed(vq, id, ctx);
1473 
1474 	last_used += vq->packed.desc_state[id].num;
1475 	if (unlikely(last_used >= vq->packed.vring.num)) {
1476 		last_used -= vq->packed.vring.num;
1477 		used_wrap_counter ^= 1;
1478 	}
1479 
1480 	last_used = (last_used | (used_wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
1481 	WRITE_ONCE(vq->last_used_idx, last_used);
1482 
1483 	/*
1484 	 * If we expect an interrupt for the next entry, tell host
1485 	 * by writing event index and flush out the write before
1486 	 * the read in the next get_buf call.
1487 	 */
1488 	if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DESC)
1489 		virtio_store_mb(vq->weak_barriers,
1490 				&vq->packed.vring.driver->off_wrap,
1491 				cpu_to_le16(vq->last_used_idx));
1492 
1493 	LAST_ADD_TIME_INVALID(vq);
1494 
1495 	END_USE(vq);
1496 	return ret;
1497 }
1498 
1499 static void virtqueue_disable_cb_packed(struct virtqueue *_vq)
1500 {
1501 	struct vring_virtqueue *vq = to_vvq(_vq);
1502 
1503 	if (vq->packed.event_flags_shadow != VRING_PACKED_EVENT_FLAG_DISABLE) {
1504 		vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1505 		vq->packed.vring.driver->flags =
1506 			cpu_to_le16(vq->packed.event_flags_shadow);
1507 	}
1508 }
1509 
1510 static unsigned int virtqueue_enable_cb_prepare_packed(struct virtqueue *_vq)
1511 {
1512 	struct vring_virtqueue *vq = to_vvq(_vq);
1513 
1514 	START_USE(vq);
1515 
1516 	/*
1517 	 * We optimistically turn back on interrupts, then check if there was
1518 	 * more to do.
1519 	 */
1520 
1521 	if (vq->event) {
1522 		vq->packed.vring.driver->off_wrap =
1523 			cpu_to_le16(vq->last_used_idx);
1524 		/*
1525 		 * We need to update event offset and event wrap
1526 		 * counter first before updating event flags.
1527 		 */
1528 		virtio_wmb(vq->weak_barriers);
1529 	}
1530 
1531 	if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1532 		vq->packed.event_flags_shadow = vq->event ?
1533 				VRING_PACKED_EVENT_FLAG_DESC :
1534 				VRING_PACKED_EVENT_FLAG_ENABLE;
1535 		vq->packed.vring.driver->flags =
1536 				cpu_to_le16(vq->packed.event_flags_shadow);
1537 	}
1538 
1539 	END_USE(vq);
1540 	return vq->last_used_idx;
1541 }
1542 
1543 static bool virtqueue_poll_packed(struct virtqueue *_vq, u16 off_wrap)
1544 {
1545 	struct vring_virtqueue *vq = to_vvq(_vq);
1546 	bool wrap_counter;
1547 	u16 used_idx;
1548 
1549 	wrap_counter = off_wrap >> VRING_PACKED_EVENT_F_WRAP_CTR;
1550 	used_idx = off_wrap & ~(1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1551 
1552 	return is_used_desc_packed(vq, used_idx, wrap_counter);
1553 }
1554 
1555 static bool virtqueue_enable_cb_delayed_packed(struct virtqueue *_vq)
1556 {
1557 	struct vring_virtqueue *vq = to_vvq(_vq);
1558 	u16 used_idx, wrap_counter, last_used_idx;
1559 	u16 bufs;
1560 
1561 	START_USE(vq);
1562 
1563 	/*
1564 	 * We optimistically turn back on interrupts, then check if there was
1565 	 * more to do.
1566 	 */
1567 
1568 	if (vq->event) {
1569 		/* TODO: tune this threshold */
1570 		bufs = (vq->packed.vring.num - vq->vq.num_free) * 3 / 4;
1571 		last_used_idx = READ_ONCE(vq->last_used_idx);
1572 		wrap_counter = packed_used_wrap_counter(last_used_idx);
1573 
1574 		used_idx = packed_last_used(last_used_idx) + bufs;
1575 		if (used_idx >= vq->packed.vring.num) {
1576 			used_idx -= vq->packed.vring.num;
1577 			wrap_counter ^= 1;
1578 		}
1579 
1580 		vq->packed.vring.driver->off_wrap = cpu_to_le16(used_idx |
1581 			(wrap_counter << VRING_PACKED_EVENT_F_WRAP_CTR));
1582 
1583 		/*
1584 		 * We need to update event offset and event wrap
1585 		 * counter first before updating event flags.
1586 		 */
1587 		virtio_wmb(vq->weak_barriers);
1588 	}
1589 
1590 	if (vq->packed.event_flags_shadow == VRING_PACKED_EVENT_FLAG_DISABLE) {
1591 		vq->packed.event_flags_shadow = vq->event ?
1592 				VRING_PACKED_EVENT_FLAG_DESC :
1593 				VRING_PACKED_EVENT_FLAG_ENABLE;
1594 		vq->packed.vring.driver->flags =
1595 				cpu_to_le16(vq->packed.event_flags_shadow);
1596 	}
1597 
1598 	/*
1599 	 * We need to update event suppression structure first
1600 	 * before re-checking for more used buffers.
1601 	 */
1602 	virtio_mb(vq->weak_barriers);
1603 
1604 	last_used_idx = READ_ONCE(vq->last_used_idx);
1605 	wrap_counter = packed_used_wrap_counter(last_used_idx);
1606 	used_idx = packed_last_used(last_used_idx);
1607 	if (is_used_desc_packed(vq, used_idx, wrap_counter)) {
1608 		END_USE(vq);
1609 		return false;
1610 	}
1611 
1612 	END_USE(vq);
1613 	return true;
1614 }
1615 
1616 static void *virtqueue_detach_unused_buf_packed(struct virtqueue *_vq)
1617 {
1618 	struct vring_virtqueue *vq = to_vvq(_vq);
1619 	unsigned int i;
1620 	void *buf;
1621 
1622 	START_USE(vq);
1623 
1624 	for (i = 0; i < vq->packed.vring.num; i++) {
1625 		if (!vq->packed.desc_state[i].data)
1626 			continue;
1627 		/* detach_buf clears data, so grab it now. */
1628 		buf = vq->packed.desc_state[i].data;
1629 		detach_buf_packed(vq, i, NULL);
1630 		END_USE(vq);
1631 		return buf;
1632 	}
1633 	/* That should have freed everything. */
1634 	BUG_ON(vq->vq.num_free != vq->packed.vring.num);
1635 
1636 	END_USE(vq);
1637 	return NULL;
1638 }
1639 
1640 static struct vring_desc_extra *vring_alloc_desc_extra(struct vring_virtqueue *vq,
1641 						       unsigned int num)
1642 {
1643 	struct vring_desc_extra *desc_extra;
1644 	unsigned int i;
1645 
1646 	desc_extra = kmalloc_array(num, sizeof(struct vring_desc_extra),
1647 				   GFP_KERNEL);
1648 	if (!desc_extra)
1649 		return NULL;
1650 
1651 	memset(desc_extra, 0, num * sizeof(struct vring_desc_extra));
1652 
1653 	for (i = 0; i < num - 1; i++)
1654 		desc_extra[i].next = i + 1;
1655 
1656 	return desc_extra;
1657 }
1658 
1659 static struct virtqueue *vring_create_virtqueue_packed(
1660 	unsigned int index,
1661 	unsigned int num,
1662 	unsigned int vring_align,
1663 	struct virtio_device *vdev,
1664 	bool weak_barriers,
1665 	bool may_reduce_num,
1666 	bool context,
1667 	bool (*notify)(struct virtqueue *),
1668 	void (*callback)(struct virtqueue *),
1669 	const char *name)
1670 {
1671 	struct vring_virtqueue *vq;
1672 	struct vring_packed_desc *ring;
1673 	struct vring_packed_desc_event *driver, *device;
1674 	dma_addr_t ring_dma_addr, driver_event_dma_addr, device_event_dma_addr;
1675 	size_t ring_size_in_bytes, event_size_in_bytes;
1676 
1677 	ring_size_in_bytes = num * sizeof(struct vring_packed_desc);
1678 
1679 	ring = vring_alloc_queue(vdev, ring_size_in_bytes,
1680 				 &ring_dma_addr,
1681 				 GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1682 	if (!ring)
1683 		goto err_ring;
1684 
1685 	event_size_in_bytes = sizeof(struct vring_packed_desc_event);
1686 
1687 	driver = vring_alloc_queue(vdev, event_size_in_bytes,
1688 				   &driver_event_dma_addr,
1689 				   GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1690 	if (!driver)
1691 		goto err_driver;
1692 
1693 	device = vring_alloc_queue(vdev, event_size_in_bytes,
1694 				   &device_event_dma_addr,
1695 				   GFP_KERNEL|__GFP_NOWARN|__GFP_ZERO);
1696 	if (!device)
1697 		goto err_device;
1698 
1699 	vq = kmalloc(sizeof(*vq), GFP_KERNEL);
1700 	if (!vq)
1701 		goto err_vq;
1702 
1703 	vq->vq.callback = callback;
1704 	vq->vq.vdev = vdev;
1705 	vq->vq.name = name;
1706 	vq->vq.num_free = num;
1707 	vq->vq.index = index;
1708 	vq->we_own_ring = true;
1709 	vq->notify = notify;
1710 	vq->weak_barriers = weak_barriers;
1711 #ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
1712 	vq->broken = true;
1713 #else
1714 	vq->broken = false;
1715 #endif
1716 	vq->last_used_idx = 0 | (1 << VRING_PACKED_EVENT_F_WRAP_CTR);
1717 	vq->event_triggered = false;
1718 	vq->num_added = 0;
1719 	vq->packed_ring = true;
1720 	vq->use_dma_api = vring_use_dma_api(vdev);
1721 #ifdef DEBUG
1722 	vq->in_use = false;
1723 	vq->last_add_time_valid = false;
1724 #endif
1725 
1726 	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
1727 		!context;
1728 	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
1729 
1730 	if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
1731 		vq->weak_barriers = false;
1732 
1733 	vq->packed.ring_dma_addr = ring_dma_addr;
1734 	vq->packed.driver_event_dma_addr = driver_event_dma_addr;
1735 	vq->packed.device_event_dma_addr = device_event_dma_addr;
1736 
1737 	vq->packed.ring_size_in_bytes = ring_size_in_bytes;
1738 	vq->packed.event_size_in_bytes = event_size_in_bytes;
1739 
1740 	vq->packed.vring.num = num;
1741 	vq->packed.vring.desc = ring;
1742 	vq->packed.vring.driver = driver;
1743 	vq->packed.vring.device = device;
1744 
1745 	vq->packed.next_avail_idx = 0;
1746 	vq->packed.avail_wrap_counter = 1;
1747 	vq->packed.event_flags_shadow = 0;
1748 	vq->packed.avail_used_flags = 1 << VRING_PACKED_DESC_F_AVAIL;
1749 
1750 	vq->packed.desc_state = kmalloc_array(num,
1751 			sizeof(struct vring_desc_state_packed),
1752 			GFP_KERNEL);
1753 	if (!vq->packed.desc_state)
1754 		goto err_desc_state;
1755 
1756 	memset(vq->packed.desc_state, 0,
1757 		num * sizeof(struct vring_desc_state_packed));
1758 
1759 	/* Put everything in free lists. */
1760 	vq->free_head = 0;
1761 
1762 	vq->packed.desc_extra = vring_alloc_desc_extra(vq, num);
1763 	if (!vq->packed.desc_extra)
1764 		goto err_desc_extra;
1765 
1766 	/* No callback?  Tell other side not to bother us. */
1767 	if (!callback) {
1768 		vq->packed.event_flags_shadow = VRING_PACKED_EVENT_FLAG_DISABLE;
1769 		vq->packed.vring.driver->flags =
1770 			cpu_to_le16(vq->packed.event_flags_shadow);
1771 	}
1772 
1773 	spin_lock(&vdev->vqs_list_lock);
1774 	list_add_tail(&vq->vq.list, &vdev->vqs);
1775 	spin_unlock(&vdev->vqs_list_lock);
1776 	return &vq->vq;
1777 
1778 err_desc_extra:
1779 	kfree(vq->packed.desc_state);
1780 err_desc_state:
1781 	kfree(vq);
1782 err_vq:
1783 	vring_free_queue(vdev, event_size_in_bytes, device, device_event_dma_addr);
1784 err_device:
1785 	vring_free_queue(vdev, event_size_in_bytes, driver, driver_event_dma_addr);
1786 err_driver:
1787 	vring_free_queue(vdev, ring_size_in_bytes, ring, ring_dma_addr);
1788 err_ring:
1789 	return NULL;
1790 }
1791 
1792 
1793 /*
1794  * Generic functions and exported symbols.
1795  */
1796 
1797 static inline int virtqueue_add(struct virtqueue *_vq,
1798 				struct scatterlist *sgs[],
1799 				unsigned int total_sg,
1800 				unsigned int out_sgs,
1801 				unsigned int in_sgs,
1802 				void *data,
1803 				void *ctx,
1804 				gfp_t gfp)
1805 {
1806 	struct vring_virtqueue *vq = to_vvq(_vq);
1807 
1808 	return vq->packed_ring ? virtqueue_add_packed(_vq, sgs, total_sg,
1809 					out_sgs, in_sgs, data, ctx, gfp) :
1810 				 virtqueue_add_split(_vq, sgs, total_sg,
1811 					out_sgs, in_sgs, data, ctx, gfp);
1812 }
1813 
1814 /**
1815  * virtqueue_add_sgs - expose buffers to other end
1816  * @_vq: the struct virtqueue we're talking about.
1817  * @sgs: array of terminated scatterlists.
1818  * @out_sgs: the number of scatterlists readable by other side
1819  * @in_sgs: the number of scatterlists which are writable (after readable ones)
1820  * @data: the token identifying the buffer.
1821  * @gfp: how to do memory allocations (if necessary).
1822  *
1823  * Caller must ensure we don't call this with other virtqueue operations
1824  * at the same time (except where noted).
1825  *
1826  * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1827  */
1828 int virtqueue_add_sgs(struct virtqueue *_vq,
1829 		      struct scatterlist *sgs[],
1830 		      unsigned int out_sgs,
1831 		      unsigned int in_sgs,
1832 		      void *data,
1833 		      gfp_t gfp)
1834 {
1835 	unsigned int i, total_sg = 0;
1836 
1837 	/* Count them first. */
1838 	for (i = 0; i < out_sgs + in_sgs; i++) {
1839 		struct scatterlist *sg;
1840 
1841 		for (sg = sgs[i]; sg; sg = sg_next(sg))
1842 			total_sg++;
1843 	}
1844 	return virtqueue_add(_vq, sgs, total_sg, out_sgs, in_sgs,
1845 			     data, NULL, gfp);
1846 }
1847 EXPORT_SYMBOL_GPL(virtqueue_add_sgs);
1848 
1849 /**
1850  * virtqueue_add_outbuf - expose output buffers to other end
1851  * @vq: the struct virtqueue we're talking about.
1852  * @sg: scatterlist (must be well-formed and terminated!)
1853  * @num: the number of entries in @sg readable by other side
1854  * @data: the token identifying the buffer.
1855  * @gfp: how to do memory allocations (if necessary).
1856  *
1857  * Caller must ensure we don't call this with other virtqueue operations
1858  * at the same time (except where noted).
1859  *
1860  * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1861  */
1862 int virtqueue_add_outbuf(struct virtqueue *vq,
1863 			 struct scatterlist *sg, unsigned int num,
1864 			 void *data,
1865 			 gfp_t gfp)
1866 {
1867 	return virtqueue_add(vq, &sg, num, 1, 0, data, NULL, gfp);
1868 }
1869 EXPORT_SYMBOL_GPL(virtqueue_add_outbuf);
1870 
1871 /**
1872  * virtqueue_add_inbuf - expose input buffers to other end
1873  * @vq: the struct virtqueue we're talking about.
1874  * @sg: scatterlist (must be well-formed and terminated!)
1875  * @num: the number of entries in @sg writable by other side
1876  * @data: the token identifying the buffer.
1877  * @gfp: how to do memory allocations (if necessary).
1878  *
1879  * Caller must ensure we don't call this with other virtqueue operations
1880  * at the same time (except where noted).
1881  *
1882  * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1883  */
1884 int virtqueue_add_inbuf(struct virtqueue *vq,
1885 			struct scatterlist *sg, unsigned int num,
1886 			void *data,
1887 			gfp_t gfp)
1888 {
1889 	return virtqueue_add(vq, &sg, num, 0, 1, data, NULL, gfp);
1890 }
1891 EXPORT_SYMBOL_GPL(virtqueue_add_inbuf);
1892 
1893 /**
1894  * virtqueue_add_inbuf_ctx - expose input buffers to other end
1895  * @vq: the struct virtqueue we're talking about.
1896  * @sg: scatterlist (must be well-formed and terminated!)
1897  * @num: the number of entries in @sg writable by other side
1898  * @data: the token identifying the buffer.
1899  * @ctx: extra context for the token
1900  * @gfp: how to do memory allocations (if necessary).
1901  *
1902  * Caller must ensure we don't call this with other virtqueue operations
1903  * at the same time (except where noted).
1904  *
1905  * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO).
1906  */
1907 int virtqueue_add_inbuf_ctx(struct virtqueue *vq,
1908 			struct scatterlist *sg, unsigned int num,
1909 			void *data,
1910 			void *ctx,
1911 			gfp_t gfp)
1912 {
1913 	return virtqueue_add(vq, &sg, num, 0, 1, data, ctx, gfp);
1914 }
1915 EXPORT_SYMBOL_GPL(virtqueue_add_inbuf_ctx);
1916 
1917 /**
1918  * virtqueue_kick_prepare - first half of split virtqueue_kick call.
1919  * @_vq: the struct virtqueue
1920  *
1921  * Instead of virtqueue_kick(), you can do:
1922  *	if (virtqueue_kick_prepare(vq))
1923  *		virtqueue_notify(vq);
1924  *
1925  * This is sometimes useful because the virtqueue_kick_prepare() needs
1926  * to be serialized, but the actual virtqueue_notify() call does not.
1927  */
1928 bool virtqueue_kick_prepare(struct virtqueue *_vq)
1929 {
1930 	struct vring_virtqueue *vq = to_vvq(_vq);
1931 
1932 	return vq->packed_ring ? virtqueue_kick_prepare_packed(_vq) :
1933 				 virtqueue_kick_prepare_split(_vq);
1934 }
1935 EXPORT_SYMBOL_GPL(virtqueue_kick_prepare);
1936 
1937 /**
1938  * virtqueue_notify - second half of split virtqueue_kick call.
1939  * @_vq: the struct virtqueue
1940  *
1941  * This does not need to be serialized.
1942  *
1943  * Returns false if host notify failed or queue is broken, otherwise true.
1944  */
1945 bool virtqueue_notify(struct virtqueue *_vq)
1946 {
1947 	struct vring_virtqueue *vq = to_vvq(_vq);
1948 
1949 	if (unlikely(vq->broken))
1950 		return false;
1951 
1952 	/* Prod other side to tell it about changes. */
1953 	if (!vq->notify(_vq)) {
1954 		vq->broken = true;
1955 		return false;
1956 	}
1957 	return true;
1958 }
1959 EXPORT_SYMBOL_GPL(virtqueue_notify);
1960 
1961 /**
1962  * virtqueue_kick - update after add_buf
1963  * @vq: the struct virtqueue
1964  *
1965  * After one or more virtqueue_add_* calls, invoke this to kick
1966  * the other side.
1967  *
1968  * Caller must ensure we don't call this with other virtqueue
1969  * operations at the same time (except where noted).
1970  *
1971  * Returns false if kick failed, otherwise true.
1972  */
1973 bool virtqueue_kick(struct virtqueue *vq)
1974 {
1975 	if (virtqueue_kick_prepare(vq))
1976 		return virtqueue_notify(vq);
1977 	return true;
1978 }
1979 EXPORT_SYMBOL_GPL(virtqueue_kick);
1980 
1981 /**
1982  * virtqueue_get_buf_ctx - get the next used buffer
1983  * @_vq: the struct virtqueue we're talking about.
1984  * @len: the length written into the buffer
1985  * @ctx: extra context for the token
1986  *
1987  * If the device wrote data into the buffer, @len will be set to the
1988  * amount written.  This means you don't need to clear the buffer
1989  * beforehand to ensure there's no data leakage in the case of short
1990  * writes.
1991  *
1992  * Caller must ensure we don't call this with other virtqueue
1993  * operations at the same time (except where noted).
1994  *
1995  * Returns NULL if there are no used buffers, or the "data" token
1996  * handed to virtqueue_add_*().
1997  */
1998 void *virtqueue_get_buf_ctx(struct virtqueue *_vq, unsigned int *len,
1999 			    void **ctx)
2000 {
2001 	struct vring_virtqueue *vq = to_vvq(_vq);
2002 
2003 	return vq->packed_ring ? virtqueue_get_buf_ctx_packed(_vq, len, ctx) :
2004 				 virtqueue_get_buf_ctx_split(_vq, len, ctx);
2005 }
2006 EXPORT_SYMBOL_GPL(virtqueue_get_buf_ctx);
2007 
2008 void *virtqueue_get_buf(struct virtqueue *_vq, unsigned int *len)
2009 {
2010 	return virtqueue_get_buf_ctx(_vq, len, NULL);
2011 }
2012 EXPORT_SYMBOL_GPL(virtqueue_get_buf);
2013 /**
2014  * virtqueue_disable_cb - disable callbacks
2015  * @_vq: the struct virtqueue we're talking about.
2016  *
2017  * Note that this is not necessarily synchronous, hence unreliable and only
2018  * useful as an optimization.
2019  *
2020  * Unlike other operations, this need not be serialized.
2021  */
2022 void virtqueue_disable_cb(struct virtqueue *_vq)
2023 {
2024 	struct vring_virtqueue *vq = to_vvq(_vq);
2025 
2026 	/* If device triggered an event already it won't trigger one again:
2027 	 * no need to disable.
2028 	 */
2029 	if (vq->event_triggered)
2030 		return;
2031 
2032 	if (vq->packed_ring)
2033 		virtqueue_disable_cb_packed(_vq);
2034 	else
2035 		virtqueue_disable_cb_split(_vq);
2036 }
2037 EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
2038 
2039 /**
2040  * virtqueue_enable_cb_prepare - restart callbacks after disable_cb
2041  * @_vq: the struct virtqueue we're talking about.
2042  *
2043  * This re-enables callbacks; it returns current queue state
2044  * in an opaque unsigned value. This value should be later tested by
2045  * virtqueue_poll, to detect a possible race between the driver checking for
2046  * more work, and enabling callbacks.
2047  *
2048  * Caller must ensure we don't call this with other virtqueue
2049  * operations at the same time (except where noted).
2050  */
2051 unsigned int virtqueue_enable_cb_prepare(struct virtqueue *_vq)
2052 {
2053 	struct vring_virtqueue *vq = to_vvq(_vq);
2054 
2055 	if (vq->event_triggered)
2056 		vq->event_triggered = false;
2057 
2058 	return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :
2059 				 virtqueue_enable_cb_prepare_split(_vq);
2060 }
2061 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
2062 
2063 /**
2064  * virtqueue_poll - query pending used buffers
2065  * @_vq: the struct virtqueue we're talking about.
2066  * @last_used_idx: virtqueue state (from call to virtqueue_enable_cb_prepare).
2067  *
2068  * Returns "true" if there are pending used buffers in the queue.
2069  *
2070  * This does not need to be serialized.
2071  */
2072 bool virtqueue_poll(struct virtqueue *_vq, unsigned int last_used_idx)
2073 {
2074 	struct vring_virtqueue *vq = to_vvq(_vq);
2075 
2076 	if (unlikely(vq->broken))
2077 		return false;
2078 
2079 	virtio_mb(vq->weak_barriers);
2080 	return vq->packed_ring ? virtqueue_poll_packed(_vq, last_used_idx) :
2081 				 virtqueue_poll_split(_vq, last_used_idx);
2082 }
2083 EXPORT_SYMBOL_GPL(virtqueue_poll);
2084 
2085 /**
2086  * virtqueue_enable_cb - restart callbacks after disable_cb.
2087  * @_vq: the struct virtqueue we're talking about.
2088  *
2089  * This re-enables callbacks; it returns "false" if there are pending
2090  * buffers in the queue, to detect a possible race between the driver
2091  * checking for more work, and enabling callbacks.
2092  *
2093  * Caller must ensure we don't call this with other virtqueue
2094  * operations at the same time (except where noted).
2095  */
2096 bool virtqueue_enable_cb(struct virtqueue *_vq)
2097 {
2098 	unsigned int last_used_idx = virtqueue_enable_cb_prepare(_vq);
2099 
2100 	return !virtqueue_poll(_vq, last_used_idx);
2101 }
2102 EXPORT_SYMBOL_GPL(virtqueue_enable_cb);
2103 
2104 /**
2105  * virtqueue_enable_cb_delayed - restart callbacks after disable_cb.
2106  * @_vq: the struct virtqueue we're talking about.
2107  *
2108  * This re-enables callbacks but hints to the other side to delay
2109  * interrupts until most of the available buffers have been processed;
2110  * it returns "false" if there are many pending buffers in the queue,
2111  * to detect a possible race between the driver checking for more work,
2112  * and enabling callbacks.
2113  *
2114  * Caller must ensure we don't call this with other virtqueue
2115  * operations at the same time (except where noted).
2116  */
2117 bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
2118 {
2119 	struct vring_virtqueue *vq = to_vvq(_vq);
2120 
2121 	if (vq->event_triggered)
2122 		vq->event_triggered = false;
2123 
2124 	return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :
2125 				 virtqueue_enable_cb_delayed_split(_vq);
2126 }
2127 EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
2128 
2129 /**
2130  * virtqueue_detach_unused_buf - detach first unused buffer
2131  * @_vq: the struct virtqueue we're talking about.
2132  *
2133  * Returns NULL or the "data" token handed to virtqueue_add_*().
2134  * This is not valid on an active queue; it is useful only for device
2135  * shutdown.
2136  */
2137 void *virtqueue_detach_unused_buf(struct virtqueue *_vq)
2138 {
2139 	struct vring_virtqueue *vq = to_vvq(_vq);
2140 
2141 	return vq->packed_ring ? virtqueue_detach_unused_buf_packed(_vq) :
2142 				 virtqueue_detach_unused_buf_split(_vq);
2143 }
2144 EXPORT_SYMBOL_GPL(virtqueue_detach_unused_buf);
2145 
2146 static inline bool more_used(const struct vring_virtqueue *vq)
2147 {
2148 	return vq->packed_ring ? more_used_packed(vq) : more_used_split(vq);
2149 }
2150 
2151 irqreturn_t vring_interrupt(int irq, void *_vq)
2152 {
2153 	struct vring_virtqueue *vq = to_vvq(_vq);
2154 
2155 	if (!more_used(vq)) {
2156 		pr_debug("virtqueue interrupt with no work for %p\n", vq);
2157 		return IRQ_NONE;
2158 	}
2159 
2160 	if (unlikely(vq->broken)) {
2161 #ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
2162 		dev_warn_once(&vq->vq.vdev->dev,
2163 			      "virtio vring IRQ raised before DRIVER_OK");
2164 		return IRQ_NONE;
2165 #else
2166 		return IRQ_HANDLED;
2167 #endif
2168 	}
2169 
2170 	/* Just a hint for performance: so it's ok that this can be racy! */
2171 	if (vq->event)
2172 		vq->event_triggered = true;
2173 
2174 	pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
2175 	if (vq->vq.callback)
2176 		vq->vq.callback(&vq->vq);
2177 
2178 	return IRQ_HANDLED;
2179 }
2180 EXPORT_SYMBOL_GPL(vring_interrupt);
2181 
2182 /* Only available for split ring */
2183 struct virtqueue *__vring_new_virtqueue(unsigned int index,
2184 					struct vring vring,
2185 					struct virtio_device *vdev,
2186 					bool weak_barriers,
2187 					bool context,
2188 					bool (*notify)(struct virtqueue *),
2189 					void (*callback)(struct virtqueue *),
2190 					const char *name)
2191 {
2192 	struct vring_virtqueue *vq;
2193 
2194 	if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2195 		return NULL;
2196 
2197 	vq = kmalloc(sizeof(*vq), GFP_KERNEL);
2198 	if (!vq)
2199 		return NULL;
2200 
2201 	vq->packed_ring = false;
2202 	vq->vq.callback = callback;
2203 	vq->vq.vdev = vdev;
2204 	vq->vq.name = name;
2205 	vq->vq.num_free = vring.num;
2206 	vq->vq.index = index;
2207 	vq->we_own_ring = false;
2208 	vq->notify = notify;
2209 	vq->weak_barriers = weak_barriers;
2210 #ifdef CONFIG_VIRTIO_HARDEN_NOTIFICATION
2211 	vq->broken = true;
2212 #else
2213 	vq->broken = false;
2214 #endif
2215 	vq->last_used_idx = 0;
2216 	vq->event_triggered = false;
2217 	vq->num_added = 0;
2218 	vq->use_dma_api = vring_use_dma_api(vdev);
2219 #ifdef DEBUG
2220 	vq->in_use = false;
2221 	vq->last_add_time_valid = false;
2222 #endif
2223 
2224 	vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&
2225 		!context;
2226 	vq->event = virtio_has_feature(vdev, VIRTIO_RING_F_EVENT_IDX);
2227 
2228 	if (virtio_has_feature(vdev, VIRTIO_F_ORDER_PLATFORM))
2229 		vq->weak_barriers = false;
2230 
2231 	vq->split.queue_dma_addr = 0;
2232 	vq->split.queue_size_in_bytes = 0;
2233 
2234 	vq->split.vring = vring;
2235 	vq->split.avail_flags_shadow = 0;
2236 	vq->split.avail_idx_shadow = 0;
2237 
2238 	/* No callback?  Tell other side not to bother us. */
2239 	if (!callback) {
2240 		vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
2241 		if (!vq->event)
2242 			vq->split.vring.avail->flags = cpu_to_virtio16(vdev,
2243 					vq->split.avail_flags_shadow);
2244 	}
2245 
2246 	vq->split.desc_state = kmalloc_array(vring.num,
2247 			sizeof(struct vring_desc_state_split), GFP_KERNEL);
2248 	if (!vq->split.desc_state)
2249 		goto err_state;
2250 
2251 	vq->split.desc_extra = vring_alloc_desc_extra(vq, vring.num);
2252 	if (!vq->split.desc_extra)
2253 		goto err_extra;
2254 
2255 	/* Put everything in free lists. */
2256 	vq->free_head = 0;
2257 	memset(vq->split.desc_state, 0, vring.num *
2258 			sizeof(struct vring_desc_state_split));
2259 
2260 	spin_lock(&vdev->vqs_list_lock);
2261 	list_add_tail(&vq->vq.list, &vdev->vqs);
2262 	spin_unlock(&vdev->vqs_list_lock);
2263 	return &vq->vq;
2264 
2265 err_extra:
2266 	kfree(vq->split.desc_state);
2267 err_state:
2268 	kfree(vq);
2269 	return NULL;
2270 }
2271 EXPORT_SYMBOL_GPL(__vring_new_virtqueue);
2272 
2273 struct virtqueue *vring_create_virtqueue(
2274 	unsigned int index,
2275 	unsigned int num,
2276 	unsigned int vring_align,
2277 	struct virtio_device *vdev,
2278 	bool weak_barriers,
2279 	bool may_reduce_num,
2280 	bool context,
2281 	bool (*notify)(struct virtqueue *),
2282 	void (*callback)(struct virtqueue *),
2283 	const char *name)
2284 {
2285 
2286 	if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2287 		return vring_create_virtqueue_packed(index, num, vring_align,
2288 				vdev, weak_barriers, may_reduce_num,
2289 				context, notify, callback, name);
2290 
2291 	return vring_create_virtqueue_split(index, num, vring_align,
2292 			vdev, weak_barriers, may_reduce_num,
2293 			context, notify, callback, name);
2294 }
2295 EXPORT_SYMBOL_GPL(vring_create_virtqueue);
2296 
2297 /* Only available for split ring */
2298 struct virtqueue *vring_new_virtqueue(unsigned int index,
2299 				      unsigned int num,
2300 				      unsigned int vring_align,
2301 				      struct virtio_device *vdev,
2302 				      bool weak_barriers,
2303 				      bool context,
2304 				      void *pages,
2305 				      bool (*notify)(struct virtqueue *vq),
2306 				      void (*callback)(struct virtqueue *vq),
2307 				      const char *name)
2308 {
2309 	struct vring vring;
2310 
2311 	if (virtio_has_feature(vdev, VIRTIO_F_RING_PACKED))
2312 		return NULL;
2313 
2314 	vring_init(&vring, num, pages, vring_align);
2315 	return __vring_new_virtqueue(index, vring, vdev, weak_barriers, context,
2316 				     notify, callback, name);
2317 }
2318 EXPORT_SYMBOL_GPL(vring_new_virtqueue);
2319 
2320 void vring_del_virtqueue(struct virtqueue *_vq)
2321 {
2322 	struct vring_virtqueue *vq = to_vvq(_vq);
2323 
2324 	spin_lock(&vq->vq.vdev->vqs_list_lock);
2325 	list_del(&_vq->list);
2326 	spin_unlock(&vq->vq.vdev->vqs_list_lock);
2327 
2328 	if (vq->we_own_ring) {
2329 		if (vq->packed_ring) {
2330 			vring_free_queue(vq->vq.vdev,
2331 					 vq->packed.ring_size_in_bytes,
2332 					 vq->packed.vring.desc,
2333 					 vq->packed.ring_dma_addr);
2334 
2335 			vring_free_queue(vq->vq.vdev,
2336 					 vq->packed.event_size_in_bytes,
2337 					 vq->packed.vring.driver,
2338 					 vq->packed.driver_event_dma_addr);
2339 
2340 			vring_free_queue(vq->vq.vdev,
2341 					 vq->packed.event_size_in_bytes,
2342 					 vq->packed.vring.device,
2343 					 vq->packed.device_event_dma_addr);
2344 
2345 			kfree(vq->packed.desc_state);
2346 			kfree(vq->packed.desc_extra);
2347 		} else {
2348 			vring_free_queue(vq->vq.vdev,
2349 					 vq->split.queue_size_in_bytes,
2350 					 vq->split.vring.desc,
2351 					 vq->split.queue_dma_addr);
2352 		}
2353 	}
2354 	if (!vq->packed_ring) {
2355 		kfree(vq->split.desc_state);
2356 		kfree(vq->split.desc_extra);
2357 	}
2358 	kfree(vq);
2359 }
2360 EXPORT_SYMBOL_GPL(vring_del_virtqueue);
2361 
2362 /* Manipulates transport-specific feature bits. */
2363 void vring_transport_features(struct virtio_device *vdev)
2364 {
2365 	unsigned int i;
2366 
2367 	for (i = VIRTIO_TRANSPORT_F_START; i < VIRTIO_TRANSPORT_F_END; i++) {
2368 		switch (i) {
2369 		case VIRTIO_RING_F_INDIRECT_DESC:
2370 			break;
2371 		case VIRTIO_RING_F_EVENT_IDX:
2372 			break;
2373 		case VIRTIO_F_VERSION_1:
2374 			break;
2375 		case VIRTIO_F_ACCESS_PLATFORM:
2376 			break;
2377 		case VIRTIO_F_RING_PACKED:
2378 			break;
2379 		case VIRTIO_F_ORDER_PLATFORM:
2380 			break;
2381 		default:
2382 			/* We don't understand this bit. */
2383 			__virtio_clear_bit(vdev, i);
2384 		}
2385 	}
2386 }
2387 EXPORT_SYMBOL_GPL(vring_transport_features);
2388 
2389 /**
2390  * virtqueue_get_vring_size - return the size of the virtqueue's vring
2391  * @_vq: the struct virtqueue containing the vring of interest.
2392  *
2393  * Returns the size of the vring.  This is mainly used for boasting to
2394  * userspace.  Unlike other operations, this need not be serialized.
2395  */
2396 unsigned int virtqueue_get_vring_size(struct virtqueue *_vq)
2397 {
2398 
2399 	struct vring_virtqueue *vq = to_vvq(_vq);
2400 
2401 	return vq->packed_ring ? vq->packed.vring.num : vq->split.vring.num;
2402 }
2403 EXPORT_SYMBOL_GPL(virtqueue_get_vring_size);
2404 
2405 bool virtqueue_is_broken(struct virtqueue *_vq)
2406 {
2407 	struct vring_virtqueue *vq = to_vvq(_vq);
2408 
2409 	return READ_ONCE(vq->broken);
2410 }
2411 EXPORT_SYMBOL_GPL(virtqueue_is_broken);
2412 
2413 /*
2414  * This should prevent the device from being used, allowing drivers to
2415  * recover.  You may need to grab appropriate locks to flush.
2416  */
2417 void virtio_break_device(struct virtio_device *dev)
2418 {
2419 	struct virtqueue *_vq;
2420 
2421 	spin_lock(&dev->vqs_list_lock);
2422 	list_for_each_entry(_vq, &dev->vqs, list) {
2423 		struct vring_virtqueue *vq = to_vvq(_vq);
2424 
2425 		/* Pairs with READ_ONCE() in virtqueue_is_broken(). */
2426 		WRITE_ONCE(vq->broken, true);
2427 	}
2428 	spin_unlock(&dev->vqs_list_lock);
2429 }
2430 EXPORT_SYMBOL_GPL(virtio_break_device);
2431 
2432 /*
2433  * This should allow the device to be used by the driver. You may
2434  * need to grab appropriate locks to flush the write to
2435  * vq->broken. This should only be used in some specific case e.g
2436  * (probing and restoring). This function should only be called by the
2437  * core, not directly by the driver.
2438  */
2439 void __virtio_unbreak_device(struct virtio_device *dev)
2440 {
2441 	struct virtqueue *_vq;
2442 
2443 	spin_lock(&dev->vqs_list_lock);
2444 	list_for_each_entry(_vq, &dev->vqs, list) {
2445 		struct vring_virtqueue *vq = to_vvq(_vq);
2446 
2447 		/* Pairs with READ_ONCE() in virtqueue_is_broken(). */
2448 		WRITE_ONCE(vq->broken, false);
2449 	}
2450 	spin_unlock(&dev->vqs_list_lock);
2451 }
2452 EXPORT_SYMBOL_GPL(__virtio_unbreak_device);
2453 
2454 dma_addr_t virtqueue_get_desc_addr(struct virtqueue *_vq)
2455 {
2456 	struct vring_virtqueue *vq = to_vvq(_vq);
2457 
2458 	BUG_ON(!vq->we_own_ring);
2459 
2460 	if (vq->packed_ring)
2461 		return vq->packed.ring_dma_addr;
2462 
2463 	return vq->split.queue_dma_addr;
2464 }
2465 EXPORT_SYMBOL_GPL(virtqueue_get_desc_addr);
2466 
2467 dma_addr_t virtqueue_get_avail_addr(struct virtqueue *_vq)
2468 {
2469 	struct vring_virtqueue *vq = to_vvq(_vq);
2470 
2471 	BUG_ON(!vq->we_own_ring);
2472 
2473 	if (vq->packed_ring)
2474 		return vq->packed.driver_event_dma_addr;
2475 
2476 	return vq->split.queue_dma_addr +
2477 		((char *)vq->split.vring.avail - (char *)vq->split.vring.desc);
2478 }
2479 EXPORT_SYMBOL_GPL(virtqueue_get_avail_addr);
2480 
2481 dma_addr_t virtqueue_get_used_addr(struct virtqueue *_vq)
2482 {
2483 	struct vring_virtqueue *vq = to_vvq(_vq);
2484 
2485 	BUG_ON(!vq->we_own_ring);
2486 
2487 	if (vq->packed_ring)
2488 		return vq->packed.device_event_dma_addr;
2489 
2490 	return vq->split.queue_dma_addr +
2491 		((char *)vq->split.vring.used - (char *)vq->split.vring.desc);
2492 }
2493 EXPORT_SYMBOL_GPL(virtqueue_get_used_addr);
2494 
2495 /* Only available for split ring */
2496 const struct vring *virtqueue_get_vring(struct virtqueue *vq)
2497 {
2498 	return &to_vvq(vq)->split.vring;
2499 }
2500 EXPORT_SYMBOL_GPL(virtqueue_get_vring);
2501 
2502 MODULE_LICENSE("GPL");
2503