virtio_ring.c (31c11db6bd93b0c051d2c835da4fa9bba636cfdb) virtio_ring.c (8d622d21d24803408b256d96463eac4574dcf067)
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* Virtio ring implementation.
3 *
4 * Copyright 2007 Rusty Russell IBM Corporation
5 */
6#include <linux/virtio.h>
7#include <linux/virtio_ring.h>
8#include <linux/virtio_config.h>

--- 99 unchanged lines hidden (view full) ---

108 /* Head of free buffer list. */
109 unsigned int free_head;
110 /* Number we've added since last sync. */
111 unsigned int num_added;
112
113 /* Last used index we've seen. */
114 u16 last_used_idx;
115
1// SPDX-License-Identifier: GPL-2.0-or-later
2/* Virtio ring implementation.
3 *
4 * Copyright 2007 Rusty Russell IBM Corporation
5 */
6#include <linux/virtio.h>
7#include <linux/virtio_ring.h>
8#include <linux/virtio_config.h>

--- 99 unchanged lines hidden (view full) ---

108 /* Head of free buffer list. */
109 unsigned int free_head;
110 /* Number we've added since last sync. */
111 unsigned int num_added;
112
113 /* Last used index we've seen. */
114 u16 last_used_idx;
115
116 /* Hint for event idx: already triggered no need to disable. */
117 bool event_triggered;
118
116 union {
117 /* Available for split ring */
118 struct {
119 /* Actual memory layout for this queue. */
120 struct vring vring;
121
122 /* Last written value to avail->flags */
123 u16 avail_flags_shadow;

--- 610 unchanged lines hidden (view full) ---

734}
735
736static void virtqueue_disable_cb_split(struct virtqueue *_vq)
737{
738 struct vring_virtqueue *vq = to_vvq(_vq);
739
740 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
741 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
119 union {
120 /* Available for split ring */
121 struct {
122 /* Actual memory layout for this queue. */
123 struct vring vring;
124
125 /* Last written value to avail->flags */
126 u16 avail_flags_shadow;

--- 610 unchanged lines hidden (view full) ---

737}
738
739static void virtqueue_disable_cb_split(struct virtqueue *_vq)
740{
741 struct vring_virtqueue *vq = to_vvq(_vq);
742
743 if (!(vq->split.avail_flags_shadow & VRING_AVAIL_F_NO_INTERRUPT)) {
744 vq->split.avail_flags_shadow |= VRING_AVAIL_F_NO_INTERRUPT;
742 if (!vq->event)
745 if (vq->event)
746 /* TODO: this is a hack. Figure out a cleaner value to write. */
747 vring_used_event(&vq->split.vring) = 0x0;
748 else
743 vq->split.vring.avail->flags =
744 cpu_to_virtio16(_vq->vdev,
745 vq->split.avail_flags_shadow);
746 }
747}
748
749static unsigned virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
750{

--- 849 unchanged lines hidden (view full) ---

1600 vq->vq.name = name;
1601 vq->vq.num_free = num;
1602 vq->vq.index = index;
1603 vq->we_own_ring = true;
1604 vq->notify = notify;
1605 vq->weak_barriers = weak_barriers;
1606 vq->broken = false;
1607 vq->last_used_idx = 0;
749 vq->split.vring.avail->flags =
750 cpu_to_virtio16(_vq->vdev,
751 vq->split.avail_flags_shadow);
752 }
753}
754
755static unsigned virtqueue_enable_cb_prepare_split(struct virtqueue *_vq)
756{

--- 849 unchanged lines hidden (view full) ---

1606 vq->vq.name = name;
1607 vq->vq.num_free = num;
1608 vq->vq.index = index;
1609 vq->we_own_ring = true;
1610 vq->notify = notify;
1611 vq->weak_barriers = weak_barriers;
1612 vq->broken = false;
1613 vq->last_used_idx = 0;
1614 vq->event_triggered = false;
1608 vq->num_added = 0;
1609 vq->packed_ring = true;
1610 vq->use_dma_api = vring_use_dma_api(vdev);
1611#ifdef DEBUG
1612 vq->in_use = false;
1613 vq->last_add_time_valid = false;
1614#endif
1615

--- 298 unchanged lines hidden (view full) ---

1914 * useful as an optimization.
1915 *
1916 * Unlike other operations, this need not be serialized.
1917 */
1918void virtqueue_disable_cb(struct virtqueue *_vq)
1919{
1920 struct vring_virtqueue *vq = to_vvq(_vq);
1921
1615 vq->num_added = 0;
1616 vq->packed_ring = true;
1617 vq->use_dma_api = vring_use_dma_api(vdev);
1618#ifdef DEBUG
1619 vq->in_use = false;
1620 vq->last_add_time_valid = false;
1621#endif
1622

--- 298 unchanged lines hidden (view full) ---

1921 * useful as an optimization.
1922 *
1923 * Unlike other operations, this need not be serialized.
1924 */
1925void virtqueue_disable_cb(struct virtqueue *_vq)
1926{
1927 struct vring_virtqueue *vq = to_vvq(_vq);
1928
1929 /* If device triggered an event already it won't trigger one again:
1930 * no need to disable.
1931 */
1932 if (vq->event_triggered)
1933 return;
1934
1922 if (vq->packed_ring)
1923 virtqueue_disable_cb_packed(_vq);
1924 else
1925 virtqueue_disable_cb_split(_vq);
1926}
1927EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
1928
1929/**

--- 7 unchanged lines hidden (view full) ---

1937 *
1938 * Caller must ensure we don't call this with other virtqueue
1939 * operations at the same time (except where noted).
1940 */
1941unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq)
1942{
1943 struct vring_virtqueue *vq = to_vvq(_vq);
1944
1935 if (vq->packed_ring)
1936 virtqueue_disable_cb_packed(_vq);
1937 else
1938 virtqueue_disable_cb_split(_vq);
1939}
1940EXPORT_SYMBOL_GPL(virtqueue_disable_cb);
1941
1942/**

--- 7 unchanged lines hidden (view full) ---

1950 *
1951 * Caller must ensure we don't call this with other virtqueue
1952 * operations at the same time (except where noted).
1953 */
1954unsigned virtqueue_enable_cb_prepare(struct virtqueue *_vq)
1955{
1956 struct vring_virtqueue *vq = to_vvq(_vq);
1957
1958 if (vq->event_triggered)
1959 vq->event_triggered = false;
1960
1945 return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :
1946 virtqueue_enable_cb_prepare_split(_vq);
1947}
1948EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
1949
1950/**
1951 * virtqueue_poll - query pending used buffers
1952 * @_vq: the struct virtqueue we're talking about.

--- 47 unchanged lines hidden (view full) ---

2000 *
2001 * Caller must ensure we don't call this with other virtqueue
2002 * operations at the same time (except where noted).
2003 */
2004bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
2005{
2006 struct vring_virtqueue *vq = to_vvq(_vq);
2007
1961 return vq->packed_ring ? virtqueue_enable_cb_prepare_packed(_vq) :
1962 virtqueue_enable_cb_prepare_split(_vq);
1963}
1964EXPORT_SYMBOL_GPL(virtqueue_enable_cb_prepare);
1965
1966/**
1967 * virtqueue_poll - query pending used buffers
1968 * @_vq: the struct virtqueue we're talking about.

--- 47 unchanged lines hidden (view full) ---

2016 *
2017 * Caller must ensure we don't call this with other virtqueue
2018 * operations at the same time (except where noted).
2019 */
2020bool virtqueue_enable_cb_delayed(struct virtqueue *_vq)
2021{
2022 struct vring_virtqueue *vq = to_vvq(_vq);
2023
2024 if (vq->event_triggered)
2025 vq->event_triggered = false;
2026
2008 return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :
2009 virtqueue_enable_cb_delayed_split(_vq);
2010}
2011EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
2012
2013/**
2014 * virtqueue_detach_unused_buf - detach first unused buffer
2015 * @_vq: the struct virtqueue we're talking about.

--- 23 unchanged lines hidden (view full) ---

2039 if (!more_used(vq)) {
2040 pr_debug("virtqueue interrupt with no work for %p\n", vq);
2041 return IRQ_NONE;
2042 }
2043
2044 if (unlikely(vq->broken))
2045 return IRQ_HANDLED;
2046
2027 return vq->packed_ring ? virtqueue_enable_cb_delayed_packed(_vq) :
2028 virtqueue_enable_cb_delayed_split(_vq);
2029}
2030EXPORT_SYMBOL_GPL(virtqueue_enable_cb_delayed);
2031
2032/**
2033 * virtqueue_detach_unused_buf - detach first unused buffer
2034 * @_vq: the struct virtqueue we're talking about.

--- 23 unchanged lines hidden (view full) ---

2058 if (!more_used(vq)) {
2059 pr_debug("virtqueue interrupt with no work for %p\n", vq);
2060 return IRQ_NONE;
2061 }
2062
2063 if (unlikely(vq->broken))
2064 return IRQ_HANDLED;
2065
2066 /* Just a hint for performance: so it's ok that this can be racy! */
2067 if (vq->event)
2068 vq->event_triggered = true;
2069
2047 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
2048 if (vq->vq.callback)
2049 vq->vq.callback(&vq->vq);
2050
2051 return IRQ_HANDLED;
2052}
2053EXPORT_SYMBOL_GPL(vring_interrupt);
2054

--- 23 unchanged lines hidden (view full) ---

2078 vq->vq.name = name;
2079 vq->vq.num_free = vring.num;
2080 vq->vq.index = index;
2081 vq->we_own_ring = false;
2082 vq->notify = notify;
2083 vq->weak_barriers = weak_barriers;
2084 vq->broken = false;
2085 vq->last_used_idx = 0;
2070 pr_debug("virtqueue callback for %p (%p)\n", vq, vq->vq.callback);
2071 if (vq->vq.callback)
2072 vq->vq.callback(&vq->vq);
2073
2074 return IRQ_HANDLED;
2075}
2076EXPORT_SYMBOL_GPL(vring_interrupt);
2077

--- 23 unchanged lines hidden (view full) ---

2101 vq->vq.name = name;
2102 vq->vq.num_free = vring.num;
2103 vq->vq.index = index;
2104 vq->we_own_ring = false;
2105 vq->notify = notify;
2106 vq->weak_barriers = weak_barriers;
2107 vq->broken = false;
2108 vq->last_used_idx = 0;
2109 vq->event_triggered = false;
2086 vq->num_added = 0;
2087 vq->use_dma_api = vring_use_dma_api(vdev);
2088#ifdef DEBUG
2089 vq->in_use = false;
2090 vq->last_add_time_valid = false;
2091#endif
2092
2093 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&

--- 239 unchanged lines hidden --- 		2110 vq->num_added = 0;
2111 vq->use_dma_api = vring_use_dma_api(vdev);
2112#ifdef DEBUG
2113 vq->in_use = false;
2114 vq->last_add_time_valid = false;
2115#endif
2116
2117 vq->indirect = virtio_has_feature(vdev, VIRTIO_RING_F_INDIRECT_DESC) &&

--- 239 unchanged lines hidden ---