1 /* SPDX-License-Identifier: BSD-3-Clause */ 2 /* 3 * Copyright (C) 2018, Tuomas Tynkkynen <tuomas.tynkkynen@iki.fi> 4 * Copyright (C) 2018, Bin Meng <bmeng.cn@gmail.com> 5 * 6 * From Linux kernel include/uapi/linux/virtio_ring.h 7 */ 8 9 #ifndef _LINUX_VIRTIO_RING_H 10 #define _LINUX_VIRTIO_RING_H 11 12 #include <virtio_types.h> 13 14 /* This marks a buffer as continuing via the next field */ 15 #define VRING_DESC_F_NEXT 1 16 /* This marks a buffer as write-only (otherwise read-only) */ 17 #define VRING_DESC_F_WRITE 2 18 /* This means the buffer contains a list of buffer descriptors */ 19 #define VRING_DESC_F_INDIRECT 4 20 21 /* 22 * The Host uses this in used->flags to advise the Guest: don't kick me when 23 * you add a buffer. It's unreliable, so it's simply an optimization. Guest 24 * will still kick if it's out of buffers. 25 */ 26 #define VRING_USED_F_NO_NOTIFY 1 27 28 /* 29 * The Guest uses this in avail->flags to advise the Host: don't interrupt me 30 * when you consume a buffer. It's unreliable, so it's simply an optimization. 31 */ 32 #define VRING_AVAIL_F_NO_INTERRUPT 1 33 34 /* We support indirect buffer descriptors */ 35 #define VIRTIO_RING_F_INDIRECT_DESC 28 36 37 /* 38 * The Guest publishes the used index for which it expects an interrupt 39 * at the end of the avail ring. Host should ignore the avail->flags field. 40 * 41 * The Host publishes the avail index for which it expects a kick 42 * at the end of the used ring. Guest should ignore the used->flags field. 43 */ 44 #define VIRTIO_RING_F_EVENT_IDX 29 45 46 /* Virtio ring descriptors: 16 bytes. These can chain together via "next". */ 47 struct vring_desc { 48 /* Address (guest-physical) */ 49 __virtio64 addr; 50 /* Length */ 51 __virtio32 len; 52 /* The flags as indicated above */ 53 __virtio16 flags; 54 /* We chain unused descriptors via this, too */ 55 __virtio16 next; 56 }; 57 58 struct vring_avail { 59 __virtio16 flags; 60 __virtio16 idx; 61 __virtio16 ring[]; 62 }; 63 64 struct vring_used_elem { 65 /* Index of start of used descriptor chain */ 66 __virtio32 id; 67 /* Total length of the descriptor chain which was used (written to) */ 68 __virtio32 len; 69 }; 70 71 struct vring_used { 72 __virtio16 flags; 73 __virtio16 idx; 74 struct vring_used_elem ring[]; 75 }; 76 77 struct vring { 78 unsigned int num; 79 struct vring_desc *desc; 80 struct vring_avail *avail; 81 struct vring_used *used; 82 }; 83 84 /** 85 * virtqueue - a queue to register buffers for sending or receiving. 86 * 87 * @list: the chain of virtqueues for this device 88 * @vdev: the virtio device this queue was created for 89 * @index: the zero-based ordinal number for this queue 90 * @num_free: number of elements we expect to be able to fit 91 * @vring: actual memory layout for this queue 92 * @event: host publishes avail event idx 93 * @free_head: head of free buffer list 94 * @num_added: number we've added since last sync 95 * @last_used_idx: last used index we've seen 96 * @avail_flags_shadow: last written value to avail->flags 97 * @avail_idx_shadow: last written value to avail->idx in guest byte order 98 */ 99 struct virtqueue { 100 struct list_head list; 101 struct udevice *vdev; 102 unsigned int index; 103 unsigned int num_free; 104 struct vring vring; 105 bool event; 106 unsigned int free_head; 107 unsigned int num_added; 108 u16 last_used_idx; 109 u16 avail_flags_shadow; 110 u16 avail_idx_shadow; 111 }; 112 113 /* 114 * Alignment requirements for vring elements. 115 * When using pre-virtio 1.0 layout, these fall out naturally. 116 */ 117 #define VRING_AVAIL_ALIGN_SIZE 2 118 #define VRING_USED_ALIGN_SIZE 4 119 #define VRING_DESC_ALIGN_SIZE 16 120 121 /* 122 * We publish the used event index at the end of the available ring, 123 * and vice versa. They are at the end for backwards compatibility. 124 */ 125 #define vring_used_event(vr) ((vr)->avail->ring[(vr)->num]) 126 #define vring_avail_event(vr) (*(__virtio16 *)&(vr)->used->ring[(vr)->num]) 127 128 static inline void vring_init(struct vring *vr, unsigned int num, void *p, 129 unsigned long align) 130 { 131 vr->num = num; 132 vr->desc = p; 133 vr->avail = p + num * sizeof(struct vring_desc); 134 vr->used = (void *)(((uintptr_t)&vr->avail->ring[num] + 135 sizeof(__virtio16) + align - 1) & ~(align - 1)); 136 } 137 138 static inline unsigned int vring_size(unsigned int num, unsigned long align) 139 { 140 return ((sizeof(struct vring_desc) * num + 141 sizeof(__virtio16) * (3 + num) + align - 1) & ~(align - 1)) + 142 sizeof(__virtio16) * 3 + sizeof(struct vring_used_elem) * num; 143 } 144 145 /* 146 * The following is used with USED_EVENT_IDX and AVAIL_EVENT_IDX. 147 * Assuming a given event_idx value from the other side, if we have just 148 * incremented index from old to new_idx, should we trigger an event? 149 */ 150 static inline int vring_need_event(__u16 event_idx, __u16 new_idx, __u16 old) 151 { 152 /* 153 * Note: Xen has similar logic for notification hold-off 154 * in include/xen/interface/io/ring.h with req_event and req_prod 155 * corresponding to event_idx + 1 and new_idx respectively. 156 * Note also that req_event and req_prod in Xen start at 1, 157 * event indexes in virtio start at 0. 158 */ 159 return (__u16)(new_idx - event_idx - 1) < (__u16)(new_idx - old); 160 } 161 162 struct virtio_sg; 163 164 /** 165 * virtqueue_add - expose buffers to other end 166 * 167 * @vq: the struct virtqueue we're talking about 168 * @sgs: array of terminated scatterlists 169 * @out_sgs: the number of scatterlists readable by other side 170 * @in_sgs: the number of scatterlists which are writable 171 * (after readable ones) 172 * 173 * Caller must ensure we don't call this with other virtqueue operations 174 * at the same time (except where noted). 175 * 176 * Returns zero or a negative error (ie. ENOSPC, ENOMEM, EIO). 177 */ 178 int virtqueue_add(struct virtqueue *vq, struct virtio_sg *sgs[], 179 unsigned int out_sgs, unsigned int in_sgs); 180 181 /** 182 * virtqueue_kick - update after add_buf 183 * 184 * @vq: the struct virtqueue 185 * 186 * After one or more virtqueue_add() calls, invoke this to kick 187 * the other side. 188 * 189 * Caller must ensure we don't call this with other virtqueue 190 * operations at the same time (except where noted). 191 */ 192 void virtqueue_kick(struct virtqueue *vq); 193 194 /** 195 * virtqueue_get_buf - get the next used buffer 196 * 197 * @vq: the struct virtqueue we're talking about 198 * @len: the length written into the buffer 199 * 200 * If the device wrote data into the buffer, @len will be set to the 201 * amount written. This means you don't need to clear the buffer 202 * beforehand to ensure there's no data leakage in the case of short 203 * writes. 204 * 205 * Caller must ensure we don't call this with other virtqueue 206 * operations at the same time (except where noted). 207 * 208 * Returns NULL if there are no used buffers, or the memory buffer 209 * handed to virtqueue_add_*(). 210 */ 211 void *virtqueue_get_buf(struct virtqueue *vq, unsigned int *len); 212 213 /** 214 * vring_create_virtqueue - create a virtqueue for a virtio device 215 * 216 * @index: the index of the queue 217 * @num: number of elements of the queue 218 * @vring_align:the alignment requirement of the descriptor ring 219 * @udev: the virtio transport udevice 220 * @return: the virtqueue pointer or NULL if failed 221 * 222 * This creates a virtqueue and allocates the descriptor ring for a virtio 223 * device. The caller should query virtqueue_get_ring_size() to learn the 224 * actual size of the ring. 225 * 226 * This API is supposed to be called by the virtio transport driver in the 227 * virtio find_vqs() uclass method. 228 */ 229 struct virtqueue *vring_create_virtqueue(unsigned int index, unsigned int num, 230 unsigned int vring_align, 231 struct udevice *udev); 232 233 /** 234 * vring_del_virtqueue - destroy a virtqueue 235 * 236 * @vq: the struct virtqueue we're talking about 237 * 238 * This destroys a virtqueue. If created with vring_create_virtqueue(), 239 * this also frees the descriptor ring. 240 * 241 * This API is supposed to be called by the virtio transport driver in the 242 * virtio del_vqs() uclass method. 243 */ 244 void vring_del_virtqueue(struct virtqueue *vq); 245 246 /** 247 * virtqueue_get_vring_size - get the size of the virtqueue's vring 248 * 249 * @vq: the struct virtqueue containing the vring of interest 250 * @return: the size of the vring in a virtqueue. 251 */ 252 unsigned int virtqueue_get_vring_size(struct virtqueue *vq); 253 254 /** 255 * virtqueue_get_desc_addr - get the vring descriptor table address 256 * 257 * @vq: the struct virtqueue containing the vring of interest 258 * @return: the descriptor table address of the vring in a virtqueue. 259 */ 260 ulong virtqueue_get_desc_addr(struct virtqueue *vq); 261 262 /** 263 * virtqueue_get_avail_addr - get the vring available ring address 264 * 265 * @vq: the struct virtqueue containing the vring of interest 266 * @return: the available ring address of the vring in a virtqueue. 267 */ 268 ulong virtqueue_get_avail_addr(struct virtqueue *vq); 269 270 /** 271 * virtqueue_get_used_addr - get the vring used ring address 272 * 273 * @vq: the struct virtqueue containing the vring of interest 274 * @return: the used ring address of the vring in a virtqueue. 275 */ 276 ulong virtqueue_get_used_addr(struct virtqueue *vq); 277 278 /** 279 * virtqueue_poll - query pending used buffers 280 * 281 * @vq: the struct virtqueue we're talking about 282 * @last_used_idx: virtqueue last used index 283 * 284 * Returns "true" if there are pending used buffers in the queue. 285 */ 286 bool virtqueue_poll(struct virtqueue *vq, u16 last_used_idx); 287 288 /** 289 * virtqueue_dump - dump the virtqueue for debugging 290 * 291 * @vq: the struct virtqueue we're talking about 292 * 293 * Caller must ensure we don't call this with other virtqueue operations 294 * at the same time (except where noted). 295 */ 296 void virtqueue_dump(struct virtqueue *vq); 297 298 /* 299 * Barriers in virtio are tricky. Since we are not in a hyperviosr/guest 300 * scenario, having these as nops is enough to work as expected. 301 */ 302 303 static inline void virtio_mb(void) 304 { 305 } 306 307 static inline void virtio_rmb(void) 308 { 309 } 310 311 static inline void virtio_wmb(void) 312 { 313 } 314 315 static inline void virtio_store_mb(__virtio16 *p, __virtio16 v) 316 { 317 WRITE_ONCE(*p, v); 318 } 319 320 #endif /* _LINUX_VIRTIO_RING_H */ 321