1f87d0fbbSRusty Russell /* 2f87d0fbbSRusty Russell * Helpers for the host side of a virtio ring. 3f87d0fbbSRusty Russell * 4f87d0fbbSRusty Russell * Since these may be in userspace, we use (inline) accessors. 5f87d0fbbSRusty Russell */ 69d1b972fSMark Rutland #include <linux/compiler.h> 7f558a845SDave Jones #include <linux/module.h> 8f87d0fbbSRusty Russell #include <linux/vringh.h> 9f87d0fbbSRusty Russell #include <linux/virtio_ring.h> 10f87d0fbbSRusty Russell #include <linux/kernel.h> 11f87d0fbbSRusty Russell #include <linux/ratelimit.h> 12f87d0fbbSRusty Russell #include <linux/uaccess.h> 13f87d0fbbSRusty Russell #include <linux/slab.h> 14f87d0fbbSRusty Russell #include <linux/export.h> 15b9f7ac8cSMichael S. Tsirkin #include <uapi/linux/virtio_config.h> 16f87d0fbbSRusty Russell 17f87d0fbbSRusty Russell static __printf(1,2) __cold void vringh_bad(const char *fmt, ...) 18f87d0fbbSRusty Russell { 19f87d0fbbSRusty Russell static DEFINE_RATELIMIT_STATE(vringh_rs, 20f87d0fbbSRusty Russell DEFAULT_RATELIMIT_INTERVAL, 21f87d0fbbSRusty Russell DEFAULT_RATELIMIT_BURST); 22f87d0fbbSRusty Russell if (__ratelimit(&vringh_rs)) { 23f87d0fbbSRusty Russell va_list ap; 24f87d0fbbSRusty Russell va_start(ap, fmt); 25f87d0fbbSRusty Russell printk(KERN_NOTICE "vringh:"); 26f87d0fbbSRusty Russell vprintk(fmt, ap); 27f87d0fbbSRusty Russell va_end(ap); 28f87d0fbbSRusty Russell } 29f87d0fbbSRusty Russell } 30f87d0fbbSRusty Russell 31f87d0fbbSRusty Russell /* Returns vring->num if empty, -ve on error. */ 32f87d0fbbSRusty Russell static inline int __vringh_get_head(const struct vringh *vrh, 33b9f7ac8cSMichael S. Tsirkin int (*getu16)(const struct vringh *vrh, 34b9f7ac8cSMichael S. Tsirkin u16 *val, const __virtio16 *p), 35f87d0fbbSRusty Russell u16 *last_avail_idx) 36f87d0fbbSRusty Russell { 37f87d0fbbSRusty Russell u16 avail_idx, i, head; 38f87d0fbbSRusty Russell int err; 39f87d0fbbSRusty Russell 40b9f7ac8cSMichael S. Tsirkin err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx); 41f87d0fbbSRusty Russell if (err) { 42f87d0fbbSRusty Russell vringh_bad("Failed to access avail idx at %p", 43f87d0fbbSRusty Russell &vrh->vring.avail->idx); 44f87d0fbbSRusty Russell return err; 45f87d0fbbSRusty Russell } 46f87d0fbbSRusty Russell 47f87d0fbbSRusty Russell if (*last_avail_idx == avail_idx) 48f87d0fbbSRusty Russell return vrh->vring.num; 49f87d0fbbSRusty Russell 50f87d0fbbSRusty Russell /* Only get avail ring entries after they have been exposed by guest. */ 51f87d0fbbSRusty Russell virtio_rmb(vrh->weak_barriers); 52f87d0fbbSRusty Russell 53f87d0fbbSRusty Russell i = *last_avail_idx & (vrh->vring.num - 1); 54f87d0fbbSRusty Russell 55b9f7ac8cSMichael S. Tsirkin err = getu16(vrh, &head, &vrh->vring.avail->ring[i]); 56f87d0fbbSRusty Russell if (err) { 57f87d0fbbSRusty Russell vringh_bad("Failed to read head: idx %d address %p", 58f87d0fbbSRusty Russell *last_avail_idx, &vrh->vring.avail->ring[i]); 59f87d0fbbSRusty Russell return err; 60f87d0fbbSRusty Russell } 61f87d0fbbSRusty Russell 62f87d0fbbSRusty Russell if (head >= vrh->vring.num) { 63f87d0fbbSRusty Russell vringh_bad("Guest says index %u > %u is available", 64f87d0fbbSRusty Russell head, vrh->vring.num); 65f87d0fbbSRusty Russell return -EINVAL; 66f87d0fbbSRusty Russell } 67f87d0fbbSRusty Russell 68f87d0fbbSRusty Russell (*last_avail_idx)++; 69f87d0fbbSRusty Russell return head; 70f87d0fbbSRusty Russell } 71f87d0fbbSRusty Russell 72f87d0fbbSRusty Russell /* Copy some bytes to/from the iovec. Returns num copied. */ 73f87d0fbbSRusty Russell static inline ssize_t vringh_iov_xfer(struct vringh_kiov *iov, 74f87d0fbbSRusty Russell void *ptr, size_t len, 75f87d0fbbSRusty Russell int (*xfer)(void *addr, void *ptr, 76f87d0fbbSRusty Russell size_t len)) 77f87d0fbbSRusty Russell { 78f87d0fbbSRusty Russell int err, done = 0; 79f87d0fbbSRusty Russell 80f87d0fbbSRusty Russell while (len && iov->i < iov->used) { 81f87d0fbbSRusty Russell size_t partlen; 82f87d0fbbSRusty Russell 83f87d0fbbSRusty Russell partlen = min(iov->iov[iov->i].iov_len, len); 84f87d0fbbSRusty Russell err = xfer(iov->iov[iov->i].iov_base, ptr, partlen); 85f87d0fbbSRusty Russell if (err) 86f87d0fbbSRusty Russell return err; 87f87d0fbbSRusty Russell done += partlen; 88f87d0fbbSRusty Russell len -= partlen; 89f87d0fbbSRusty Russell ptr += partlen; 90f87d0fbbSRusty Russell iov->consumed += partlen; 91f87d0fbbSRusty Russell iov->iov[iov->i].iov_len -= partlen; 92f87d0fbbSRusty Russell iov->iov[iov->i].iov_base += partlen; 93f87d0fbbSRusty Russell 94f87d0fbbSRusty Russell if (!iov->iov[iov->i].iov_len) { 95f87d0fbbSRusty Russell /* Fix up old iov element then increment. */ 96f87d0fbbSRusty Russell iov->iov[iov->i].iov_len = iov->consumed; 97f87d0fbbSRusty Russell iov->iov[iov->i].iov_base -= iov->consumed; 98f87d0fbbSRusty Russell 99f87d0fbbSRusty Russell iov->consumed = 0; 100f87d0fbbSRusty Russell iov->i++; 101f87d0fbbSRusty Russell } 102f87d0fbbSRusty Russell } 103f87d0fbbSRusty Russell return done; 104f87d0fbbSRusty Russell } 105f87d0fbbSRusty Russell 106f87d0fbbSRusty Russell /* May reduce *len if range is shorter. */ 107f87d0fbbSRusty Russell static inline bool range_check(struct vringh *vrh, u64 addr, size_t *len, 108f87d0fbbSRusty Russell struct vringh_range *range, 109f87d0fbbSRusty Russell bool (*getrange)(struct vringh *, 110f87d0fbbSRusty Russell u64, struct vringh_range *)) 111f87d0fbbSRusty Russell { 112f87d0fbbSRusty Russell if (addr < range->start || addr > range->end_incl) { 113f87d0fbbSRusty Russell if (!getrange(vrh, addr, range)) 114f87d0fbbSRusty Russell return false; 115f87d0fbbSRusty Russell } 116f87d0fbbSRusty Russell BUG_ON(addr < range->start || addr > range->end_incl); 117f87d0fbbSRusty Russell 118f87d0fbbSRusty Russell /* To end of memory? */ 119f87d0fbbSRusty Russell if (unlikely(addr + *len == 0)) { 120f87d0fbbSRusty Russell if (range->end_incl == -1ULL) 121f87d0fbbSRusty Russell return true; 122f87d0fbbSRusty Russell goto truncate; 123f87d0fbbSRusty Russell } 124f87d0fbbSRusty Russell 125f87d0fbbSRusty Russell /* Otherwise, don't wrap. */ 126f87d0fbbSRusty Russell if (addr + *len < addr) { 127f87d0fbbSRusty Russell vringh_bad("Wrapping descriptor %zu@0x%llx", 128f87d0fbbSRusty Russell *len, (unsigned long long)addr); 129f87d0fbbSRusty Russell return false; 130f87d0fbbSRusty Russell } 131f87d0fbbSRusty Russell 132f87d0fbbSRusty Russell if (unlikely(addr + *len - 1 > range->end_incl)) 133f87d0fbbSRusty Russell goto truncate; 134f87d0fbbSRusty Russell return true; 135f87d0fbbSRusty Russell 136f87d0fbbSRusty Russell truncate: 137f87d0fbbSRusty Russell *len = range->end_incl + 1 - addr; 138f87d0fbbSRusty Russell return true; 139f87d0fbbSRusty Russell } 140f87d0fbbSRusty Russell 141f87d0fbbSRusty Russell static inline bool no_range_check(struct vringh *vrh, u64 addr, size_t *len, 142f87d0fbbSRusty Russell struct vringh_range *range, 143f87d0fbbSRusty Russell bool (*getrange)(struct vringh *, 144f87d0fbbSRusty Russell u64, struct vringh_range *)) 145f87d0fbbSRusty Russell { 146f87d0fbbSRusty Russell return true; 147f87d0fbbSRusty Russell } 148f87d0fbbSRusty Russell 149f87d0fbbSRusty Russell /* No reason for this code to be inline. */ 150b9f7ac8cSMichael S. Tsirkin static int move_to_indirect(const struct vringh *vrh, 151b9f7ac8cSMichael S. Tsirkin int *up_next, u16 *i, void *addr, 152f87d0fbbSRusty Russell const struct vring_desc *desc, 153f87d0fbbSRusty Russell struct vring_desc **descs, int *desc_max) 154f87d0fbbSRusty Russell { 155b9f7ac8cSMichael S. Tsirkin u32 len; 156b9f7ac8cSMichael S. Tsirkin 157f87d0fbbSRusty Russell /* Indirect tables can't have indirect. */ 158f87d0fbbSRusty Russell if (*up_next != -1) { 159f87d0fbbSRusty Russell vringh_bad("Multilevel indirect %u->%u", *up_next, *i); 160f87d0fbbSRusty Russell return -EINVAL; 161f87d0fbbSRusty Russell } 162f87d0fbbSRusty Russell 163b9f7ac8cSMichael S. Tsirkin len = vringh32_to_cpu(vrh, desc->len); 164b9f7ac8cSMichael S. Tsirkin if (unlikely(len % sizeof(struct vring_desc))) { 165f87d0fbbSRusty Russell vringh_bad("Strange indirect len %u", desc->len); 166f87d0fbbSRusty Russell return -EINVAL; 167f87d0fbbSRusty Russell } 168f87d0fbbSRusty Russell 169f87d0fbbSRusty Russell /* We will check this when we follow it! */ 170b9f7ac8cSMichael S. Tsirkin if (desc->flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT)) 171b9f7ac8cSMichael S. Tsirkin *up_next = vringh16_to_cpu(vrh, desc->next); 172f87d0fbbSRusty Russell else 173f87d0fbbSRusty Russell *up_next = -2; 174f87d0fbbSRusty Russell *descs = addr; 175b9f7ac8cSMichael S. Tsirkin *desc_max = len / sizeof(struct vring_desc); 176f87d0fbbSRusty Russell 177f87d0fbbSRusty Russell /* Now, start at the first indirect. */ 178f87d0fbbSRusty Russell *i = 0; 179f87d0fbbSRusty Russell return 0; 180f87d0fbbSRusty Russell } 181f87d0fbbSRusty Russell 182f87d0fbbSRusty Russell static int resize_iovec(struct vringh_kiov *iov, gfp_t gfp) 183f87d0fbbSRusty Russell { 184f87d0fbbSRusty Russell struct kvec *new; 185f87d0fbbSRusty Russell unsigned int flag, new_num = (iov->max_num & ~VRINGH_IOV_ALLOCATED) * 2; 186f87d0fbbSRusty Russell 187f87d0fbbSRusty Russell if (new_num < 8) 188f87d0fbbSRusty Russell new_num = 8; 189f87d0fbbSRusty Russell 190f87d0fbbSRusty Russell flag = (iov->max_num & VRINGH_IOV_ALLOCATED); 191f87d0fbbSRusty Russell if (flag) 192f87d0fbbSRusty Russell new = krealloc(iov->iov, new_num * sizeof(struct iovec), gfp); 193f87d0fbbSRusty Russell else { 194f87d0fbbSRusty Russell new = kmalloc(new_num * sizeof(struct iovec), gfp); 195f87d0fbbSRusty Russell if (new) { 196f87d0fbbSRusty Russell memcpy(new, iov->iov, 197f87d0fbbSRusty Russell iov->max_num * sizeof(struct iovec)); 198f87d0fbbSRusty Russell flag = VRINGH_IOV_ALLOCATED; 199f87d0fbbSRusty Russell } 200f87d0fbbSRusty Russell } 201f87d0fbbSRusty Russell if (!new) 202f87d0fbbSRusty Russell return -ENOMEM; 203f87d0fbbSRusty Russell iov->iov = new; 204f87d0fbbSRusty Russell iov->max_num = (new_num | flag); 205f87d0fbbSRusty Russell return 0; 206f87d0fbbSRusty Russell } 207f87d0fbbSRusty Russell 208f87d0fbbSRusty Russell static u16 __cold return_from_indirect(const struct vringh *vrh, int *up_next, 209f87d0fbbSRusty Russell struct vring_desc **descs, int *desc_max) 210f87d0fbbSRusty Russell { 211f87d0fbbSRusty Russell u16 i = *up_next; 212f87d0fbbSRusty Russell 213f87d0fbbSRusty Russell *up_next = -1; 214f87d0fbbSRusty Russell *descs = vrh->vring.desc; 215f87d0fbbSRusty Russell *desc_max = vrh->vring.num; 216f87d0fbbSRusty Russell return i; 217f87d0fbbSRusty Russell } 218f87d0fbbSRusty Russell 219f87d0fbbSRusty Russell static int slow_copy(struct vringh *vrh, void *dst, const void *src, 220f87d0fbbSRusty Russell bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len, 221f87d0fbbSRusty Russell struct vringh_range *range, 222f87d0fbbSRusty Russell bool (*getrange)(struct vringh *vrh, 223f87d0fbbSRusty Russell u64, 224f87d0fbbSRusty Russell struct vringh_range *)), 225f87d0fbbSRusty Russell bool (*getrange)(struct vringh *vrh, 226f87d0fbbSRusty Russell u64 addr, 227f87d0fbbSRusty Russell struct vringh_range *r), 228f87d0fbbSRusty Russell struct vringh_range *range, 229f87d0fbbSRusty Russell int (*copy)(void *dst, const void *src, size_t len)) 230f87d0fbbSRusty Russell { 231f87d0fbbSRusty Russell size_t part, len = sizeof(struct vring_desc); 232f87d0fbbSRusty Russell 233f87d0fbbSRusty Russell do { 234f87d0fbbSRusty Russell u64 addr; 235f87d0fbbSRusty Russell int err; 236f87d0fbbSRusty Russell 237f87d0fbbSRusty Russell part = len; 238f87d0fbbSRusty Russell addr = (u64)(unsigned long)src - range->offset; 239f87d0fbbSRusty Russell 240f87d0fbbSRusty Russell if (!rcheck(vrh, addr, &part, range, getrange)) 241f87d0fbbSRusty Russell return -EINVAL; 242f87d0fbbSRusty Russell 243f87d0fbbSRusty Russell err = copy(dst, src, part); 244f87d0fbbSRusty Russell if (err) 245f87d0fbbSRusty Russell return err; 246f87d0fbbSRusty Russell 247f87d0fbbSRusty Russell dst += part; 248f87d0fbbSRusty Russell src += part; 249f87d0fbbSRusty Russell len -= part; 250f87d0fbbSRusty Russell } while (len); 251f87d0fbbSRusty Russell return 0; 252f87d0fbbSRusty Russell } 253f87d0fbbSRusty Russell 254f87d0fbbSRusty Russell static inline int 255f87d0fbbSRusty Russell __vringh_iov(struct vringh *vrh, u16 i, 256f87d0fbbSRusty Russell struct vringh_kiov *riov, 257f87d0fbbSRusty Russell struct vringh_kiov *wiov, 258f87d0fbbSRusty Russell bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len, 259f87d0fbbSRusty Russell struct vringh_range *range, 260f87d0fbbSRusty Russell bool (*getrange)(struct vringh *, u64, 261f87d0fbbSRusty Russell struct vringh_range *)), 262f87d0fbbSRusty Russell bool (*getrange)(struct vringh *, u64, struct vringh_range *), 263f87d0fbbSRusty Russell gfp_t gfp, 264f87d0fbbSRusty Russell int (*copy)(void *dst, const void *src, size_t len)) 265f87d0fbbSRusty Russell { 266f87d0fbbSRusty Russell int err, count = 0, up_next, desc_max; 267f87d0fbbSRusty Russell struct vring_desc desc, *descs; 268f87d0fbbSRusty Russell struct vringh_range range = { -1ULL, 0 }, slowrange; 269f87d0fbbSRusty Russell bool slow = false; 270f87d0fbbSRusty Russell 271f87d0fbbSRusty Russell /* We start traversing vring's descriptor table. */ 272f87d0fbbSRusty Russell descs = vrh->vring.desc; 273f87d0fbbSRusty Russell desc_max = vrh->vring.num; 274f87d0fbbSRusty Russell up_next = -1; 275f87d0fbbSRusty Russell 276f87d0fbbSRusty Russell if (riov) 277f87d0fbbSRusty Russell riov->i = riov->used = 0; 278f87d0fbbSRusty Russell else if (wiov) 279f87d0fbbSRusty Russell wiov->i = wiov->used = 0; 280f87d0fbbSRusty Russell else 281f87d0fbbSRusty Russell /* You must want something! */ 282f87d0fbbSRusty Russell BUG(); 283f87d0fbbSRusty Russell 284f87d0fbbSRusty Russell for (;;) { 285f87d0fbbSRusty Russell void *addr; 286f87d0fbbSRusty Russell struct vringh_kiov *iov; 287f87d0fbbSRusty Russell size_t len; 288f87d0fbbSRusty Russell 289f87d0fbbSRusty Russell if (unlikely(slow)) 290f87d0fbbSRusty Russell err = slow_copy(vrh, &desc, &descs[i], rcheck, getrange, 291f87d0fbbSRusty Russell &slowrange, copy); 292f87d0fbbSRusty Russell else 293f87d0fbbSRusty Russell err = copy(&desc, &descs[i], sizeof(desc)); 294f87d0fbbSRusty Russell if (unlikely(err)) 295f87d0fbbSRusty Russell goto fail; 296f87d0fbbSRusty Russell 297b9f7ac8cSMichael S. Tsirkin if (unlikely(desc.flags & 298b9f7ac8cSMichael S. Tsirkin cpu_to_vringh16(vrh, VRING_DESC_F_INDIRECT))) { 299b9f7ac8cSMichael S. Tsirkin u64 a = vringh64_to_cpu(vrh, desc.addr); 300b9f7ac8cSMichael S. Tsirkin 301f87d0fbbSRusty Russell /* Make sure it's OK, and get offset. */ 302b9f7ac8cSMichael S. Tsirkin len = vringh32_to_cpu(vrh, desc.len); 303b9f7ac8cSMichael S. Tsirkin if (!rcheck(vrh, a, &len, &range, getrange)) { 304f87d0fbbSRusty Russell err = -EINVAL; 305f87d0fbbSRusty Russell goto fail; 306f87d0fbbSRusty Russell } 307f87d0fbbSRusty Russell 308b9f7ac8cSMichael S. Tsirkin if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) { 309f87d0fbbSRusty Russell slow = true; 310f87d0fbbSRusty Russell /* We need to save this range to use offset */ 311f87d0fbbSRusty Russell slowrange = range; 312f87d0fbbSRusty Russell } 313f87d0fbbSRusty Russell 314b9f7ac8cSMichael S. Tsirkin addr = (void *)(long)(a + range.offset); 315b9f7ac8cSMichael S. Tsirkin err = move_to_indirect(vrh, &up_next, &i, addr, &desc, 316f87d0fbbSRusty Russell &descs, &desc_max); 317f87d0fbbSRusty Russell if (err) 318f87d0fbbSRusty Russell goto fail; 319f87d0fbbSRusty Russell continue; 320f87d0fbbSRusty Russell } 321f87d0fbbSRusty Russell 322f87d0fbbSRusty Russell if (count++ == vrh->vring.num) { 323f87d0fbbSRusty Russell vringh_bad("Descriptor loop in %p", descs); 324f87d0fbbSRusty Russell err = -ELOOP; 325f87d0fbbSRusty Russell goto fail; 326f87d0fbbSRusty Russell } 327f87d0fbbSRusty Russell 328b9f7ac8cSMichael S. Tsirkin if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_WRITE)) 329f87d0fbbSRusty Russell iov = wiov; 330f87d0fbbSRusty Russell else { 331f87d0fbbSRusty Russell iov = riov; 332f87d0fbbSRusty Russell if (unlikely(wiov && wiov->i)) { 333f87d0fbbSRusty Russell vringh_bad("Readable desc %p after writable", 334f87d0fbbSRusty Russell &descs[i]); 335f87d0fbbSRusty Russell err = -EINVAL; 336f87d0fbbSRusty Russell goto fail; 337f87d0fbbSRusty Russell } 338f87d0fbbSRusty Russell } 339f87d0fbbSRusty Russell 340f87d0fbbSRusty Russell if (!iov) { 341f87d0fbbSRusty Russell vringh_bad("Unexpected %s desc", 342f87d0fbbSRusty Russell !wiov ? "writable" : "readable"); 343f87d0fbbSRusty Russell err = -EPROTO; 344f87d0fbbSRusty Russell goto fail; 345f87d0fbbSRusty Russell } 346f87d0fbbSRusty Russell 347f87d0fbbSRusty Russell again: 348f87d0fbbSRusty Russell /* Make sure it's OK, and get offset. */ 349b9f7ac8cSMichael S. Tsirkin len = vringh32_to_cpu(vrh, desc.len); 350b9f7ac8cSMichael S. Tsirkin if (!rcheck(vrh, vringh64_to_cpu(vrh, desc.addr), &len, &range, 351b9f7ac8cSMichael S. Tsirkin getrange)) { 352f87d0fbbSRusty Russell err = -EINVAL; 353f87d0fbbSRusty Russell goto fail; 354f87d0fbbSRusty Russell } 355b9f7ac8cSMichael S. Tsirkin addr = (void *)(unsigned long)(vringh64_to_cpu(vrh, desc.addr) + 356b9f7ac8cSMichael S. Tsirkin range.offset); 357f87d0fbbSRusty Russell 358f87d0fbbSRusty Russell if (unlikely(iov->used == (iov->max_num & ~VRINGH_IOV_ALLOCATED))) { 359f87d0fbbSRusty Russell err = resize_iovec(iov, gfp); 360f87d0fbbSRusty Russell if (err) 361f87d0fbbSRusty Russell goto fail; 362f87d0fbbSRusty Russell } 363f87d0fbbSRusty Russell 364f87d0fbbSRusty Russell iov->iov[iov->used].iov_base = addr; 365f87d0fbbSRusty Russell iov->iov[iov->used].iov_len = len; 366f87d0fbbSRusty Russell iov->used++; 367f87d0fbbSRusty Russell 368b9f7ac8cSMichael S. Tsirkin if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) { 369b9f7ac8cSMichael S. Tsirkin desc.len = cpu_to_vringh32(vrh, 370b9f7ac8cSMichael S. Tsirkin vringh32_to_cpu(vrh, desc.len) - len); 371b9f7ac8cSMichael S. Tsirkin desc.addr = cpu_to_vringh64(vrh, 372b9f7ac8cSMichael S. Tsirkin vringh64_to_cpu(vrh, desc.addr) + len); 373f87d0fbbSRusty Russell goto again; 374f87d0fbbSRusty Russell } 375f87d0fbbSRusty Russell 376b9f7ac8cSMichael S. Tsirkin if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT)) { 377b9f7ac8cSMichael S. Tsirkin i = vringh16_to_cpu(vrh, desc.next); 378f87d0fbbSRusty Russell } else { 379f87d0fbbSRusty Russell /* Just in case we need to finish traversing above. */ 380f87d0fbbSRusty Russell if (unlikely(up_next > 0)) { 381f87d0fbbSRusty Russell i = return_from_indirect(vrh, &up_next, 382f87d0fbbSRusty Russell &descs, &desc_max); 383f87d0fbbSRusty Russell slow = false; 384f87d0fbbSRusty Russell } else 385f87d0fbbSRusty Russell break; 386f87d0fbbSRusty Russell } 387f87d0fbbSRusty Russell 388f87d0fbbSRusty Russell if (i >= desc_max) { 389f87d0fbbSRusty Russell vringh_bad("Chained index %u > %u", i, desc_max); 390f87d0fbbSRusty Russell err = -EINVAL; 391f87d0fbbSRusty Russell goto fail; 392f87d0fbbSRusty Russell } 393f87d0fbbSRusty Russell } 394f87d0fbbSRusty Russell 395f87d0fbbSRusty Russell return 0; 396f87d0fbbSRusty Russell 397f87d0fbbSRusty Russell fail: 398f87d0fbbSRusty Russell return err; 399f87d0fbbSRusty Russell } 400f87d0fbbSRusty Russell 401f87d0fbbSRusty Russell static inline int __vringh_complete(struct vringh *vrh, 402f87d0fbbSRusty Russell const struct vring_used_elem *used, 403f87d0fbbSRusty Russell unsigned int num_used, 404b9f7ac8cSMichael S. Tsirkin int (*putu16)(const struct vringh *vrh, 405b9f7ac8cSMichael S. Tsirkin __virtio16 *p, u16 val), 406f87d0fbbSRusty Russell int (*putused)(struct vring_used_elem *dst, 407f87d0fbbSRusty Russell const struct vring_used_elem 408f87d0fbbSRusty Russell *src, unsigned num)) 409f87d0fbbSRusty Russell { 410f87d0fbbSRusty Russell struct vring_used *used_ring; 411f87d0fbbSRusty Russell int err; 412f87d0fbbSRusty Russell u16 used_idx, off; 413f87d0fbbSRusty Russell 414f87d0fbbSRusty Russell used_ring = vrh->vring.used; 415f87d0fbbSRusty Russell used_idx = vrh->last_used_idx + vrh->completed; 416f87d0fbbSRusty Russell 417f87d0fbbSRusty Russell off = used_idx % vrh->vring.num; 418f87d0fbbSRusty Russell 419f87d0fbbSRusty Russell /* Compiler knows num_used == 1 sometimes, hence extra check */ 420f87d0fbbSRusty Russell if (num_used > 1 && unlikely(off + num_used >= vrh->vring.num)) { 421f87d0fbbSRusty Russell u16 part = vrh->vring.num - off; 422f87d0fbbSRusty Russell err = putused(&used_ring->ring[off], used, part); 423f87d0fbbSRusty Russell if (!err) 424f87d0fbbSRusty Russell err = putused(&used_ring->ring[0], used + part, 425f87d0fbbSRusty Russell num_used - part); 426f87d0fbbSRusty Russell } else 427f87d0fbbSRusty Russell err = putused(&used_ring->ring[off], used, num_used); 428f87d0fbbSRusty Russell 429f87d0fbbSRusty Russell if (err) { 430f87d0fbbSRusty Russell vringh_bad("Failed to write %u used entries %u at %p", 431f87d0fbbSRusty Russell num_used, off, &used_ring->ring[off]); 432f87d0fbbSRusty Russell return err; 433f87d0fbbSRusty Russell } 434f87d0fbbSRusty Russell 435f87d0fbbSRusty Russell /* Make sure buffer is written before we update index. */ 436f87d0fbbSRusty Russell virtio_wmb(vrh->weak_barriers); 437f87d0fbbSRusty Russell 438b9f7ac8cSMichael S. Tsirkin err = putu16(vrh, &vrh->vring.used->idx, used_idx + num_used); 439f87d0fbbSRusty Russell if (err) { 440f87d0fbbSRusty Russell vringh_bad("Failed to update used index at %p", 441f87d0fbbSRusty Russell &vrh->vring.used->idx); 442f87d0fbbSRusty Russell return err; 443f87d0fbbSRusty Russell } 444f87d0fbbSRusty Russell 445f87d0fbbSRusty Russell vrh->completed += num_used; 446f87d0fbbSRusty Russell return 0; 447f87d0fbbSRusty Russell } 448f87d0fbbSRusty Russell 449f87d0fbbSRusty Russell 450f87d0fbbSRusty Russell static inline int __vringh_need_notify(struct vringh *vrh, 451b9f7ac8cSMichael S. Tsirkin int (*getu16)(const struct vringh *vrh, 452b9f7ac8cSMichael S. Tsirkin u16 *val, 453b9f7ac8cSMichael S. Tsirkin const __virtio16 *p)) 454f87d0fbbSRusty Russell { 455f87d0fbbSRusty Russell bool notify; 456f87d0fbbSRusty Russell u16 used_event; 457f87d0fbbSRusty Russell int err; 458f87d0fbbSRusty Russell 459f87d0fbbSRusty Russell /* Flush out used index update. This is paired with the 460f87d0fbbSRusty Russell * barrier that the Guest executes when enabling 461f87d0fbbSRusty Russell * interrupts. */ 462f87d0fbbSRusty Russell virtio_mb(vrh->weak_barriers); 463f87d0fbbSRusty Russell 464f87d0fbbSRusty Russell /* Old-style, without event indices. */ 465f87d0fbbSRusty Russell if (!vrh->event_indices) { 466f87d0fbbSRusty Russell u16 flags; 467b9f7ac8cSMichael S. Tsirkin err = getu16(vrh, &flags, &vrh->vring.avail->flags); 468f87d0fbbSRusty Russell if (err) { 469f87d0fbbSRusty Russell vringh_bad("Failed to get flags at %p", 470f87d0fbbSRusty Russell &vrh->vring.avail->flags); 471f87d0fbbSRusty Russell return err; 472f87d0fbbSRusty Russell } 473f87d0fbbSRusty Russell return (!(flags & VRING_AVAIL_F_NO_INTERRUPT)); 474f87d0fbbSRusty Russell } 475f87d0fbbSRusty Russell 476f87d0fbbSRusty Russell /* Modern: we know when other side wants to know. */ 477b9f7ac8cSMichael S. Tsirkin err = getu16(vrh, &used_event, &vring_used_event(&vrh->vring)); 478f87d0fbbSRusty Russell if (err) { 479f87d0fbbSRusty Russell vringh_bad("Failed to get used event idx at %p", 480f87d0fbbSRusty Russell &vring_used_event(&vrh->vring)); 481f87d0fbbSRusty Russell return err; 482f87d0fbbSRusty Russell } 483f87d0fbbSRusty Russell 484f87d0fbbSRusty Russell /* Just in case we added so many that we wrap. */ 485f87d0fbbSRusty Russell if (unlikely(vrh->completed > 0xffff)) 486f87d0fbbSRusty Russell notify = true; 487f87d0fbbSRusty Russell else 488f87d0fbbSRusty Russell notify = vring_need_event(used_event, 489f87d0fbbSRusty Russell vrh->last_used_idx + vrh->completed, 490f87d0fbbSRusty Russell vrh->last_used_idx); 491f87d0fbbSRusty Russell 492f87d0fbbSRusty Russell vrh->last_used_idx += vrh->completed; 493f87d0fbbSRusty Russell vrh->completed = 0; 494f87d0fbbSRusty Russell return notify; 495f87d0fbbSRusty Russell } 496f87d0fbbSRusty Russell 497f87d0fbbSRusty Russell static inline bool __vringh_notify_enable(struct vringh *vrh, 498b9f7ac8cSMichael S. Tsirkin int (*getu16)(const struct vringh *vrh, 499b9f7ac8cSMichael S. Tsirkin u16 *val, const __virtio16 *p), 500b9f7ac8cSMichael S. Tsirkin int (*putu16)(const struct vringh *vrh, 501b9f7ac8cSMichael S. Tsirkin __virtio16 *p, u16 val)) 502f87d0fbbSRusty Russell { 503f87d0fbbSRusty Russell u16 avail; 504f87d0fbbSRusty Russell 505f87d0fbbSRusty Russell if (!vrh->event_indices) { 506f87d0fbbSRusty Russell /* Old-school; update flags. */ 507b9f7ac8cSMichael S. Tsirkin if (putu16(vrh, &vrh->vring.used->flags, 0) != 0) { 508f87d0fbbSRusty Russell vringh_bad("Clearing used flags %p", 509f87d0fbbSRusty Russell &vrh->vring.used->flags); 510f87d0fbbSRusty Russell return true; 511f87d0fbbSRusty Russell } 512f87d0fbbSRusty Russell } else { 513b9f7ac8cSMichael S. Tsirkin if (putu16(vrh, &vring_avail_event(&vrh->vring), 514f87d0fbbSRusty Russell vrh->last_avail_idx) != 0) { 515f87d0fbbSRusty Russell vringh_bad("Updating avail event index %p", 516f87d0fbbSRusty Russell &vring_avail_event(&vrh->vring)); 517f87d0fbbSRusty Russell return true; 518f87d0fbbSRusty Russell } 519f87d0fbbSRusty Russell } 520f87d0fbbSRusty Russell 521f87d0fbbSRusty Russell /* They could have slipped one in as we were doing that: make 522f87d0fbbSRusty Russell * sure it's written, then check again. */ 523f87d0fbbSRusty Russell virtio_mb(vrh->weak_barriers); 524f87d0fbbSRusty Russell 525b9f7ac8cSMichael S. Tsirkin if (getu16(vrh, &avail, &vrh->vring.avail->idx) != 0) { 526f87d0fbbSRusty Russell vringh_bad("Failed to check avail idx at %p", 527f87d0fbbSRusty Russell &vrh->vring.avail->idx); 528f87d0fbbSRusty Russell return true; 529f87d0fbbSRusty Russell } 530f87d0fbbSRusty Russell 531f87d0fbbSRusty Russell /* This is unlikely, so we just leave notifications enabled 532f87d0fbbSRusty Russell * (if we're using event_indices, we'll only get one 533f87d0fbbSRusty Russell * notification anyway). */ 534f87d0fbbSRusty Russell return avail == vrh->last_avail_idx; 535f87d0fbbSRusty Russell } 536f87d0fbbSRusty Russell 537f87d0fbbSRusty Russell static inline void __vringh_notify_disable(struct vringh *vrh, 538b9f7ac8cSMichael S. Tsirkin int (*putu16)(const struct vringh *vrh, 539b9f7ac8cSMichael S. Tsirkin __virtio16 *p, u16 val)) 540f87d0fbbSRusty Russell { 541f87d0fbbSRusty Russell if (!vrh->event_indices) { 542f87d0fbbSRusty Russell /* Old-school; update flags. */ 543b9f7ac8cSMichael S. Tsirkin if (putu16(vrh, &vrh->vring.used->flags, 544b9f7ac8cSMichael S. Tsirkin VRING_USED_F_NO_NOTIFY)) { 545f87d0fbbSRusty Russell vringh_bad("Setting used flags %p", 546f87d0fbbSRusty Russell &vrh->vring.used->flags); 547f87d0fbbSRusty Russell } 548f87d0fbbSRusty Russell } 549f87d0fbbSRusty Russell } 550f87d0fbbSRusty Russell 551f87d0fbbSRusty Russell /* Userspace access helpers: in this case, addresses are really userspace. */ 552b9f7ac8cSMichael S. Tsirkin static inline int getu16_user(const struct vringh *vrh, u16 *val, const __virtio16 *p) 553f87d0fbbSRusty Russell { 554b9f7ac8cSMichael S. Tsirkin __virtio16 v = 0; 555b9f7ac8cSMichael S. Tsirkin int rc = get_user(v, (__force __virtio16 __user *)p); 556b9f7ac8cSMichael S. Tsirkin *val = vringh16_to_cpu(vrh, v); 557b9f7ac8cSMichael S. Tsirkin return rc; 558f87d0fbbSRusty Russell } 559f87d0fbbSRusty Russell 560b9f7ac8cSMichael S. Tsirkin static inline int putu16_user(const struct vringh *vrh, __virtio16 *p, u16 val) 561f87d0fbbSRusty Russell { 562b9f7ac8cSMichael S. Tsirkin __virtio16 v = cpu_to_vringh16(vrh, val); 563b9f7ac8cSMichael S. Tsirkin return put_user(v, (__force __virtio16 __user *)p); 564f87d0fbbSRusty Russell } 565f87d0fbbSRusty Russell 566f87d0fbbSRusty Russell static inline int copydesc_user(void *dst, const void *src, size_t len) 567f87d0fbbSRusty Russell { 568f87d0fbbSRusty Russell return copy_from_user(dst, (__force void __user *)src, len) ? 569f87d0fbbSRusty Russell -EFAULT : 0; 570f87d0fbbSRusty Russell } 571f87d0fbbSRusty Russell 572f87d0fbbSRusty Russell static inline int putused_user(struct vring_used_elem *dst, 573f87d0fbbSRusty Russell const struct vring_used_elem *src, 574f87d0fbbSRusty Russell unsigned int num) 575f87d0fbbSRusty Russell { 576f87d0fbbSRusty Russell return copy_to_user((__force void __user *)dst, src, 577f87d0fbbSRusty Russell sizeof(*dst) * num) ? -EFAULT : 0; 578f87d0fbbSRusty Russell } 579f87d0fbbSRusty Russell 580f87d0fbbSRusty Russell static inline int xfer_from_user(void *src, void *dst, size_t len) 581f87d0fbbSRusty Russell { 582f87d0fbbSRusty Russell return copy_from_user(dst, (__force void __user *)src, len) ? 583f87d0fbbSRusty Russell -EFAULT : 0; 584f87d0fbbSRusty Russell } 585f87d0fbbSRusty Russell 586f87d0fbbSRusty Russell static inline int xfer_to_user(void *dst, void *src, size_t len) 587f87d0fbbSRusty Russell { 588f87d0fbbSRusty Russell return copy_to_user((__force void __user *)dst, src, len) ? 589f87d0fbbSRusty Russell -EFAULT : 0; 590f87d0fbbSRusty Russell } 591f87d0fbbSRusty Russell 592f87d0fbbSRusty Russell /** 593f87d0fbbSRusty Russell * vringh_init_user - initialize a vringh for a userspace vring. 594f87d0fbbSRusty Russell * @vrh: the vringh to initialize. 595f87d0fbbSRusty Russell * @features: the feature bits for this ring. 596f87d0fbbSRusty Russell * @num: the number of elements. 597f87d0fbbSRusty Russell * @weak_barriers: true if we only need memory barriers, not I/O. 598f87d0fbbSRusty Russell * @desc: the userpace descriptor pointer. 599f87d0fbbSRusty Russell * @avail: the userpace avail pointer. 600f87d0fbbSRusty Russell * @used: the userpace used pointer. 601f87d0fbbSRusty Russell * 602f87d0fbbSRusty Russell * Returns an error if num is invalid: you should check pointers 603f87d0fbbSRusty Russell * yourself! 604f87d0fbbSRusty Russell */ 605b97a8a90SMichael S. Tsirkin int vringh_init_user(struct vringh *vrh, u64 features, 606f87d0fbbSRusty Russell unsigned int num, bool weak_barriers, 607f87d0fbbSRusty Russell struct vring_desc __user *desc, 608f87d0fbbSRusty Russell struct vring_avail __user *avail, 609f87d0fbbSRusty Russell struct vring_used __user *used) 610f87d0fbbSRusty Russell { 611f87d0fbbSRusty Russell /* Sane power of 2 please! */ 612f87d0fbbSRusty Russell if (!num || num > 0xffff || (num & (num - 1))) { 613f87d0fbbSRusty Russell vringh_bad("Bad ring size %u", num); 614f87d0fbbSRusty Russell return -EINVAL; 615f87d0fbbSRusty Russell } 616f87d0fbbSRusty Russell 617b9f7ac8cSMichael S. Tsirkin vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1)); 618f87d0fbbSRusty Russell vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX)); 619f87d0fbbSRusty Russell vrh->weak_barriers = weak_barriers; 620f87d0fbbSRusty Russell vrh->completed = 0; 621f87d0fbbSRusty Russell vrh->last_avail_idx = 0; 622f87d0fbbSRusty Russell vrh->last_used_idx = 0; 623f87d0fbbSRusty Russell vrh->vring.num = num; 624f87d0fbbSRusty Russell /* vring expects kernel addresses, but only used via accessors. */ 625f87d0fbbSRusty Russell vrh->vring.desc = (__force struct vring_desc *)desc; 626f87d0fbbSRusty Russell vrh->vring.avail = (__force struct vring_avail *)avail; 627f87d0fbbSRusty Russell vrh->vring.used = (__force struct vring_used *)used; 628f87d0fbbSRusty Russell return 0; 629f87d0fbbSRusty Russell } 630f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_init_user); 631f87d0fbbSRusty Russell 632f87d0fbbSRusty Russell /** 633f87d0fbbSRusty Russell * vringh_getdesc_user - get next available descriptor from userspace ring. 634f87d0fbbSRusty Russell * @vrh: the userspace vring. 635f87d0fbbSRusty Russell * @riov: where to put the readable descriptors (or NULL) 636f87d0fbbSRusty Russell * @wiov: where to put the writable descriptors (or NULL) 637f87d0fbbSRusty Russell * @getrange: function to call to check ranges. 638f87d0fbbSRusty Russell * @head: head index we received, for passing to vringh_complete_user(). 639f87d0fbbSRusty Russell * 640f87d0fbbSRusty Russell * Returns 0 if there was no descriptor, 1 if there was, or -errno. 641f87d0fbbSRusty Russell * 642f87d0fbbSRusty Russell * Note that on error return, you can tell the difference between an 643f87d0fbbSRusty Russell * invalid ring and a single invalid descriptor: in the former case, 644f87d0fbbSRusty Russell * *head will be vrh->vring.num. You may be able to ignore an invalid 645f87d0fbbSRusty Russell * descriptor, but there's not much you can do with an invalid ring. 646f87d0fbbSRusty Russell * 647f87d0fbbSRusty Russell * Note that you may need to clean up riov and wiov, even on error! 648f87d0fbbSRusty Russell */ 649f87d0fbbSRusty Russell int vringh_getdesc_user(struct vringh *vrh, 650f87d0fbbSRusty Russell struct vringh_iov *riov, 651f87d0fbbSRusty Russell struct vringh_iov *wiov, 652f87d0fbbSRusty Russell bool (*getrange)(struct vringh *vrh, 653f87d0fbbSRusty Russell u64 addr, struct vringh_range *r), 654f87d0fbbSRusty Russell u16 *head) 655f87d0fbbSRusty Russell { 656f87d0fbbSRusty Russell int err; 657f87d0fbbSRusty Russell 658f87d0fbbSRusty Russell *head = vrh->vring.num; 659f87d0fbbSRusty Russell err = __vringh_get_head(vrh, getu16_user, &vrh->last_avail_idx); 660f87d0fbbSRusty Russell if (err < 0) 661f87d0fbbSRusty Russell return err; 662f87d0fbbSRusty Russell 663f87d0fbbSRusty Russell /* Empty... */ 664f87d0fbbSRusty Russell if (err == vrh->vring.num) 665f87d0fbbSRusty Russell return 0; 666f87d0fbbSRusty Russell 667f87d0fbbSRusty Russell /* We need the layouts to be the identical for this to work */ 668f87d0fbbSRusty Russell BUILD_BUG_ON(sizeof(struct vringh_kiov) != sizeof(struct vringh_iov)); 669f87d0fbbSRusty Russell BUILD_BUG_ON(offsetof(struct vringh_kiov, iov) != 670f87d0fbbSRusty Russell offsetof(struct vringh_iov, iov)); 671f87d0fbbSRusty Russell BUILD_BUG_ON(offsetof(struct vringh_kiov, i) != 672f87d0fbbSRusty Russell offsetof(struct vringh_iov, i)); 673f87d0fbbSRusty Russell BUILD_BUG_ON(offsetof(struct vringh_kiov, used) != 674f87d0fbbSRusty Russell offsetof(struct vringh_iov, used)); 675f87d0fbbSRusty Russell BUILD_BUG_ON(offsetof(struct vringh_kiov, max_num) != 676f87d0fbbSRusty Russell offsetof(struct vringh_iov, max_num)); 677f87d0fbbSRusty Russell BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec)); 678f87d0fbbSRusty Russell BUILD_BUG_ON(offsetof(struct iovec, iov_base) != 679f87d0fbbSRusty Russell offsetof(struct kvec, iov_base)); 680f87d0fbbSRusty Russell BUILD_BUG_ON(offsetof(struct iovec, iov_len) != 681f87d0fbbSRusty Russell offsetof(struct kvec, iov_len)); 682f87d0fbbSRusty Russell BUILD_BUG_ON(sizeof(((struct iovec *)NULL)->iov_base) 683f87d0fbbSRusty Russell != sizeof(((struct kvec *)NULL)->iov_base)); 684f87d0fbbSRusty Russell BUILD_BUG_ON(sizeof(((struct iovec *)NULL)->iov_len) 685f87d0fbbSRusty Russell != sizeof(((struct kvec *)NULL)->iov_len)); 686f87d0fbbSRusty Russell 687f87d0fbbSRusty Russell *head = err; 688f87d0fbbSRusty Russell err = __vringh_iov(vrh, *head, (struct vringh_kiov *)riov, 689f87d0fbbSRusty Russell (struct vringh_kiov *)wiov, 690f87d0fbbSRusty Russell range_check, getrange, GFP_KERNEL, copydesc_user); 691f87d0fbbSRusty Russell if (err) 692f87d0fbbSRusty Russell return err; 693f87d0fbbSRusty Russell 694f87d0fbbSRusty Russell return 1; 695f87d0fbbSRusty Russell } 696f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_getdesc_user); 697f87d0fbbSRusty Russell 698f87d0fbbSRusty Russell /** 699f87d0fbbSRusty Russell * vringh_iov_pull_user - copy bytes from vring_iov. 700f87d0fbbSRusty Russell * @riov: the riov as passed to vringh_getdesc_user() (updated as we consume) 701f87d0fbbSRusty Russell * @dst: the place to copy. 702f87d0fbbSRusty Russell * @len: the maximum length to copy. 703f87d0fbbSRusty Russell * 704f87d0fbbSRusty Russell * Returns the bytes copied <= len or a negative errno. 705f87d0fbbSRusty Russell */ 706f87d0fbbSRusty Russell ssize_t vringh_iov_pull_user(struct vringh_iov *riov, void *dst, size_t len) 707f87d0fbbSRusty Russell { 708f87d0fbbSRusty Russell return vringh_iov_xfer((struct vringh_kiov *)riov, 709f87d0fbbSRusty Russell dst, len, xfer_from_user); 710f87d0fbbSRusty Russell } 711f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_iov_pull_user); 712f87d0fbbSRusty Russell 713f87d0fbbSRusty Russell /** 714f87d0fbbSRusty Russell * vringh_iov_push_user - copy bytes into vring_iov. 715f87d0fbbSRusty Russell * @wiov: the wiov as passed to vringh_getdesc_user() (updated as we consume) 716f87d0fbbSRusty Russell * @dst: the place to copy. 717f87d0fbbSRusty Russell * @len: the maximum length to copy. 718f87d0fbbSRusty Russell * 719f87d0fbbSRusty Russell * Returns the bytes copied <= len or a negative errno. 720f87d0fbbSRusty Russell */ 721f87d0fbbSRusty Russell ssize_t vringh_iov_push_user(struct vringh_iov *wiov, 722f87d0fbbSRusty Russell const void *src, size_t len) 723f87d0fbbSRusty Russell { 724f87d0fbbSRusty Russell return vringh_iov_xfer((struct vringh_kiov *)wiov, 725f87d0fbbSRusty Russell (void *)src, len, xfer_to_user); 726f87d0fbbSRusty Russell } 727f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_iov_push_user); 728f87d0fbbSRusty Russell 729f87d0fbbSRusty Russell /** 730f87d0fbbSRusty Russell * vringh_abandon_user - we've decided not to handle the descriptor(s). 731f87d0fbbSRusty Russell * @vrh: the vring. 732f87d0fbbSRusty Russell * @num: the number of descriptors to put back (ie. num 733f87d0fbbSRusty Russell * vringh_get_user() to undo). 734f87d0fbbSRusty Russell * 735f87d0fbbSRusty Russell * The next vringh_get_user() will return the old descriptor(s) again. 736f87d0fbbSRusty Russell */ 737f87d0fbbSRusty Russell void vringh_abandon_user(struct vringh *vrh, unsigned int num) 738f87d0fbbSRusty Russell { 739f87d0fbbSRusty Russell /* We only update vring_avail_event(vr) when we want to be notified, 740f87d0fbbSRusty Russell * so we haven't changed that yet. */ 741f87d0fbbSRusty Russell vrh->last_avail_idx -= num; 742f87d0fbbSRusty Russell } 743f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_abandon_user); 744f87d0fbbSRusty Russell 745f87d0fbbSRusty Russell /** 746f87d0fbbSRusty Russell * vringh_complete_user - we've finished with descriptor, publish it. 747f87d0fbbSRusty Russell * @vrh: the vring. 748f87d0fbbSRusty Russell * @head: the head as filled in by vringh_getdesc_user. 749f87d0fbbSRusty Russell * @len: the length of data we have written. 750f87d0fbbSRusty Russell * 751f87d0fbbSRusty Russell * You should check vringh_need_notify_user() after one or more calls 752f87d0fbbSRusty Russell * to this function. 753f87d0fbbSRusty Russell */ 754f87d0fbbSRusty Russell int vringh_complete_user(struct vringh *vrh, u16 head, u32 len) 755f87d0fbbSRusty Russell { 756f87d0fbbSRusty Russell struct vring_used_elem used; 757f87d0fbbSRusty Russell 758b9f7ac8cSMichael S. Tsirkin used.id = cpu_to_vringh32(vrh, head); 759b9f7ac8cSMichael S. Tsirkin used.len = cpu_to_vringh32(vrh, len); 760f87d0fbbSRusty Russell return __vringh_complete(vrh, &used, 1, putu16_user, putused_user); 761f87d0fbbSRusty Russell } 762f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_complete_user); 763f87d0fbbSRusty Russell 764f87d0fbbSRusty Russell /** 765f87d0fbbSRusty Russell * vringh_complete_multi_user - we've finished with many descriptors. 766f87d0fbbSRusty Russell * @vrh: the vring. 767f87d0fbbSRusty Russell * @used: the head, length pairs. 768f87d0fbbSRusty Russell * @num_used: the number of used elements. 769f87d0fbbSRusty Russell * 770f87d0fbbSRusty Russell * You should check vringh_need_notify_user() after one or more calls 771f87d0fbbSRusty Russell * to this function. 772f87d0fbbSRusty Russell */ 773f87d0fbbSRusty Russell int vringh_complete_multi_user(struct vringh *vrh, 774f87d0fbbSRusty Russell const struct vring_used_elem used[], 775f87d0fbbSRusty Russell unsigned num_used) 776f87d0fbbSRusty Russell { 777f87d0fbbSRusty Russell return __vringh_complete(vrh, used, num_used, 778f87d0fbbSRusty Russell putu16_user, putused_user); 779f87d0fbbSRusty Russell } 780f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_complete_multi_user); 781f87d0fbbSRusty Russell 782f87d0fbbSRusty Russell /** 783f87d0fbbSRusty Russell * vringh_notify_enable_user - we want to know if something changes. 784f87d0fbbSRusty Russell * @vrh: the vring. 785f87d0fbbSRusty Russell * 786f87d0fbbSRusty Russell * This always enables notifications, but returns false if there are 787f87d0fbbSRusty Russell * now more buffers available in the vring. 788f87d0fbbSRusty Russell */ 789f87d0fbbSRusty Russell bool vringh_notify_enable_user(struct vringh *vrh) 790f87d0fbbSRusty Russell { 791f87d0fbbSRusty Russell return __vringh_notify_enable(vrh, getu16_user, putu16_user); 792f87d0fbbSRusty Russell } 793f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_notify_enable_user); 794f87d0fbbSRusty Russell 795f87d0fbbSRusty Russell /** 796f87d0fbbSRusty Russell * vringh_notify_disable_user - don't tell us if something changes. 797f87d0fbbSRusty Russell * @vrh: the vring. 798f87d0fbbSRusty Russell * 799f87d0fbbSRusty Russell * This is our normal running state: we disable and then only enable when 800f87d0fbbSRusty Russell * we're going to sleep. 801f87d0fbbSRusty Russell */ 802f87d0fbbSRusty Russell void vringh_notify_disable_user(struct vringh *vrh) 803f87d0fbbSRusty Russell { 804f87d0fbbSRusty Russell __vringh_notify_disable(vrh, putu16_user); 805f87d0fbbSRusty Russell } 806f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_notify_disable_user); 807f87d0fbbSRusty Russell 808f87d0fbbSRusty Russell /** 809f87d0fbbSRusty Russell * vringh_need_notify_user - must we tell the other side about used buffers? 810f87d0fbbSRusty Russell * @vrh: the vring we've called vringh_complete_user() on. 811f87d0fbbSRusty Russell * 812f87d0fbbSRusty Russell * Returns -errno or 0 if we don't need to tell the other side, 1 if we do. 813f87d0fbbSRusty Russell */ 814f87d0fbbSRusty Russell int vringh_need_notify_user(struct vringh *vrh) 815f87d0fbbSRusty Russell { 816f87d0fbbSRusty Russell return __vringh_need_notify(vrh, getu16_user); 817f87d0fbbSRusty Russell } 818f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_need_notify_user); 819f87d0fbbSRusty Russell 820f87d0fbbSRusty Russell /* Kernelspace access helpers. */ 821b9f7ac8cSMichael S. Tsirkin static inline int getu16_kern(const struct vringh *vrh, 822b9f7ac8cSMichael S. Tsirkin u16 *val, const __virtio16 *p) 823f87d0fbbSRusty Russell { 8249d1b972fSMark Rutland *val = vringh16_to_cpu(vrh, READ_ONCE(*p)); 825f87d0fbbSRusty Russell return 0; 826f87d0fbbSRusty Russell } 827f87d0fbbSRusty Russell 828b9f7ac8cSMichael S. Tsirkin static inline int putu16_kern(const struct vringh *vrh, __virtio16 *p, u16 val) 829f87d0fbbSRusty Russell { 8309d1b972fSMark Rutland WRITE_ONCE(*p, cpu_to_vringh16(vrh, val)); 831f87d0fbbSRusty Russell return 0; 832f87d0fbbSRusty Russell } 833f87d0fbbSRusty Russell 834f87d0fbbSRusty Russell static inline int copydesc_kern(void *dst, const void *src, size_t len) 835f87d0fbbSRusty Russell { 836f87d0fbbSRusty Russell memcpy(dst, src, len); 837f87d0fbbSRusty Russell return 0; 838f87d0fbbSRusty Russell } 839f87d0fbbSRusty Russell 840f87d0fbbSRusty Russell static inline int putused_kern(struct vring_used_elem *dst, 841f87d0fbbSRusty Russell const struct vring_used_elem *src, 842f87d0fbbSRusty Russell unsigned int num) 843f87d0fbbSRusty Russell { 844f87d0fbbSRusty Russell memcpy(dst, src, num * sizeof(*dst)); 845f87d0fbbSRusty Russell return 0; 846f87d0fbbSRusty Russell } 847f87d0fbbSRusty Russell 848f87d0fbbSRusty Russell static inline int xfer_kern(void *src, void *dst, size_t len) 849f87d0fbbSRusty Russell { 850f87d0fbbSRusty Russell memcpy(dst, src, len); 851f87d0fbbSRusty Russell return 0; 852f87d0fbbSRusty Russell } 853f87d0fbbSRusty Russell 854f87d0fbbSRusty Russell /** 855f87d0fbbSRusty Russell * vringh_init_kern - initialize a vringh for a kernelspace vring. 856f87d0fbbSRusty Russell * @vrh: the vringh to initialize. 857f87d0fbbSRusty Russell * @features: the feature bits for this ring. 858f87d0fbbSRusty Russell * @num: the number of elements. 859f87d0fbbSRusty Russell * @weak_barriers: true if we only need memory barriers, not I/O. 860f87d0fbbSRusty Russell * @desc: the userpace descriptor pointer. 861f87d0fbbSRusty Russell * @avail: the userpace avail pointer. 862f87d0fbbSRusty Russell * @used: the userpace used pointer. 863f87d0fbbSRusty Russell * 864f87d0fbbSRusty Russell * Returns an error if num is invalid. 865f87d0fbbSRusty Russell */ 866b97a8a90SMichael S. Tsirkin int vringh_init_kern(struct vringh *vrh, u64 features, 867f87d0fbbSRusty Russell unsigned int num, bool weak_barriers, 868f87d0fbbSRusty Russell struct vring_desc *desc, 869f87d0fbbSRusty Russell struct vring_avail *avail, 870f87d0fbbSRusty Russell struct vring_used *used) 871f87d0fbbSRusty Russell { 872f87d0fbbSRusty Russell /* Sane power of 2 please! */ 873f87d0fbbSRusty Russell if (!num || num > 0xffff || (num & (num - 1))) { 874f87d0fbbSRusty Russell vringh_bad("Bad ring size %u", num); 875f87d0fbbSRusty Russell return -EINVAL; 876f87d0fbbSRusty Russell } 877f87d0fbbSRusty Russell 878b9f7ac8cSMichael S. Tsirkin vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1)); 879f87d0fbbSRusty Russell vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX)); 880f87d0fbbSRusty Russell vrh->weak_barriers = weak_barriers; 881f87d0fbbSRusty Russell vrh->completed = 0; 882f87d0fbbSRusty Russell vrh->last_avail_idx = 0; 883f87d0fbbSRusty Russell vrh->last_used_idx = 0; 884f87d0fbbSRusty Russell vrh->vring.num = num; 885f87d0fbbSRusty Russell vrh->vring.desc = desc; 886f87d0fbbSRusty Russell vrh->vring.avail = avail; 887f87d0fbbSRusty Russell vrh->vring.used = used; 888f87d0fbbSRusty Russell return 0; 889f87d0fbbSRusty Russell } 890f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_init_kern); 891f87d0fbbSRusty Russell 892f87d0fbbSRusty Russell /** 893f87d0fbbSRusty Russell * vringh_getdesc_kern - get next available descriptor from kernelspace ring. 894f87d0fbbSRusty Russell * @vrh: the kernelspace vring. 895f87d0fbbSRusty Russell * @riov: where to put the readable descriptors (or NULL) 896f87d0fbbSRusty Russell * @wiov: where to put the writable descriptors (or NULL) 897f87d0fbbSRusty Russell * @head: head index we received, for passing to vringh_complete_kern(). 898f87d0fbbSRusty Russell * @gfp: flags for allocating larger riov/wiov. 899f87d0fbbSRusty Russell * 900f87d0fbbSRusty Russell * Returns 0 if there was no descriptor, 1 if there was, or -errno. 901f87d0fbbSRusty Russell * 902f87d0fbbSRusty Russell * Note that on error return, you can tell the difference between an 903f87d0fbbSRusty Russell * invalid ring and a single invalid descriptor: in the former case, 904f87d0fbbSRusty Russell * *head will be vrh->vring.num. You may be able to ignore an invalid 905f87d0fbbSRusty Russell * descriptor, but there's not much you can do with an invalid ring. 906f87d0fbbSRusty Russell * 907f87d0fbbSRusty Russell * Note that you may need to clean up riov and wiov, even on error! 908f87d0fbbSRusty Russell */ 909f87d0fbbSRusty Russell int vringh_getdesc_kern(struct vringh *vrh, 910f87d0fbbSRusty Russell struct vringh_kiov *riov, 911f87d0fbbSRusty Russell struct vringh_kiov *wiov, 912f87d0fbbSRusty Russell u16 *head, 913f87d0fbbSRusty Russell gfp_t gfp) 914f87d0fbbSRusty Russell { 915f87d0fbbSRusty Russell int err; 916f87d0fbbSRusty Russell 917f87d0fbbSRusty Russell err = __vringh_get_head(vrh, getu16_kern, &vrh->last_avail_idx); 918f87d0fbbSRusty Russell if (err < 0) 919f87d0fbbSRusty Russell return err; 920f87d0fbbSRusty Russell 921f87d0fbbSRusty Russell /* Empty... */ 922f87d0fbbSRusty Russell if (err == vrh->vring.num) 923f87d0fbbSRusty Russell return 0; 924f87d0fbbSRusty Russell 925f87d0fbbSRusty Russell *head = err; 926f87d0fbbSRusty Russell err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL, 927f87d0fbbSRusty Russell gfp, copydesc_kern); 928f87d0fbbSRusty Russell if (err) 929f87d0fbbSRusty Russell return err; 930f87d0fbbSRusty Russell 931f87d0fbbSRusty Russell return 1; 932f87d0fbbSRusty Russell } 933f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_getdesc_kern); 934f87d0fbbSRusty Russell 935f87d0fbbSRusty Russell /** 936f87d0fbbSRusty Russell * vringh_iov_pull_kern - copy bytes from vring_iov. 937f87d0fbbSRusty Russell * @riov: the riov as passed to vringh_getdesc_kern() (updated as we consume) 938f87d0fbbSRusty Russell * @dst: the place to copy. 939f87d0fbbSRusty Russell * @len: the maximum length to copy. 940f87d0fbbSRusty Russell * 941f87d0fbbSRusty Russell * Returns the bytes copied <= len or a negative errno. 942f87d0fbbSRusty Russell */ 943f87d0fbbSRusty Russell ssize_t vringh_iov_pull_kern(struct vringh_kiov *riov, void *dst, size_t len) 944f87d0fbbSRusty Russell { 945f87d0fbbSRusty Russell return vringh_iov_xfer(riov, dst, len, xfer_kern); 946f87d0fbbSRusty Russell } 947f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_iov_pull_kern); 948f87d0fbbSRusty Russell 949f87d0fbbSRusty Russell /** 950f87d0fbbSRusty Russell * vringh_iov_push_kern - copy bytes into vring_iov. 951f87d0fbbSRusty Russell * @wiov: the wiov as passed to vringh_getdesc_kern() (updated as we consume) 952f87d0fbbSRusty Russell * @dst: the place to copy. 953f87d0fbbSRusty Russell * @len: the maximum length to copy. 954f87d0fbbSRusty Russell * 955f87d0fbbSRusty Russell * Returns the bytes copied <= len or a negative errno. 956f87d0fbbSRusty Russell */ 957f87d0fbbSRusty Russell ssize_t vringh_iov_push_kern(struct vringh_kiov *wiov, 958f87d0fbbSRusty Russell const void *src, size_t len) 959f87d0fbbSRusty Russell { 960f87d0fbbSRusty Russell return vringh_iov_xfer(wiov, (void *)src, len, xfer_kern); 961f87d0fbbSRusty Russell } 962f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_iov_push_kern); 963f87d0fbbSRusty Russell 964f87d0fbbSRusty Russell /** 965f87d0fbbSRusty Russell * vringh_abandon_kern - we've decided not to handle the descriptor(s). 966f87d0fbbSRusty Russell * @vrh: the vring. 967f87d0fbbSRusty Russell * @num: the number of descriptors to put back (ie. num 968f87d0fbbSRusty Russell * vringh_get_kern() to undo). 969f87d0fbbSRusty Russell * 970f87d0fbbSRusty Russell * The next vringh_get_kern() will return the old descriptor(s) again. 971f87d0fbbSRusty Russell */ 972f87d0fbbSRusty Russell void vringh_abandon_kern(struct vringh *vrh, unsigned int num) 973f87d0fbbSRusty Russell { 974f87d0fbbSRusty Russell /* We only update vring_avail_event(vr) when we want to be notified, 975f87d0fbbSRusty Russell * so we haven't changed that yet. */ 976f87d0fbbSRusty Russell vrh->last_avail_idx -= num; 977f87d0fbbSRusty Russell } 978f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_abandon_kern); 979f87d0fbbSRusty Russell 980f87d0fbbSRusty Russell /** 981f87d0fbbSRusty Russell * vringh_complete_kern - we've finished with descriptor, publish it. 982f87d0fbbSRusty Russell * @vrh: the vring. 983f87d0fbbSRusty Russell * @head: the head as filled in by vringh_getdesc_kern. 984f87d0fbbSRusty Russell * @len: the length of data we have written. 985f87d0fbbSRusty Russell * 986f87d0fbbSRusty Russell * You should check vringh_need_notify_kern() after one or more calls 987f87d0fbbSRusty Russell * to this function. 988f87d0fbbSRusty Russell */ 989f87d0fbbSRusty Russell int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len) 990f87d0fbbSRusty Russell { 991f87d0fbbSRusty Russell struct vring_used_elem used; 992f87d0fbbSRusty Russell 993b9f7ac8cSMichael S. Tsirkin used.id = cpu_to_vringh32(vrh, head); 994b9f7ac8cSMichael S. Tsirkin used.len = cpu_to_vringh32(vrh, len); 995f87d0fbbSRusty Russell 996f87d0fbbSRusty Russell return __vringh_complete(vrh, &used, 1, putu16_kern, putused_kern); 997f87d0fbbSRusty Russell } 998f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_complete_kern); 999f87d0fbbSRusty Russell 1000f87d0fbbSRusty Russell /** 1001f87d0fbbSRusty Russell * vringh_notify_enable_kern - we want to know if something changes. 1002f87d0fbbSRusty Russell * @vrh: the vring. 1003f87d0fbbSRusty Russell * 1004f87d0fbbSRusty Russell * This always enables notifications, but returns false if there are 1005f87d0fbbSRusty Russell * now more buffers available in the vring. 1006f87d0fbbSRusty Russell */ 1007f87d0fbbSRusty Russell bool vringh_notify_enable_kern(struct vringh *vrh) 1008f87d0fbbSRusty Russell { 1009f87d0fbbSRusty Russell return __vringh_notify_enable(vrh, getu16_kern, putu16_kern); 1010f87d0fbbSRusty Russell } 1011f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_notify_enable_kern); 1012f87d0fbbSRusty Russell 1013f87d0fbbSRusty Russell /** 1014f87d0fbbSRusty Russell * vringh_notify_disable_kern - don't tell us if something changes. 1015f87d0fbbSRusty Russell * @vrh: the vring. 1016f87d0fbbSRusty Russell * 1017f87d0fbbSRusty Russell * This is our normal running state: we disable and then only enable when 1018f87d0fbbSRusty Russell * we're going to sleep. 1019f87d0fbbSRusty Russell */ 1020f87d0fbbSRusty Russell void vringh_notify_disable_kern(struct vringh *vrh) 1021f87d0fbbSRusty Russell { 1022f87d0fbbSRusty Russell __vringh_notify_disable(vrh, putu16_kern); 1023f87d0fbbSRusty Russell } 1024f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_notify_disable_kern); 1025f87d0fbbSRusty Russell 1026f87d0fbbSRusty Russell /** 1027f87d0fbbSRusty Russell * vringh_need_notify_kern - must we tell the other side about used buffers? 1028f87d0fbbSRusty Russell * @vrh: the vring we've called vringh_complete_kern() on. 1029f87d0fbbSRusty Russell * 1030f87d0fbbSRusty Russell * Returns -errno or 0 if we don't need to tell the other side, 1 if we do. 1031f87d0fbbSRusty Russell */ 1032f87d0fbbSRusty Russell int vringh_need_notify_kern(struct vringh *vrh) 1033f87d0fbbSRusty Russell { 1034f87d0fbbSRusty Russell return __vringh_need_notify(vrh, getu16_kern); 1035f87d0fbbSRusty Russell } 1036f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_need_notify_kern); 1037f558a845SDave Jones 1038f558a845SDave Jones MODULE_LICENSE("GPL"); 1039