109c434b8SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only 2f87d0fbbSRusty Russell /* 3f87d0fbbSRusty Russell * Helpers for the host side of a virtio ring. 4f87d0fbbSRusty Russell * 5f87d0fbbSRusty Russell * Since these may be in userspace, we use (inline) accessors. 6f87d0fbbSRusty Russell */ 79d1b972fSMark Rutland #include <linux/compiler.h> 8f558a845SDave Jones #include <linux/module.h> 9f87d0fbbSRusty Russell #include <linux/vringh.h> 10f87d0fbbSRusty Russell #include <linux/virtio_ring.h> 11f87d0fbbSRusty Russell #include <linux/kernel.h> 12f87d0fbbSRusty Russell #include <linux/ratelimit.h> 13f87d0fbbSRusty Russell #include <linux/uaccess.h> 14f87d0fbbSRusty Russell #include <linux/slab.h> 15f87d0fbbSRusty Russell #include <linux/export.h> 163302363aSMichael S. Tsirkin #if IS_REACHABLE(CONFIG_VHOST_IOTLB) 179ad9c49cSJason Wang #include <linux/bvec.h> 189ad9c49cSJason Wang #include <linux/highmem.h> 199ad9c49cSJason Wang #include <linux/vhost_iotlb.h> 203302363aSMichael S. Tsirkin #endif 21b9f7ac8cSMichael S. Tsirkin #include <uapi/linux/virtio_config.h> 22f87d0fbbSRusty Russell 23f87d0fbbSRusty Russell static __printf(1,2) __cold void vringh_bad(const char *fmt, ...) 24f87d0fbbSRusty Russell { 25f87d0fbbSRusty Russell static DEFINE_RATELIMIT_STATE(vringh_rs, 26f87d0fbbSRusty Russell DEFAULT_RATELIMIT_INTERVAL, 27f87d0fbbSRusty Russell DEFAULT_RATELIMIT_BURST); 28f87d0fbbSRusty Russell if (__ratelimit(&vringh_rs)) { 29f87d0fbbSRusty Russell va_list ap; 30f87d0fbbSRusty Russell va_start(ap, fmt); 31f87d0fbbSRusty Russell printk(KERN_NOTICE "vringh:"); 32f87d0fbbSRusty Russell vprintk(fmt, ap); 33f87d0fbbSRusty Russell va_end(ap); 34f87d0fbbSRusty Russell } 35f87d0fbbSRusty Russell } 36f87d0fbbSRusty Russell 37f87d0fbbSRusty Russell /* Returns vring->num if empty, -ve on error. */ 38f87d0fbbSRusty Russell static inline int __vringh_get_head(const struct vringh *vrh, 39b9f7ac8cSMichael S. Tsirkin int (*getu16)(const struct vringh *vrh, 40b9f7ac8cSMichael S. Tsirkin u16 *val, const __virtio16 *p), 41f87d0fbbSRusty Russell u16 *last_avail_idx) 42f87d0fbbSRusty Russell { 43f87d0fbbSRusty Russell u16 avail_idx, i, head; 44f87d0fbbSRusty Russell int err; 45f87d0fbbSRusty Russell 46b9f7ac8cSMichael S. Tsirkin err = getu16(vrh, &avail_idx, &vrh->vring.avail->idx); 47f87d0fbbSRusty Russell if (err) { 48f87d0fbbSRusty Russell vringh_bad("Failed to access avail idx at %p", 49f87d0fbbSRusty Russell &vrh->vring.avail->idx); 50f87d0fbbSRusty Russell return err; 51f87d0fbbSRusty Russell } 52f87d0fbbSRusty Russell 53f87d0fbbSRusty Russell if (*last_avail_idx == avail_idx) 54f87d0fbbSRusty Russell return vrh->vring.num; 55f87d0fbbSRusty Russell 56f87d0fbbSRusty Russell /* Only get avail ring entries after they have been exposed by guest. */ 57f87d0fbbSRusty Russell virtio_rmb(vrh->weak_barriers); 58f87d0fbbSRusty Russell 59f87d0fbbSRusty Russell i = *last_avail_idx & (vrh->vring.num - 1); 60f87d0fbbSRusty Russell 61b9f7ac8cSMichael S. Tsirkin err = getu16(vrh, &head, &vrh->vring.avail->ring[i]); 62f87d0fbbSRusty Russell if (err) { 63f87d0fbbSRusty Russell vringh_bad("Failed to read head: idx %d address %p", 64f87d0fbbSRusty Russell *last_avail_idx, &vrh->vring.avail->ring[i]); 65f87d0fbbSRusty Russell return err; 66f87d0fbbSRusty Russell } 67f87d0fbbSRusty Russell 68f87d0fbbSRusty Russell if (head >= vrh->vring.num) { 69f87d0fbbSRusty Russell vringh_bad("Guest says index %u > %u is available", 70f87d0fbbSRusty Russell head, vrh->vring.num); 71f87d0fbbSRusty Russell return -EINVAL; 72f87d0fbbSRusty Russell } 73f87d0fbbSRusty Russell 74f87d0fbbSRusty Russell (*last_avail_idx)++; 75f87d0fbbSRusty Russell return head; 76f87d0fbbSRusty Russell } 77f87d0fbbSRusty Russell 78b8c06ad4SStefano Garzarella /** 79b8c06ad4SStefano Garzarella * vringh_kiov_advance - skip bytes from vring_kiov 80b8c06ad4SStefano Garzarella * @iov: an iov passed to vringh_getdesc_*() (updated as we consume) 81b8c06ad4SStefano Garzarella * @len: the maximum length to advance 82b8c06ad4SStefano Garzarella */ 83b8c06ad4SStefano Garzarella void vringh_kiov_advance(struct vringh_kiov *iov, size_t len) 84b8c06ad4SStefano Garzarella { 85b8c06ad4SStefano Garzarella while (len && iov->i < iov->used) { 86b8c06ad4SStefano Garzarella size_t partlen = min(iov->iov[iov->i].iov_len, len); 87b8c06ad4SStefano Garzarella 88b8c06ad4SStefano Garzarella iov->consumed += partlen; 89b8c06ad4SStefano Garzarella iov->iov[iov->i].iov_len -= partlen; 90b8c06ad4SStefano Garzarella iov->iov[iov->i].iov_base += partlen; 91b8c06ad4SStefano Garzarella 92b8c06ad4SStefano Garzarella if (!iov->iov[iov->i].iov_len) { 93b8c06ad4SStefano Garzarella /* Fix up old iov element then increment. */ 94b8c06ad4SStefano Garzarella iov->iov[iov->i].iov_len = iov->consumed; 95b8c06ad4SStefano Garzarella iov->iov[iov->i].iov_base -= iov->consumed; 96b8c06ad4SStefano Garzarella 97b8c06ad4SStefano Garzarella iov->consumed = 0; 98b8c06ad4SStefano Garzarella iov->i++; 99b8c06ad4SStefano Garzarella } 100b8c06ad4SStefano Garzarella 101b8c06ad4SStefano Garzarella len -= partlen; 102b8c06ad4SStefano Garzarella } 103b8c06ad4SStefano Garzarella } 104b8c06ad4SStefano Garzarella EXPORT_SYMBOL(vringh_kiov_advance); 105b8c06ad4SStefano Garzarella 106f87d0fbbSRusty Russell /* Copy some bytes to/from the iovec. Returns num copied. */ 1079ad9c49cSJason Wang static inline ssize_t vringh_iov_xfer(struct vringh *vrh, 1089ad9c49cSJason Wang struct vringh_kiov *iov, 109f87d0fbbSRusty Russell void *ptr, size_t len, 1109ad9c49cSJason Wang int (*xfer)(const struct vringh *vrh, 1119ad9c49cSJason Wang void *addr, void *ptr, 112f87d0fbbSRusty Russell size_t len)) 113f87d0fbbSRusty Russell { 114f87d0fbbSRusty Russell int err, done = 0; 115f87d0fbbSRusty Russell 116f87d0fbbSRusty Russell while (len && iov->i < iov->used) { 117f87d0fbbSRusty Russell size_t partlen; 118f87d0fbbSRusty Russell 119f87d0fbbSRusty Russell partlen = min(iov->iov[iov->i].iov_len, len); 1209ad9c49cSJason Wang err = xfer(vrh, iov->iov[iov->i].iov_base, ptr, partlen); 121f87d0fbbSRusty Russell if (err) 122f87d0fbbSRusty Russell return err; 123f87d0fbbSRusty Russell done += partlen; 124f87d0fbbSRusty Russell len -= partlen; 125f87d0fbbSRusty Russell ptr += partlen; 126f87d0fbbSRusty Russell 127b8c06ad4SStefano Garzarella vringh_kiov_advance(iov, partlen); 128f87d0fbbSRusty Russell } 129f87d0fbbSRusty Russell return done; 130f87d0fbbSRusty Russell } 131f87d0fbbSRusty Russell 132f87d0fbbSRusty Russell /* May reduce *len if range is shorter. */ 133f87d0fbbSRusty Russell static inline bool range_check(struct vringh *vrh, u64 addr, size_t *len, 134f87d0fbbSRusty Russell struct vringh_range *range, 135f87d0fbbSRusty Russell bool (*getrange)(struct vringh *, 136f87d0fbbSRusty Russell u64, struct vringh_range *)) 137f87d0fbbSRusty Russell { 138f87d0fbbSRusty Russell if (addr < range->start || addr > range->end_incl) { 139f87d0fbbSRusty Russell if (!getrange(vrh, addr, range)) 140f87d0fbbSRusty Russell return false; 141f87d0fbbSRusty Russell } 142f87d0fbbSRusty Russell BUG_ON(addr < range->start || addr > range->end_incl); 143f87d0fbbSRusty Russell 144f87d0fbbSRusty Russell /* To end of memory? */ 145f87d0fbbSRusty Russell if (unlikely(addr + *len == 0)) { 146f87d0fbbSRusty Russell if (range->end_incl == -1ULL) 147f87d0fbbSRusty Russell return true; 148f87d0fbbSRusty Russell goto truncate; 149f87d0fbbSRusty Russell } 150f87d0fbbSRusty Russell 151f87d0fbbSRusty Russell /* Otherwise, don't wrap. */ 152f87d0fbbSRusty Russell if (addr + *len < addr) { 153f87d0fbbSRusty Russell vringh_bad("Wrapping descriptor %zu@0x%llx", 154f87d0fbbSRusty Russell *len, (unsigned long long)addr); 155f87d0fbbSRusty Russell return false; 156f87d0fbbSRusty Russell } 157f87d0fbbSRusty Russell 158f87d0fbbSRusty Russell if (unlikely(addr + *len - 1 > range->end_incl)) 159f87d0fbbSRusty Russell goto truncate; 160f87d0fbbSRusty Russell return true; 161f87d0fbbSRusty Russell 162f87d0fbbSRusty Russell truncate: 163f87d0fbbSRusty Russell *len = range->end_incl + 1 - addr; 164f87d0fbbSRusty Russell return true; 165f87d0fbbSRusty Russell } 166f87d0fbbSRusty Russell 167f87d0fbbSRusty Russell static inline bool no_range_check(struct vringh *vrh, u64 addr, size_t *len, 168f87d0fbbSRusty Russell struct vringh_range *range, 169f87d0fbbSRusty Russell bool (*getrange)(struct vringh *, 170f87d0fbbSRusty Russell u64, struct vringh_range *)) 171f87d0fbbSRusty Russell { 172f87d0fbbSRusty Russell return true; 173f87d0fbbSRusty Russell } 174f87d0fbbSRusty Russell 175f87d0fbbSRusty Russell /* No reason for this code to be inline. */ 176b9f7ac8cSMichael S. Tsirkin static int move_to_indirect(const struct vringh *vrh, 177b9f7ac8cSMichael S. Tsirkin int *up_next, u16 *i, void *addr, 178f87d0fbbSRusty Russell const struct vring_desc *desc, 179f87d0fbbSRusty Russell struct vring_desc **descs, int *desc_max) 180f87d0fbbSRusty Russell { 181b9f7ac8cSMichael S. Tsirkin u32 len; 182b9f7ac8cSMichael S. Tsirkin 183f87d0fbbSRusty Russell /* Indirect tables can't have indirect. */ 184f87d0fbbSRusty Russell if (*up_next != -1) { 185f87d0fbbSRusty Russell vringh_bad("Multilevel indirect %u->%u", *up_next, *i); 186f87d0fbbSRusty Russell return -EINVAL; 187f87d0fbbSRusty Russell } 188f87d0fbbSRusty Russell 189b9f7ac8cSMichael S. Tsirkin len = vringh32_to_cpu(vrh, desc->len); 190b9f7ac8cSMichael S. Tsirkin if (unlikely(len % sizeof(struct vring_desc))) { 191f87d0fbbSRusty Russell vringh_bad("Strange indirect len %u", desc->len); 192f87d0fbbSRusty Russell return -EINVAL; 193f87d0fbbSRusty Russell } 194f87d0fbbSRusty Russell 195f87d0fbbSRusty Russell /* We will check this when we follow it! */ 196b9f7ac8cSMichael S. Tsirkin if (desc->flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT)) 197b9f7ac8cSMichael S. Tsirkin *up_next = vringh16_to_cpu(vrh, desc->next); 198f87d0fbbSRusty Russell else 199f87d0fbbSRusty Russell *up_next = -2; 200f87d0fbbSRusty Russell *descs = addr; 201b9f7ac8cSMichael S. Tsirkin *desc_max = len / sizeof(struct vring_desc); 202f87d0fbbSRusty Russell 203f87d0fbbSRusty Russell /* Now, start at the first indirect. */ 204f87d0fbbSRusty Russell *i = 0; 205f87d0fbbSRusty Russell return 0; 206f87d0fbbSRusty Russell } 207f87d0fbbSRusty Russell 208f87d0fbbSRusty Russell static int resize_iovec(struct vringh_kiov *iov, gfp_t gfp) 209f87d0fbbSRusty Russell { 210f87d0fbbSRusty Russell struct kvec *new; 211f87d0fbbSRusty Russell unsigned int flag, new_num = (iov->max_num & ~VRINGH_IOV_ALLOCATED) * 2; 212f87d0fbbSRusty Russell 213f87d0fbbSRusty Russell if (new_num < 8) 214f87d0fbbSRusty Russell new_num = 8; 215f87d0fbbSRusty Russell 216f87d0fbbSRusty Russell flag = (iov->max_num & VRINGH_IOV_ALLOCATED); 217f87d0fbbSRusty Russell if (flag) 2183a999748SBartosz Golaszewski new = krealloc_array(iov->iov, new_num, 2193a999748SBartosz Golaszewski sizeof(struct iovec), gfp); 220f87d0fbbSRusty Russell else { 2216da2ec56SKees Cook new = kmalloc_array(new_num, sizeof(struct iovec), gfp); 222f87d0fbbSRusty Russell if (new) { 223f87d0fbbSRusty Russell memcpy(new, iov->iov, 224f87d0fbbSRusty Russell iov->max_num * sizeof(struct iovec)); 225f87d0fbbSRusty Russell flag = VRINGH_IOV_ALLOCATED; 226f87d0fbbSRusty Russell } 227f87d0fbbSRusty Russell } 228f87d0fbbSRusty Russell if (!new) 229f87d0fbbSRusty Russell return -ENOMEM; 230f87d0fbbSRusty Russell iov->iov = new; 231f87d0fbbSRusty Russell iov->max_num = (new_num | flag); 232f87d0fbbSRusty Russell return 0; 233f87d0fbbSRusty Russell } 234f87d0fbbSRusty Russell 235f87d0fbbSRusty Russell static u16 __cold return_from_indirect(const struct vringh *vrh, int *up_next, 236f87d0fbbSRusty Russell struct vring_desc **descs, int *desc_max) 237f87d0fbbSRusty Russell { 238f87d0fbbSRusty Russell u16 i = *up_next; 239f87d0fbbSRusty Russell 240f87d0fbbSRusty Russell *up_next = -1; 241f87d0fbbSRusty Russell *descs = vrh->vring.desc; 242f87d0fbbSRusty Russell *desc_max = vrh->vring.num; 243f87d0fbbSRusty Russell return i; 244f87d0fbbSRusty Russell } 245f87d0fbbSRusty Russell 246f87d0fbbSRusty Russell static int slow_copy(struct vringh *vrh, void *dst, const void *src, 247f87d0fbbSRusty Russell bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len, 248f87d0fbbSRusty Russell struct vringh_range *range, 249f87d0fbbSRusty Russell bool (*getrange)(struct vringh *vrh, 250f87d0fbbSRusty Russell u64, 251f87d0fbbSRusty Russell struct vringh_range *)), 252f87d0fbbSRusty Russell bool (*getrange)(struct vringh *vrh, 253f87d0fbbSRusty Russell u64 addr, 254f87d0fbbSRusty Russell struct vringh_range *r), 255f87d0fbbSRusty Russell struct vringh_range *range, 2569ad9c49cSJason Wang int (*copy)(const struct vringh *vrh, 2579ad9c49cSJason Wang void *dst, const void *src, size_t len)) 258f87d0fbbSRusty Russell { 259f87d0fbbSRusty Russell size_t part, len = sizeof(struct vring_desc); 260f87d0fbbSRusty Russell 261f87d0fbbSRusty Russell do { 262f87d0fbbSRusty Russell u64 addr; 263f87d0fbbSRusty Russell int err; 264f87d0fbbSRusty Russell 265f87d0fbbSRusty Russell part = len; 266f87d0fbbSRusty Russell addr = (u64)(unsigned long)src - range->offset; 267f87d0fbbSRusty Russell 268f87d0fbbSRusty Russell if (!rcheck(vrh, addr, &part, range, getrange)) 269f87d0fbbSRusty Russell return -EINVAL; 270f87d0fbbSRusty Russell 2719ad9c49cSJason Wang err = copy(vrh, dst, src, part); 272f87d0fbbSRusty Russell if (err) 273f87d0fbbSRusty Russell return err; 274f87d0fbbSRusty Russell 275f87d0fbbSRusty Russell dst += part; 276f87d0fbbSRusty Russell src += part; 277f87d0fbbSRusty Russell len -= part; 278f87d0fbbSRusty Russell } while (len); 279f87d0fbbSRusty Russell return 0; 280f87d0fbbSRusty Russell } 281f87d0fbbSRusty Russell 282f87d0fbbSRusty Russell static inline int 283f87d0fbbSRusty Russell __vringh_iov(struct vringh *vrh, u16 i, 284f87d0fbbSRusty Russell struct vringh_kiov *riov, 285f87d0fbbSRusty Russell struct vringh_kiov *wiov, 286f87d0fbbSRusty Russell bool (*rcheck)(struct vringh *vrh, u64 addr, size_t *len, 287f87d0fbbSRusty Russell struct vringh_range *range, 288f87d0fbbSRusty Russell bool (*getrange)(struct vringh *, u64, 289f87d0fbbSRusty Russell struct vringh_range *)), 290f87d0fbbSRusty Russell bool (*getrange)(struct vringh *, u64, struct vringh_range *), 291f87d0fbbSRusty Russell gfp_t gfp, 2929ad9c49cSJason Wang int (*copy)(const struct vringh *vrh, 2939ad9c49cSJason Wang void *dst, const void *src, size_t len)) 294f87d0fbbSRusty Russell { 295dbd29e07SXie Yongji int err, count = 0, indirect_count = 0, up_next, desc_max; 296f87d0fbbSRusty Russell struct vring_desc desc, *descs; 297f87d0fbbSRusty Russell struct vringh_range range = { -1ULL, 0 }, slowrange; 298f87d0fbbSRusty Russell bool slow = false; 299f87d0fbbSRusty Russell 300f87d0fbbSRusty Russell /* We start traversing vring's descriptor table. */ 301f87d0fbbSRusty Russell descs = vrh->vring.desc; 302f87d0fbbSRusty Russell desc_max = vrh->vring.num; 303f87d0fbbSRusty Russell up_next = -1; 304f87d0fbbSRusty Russell 3055745bcfbSStefano Garzarella /* You must want something! */ 3065745bcfbSStefano Garzarella if (WARN_ON(!riov && !wiov)) 3075745bcfbSStefano Garzarella return -EINVAL; 3085745bcfbSStefano Garzarella 309f87d0fbbSRusty Russell if (riov) 310bbc2c372SStefano Garzarella riov->i = riov->used = riov->consumed = 0; 3115745bcfbSStefano Garzarella if (wiov) 312bbc2c372SStefano Garzarella wiov->i = wiov->used = wiov->consumed = 0; 313f87d0fbbSRusty Russell 314f87d0fbbSRusty Russell for (;;) { 315f87d0fbbSRusty Russell void *addr; 316f87d0fbbSRusty Russell struct vringh_kiov *iov; 317f87d0fbbSRusty Russell size_t len; 318f87d0fbbSRusty Russell 319f87d0fbbSRusty Russell if (unlikely(slow)) 320f87d0fbbSRusty Russell err = slow_copy(vrh, &desc, &descs[i], rcheck, getrange, 321f87d0fbbSRusty Russell &slowrange, copy); 322f87d0fbbSRusty Russell else 3239ad9c49cSJason Wang err = copy(vrh, &desc, &descs[i], sizeof(desc)); 324f87d0fbbSRusty Russell if (unlikely(err)) 325f87d0fbbSRusty Russell goto fail; 326f87d0fbbSRusty Russell 327b9f7ac8cSMichael S. Tsirkin if (unlikely(desc.flags & 328b9f7ac8cSMichael S. Tsirkin cpu_to_vringh16(vrh, VRING_DESC_F_INDIRECT))) { 329b9f7ac8cSMichael S. Tsirkin u64 a = vringh64_to_cpu(vrh, desc.addr); 330b9f7ac8cSMichael S. Tsirkin 331f87d0fbbSRusty Russell /* Make sure it's OK, and get offset. */ 332b9f7ac8cSMichael S. Tsirkin len = vringh32_to_cpu(vrh, desc.len); 333b9f7ac8cSMichael S. Tsirkin if (!rcheck(vrh, a, &len, &range, getrange)) { 334f87d0fbbSRusty Russell err = -EINVAL; 335f87d0fbbSRusty Russell goto fail; 336f87d0fbbSRusty Russell } 337f87d0fbbSRusty Russell 338b9f7ac8cSMichael S. Tsirkin if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) { 339f87d0fbbSRusty Russell slow = true; 340f87d0fbbSRusty Russell /* We need to save this range to use offset */ 341f87d0fbbSRusty Russell slowrange = range; 342f87d0fbbSRusty Russell } 343f87d0fbbSRusty Russell 344b9f7ac8cSMichael S. Tsirkin addr = (void *)(long)(a + range.offset); 345b9f7ac8cSMichael S. Tsirkin err = move_to_indirect(vrh, &up_next, &i, addr, &desc, 346f87d0fbbSRusty Russell &descs, &desc_max); 347f87d0fbbSRusty Russell if (err) 348f87d0fbbSRusty Russell goto fail; 349f87d0fbbSRusty Russell continue; 350f87d0fbbSRusty Russell } 351f87d0fbbSRusty Russell 352dbd29e07SXie Yongji if (up_next == -1) 353dbd29e07SXie Yongji count++; 354dbd29e07SXie Yongji else 355dbd29e07SXie Yongji indirect_count++; 356dbd29e07SXie Yongji 357dbd29e07SXie Yongji if (count > vrh->vring.num || indirect_count > desc_max) { 358f87d0fbbSRusty Russell vringh_bad("Descriptor loop in %p", descs); 359f87d0fbbSRusty Russell err = -ELOOP; 360f87d0fbbSRusty Russell goto fail; 361f87d0fbbSRusty Russell } 362f87d0fbbSRusty Russell 363b9f7ac8cSMichael S. Tsirkin if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_WRITE)) 364f87d0fbbSRusty Russell iov = wiov; 365f87d0fbbSRusty Russell else { 366f87d0fbbSRusty Russell iov = riov; 367e74cfa91SNeeraj Upadhyay if (unlikely(wiov && wiov->used)) { 368f87d0fbbSRusty Russell vringh_bad("Readable desc %p after writable", 369f87d0fbbSRusty Russell &descs[i]); 370f87d0fbbSRusty Russell err = -EINVAL; 371f87d0fbbSRusty Russell goto fail; 372f87d0fbbSRusty Russell } 373f87d0fbbSRusty Russell } 374f87d0fbbSRusty Russell 375f87d0fbbSRusty Russell if (!iov) { 376f87d0fbbSRusty Russell vringh_bad("Unexpected %s desc", 377f87d0fbbSRusty Russell !wiov ? "writable" : "readable"); 378f87d0fbbSRusty Russell err = -EPROTO; 379f87d0fbbSRusty Russell goto fail; 380f87d0fbbSRusty Russell } 381f87d0fbbSRusty Russell 382f87d0fbbSRusty Russell again: 383f87d0fbbSRusty Russell /* Make sure it's OK, and get offset. */ 384b9f7ac8cSMichael S. Tsirkin len = vringh32_to_cpu(vrh, desc.len); 385b9f7ac8cSMichael S. Tsirkin if (!rcheck(vrh, vringh64_to_cpu(vrh, desc.addr), &len, &range, 386b9f7ac8cSMichael S. Tsirkin getrange)) { 387f87d0fbbSRusty Russell err = -EINVAL; 388f87d0fbbSRusty Russell goto fail; 389f87d0fbbSRusty Russell } 390b9f7ac8cSMichael S. Tsirkin addr = (void *)(unsigned long)(vringh64_to_cpu(vrh, desc.addr) + 391b9f7ac8cSMichael S. Tsirkin range.offset); 392f87d0fbbSRusty Russell 393f87d0fbbSRusty Russell if (unlikely(iov->used == (iov->max_num & ~VRINGH_IOV_ALLOCATED))) { 394f87d0fbbSRusty Russell err = resize_iovec(iov, gfp); 395f87d0fbbSRusty Russell if (err) 396f87d0fbbSRusty Russell goto fail; 397f87d0fbbSRusty Russell } 398f87d0fbbSRusty Russell 399f87d0fbbSRusty Russell iov->iov[iov->used].iov_base = addr; 400f87d0fbbSRusty Russell iov->iov[iov->used].iov_len = len; 401f87d0fbbSRusty Russell iov->used++; 402f87d0fbbSRusty Russell 403b9f7ac8cSMichael S. Tsirkin if (unlikely(len != vringh32_to_cpu(vrh, desc.len))) { 404b9f7ac8cSMichael S. Tsirkin desc.len = cpu_to_vringh32(vrh, 405b9f7ac8cSMichael S. Tsirkin vringh32_to_cpu(vrh, desc.len) - len); 406b9f7ac8cSMichael S. Tsirkin desc.addr = cpu_to_vringh64(vrh, 407b9f7ac8cSMichael S. Tsirkin vringh64_to_cpu(vrh, desc.addr) + len); 408f87d0fbbSRusty Russell goto again; 409f87d0fbbSRusty Russell } 410f87d0fbbSRusty Russell 411b9f7ac8cSMichael S. Tsirkin if (desc.flags & cpu_to_vringh16(vrh, VRING_DESC_F_NEXT)) { 412b9f7ac8cSMichael S. Tsirkin i = vringh16_to_cpu(vrh, desc.next); 413f87d0fbbSRusty Russell } else { 414f87d0fbbSRusty Russell /* Just in case we need to finish traversing above. */ 415f87d0fbbSRusty Russell if (unlikely(up_next > 0)) { 416f87d0fbbSRusty Russell i = return_from_indirect(vrh, &up_next, 417f87d0fbbSRusty Russell &descs, &desc_max); 418f87d0fbbSRusty Russell slow = false; 419dbd29e07SXie Yongji indirect_count = 0; 420f87d0fbbSRusty Russell } else 421f87d0fbbSRusty Russell break; 422f87d0fbbSRusty Russell } 423f87d0fbbSRusty Russell 424f87d0fbbSRusty Russell if (i >= desc_max) { 425f87d0fbbSRusty Russell vringh_bad("Chained index %u > %u", i, desc_max); 426f87d0fbbSRusty Russell err = -EINVAL; 427f87d0fbbSRusty Russell goto fail; 428f87d0fbbSRusty Russell } 429f87d0fbbSRusty Russell } 430f87d0fbbSRusty Russell 431f87d0fbbSRusty Russell return 0; 432f87d0fbbSRusty Russell 433f87d0fbbSRusty Russell fail: 434f87d0fbbSRusty Russell return err; 435f87d0fbbSRusty Russell } 436f87d0fbbSRusty Russell 437f87d0fbbSRusty Russell static inline int __vringh_complete(struct vringh *vrh, 438f87d0fbbSRusty Russell const struct vring_used_elem *used, 439f87d0fbbSRusty Russell unsigned int num_used, 440b9f7ac8cSMichael S. Tsirkin int (*putu16)(const struct vringh *vrh, 441b9f7ac8cSMichael S. Tsirkin __virtio16 *p, u16 val), 4429ad9c49cSJason Wang int (*putused)(const struct vringh *vrh, 4439ad9c49cSJason Wang struct vring_used_elem *dst, 444f87d0fbbSRusty Russell const struct vring_used_elem 445f87d0fbbSRusty Russell *src, unsigned num)) 446f87d0fbbSRusty Russell { 447f87d0fbbSRusty Russell struct vring_used *used_ring; 448f87d0fbbSRusty Russell int err; 449f87d0fbbSRusty Russell u16 used_idx, off; 450f87d0fbbSRusty Russell 451f87d0fbbSRusty Russell used_ring = vrh->vring.used; 452f87d0fbbSRusty Russell used_idx = vrh->last_used_idx + vrh->completed; 453f87d0fbbSRusty Russell 454f87d0fbbSRusty Russell off = used_idx % vrh->vring.num; 455f87d0fbbSRusty Russell 456f87d0fbbSRusty Russell /* Compiler knows num_used == 1 sometimes, hence extra check */ 457f87d0fbbSRusty Russell if (num_used > 1 && unlikely(off + num_used >= vrh->vring.num)) { 458f87d0fbbSRusty Russell u16 part = vrh->vring.num - off; 4599ad9c49cSJason Wang err = putused(vrh, &used_ring->ring[off], used, part); 460f87d0fbbSRusty Russell if (!err) 4619ad9c49cSJason Wang err = putused(vrh, &used_ring->ring[0], used + part, 462f87d0fbbSRusty Russell num_used - part); 463f87d0fbbSRusty Russell } else 4649ad9c49cSJason Wang err = putused(vrh, &used_ring->ring[off], used, num_used); 465f87d0fbbSRusty Russell 466f87d0fbbSRusty Russell if (err) { 467f87d0fbbSRusty Russell vringh_bad("Failed to write %u used entries %u at %p", 468f87d0fbbSRusty Russell num_used, off, &used_ring->ring[off]); 469f87d0fbbSRusty Russell return err; 470f87d0fbbSRusty Russell } 471f87d0fbbSRusty Russell 472f87d0fbbSRusty Russell /* Make sure buffer is written before we update index. */ 473f87d0fbbSRusty Russell virtio_wmb(vrh->weak_barriers); 474f87d0fbbSRusty Russell 475b9f7ac8cSMichael S. Tsirkin err = putu16(vrh, &vrh->vring.used->idx, used_idx + num_used); 476f87d0fbbSRusty Russell if (err) { 477f87d0fbbSRusty Russell vringh_bad("Failed to update used index at %p", 478f87d0fbbSRusty Russell &vrh->vring.used->idx); 479f87d0fbbSRusty Russell return err; 480f87d0fbbSRusty Russell } 481f87d0fbbSRusty Russell 482f87d0fbbSRusty Russell vrh->completed += num_used; 483f87d0fbbSRusty Russell return 0; 484f87d0fbbSRusty Russell } 485f87d0fbbSRusty Russell 486f87d0fbbSRusty Russell 487f87d0fbbSRusty Russell static inline int __vringh_need_notify(struct vringh *vrh, 488b9f7ac8cSMichael S. Tsirkin int (*getu16)(const struct vringh *vrh, 489b9f7ac8cSMichael S. Tsirkin u16 *val, 490b9f7ac8cSMichael S. Tsirkin const __virtio16 *p)) 491f87d0fbbSRusty Russell { 492f87d0fbbSRusty Russell bool notify; 493f87d0fbbSRusty Russell u16 used_event; 494f87d0fbbSRusty Russell int err; 495f87d0fbbSRusty Russell 496f87d0fbbSRusty Russell /* Flush out used index update. This is paired with the 497f87d0fbbSRusty Russell * barrier that the Guest executes when enabling 498f87d0fbbSRusty Russell * interrupts. */ 499f87d0fbbSRusty Russell virtio_mb(vrh->weak_barriers); 500f87d0fbbSRusty Russell 501f87d0fbbSRusty Russell /* Old-style, without event indices. */ 502f87d0fbbSRusty Russell if (!vrh->event_indices) { 503f87d0fbbSRusty Russell u16 flags; 504b9f7ac8cSMichael S. Tsirkin err = getu16(vrh, &flags, &vrh->vring.avail->flags); 505f87d0fbbSRusty Russell if (err) { 506f87d0fbbSRusty Russell vringh_bad("Failed to get flags at %p", 507f87d0fbbSRusty Russell &vrh->vring.avail->flags); 508f87d0fbbSRusty Russell return err; 509f87d0fbbSRusty Russell } 510f87d0fbbSRusty Russell return (!(flags & VRING_AVAIL_F_NO_INTERRUPT)); 511f87d0fbbSRusty Russell } 512f87d0fbbSRusty Russell 513f87d0fbbSRusty Russell /* Modern: we know when other side wants to know. */ 514b9f7ac8cSMichael S. Tsirkin err = getu16(vrh, &used_event, &vring_used_event(&vrh->vring)); 515f87d0fbbSRusty Russell if (err) { 516f87d0fbbSRusty Russell vringh_bad("Failed to get used event idx at %p", 517f87d0fbbSRusty Russell &vring_used_event(&vrh->vring)); 518f87d0fbbSRusty Russell return err; 519f87d0fbbSRusty Russell } 520f87d0fbbSRusty Russell 521f87d0fbbSRusty Russell /* Just in case we added so many that we wrap. */ 522f87d0fbbSRusty Russell if (unlikely(vrh->completed > 0xffff)) 523f87d0fbbSRusty Russell notify = true; 524f87d0fbbSRusty Russell else 525f87d0fbbSRusty Russell notify = vring_need_event(used_event, 526f87d0fbbSRusty Russell vrh->last_used_idx + vrh->completed, 527f87d0fbbSRusty Russell vrh->last_used_idx); 528f87d0fbbSRusty Russell 529f87d0fbbSRusty Russell vrh->last_used_idx += vrh->completed; 530f87d0fbbSRusty Russell vrh->completed = 0; 531f87d0fbbSRusty Russell return notify; 532f87d0fbbSRusty Russell } 533f87d0fbbSRusty Russell 534f87d0fbbSRusty Russell static inline bool __vringh_notify_enable(struct vringh *vrh, 535b9f7ac8cSMichael S. Tsirkin int (*getu16)(const struct vringh *vrh, 536b9f7ac8cSMichael S. Tsirkin u16 *val, const __virtio16 *p), 537b9f7ac8cSMichael S. Tsirkin int (*putu16)(const struct vringh *vrh, 538b9f7ac8cSMichael S. Tsirkin __virtio16 *p, u16 val)) 539f87d0fbbSRusty Russell { 540f87d0fbbSRusty Russell u16 avail; 541f87d0fbbSRusty Russell 542f87d0fbbSRusty Russell if (!vrh->event_indices) { 543f87d0fbbSRusty Russell /* Old-school; update flags. */ 544b9f7ac8cSMichael S. Tsirkin if (putu16(vrh, &vrh->vring.used->flags, 0) != 0) { 545f87d0fbbSRusty Russell vringh_bad("Clearing used flags %p", 546f87d0fbbSRusty Russell &vrh->vring.used->flags); 547f87d0fbbSRusty Russell return true; 548f87d0fbbSRusty Russell } 549f87d0fbbSRusty Russell } else { 550b9f7ac8cSMichael S. Tsirkin if (putu16(vrh, &vring_avail_event(&vrh->vring), 551f87d0fbbSRusty Russell vrh->last_avail_idx) != 0) { 552f87d0fbbSRusty Russell vringh_bad("Updating avail event index %p", 553f87d0fbbSRusty Russell &vring_avail_event(&vrh->vring)); 554f87d0fbbSRusty Russell return true; 555f87d0fbbSRusty Russell } 556f87d0fbbSRusty Russell } 557f87d0fbbSRusty Russell 558f87d0fbbSRusty Russell /* They could have slipped one in as we were doing that: make 559f87d0fbbSRusty Russell * sure it's written, then check again. */ 560f87d0fbbSRusty Russell virtio_mb(vrh->weak_barriers); 561f87d0fbbSRusty Russell 562b9f7ac8cSMichael S. Tsirkin if (getu16(vrh, &avail, &vrh->vring.avail->idx) != 0) { 563f87d0fbbSRusty Russell vringh_bad("Failed to check avail idx at %p", 564f87d0fbbSRusty Russell &vrh->vring.avail->idx); 565f87d0fbbSRusty Russell return true; 566f87d0fbbSRusty Russell } 567f87d0fbbSRusty Russell 568f87d0fbbSRusty Russell /* This is unlikely, so we just leave notifications enabled 569f87d0fbbSRusty Russell * (if we're using event_indices, we'll only get one 570f87d0fbbSRusty Russell * notification anyway). */ 571f87d0fbbSRusty Russell return avail == vrh->last_avail_idx; 572f87d0fbbSRusty Russell } 573f87d0fbbSRusty Russell 574f87d0fbbSRusty Russell static inline void __vringh_notify_disable(struct vringh *vrh, 575b9f7ac8cSMichael S. Tsirkin int (*putu16)(const struct vringh *vrh, 576b9f7ac8cSMichael S. Tsirkin __virtio16 *p, u16 val)) 577f87d0fbbSRusty Russell { 578f87d0fbbSRusty Russell if (!vrh->event_indices) { 579f87d0fbbSRusty Russell /* Old-school; update flags. */ 580b9f7ac8cSMichael S. Tsirkin if (putu16(vrh, &vrh->vring.used->flags, 581b9f7ac8cSMichael S. Tsirkin VRING_USED_F_NO_NOTIFY)) { 582f87d0fbbSRusty Russell vringh_bad("Setting used flags %p", 583f87d0fbbSRusty Russell &vrh->vring.used->flags); 584f87d0fbbSRusty Russell } 585f87d0fbbSRusty Russell } 586f87d0fbbSRusty Russell } 587f87d0fbbSRusty Russell 588f87d0fbbSRusty Russell /* Userspace access helpers: in this case, addresses are really userspace. */ 589b9f7ac8cSMichael S. Tsirkin static inline int getu16_user(const struct vringh *vrh, u16 *val, const __virtio16 *p) 590f87d0fbbSRusty Russell { 591b9f7ac8cSMichael S. Tsirkin __virtio16 v = 0; 592b9f7ac8cSMichael S. Tsirkin int rc = get_user(v, (__force __virtio16 __user *)p); 593b9f7ac8cSMichael S. Tsirkin *val = vringh16_to_cpu(vrh, v); 594b9f7ac8cSMichael S. Tsirkin return rc; 595f87d0fbbSRusty Russell } 596f87d0fbbSRusty Russell 597b9f7ac8cSMichael S. Tsirkin static inline int putu16_user(const struct vringh *vrh, __virtio16 *p, u16 val) 598f87d0fbbSRusty Russell { 599b9f7ac8cSMichael S. Tsirkin __virtio16 v = cpu_to_vringh16(vrh, val); 600b9f7ac8cSMichael S. Tsirkin return put_user(v, (__force __virtio16 __user *)p); 601f87d0fbbSRusty Russell } 602f87d0fbbSRusty Russell 6039ad9c49cSJason Wang static inline int copydesc_user(const struct vringh *vrh, 6049ad9c49cSJason Wang void *dst, const void *src, size_t len) 605f87d0fbbSRusty Russell { 606f87d0fbbSRusty Russell return copy_from_user(dst, (__force void __user *)src, len) ? 607f87d0fbbSRusty Russell -EFAULT : 0; 608f87d0fbbSRusty Russell } 609f87d0fbbSRusty Russell 6109ad9c49cSJason Wang static inline int putused_user(const struct vringh *vrh, 6119ad9c49cSJason Wang struct vring_used_elem *dst, 612f87d0fbbSRusty Russell const struct vring_used_elem *src, 613f87d0fbbSRusty Russell unsigned int num) 614f87d0fbbSRusty Russell { 615f87d0fbbSRusty Russell return copy_to_user((__force void __user *)dst, src, 616f87d0fbbSRusty Russell sizeof(*dst) * num) ? -EFAULT : 0; 617f87d0fbbSRusty Russell } 618f87d0fbbSRusty Russell 6199ad9c49cSJason Wang static inline int xfer_from_user(const struct vringh *vrh, void *src, 6209ad9c49cSJason Wang void *dst, size_t len) 621f87d0fbbSRusty Russell { 622f87d0fbbSRusty Russell return copy_from_user(dst, (__force void __user *)src, len) ? 623f87d0fbbSRusty Russell -EFAULT : 0; 624f87d0fbbSRusty Russell } 625f87d0fbbSRusty Russell 6269ad9c49cSJason Wang static inline int xfer_to_user(const struct vringh *vrh, 6279ad9c49cSJason Wang void *dst, void *src, size_t len) 628f87d0fbbSRusty Russell { 629f87d0fbbSRusty Russell return copy_to_user((__force void __user *)dst, src, len) ? 630f87d0fbbSRusty Russell -EFAULT : 0; 631f87d0fbbSRusty Russell } 632f87d0fbbSRusty Russell 633f87d0fbbSRusty Russell /** 634f87d0fbbSRusty Russell * vringh_init_user - initialize a vringh for a userspace vring. 635f87d0fbbSRusty Russell * @vrh: the vringh to initialize. 636f87d0fbbSRusty Russell * @features: the feature bits for this ring. 637f87d0fbbSRusty Russell * @num: the number of elements. 638f87d0fbbSRusty Russell * @weak_barriers: true if we only need memory barriers, not I/O. 639905233afSStefano Garzarella * @desc: the userspace descriptor pointer. 640905233afSStefano Garzarella * @avail: the userspace avail pointer. 641905233afSStefano Garzarella * @used: the userspace used pointer. 642f87d0fbbSRusty Russell * 643f87d0fbbSRusty Russell * Returns an error if num is invalid: you should check pointers 644f87d0fbbSRusty Russell * yourself! 645f87d0fbbSRusty Russell */ 646b97a8a90SMichael S. Tsirkin int vringh_init_user(struct vringh *vrh, u64 features, 647f87d0fbbSRusty Russell unsigned int num, bool weak_barriers, 648a865e420SMichael S. Tsirkin vring_desc_t __user *desc, 649a865e420SMichael S. Tsirkin vring_avail_t __user *avail, 650a865e420SMichael S. Tsirkin vring_used_t __user *used) 651f87d0fbbSRusty Russell { 652f87d0fbbSRusty Russell /* Sane power of 2 please! */ 653f87d0fbbSRusty Russell if (!num || num > 0xffff || (num & (num - 1))) { 654f87d0fbbSRusty Russell vringh_bad("Bad ring size %u", num); 655f87d0fbbSRusty Russell return -EINVAL; 656f87d0fbbSRusty Russell } 657f87d0fbbSRusty Russell 658b9f7ac8cSMichael S. Tsirkin vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1)); 659f87d0fbbSRusty Russell vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX)); 660f87d0fbbSRusty Russell vrh->weak_barriers = weak_barriers; 661f87d0fbbSRusty Russell vrh->completed = 0; 662f87d0fbbSRusty Russell vrh->last_avail_idx = 0; 663f87d0fbbSRusty Russell vrh->last_used_idx = 0; 664f87d0fbbSRusty Russell vrh->vring.num = num; 665f87d0fbbSRusty Russell /* vring expects kernel addresses, but only used via accessors. */ 666f87d0fbbSRusty Russell vrh->vring.desc = (__force struct vring_desc *)desc; 667f87d0fbbSRusty Russell vrh->vring.avail = (__force struct vring_avail *)avail; 668f87d0fbbSRusty Russell vrh->vring.used = (__force struct vring_used *)used; 669f87d0fbbSRusty Russell return 0; 670f87d0fbbSRusty Russell } 671f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_init_user); 672f87d0fbbSRusty Russell 673f87d0fbbSRusty Russell /** 674f87d0fbbSRusty Russell * vringh_getdesc_user - get next available descriptor from userspace ring. 675f87d0fbbSRusty Russell * @vrh: the userspace vring. 676f87d0fbbSRusty Russell * @riov: where to put the readable descriptors (or NULL) 677f87d0fbbSRusty Russell * @wiov: where to put the writable descriptors (or NULL) 678f87d0fbbSRusty Russell * @getrange: function to call to check ranges. 679f87d0fbbSRusty Russell * @head: head index we received, for passing to vringh_complete_user(). 680f87d0fbbSRusty Russell * 681f87d0fbbSRusty Russell * Returns 0 if there was no descriptor, 1 if there was, or -errno. 682f87d0fbbSRusty Russell * 683f87d0fbbSRusty Russell * Note that on error return, you can tell the difference between an 684f87d0fbbSRusty Russell * invalid ring and a single invalid descriptor: in the former case, 685f87d0fbbSRusty Russell * *head will be vrh->vring.num. You may be able to ignore an invalid 686f87d0fbbSRusty Russell * descriptor, but there's not much you can do with an invalid ring. 687f87d0fbbSRusty Russell * 68869c13c58SStefano Garzarella * Note that you can reuse riov and wiov with subsequent calls. Content is 68969c13c58SStefano Garzarella * overwritten and memory reallocated if more space is needed. 69069c13c58SStefano Garzarella * When you don't have to use riov and wiov anymore, you should clean up them 69169c13c58SStefano Garzarella * calling vringh_iov_cleanup() to release the memory, even on error! 692f87d0fbbSRusty Russell */ 693f87d0fbbSRusty Russell int vringh_getdesc_user(struct vringh *vrh, 694f87d0fbbSRusty Russell struct vringh_iov *riov, 695f87d0fbbSRusty Russell struct vringh_iov *wiov, 696f87d0fbbSRusty Russell bool (*getrange)(struct vringh *vrh, 697f87d0fbbSRusty Russell u64 addr, struct vringh_range *r), 698f87d0fbbSRusty Russell u16 *head) 699f87d0fbbSRusty Russell { 700f87d0fbbSRusty Russell int err; 701f87d0fbbSRusty Russell 702f87d0fbbSRusty Russell *head = vrh->vring.num; 703f87d0fbbSRusty Russell err = __vringh_get_head(vrh, getu16_user, &vrh->last_avail_idx); 704f87d0fbbSRusty Russell if (err < 0) 705f87d0fbbSRusty Russell return err; 706f87d0fbbSRusty Russell 707f87d0fbbSRusty Russell /* Empty... */ 708f87d0fbbSRusty Russell if (err == vrh->vring.num) 709f87d0fbbSRusty Russell return 0; 710f87d0fbbSRusty Russell 711f87d0fbbSRusty Russell /* We need the layouts to be the identical for this to work */ 712f87d0fbbSRusty Russell BUILD_BUG_ON(sizeof(struct vringh_kiov) != sizeof(struct vringh_iov)); 713f87d0fbbSRusty Russell BUILD_BUG_ON(offsetof(struct vringh_kiov, iov) != 714f87d0fbbSRusty Russell offsetof(struct vringh_iov, iov)); 715f87d0fbbSRusty Russell BUILD_BUG_ON(offsetof(struct vringh_kiov, i) != 716f87d0fbbSRusty Russell offsetof(struct vringh_iov, i)); 717f87d0fbbSRusty Russell BUILD_BUG_ON(offsetof(struct vringh_kiov, used) != 718f87d0fbbSRusty Russell offsetof(struct vringh_iov, used)); 719f87d0fbbSRusty Russell BUILD_BUG_ON(offsetof(struct vringh_kiov, max_num) != 720f87d0fbbSRusty Russell offsetof(struct vringh_iov, max_num)); 721f87d0fbbSRusty Russell BUILD_BUG_ON(sizeof(struct iovec) != sizeof(struct kvec)); 722f87d0fbbSRusty Russell BUILD_BUG_ON(offsetof(struct iovec, iov_base) != 723f87d0fbbSRusty Russell offsetof(struct kvec, iov_base)); 724f87d0fbbSRusty Russell BUILD_BUG_ON(offsetof(struct iovec, iov_len) != 725f87d0fbbSRusty Russell offsetof(struct kvec, iov_len)); 726f87d0fbbSRusty Russell BUILD_BUG_ON(sizeof(((struct iovec *)NULL)->iov_base) 727f87d0fbbSRusty Russell != sizeof(((struct kvec *)NULL)->iov_base)); 728f87d0fbbSRusty Russell BUILD_BUG_ON(sizeof(((struct iovec *)NULL)->iov_len) 729f87d0fbbSRusty Russell != sizeof(((struct kvec *)NULL)->iov_len)); 730f87d0fbbSRusty Russell 731f87d0fbbSRusty Russell *head = err; 732f87d0fbbSRusty Russell err = __vringh_iov(vrh, *head, (struct vringh_kiov *)riov, 733f87d0fbbSRusty Russell (struct vringh_kiov *)wiov, 734f87d0fbbSRusty Russell range_check, getrange, GFP_KERNEL, copydesc_user); 735f87d0fbbSRusty Russell if (err) 736f87d0fbbSRusty Russell return err; 737f87d0fbbSRusty Russell 738f87d0fbbSRusty Russell return 1; 739f87d0fbbSRusty Russell } 740f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_getdesc_user); 741f87d0fbbSRusty Russell 742f87d0fbbSRusty Russell /** 743f87d0fbbSRusty Russell * vringh_iov_pull_user - copy bytes from vring_iov. 744f87d0fbbSRusty Russell * @riov: the riov as passed to vringh_getdesc_user() (updated as we consume) 745f87d0fbbSRusty Russell * @dst: the place to copy. 746f87d0fbbSRusty Russell * @len: the maximum length to copy. 747f87d0fbbSRusty Russell * 748f87d0fbbSRusty Russell * Returns the bytes copied <= len or a negative errno. 749f87d0fbbSRusty Russell */ 750f87d0fbbSRusty Russell ssize_t vringh_iov_pull_user(struct vringh_iov *riov, void *dst, size_t len) 751f87d0fbbSRusty Russell { 7529ad9c49cSJason Wang return vringh_iov_xfer(NULL, (struct vringh_kiov *)riov, 753f87d0fbbSRusty Russell dst, len, xfer_from_user); 754f87d0fbbSRusty Russell } 755f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_iov_pull_user); 756f87d0fbbSRusty Russell 757f87d0fbbSRusty Russell /** 758f87d0fbbSRusty Russell * vringh_iov_push_user - copy bytes into vring_iov. 759f87d0fbbSRusty Russell * @wiov: the wiov as passed to vringh_getdesc_user() (updated as we consume) 7608009b0f4SStefano Garzarella * @src: the place to copy from. 761f87d0fbbSRusty Russell * @len: the maximum length to copy. 762f87d0fbbSRusty Russell * 763f87d0fbbSRusty Russell * Returns the bytes copied <= len or a negative errno. 764f87d0fbbSRusty Russell */ 765f87d0fbbSRusty Russell ssize_t vringh_iov_push_user(struct vringh_iov *wiov, 766f87d0fbbSRusty Russell const void *src, size_t len) 767f87d0fbbSRusty Russell { 7689ad9c49cSJason Wang return vringh_iov_xfer(NULL, (struct vringh_kiov *)wiov, 769f87d0fbbSRusty Russell (void *)src, len, xfer_to_user); 770f87d0fbbSRusty Russell } 771f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_iov_push_user); 772f87d0fbbSRusty Russell 773f87d0fbbSRusty Russell /** 774f87d0fbbSRusty Russell * vringh_abandon_user - we've decided not to handle the descriptor(s). 775f87d0fbbSRusty Russell * @vrh: the vring. 776f87d0fbbSRusty Russell * @num: the number of descriptors to put back (ie. num 777f87d0fbbSRusty Russell * vringh_get_user() to undo). 778f87d0fbbSRusty Russell * 779f87d0fbbSRusty Russell * The next vringh_get_user() will return the old descriptor(s) again. 780f87d0fbbSRusty Russell */ 781f87d0fbbSRusty Russell void vringh_abandon_user(struct vringh *vrh, unsigned int num) 782f87d0fbbSRusty Russell { 783f87d0fbbSRusty Russell /* We only update vring_avail_event(vr) when we want to be notified, 784f87d0fbbSRusty Russell * so we haven't changed that yet. */ 785f87d0fbbSRusty Russell vrh->last_avail_idx -= num; 786f87d0fbbSRusty Russell } 787f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_abandon_user); 788f87d0fbbSRusty Russell 789f87d0fbbSRusty Russell /** 790f87d0fbbSRusty Russell * vringh_complete_user - we've finished with descriptor, publish it. 791f87d0fbbSRusty Russell * @vrh: the vring. 792f87d0fbbSRusty Russell * @head: the head as filled in by vringh_getdesc_user. 793f87d0fbbSRusty Russell * @len: the length of data we have written. 794f87d0fbbSRusty Russell * 795f87d0fbbSRusty Russell * You should check vringh_need_notify_user() after one or more calls 796f87d0fbbSRusty Russell * to this function. 797f87d0fbbSRusty Russell */ 798f87d0fbbSRusty Russell int vringh_complete_user(struct vringh *vrh, u16 head, u32 len) 799f87d0fbbSRusty Russell { 800f87d0fbbSRusty Russell struct vring_used_elem used; 801f87d0fbbSRusty Russell 802b9f7ac8cSMichael S. Tsirkin used.id = cpu_to_vringh32(vrh, head); 803b9f7ac8cSMichael S. Tsirkin used.len = cpu_to_vringh32(vrh, len); 804f87d0fbbSRusty Russell return __vringh_complete(vrh, &used, 1, putu16_user, putused_user); 805f87d0fbbSRusty Russell } 806f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_complete_user); 807f87d0fbbSRusty Russell 808f87d0fbbSRusty Russell /** 809f87d0fbbSRusty Russell * vringh_complete_multi_user - we've finished with many descriptors. 810f87d0fbbSRusty Russell * @vrh: the vring. 811f87d0fbbSRusty Russell * @used: the head, length pairs. 812f87d0fbbSRusty Russell * @num_used: the number of used elements. 813f87d0fbbSRusty Russell * 814f87d0fbbSRusty Russell * You should check vringh_need_notify_user() after one or more calls 815f87d0fbbSRusty Russell * to this function. 816f87d0fbbSRusty Russell */ 817f87d0fbbSRusty Russell int vringh_complete_multi_user(struct vringh *vrh, 818f87d0fbbSRusty Russell const struct vring_used_elem used[], 819f87d0fbbSRusty Russell unsigned num_used) 820f87d0fbbSRusty Russell { 821f87d0fbbSRusty Russell return __vringh_complete(vrh, used, num_used, 822f87d0fbbSRusty Russell putu16_user, putused_user); 823f87d0fbbSRusty Russell } 824f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_complete_multi_user); 825f87d0fbbSRusty Russell 826f87d0fbbSRusty Russell /** 827f87d0fbbSRusty Russell * vringh_notify_enable_user - we want to know if something changes. 828f87d0fbbSRusty Russell * @vrh: the vring. 829f87d0fbbSRusty Russell * 830f87d0fbbSRusty Russell * This always enables notifications, but returns false if there are 831f87d0fbbSRusty Russell * now more buffers available in the vring. 832f87d0fbbSRusty Russell */ 833f87d0fbbSRusty Russell bool vringh_notify_enable_user(struct vringh *vrh) 834f87d0fbbSRusty Russell { 835f87d0fbbSRusty Russell return __vringh_notify_enable(vrh, getu16_user, putu16_user); 836f87d0fbbSRusty Russell } 837f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_notify_enable_user); 838f87d0fbbSRusty Russell 839f87d0fbbSRusty Russell /** 840f87d0fbbSRusty Russell * vringh_notify_disable_user - don't tell us if something changes. 841f87d0fbbSRusty Russell * @vrh: the vring. 842f87d0fbbSRusty Russell * 843f87d0fbbSRusty Russell * This is our normal running state: we disable and then only enable when 844f87d0fbbSRusty Russell * we're going to sleep. 845f87d0fbbSRusty Russell */ 846f87d0fbbSRusty Russell void vringh_notify_disable_user(struct vringh *vrh) 847f87d0fbbSRusty Russell { 848f87d0fbbSRusty Russell __vringh_notify_disable(vrh, putu16_user); 849f87d0fbbSRusty Russell } 850f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_notify_disable_user); 851f87d0fbbSRusty Russell 852f87d0fbbSRusty Russell /** 853f87d0fbbSRusty Russell * vringh_need_notify_user - must we tell the other side about used buffers? 854f87d0fbbSRusty Russell * @vrh: the vring we've called vringh_complete_user() on. 855f87d0fbbSRusty Russell * 856f87d0fbbSRusty Russell * Returns -errno or 0 if we don't need to tell the other side, 1 if we do. 857f87d0fbbSRusty Russell */ 858f87d0fbbSRusty Russell int vringh_need_notify_user(struct vringh *vrh) 859f87d0fbbSRusty Russell { 860f87d0fbbSRusty Russell return __vringh_need_notify(vrh, getu16_user); 861f87d0fbbSRusty Russell } 862f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_need_notify_user); 863f87d0fbbSRusty Russell 864f87d0fbbSRusty Russell /* Kernelspace access helpers. */ 865b9f7ac8cSMichael S. Tsirkin static inline int getu16_kern(const struct vringh *vrh, 866b9f7ac8cSMichael S. Tsirkin u16 *val, const __virtio16 *p) 867f87d0fbbSRusty Russell { 8689d1b972fSMark Rutland *val = vringh16_to_cpu(vrh, READ_ONCE(*p)); 869f87d0fbbSRusty Russell return 0; 870f87d0fbbSRusty Russell } 871f87d0fbbSRusty Russell 872b9f7ac8cSMichael S. Tsirkin static inline int putu16_kern(const struct vringh *vrh, __virtio16 *p, u16 val) 873f87d0fbbSRusty Russell { 8749d1b972fSMark Rutland WRITE_ONCE(*p, cpu_to_vringh16(vrh, val)); 875f87d0fbbSRusty Russell return 0; 876f87d0fbbSRusty Russell } 877f87d0fbbSRusty Russell 8789ad9c49cSJason Wang static inline int copydesc_kern(const struct vringh *vrh, 8799ad9c49cSJason Wang void *dst, const void *src, size_t len) 880f87d0fbbSRusty Russell { 881f87d0fbbSRusty Russell memcpy(dst, src, len); 882f87d0fbbSRusty Russell return 0; 883f87d0fbbSRusty Russell } 884f87d0fbbSRusty Russell 8859ad9c49cSJason Wang static inline int putused_kern(const struct vringh *vrh, 8869ad9c49cSJason Wang struct vring_used_elem *dst, 887f87d0fbbSRusty Russell const struct vring_used_elem *src, 888f87d0fbbSRusty Russell unsigned int num) 889f87d0fbbSRusty Russell { 890f87d0fbbSRusty Russell memcpy(dst, src, num * sizeof(*dst)); 891f87d0fbbSRusty Russell return 0; 892f87d0fbbSRusty Russell } 893f87d0fbbSRusty Russell 8949ad9c49cSJason Wang static inline int xfer_kern(const struct vringh *vrh, void *src, 8959ad9c49cSJason Wang void *dst, size_t len) 896f87d0fbbSRusty Russell { 897f87d0fbbSRusty Russell memcpy(dst, src, len); 898f87d0fbbSRusty Russell return 0; 899f87d0fbbSRusty Russell } 900f87d0fbbSRusty Russell 9019ad9c49cSJason Wang static inline int kern_xfer(const struct vringh *vrh, void *dst, 9029ad9c49cSJason Wang void *src, size_t len) 903b3683deeSJason Wang { 904b3683deeSJason Wang memcpy(dst, src, len); 905b3683deeSJason Wang return 0; 906b3683deeSJason Wang } 907b3683deeSJason Wang 908f87d0fbbSRusty Russell /** 909f87d0fbbSRusty Russell * vringh_init_kern - initialize a vringh for a kernelspace vring. 910f87d0fbbSRusty Russell * @vrh: the vringh to initialize. 911f87d0fbbSRusty Russell * @features: the feature bits for this ring. 912f87d0fbbSRusty Russell * @num: the number of elements. 913f87d0fbbSRusty Russell * @weak_barriers: true if we only need memory barriers, not I/O. 914905233afSStefano Garzarella * @desc: the userspace descriptor pointer. 915905233afSStefano Garzarella * @avail: the userspace avail pointer. 916905233afSStefano Garzarella * @used: the userspace used pointer. 917f87d0fbbSRusty Russell * 918f87d0fbbSRusty Russell * Returns an error if num is invalid. 919f87d0fbbSRusty Russell */ 920b97a8a90SMichael S. Tsirkin int vringh_init_kern(struct vringh *vrh, u64 features, 921f87d0fbbSRusty Russell unsigned int num, bool weak_barriers, 922f87d0fbbSRusty Russell struct vring_desc *desc, 923f87d0fbbSRusty Russell struct vring_avail *avail, 924f87d0fbbSRusty Russell struct vring_used *used) 925f87d0fbbSRusty Russell { 926f87d0fbbSRusty Russell /* Sane power of 2 please! */ 927f87d0fbbSRusty Russell if (!num || num > 0xffff || (num & (num - 1))) { 928f87d0fbbSRusty Russell vringh_bad("Bad ring size %u", num); 929f87d0fbbSRusty Russell return -EINVAL; 930f87d0fbbSRusty Russell } 931f87d0fbbSRusty Russell 932b9f7ac8cSMichael S. Tsirkin vrh->little_endian = (features & (1ULL << VIRTIO_F_VERSION_1)); 933f87d0fbbSRusty Russell vrh->event_indices = (features & (1 << VIRTIO_RING_F_EVENT_IDX)); 934f87d0fbbSRusty Russell vrh->weak_barriers = weak_barriers; 935f87d0fbbSRusty Russell vrh->completed = 0; 936f87d0fbbSRusty Russell vrh->last_avail_idx = 0; 937f87d0fbbSRusty Russell vrh->last_used_idx = 0; 938f87d0fbbSRusty Russell vrh->vring.num = num; 939f87d0fbbSRusty Russell vrh->vring.desc = desc; 940f87d0fbbSRusty Russell vrh->vring.avail = avail; 941f87d0fbbSRusty Russell vrh->vring.used = used; 942f87d0fbbSRusty Russell return 0; 943f87d0fbbSRusty Russell } 944f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_init_kern); 945f87d0fbbSRusty Russell 946f87d0fbbSRusty Russell /** 947f87d0fbbSRusty Russell * vringh_getdesc_kern - get next available descriptor from kernelspace ring. 948f87d0fbbSRusty Russell * @vrh: the kernelspace vring. 949f87d0fbbSRusty Russell * @riov: where to put the readable descriptors (or NULL) 950f87d0fbbSRusty Russell * @wiov: where to put the writable descriptors (or NULL) 951f87d0fbbSRusty Russell * @head: head index we received, for passing to vringh_complete_kern(). 952f87d0fbbSRusty Russell * @gfp: flags for allocating larger riov/wiov. 953f87d0fbbSRusty Russell * 954f87d0fbbSRusty Russell * Returns 0 if there was no descriptor, 1 if there was, or -errno. 955f87d0fbbSRusty Russell * 956f87d0fbbSRusty Russell * Note that on error return, you can tell the difference between an 957f87d0fbbSRusty Russell * invalid ring and a single invalid descriptor: in the former case, 958f87d0fbbSRusty Russell * *head will be vrh->vring.num. You may be able to ignore an invalid 959f87d0fbbSRusty Russell * descriptor, but there's not much you can do with an invalid ring. 960f87d0fbbSRusty Russell * 96169c13c58SStefano Garzarella * Note that you can reuse riov and wiov with subsequent calls. Content is 96269c13c58SStefano Garzarella * overwritten and memory reallocated if more space is needed. 96369c13c58SStefano Garzarella * When you don't have to use riov and wiov anymore, you should clean up them 96469c13c58SStefano Garzarella * calling vringh_kiov_cleanup() to release the memory, even on error! 965f87d0fbbSRusty Russell */ 966f87d0fbbSRusty Russell int vringh_getdesc_kern(struct vringh *vrh, 967f87d0fbbSRusty Russell struct vringh_kiov *riov, 968f87d0fbbSRusty Russell struct vringh_kiov *wiov, 969f87d0fbbSRusty Russell u16 *head, 970f87d0fbbSRusty Russell gfp_t gfp) 971f87d0fbbSRusty Russell { 972f87d0fbbSRusty Russell int err; 973f87d0fbbSRusty Russell 974f87d0fbbSRusty Russell err = __vringh_get_head(vrh, getu16_kern, &vrh->last_avail_idx); 975f87d0fbbSRusty Russell if (err < 0) 976f87d0fbbSRusty Russell return err; 977f87d0fbbSRusty Russell 978f87d0fbbSRusty Russell /* Empty... */ 979f87d0fbbSRusty Russell if (err == vrh->vring.num) 980f87d0fbbSRusty Russell return 0; 981f87d0fbbSRusty Russell 982f87d0fbbSRusty Russell *head = err; 983f87d0fbbSRusty Russell err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL, 984f87d0fbbSRusty Russell gfp, copydesc_kern); 985f87d0fbbSRusty Russell if (err) 986f87d0fbbSRusty Russell return err; 987f87d0fbbSRusty Russell 988f87d0fbbSRusty Russell return 1; 989f87d0fbbSRusty Russell } 990f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_getdesc_kern); 991f87d0fbbSRusty Russell 992f87d0fbbSRusty Russell /** 993f87d0fbbSRusty Russell * vringh_iov_pull_kern - copy bytes from vring_iov. 994f87d0fbbSRusty Russell * @riov: the riov as passed to vringh_getdesc_kern() (updated as we consume) 995f87d0fbbSRusty Russell * @dst: the place to copy. 996f87d0fbbSRusty Russell * @len: the maximum length to copy. 997f87d0fbbSRusty Russell * 998f87d0fbbSRusty Russell * Returns the bytes copied <= len or a negative errno. 999f87d0fbbSRusty Russell */ 1000f87d0fbbSRusty Russell ssize_t vringh_iov_pull_kern(struct vringh_kiov *riov, void *dst, size_t len) 1001f87d0fbbSRusty Russell { 10029ad9c49cSJason Wang return vringh_iov_xfer(NULL, riov, dst, len, xfer_kern); 1003f87d0fbbSRusty Russell } 1004f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_iov_pull_kern); 1005f87d0fbbSRusty Russell 1006f87d0fbbSRusty Russell /** 1007f87d0fbbSRusty Russell * vringh_iov_push_kern - copy bytes into vring_iov. 1008f87d0fbbSRusty Russell * @wiov: the wiov as passed to vringh_getdesc_kern() (updated as we consume) 10098009b0f4SStefano Garzarella * @src: the place to copy from. 1010f87d0fbbSRusty Russell * @len: the maximum length to copy. 1011f87d0fbbSRusty Russell * 1012f87d0fbbSRusty Russell * Returns the bytes copied <= len or a negative errno. 1013f87d0fbbSRusty Russell */ 1014f87d0fbbSRusty Russell ssize_t vringh_iov_push_kern(struct vringh_kiov *wiov, 1015f87d0fbbSRusty Russell const void *src, size_t len) 1016f87d0fbbSRusty Russell { 10179ad9c49cSJason Wang return vringh_iov_xfer(NULL, wiov, (void *)src, len, kern_xfer); 1018f87d0fbbSRusty Russell } 1019f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_iov_push_kern); 1020f87d0fbbSRusty Russell 1021f87d0fbbSRusty Russell /** 1022f87d0fbbSRusty Russell * vringh_abandon_kern - we've decided not to handle the descriptor(s). 1023f87d0fbbSRusty Russell * @vrh: the vring. 1024f87d0fbbSRusty Russell * @num: the number of descriptors to put back (ie. num 1025f87d0fbbSRusty Russell * vringh_get_kern() to undo). 1026f87d0fbbSRusty Russell * 1027f87d0fbbSRusty Russell * The next vringh_get_kern() will return the old descriptor(s) again. 1028f87d0fbbSRusty Russell */ 1029f87d0fbbSRusty Russell void vringh_abandon_kern(struct vringh *vrh, unsigned int num) 1030f87d0fbbSRusty Russell { 1031f87d0fbbSRusty Russell /* We only update vring_avail_event(vr) when we want to be notified, 1032f87d0fbbSRusty Russell * so we haven't changed that yet. */ 1033f87d0fbbSRusty Russell vrh->last_avail_idx -= num; 1034f87d0fbbSRusty Russell } 1035f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_abandon_kern); 1036f87d0fbbSRusty Russell 1037f87d0fbbSRusty Russell /** 1038f87d0fbbSRusty Russell * vringh_complete_kern - we've finished with descriptor, publish it. 1039f87d0fbbSRusty Russell * @vrh: the vring. 1040f87d0fbbSRusty Russell * @head: the head as filled in by vringh_getdesc_kern. 1041f87d0fbbSRusty Russell * @len: the length of data we have written. 1042f87d0fbbSRusty Russell * 1043f87d0fbbSRusty Russell * You should check vringh_need_notify_kern() after one or more calls 1044f87d0fbbSRusty Russell * to this function. 1045f87d0fbbSRusty Russell */ 1046f87d0fbbSRusty Russell int vringh_complete_kern(struct vringh *vrh, u16 head, u32 len) 1047f87d0fbbSRusty Russell { 1048f87d0fbbSRusty Russell struct vring_used_elem used; 1049f87d0fbbSRusty Russell 1050b9f7ac8cSMichael S. Tsirkin used.id = cpu_to_vringh32(vrh, head); 1051b9f7ac8cSMichael S. Tsirkin used.len = cpu_to_vringh32(vrh, len); 1052f87d0fbbSRusty Russell 1053f87d0fbbSRusty Russell return __vringh_complete(vrh, &used, 1, putu16_kern, putused_kern); 1054f87d0fbbSRusty Russell } 1055f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_complete_kern); 1056f87d0fbbSRusty Russell 1057f87d0fbbSRusty Russell /** 1058f87d0fbbSRusty Russell * vringh_notify_enable_kern - we want to know if something changes. 1059f87d0fbbSRusty Russell * @vrh: the vring. 1060f87d0fbbSRusty Russell * 1061f87d0fbbSRusty Russell * This always enables notifications, but returns false if there are 1062f87d0fbbSRusty Russell * now more buffers available in the vring. 1063f87d0fbbSRusty Russell */ 1064f87d0fbbSRusty Russell bool vringh_notify_enable_kern(struct vringh *vrh) 1065f87d0fbbSRusty Russell { 1066f87d0fbbSRusty Russell return __vringh_notify_enable(vrh, getu16_kern, putu16_kern); 1067f87d0fbbSRusty Russell } 1068f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_notify_enable_kern); 1069f87d0fbbSRusty Russell 1070f87d0fbbSRusty Russell /** 1071f87d0fbbSRusty Russell * vringh_notify_disable_kern - don't tell us if something changes. 1072f87d0fbbSRusty Russell * @vrh: the vring. 1073f87d0fbbSRusty Russell * 1074f87d0fbbSRusty Russell * This is our normal running state: we disable and then only enable when 1075f87d0fbbSRusty Russell * we're going to sleep. 1076f87d0fbbSRusty Russell */ 1077f87d0fbbSRusty Russell void vringh_notify_disable_kern(struct vringh *vrh) 1078f87d0fbbSRusty Russell { 1079f87d0fbbSRusty Russell __vringh_notify_disable(vrh, putu16_kern); 1080f87d0fbbSRusty Russell } 1081f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_notify_disable_kern); 1082f87d0fbbSRusty Russell 1083f87d0fbbSRusty Russell /** 1084f87d0fbbSRusty Russell * vringh_need_notify_kern - must we tell the other side about used buffers? 1085f87d0fbbSRusty Russell * @vrh: the vring we've called vringh_complete_kern() on. 1086f87d0fbbSRusty Russell * 1087f87d0fbbSRusty Russell * Returns -errno or 0 if we don't need to tell the other side, 1 if we do. 1088f87d0fbbSRusty Russell */ 1089f87d0fbbSRusty Russell int vringh_need_notify_kern(struct vringh *vrh) 1090f87d0fbbSRusty Russell { 1091f87d0fbbSRusty Russell return __vringh_need_notify(vrh, getu16_kern); 1092f87d0fbbSRusty Russell } 1093f87d0fbbSRusty Russell EXPORT_SYMBOL(vringh_need_notify_kern); 1094f558a845SDave Jones 10953302363aSMichael S. Tsirkin #if IS_REACHABLE(CONFIG_VHOST_IOTLB) 10963302363aSMichael S. Tsirkin 10979ad9c49cSJason Wang static int iotlb_translate(const struct vringh *vrh, 1098309bba39SStefano Garzarella u64 addr, u64 len, u64 *translated, 1099309bba39SStefano Garzarella struct bio_vec iov[], 11009ad9c49cSJason Wang int iov_size, u32 perm) 11019ad9c49cSJason Wang { 11029ad9c49cSJason Wang struct vhost_iotlb_map *map; 11039ad9c49cSJason Wang struct vhost_iotlb *iotlb = vrh->iotlb; 11049ad9c49cSJason Wang int ret = 0; 1105f85efa9bSStefano Garzarella u64 s = 0, last = addr + len - 1; 11069ad9c49cSJason Wang 1107f53d9910SStefano Garzarella spin_lock(vrh->iotlb_lock); 1108f53d9910SStefano Garzarella 11099ad9c49cSJason Wang while (len > s) { 11109ad9c49cSJason Wang u64 size, pa, pfn; 11119ad9c49cSJason Wang 11129ad9c49cSJason Wang if (unlikely(ret >= iov_size)) { 11139ad9c49cSJason Wang ret = -ENOBUFS; 11149ad9c49cSJason Wang break; 11159ad9c49cSJason Wang } 11169ad9c49cSJason Wang 1117f85efa9bSStefano Garzarella map = vhost_iotlb_itree_first(iotlb, addr, last); 11189ad9c49cSJason Wang if (!map || map->start > addr) { 11199ad9c49cSJason Wang ret = -EINVAL; 11209ad9c49cSJason Wang break; 11219ad9c49cSJason Wang } else if (!(map->perm & perm)) { 11229ad9c49cSJason Wang ret = -EPERM; 11239ad9c49cSJason Wang break; 11249ad9c49cSJason Wang } 11259ad9c49cSJason Wang 11269ad9c49cSJason Wang size = map->size - addr + map->start; 11279ad9c49cSJason Wang pa = map->addr + addr - map->start; 11289ad9c49cSJason Wang pfn = pa >> PAGE_SHIFT; 112958dfe140SChristoph Hellwig bvec_set_page(&iov[ret], pfn_to_page(pfn), min(len - s, size), 113058dfe140SChristoph Hellwig pa & (PAGE_SIZE - 1)); 11319ad9c49cSJason Wang s += size; 11329ad9c49cSJason Wang addr += size; 11339ad9c49cSJason Wang ++ret; 11349ad9c49cSJason Wang } 11359ad9c49cSJason Wang 1136f53d9910SStefano Garzarella spin_unlock(vrh->iotlb_lock); 1137f53d9910SStefano Garzarella 1138309bba39SStefano Garzarella if (translated) 1139309bba39SStefano Garzarella *translated = min(len, s); 1140309bba39SStefano Garzarella 11419ad9c49cSJason Wang return ret; 11429ad9c49cSJason Wang } 11439ad9c49cSJason Wang 1144*f609d6cbSStefano Garzarella #define IOTLB_IOV_STRIDE 16 1145*f609d6cbSStefano Garzarella 11469ad9c49cSJason Wang static inline int copy_from_iotlb(const struct vringh *vrh, void *dst, 11479ad9c49cSJason Wang void *src, size_t len) 11489ad9c49cSJason Wang { 1149309bba39SStefano Garzarella u64 total_translated = 0; 1150309bba39SStefano Garzarella 1151309bba39SStefano Garzarella while (total_translated < len) { 1152*f609d6cbSStefano Garzarella struct bio_vec iov[IOTLB_IOV_STRIDE]; 1153309bba39SStefano Garzarella struct iov_iter iter; 1154309bba39SStefano Garzarella u64 translated; 11559ad9c49cSJason Wang int ret; 11569ad9c49cSJason Wang 11579ad9c49cSJason Wang ret = iotlb_translate(vrh, (u64)(uintptr_t)src, 1158309bba39SStefano Garzarella len - total_translated, &translated, 1159309bba39SStefano Garzarella iov, ARRAY_SIZE(iov), VHOST_MAP_RO); 1160309bba39SStefano Garzarella if (ret == -ENOBUFS) 1161309bba39SStefano Garzarella ret = ARRAY_SIZE(iov); 1162309bba39SStefano Garzarella else if (ret < 0) 1163309bba39SStefano Garzarella return ret; 1164309bba39SStefano Garzarella 1165de4eda9dSAl Viro iov_iter_bvec(&iter, ITER_SOURCE, iov, ret, translated); 1166309bba39SStefano Garzarella 1167309bba39SStefano Garzarella ret = copy_from_iter(dst, translated, &iter); 11689ad9c49cSJason Wang if (ret < 0) 11699ad9c49cSJason Wang return ret; 11709ad9c49cSJason Wang 1171309bba39SStefano Garzarella src += translated; 1172309bba39SStefano Garzarella dst += translated; 1173309bba39SStefano Garzarella total_translated += translated; 1174309bba39SStefano Garzarella } 11759ad9c49cSJason Wang 1176309bba39SStefano Garzarella return total_translated; 11779ad9c49cSJason Wang } 11789ad9c49cSJason Wang 11799ad9c49cSJason Wang static inline int copy_to_iotlb(const struct vringh *vrh, void *dst, 11809ad9c49cSJason Wang void *src, size_t len) 11819ad9c49cSJason Wang { 1182309bba39SStefano Garzarella u64 total_translated = 0; 1183309bba39SStefano Garzarella 1184309bba39SStefano Garzarella while (total_translated < len) { 1185*f609d6cbSStefano Garzarella struct bio_vec iov[IOTLB_IOV_STRIDE]; 1186309bba39SStefano Garzarella struct iov_iter iter; 1187309bba39SStefano Garzarella u64 translated; 11889ad9c49cSJason Wang int ret; 11899ad9c49cSJason Wang 11909ad9c49cSJason Wang ret = iotlb_translate(vrh, (u64)(uintptr_t)dst, 1191309bba39SStefano Garzarella len - total_translated, &translated, 1192309bba39SStefano Garzarella iov, ARRAY_SIZE(iov), VHOST_MAP_WO); 1193309bba39SStefano Garzarella if (ret == -ENOBUFS) 1194309bba39SStefano Garzarella ret = ARRAY_SIZE(iov); 1195309bba39SStefano Garzarella else if (ret < 0) 1196309bba39SStefano Garzarella return ret; 1197309bba39SStefano Garzarella 1198de4eda9dSAl Viro iov_iter_bvec(&iter, ITER_DEST, iov, ret, translated); 1199309bba39SStefano Garzarella 1200309bba39SStefano Garzarella ret = copy_to_iter(src, translated, &iter); 12019ad9c49cSJason Wang if (ret < 0) 12029ad9c49cSJason Wang return ret; 12039ad9c49cSJason Wang 1204309bba39SStefano Garzarella src += translated; 1205309bba39SStefano Garzarella dst += translated; 1206309bba39SStefano Garzarella total_translated += translated; 1207309bba39SStefano Garzarella } 12089ad9c49cSJason Wang 1209309bba39SStefano Garzarella return total_translated; 12109ad9c49cSJason Wang } 12119ad9c49cSJason Wang 12129ad9c49cSJason Wang static inline int getu16_iotlb(const struct vringh *vrh, 12139ad9c49cSJason Wang u16 *val, const __virtio16 *p) 12149ad9c49cSJason Wang { 12159ad9c49cSJason Wang struct bio_vec iov; 12169ad9c49cSJason Wang void *kaddr, *from; 12179ad9c49cSJason Wang int ret; 12189ad9c49cSJason Wang 12199ad9c49cSJason Wang /* Atomic read is needed for getu16 */ 1220309bba39SStefano Garzarella ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL, 12219ad9c49cSJason Wang &iov, 1, VHOST_MAP_RO); 12229ad9c49cSJason Wang if (ret < 0) 12239ad9c49cSJason Wang return ret; 12249ad9c49cSJason Wang 1225c0371782SStefano Garzarella kaddr = kmap_local_page(iov.bv_page); 12269ad9c49cSJason Wang from = kaddr + iov.bv_offset; 12279ad9c49cSJason Wang *val = vringh16_to_cpu(vrh, READ_ONCE(*(__virtio16 *)from)); 1228c0371782SStefano Garzarella kunmap_local(kaddr); 12299ad9c49cSJason Wang 12309ad9c49cSJason Wang return 0; 12319ad9c49cSJason Wang } 12329ad9c49cSJason Wang 12339ad9c49cSJason Wang static inline int putu16_iotlb(const struct vringh *vrh, 12349ad9c49cSJason Wang __virtio16 *p, u16 val) 12359ad9c49cSJason Wang { 12369ad9c49cSJason Wang struct bio_vec iov; 12379ad9c49cSJason Wang void *kaddr, *to; 12389ad9c49cSJason Wang int ret; 12399ad9c49cSJason Wang 12409ad9c49cSJason Wang /* Atomic write is needed for putu16 */ 1241309bba39SStefano Garzarella ret = iotlb_translate(vrh, (u64)(uintptr_t)p, sizeof(*p), NULL, 12429ad9c49cSJason Wang &iov, 1, VHOST_MAP_WO); 12439ad9c49cSJason Wang if (ret < 0) 12449ad9c49cSJason Wang return ret; 12459ad9c49cSJason Wang 1246c0371782SStefano Garzarella kaddr = kmap_local_page(iov.bv_page); 12479ad9c49cSJason Wang to = kaddr + iov.bv_offset; 12489ad9c49cSJason Wang WRITE_ONCE(*(__virtio16 *)to, cpu_to_vringh16(vrh, val)); 1249c0371782SStefano Garzarella kunmap_local(kaddr); 12509ad9c49cSJason Wang 12519ad9c49cSJason Wang return 0; 12529ad9c49cSJason Wang } 12539ad9c49cSJason Wang 12549ad9c49cSJason Wang static inline int copydesc_iotlb(const struct vringh *vrh, 12559ad9c49cSJason Wang void *dst, const void *src, size_t len) 12569ad9c49cSJason Wang { 12579ad9c49cSJason Wang int ret; 12589ad9c49cSJason Wang 12599ad9c49cSJason Wang ret = copy_from_iotlb(vrh, dst, (void *)src, len); 12609ad9c49cSJason Wang if (ret != len) 12619ad9c49cSJason Wang return -EFAULT; 12629ad9c49cSJason Wang 12639ad9c49cSJason Wang return 0; 12649ad9c49cSJason Wang } 12659ad9c49cSJason Wang 12669ad9c49cSJason Wang static inline int xfer_from_iotlb(const struct vringh *vrh, void *src, 12679ad9c49cSJason Wang void *dst, size_t len) 12689ad9c49cSJason Wang { 12699ad9c49cSJason Wang int ret; 12709ad9c49cSJason Wang 12719ad9c49cSJason Wang ret = copy_from_iotlb(vrh, dst, src, len); 12729ad9c49cSJason Wang if (ret != len) 12739ad9c49cSJason Wang return -EFAULT; 12749ad9c49cSJason Wang 12759ad9c49cSJason Wang return 0; 12769ad9c49cSJason Wang } 12779ad9c49cSJason Wang 12789ad9c49cSJason Wang static inline int xfer_to_iotlb(const struct vringh *vrh, 12799ad9c49cSJason Wang void *dst, void *src, size_t len) 12809ad9c49cSJason Wang { 12819ad9c49cSJason Wang int ret; 12829ad9c49cSJason Wang 12839ad9c49cSJason Wang ret = copy_to_iotlb(vrh, dst, src, len); 12849ad9c49cSJason Wang if (ret != len) 12859ad9c49cSJason Wang return -EFAULT; 12869ad9c49cSJason Wang 12879ad9c49cSJason Wang return 0; 12889ad9c49cSJason Wang } 12899ad9c49cSJason Wang 12909ad9c49cSJason Wang static inline int putused_iotlb(const struct vringh *vrh, 12919ad9c49cSJason Wang struct vring_used_elem *dst, 12929ad9c49cSJason Wang const struct vring_used_elem *src, 12939ad9c49cSJason Wang unsigned int num) 12949ad9c49cSJason Wang { 12959ad9c49cSJason Wang int size = num * sizeof(*dst); 12969ad9c49cSJason Wang int ret; 12979ad9c49cSJason Wang 12989ad9c49cSJason Wang ret = copy_to_iotlb(vrh, dst, (void *)src, num * sizeof(*dst)); 12999ad9c49cSJason Wang if (ret != size) 13009ad9c49cSJason Wang return -EFAULT; 13019ad9c49cSJason Wang 13029ad9c49cSJason Wang return 0; 13039ad9c49cSJason Wang } 13049ad9c49cSJason Wang 13059ad9c49cSJason Wang /** 13069ad9c49cSJason Wang * vringh_init_iotlb - initialize a vringh for a ring with IOTLB. 13079ad9c49cSJason Wang * @vrh: the vringh to initialize. 13089ad9c49cSJason Wang * @features: the feature bits for this ring. 13099ad9c49cSJason Wang * @num: the number of elements. 13109ad9c49cSJason Wang * @weak_barriers: true if we only need memory barriers, not I/O. 1311905233afSStefano Garzarella * @desc: the userspace descriptor pointer. 1312905233afSStefano Garzarella * @avail: the userspace avail pointer. 1313905233afSStefano Garzarella * @used: the userspace used pointer. 13149ad9c49cSJason Wang * 13159ad9c49cSJason Wang * Returns an error if num is invalid. 13169ad9c49cSJason Wang */ 13179ad9c49cSJason Wang int vringh_init_iotlb(struct vringh *vrh, u64 features, 13189ad9c49cSJason Wang unsigned int num, bool weak_barriers, 13199ad9c49cSJason Wang struct vring_desc *desc, 13209ad9c49cSJason Wang struct vring_avail *avail, 13219ad9c49cSJason Wang struct vring_used *used) 13229ad9c49cSJason Wang { 13239ad9c49cSJason Wang return vringh_init_kern(vrh, features, num, weak_barriers, 13249ad9c49cSJason Wang desc, avail, used); 13259ad9c49cSJason Wang } 13269ad9c49cSJason Wang EXPORT_SYMBOL(vringh_init_iotlb); 13279ad9c49cSJason Wang 13289ad9c49cSJason Wang /** 13299ad9c49cSJason Wang * vringh_set_iotlb - initialize a vringh for a ring with IOTLB. 13309ad9c49cSJason Wang * @vrh: the vring 13319ad9c49cSJason Wang * @iotlb: iotlb associated with this vring 1332f53d9910SStefano Garzarella * @iotlb_lock: spinlock to synchronize the iotlb accesses 13339ad9c49cSJason Wang */ 1334f53d9910SStefano Garzarella void vringh_set_iotlb(struct vringh *vrh, struct vhost_iotlb *iotlb, 1335f53d9910SStefano Garzarella spinlock_t *iotlb_lock) 13369ad9c49cSJason Wang { 13379ad9c49cSJason Wang vrh->iotlb = iotlb; 1338f53d9910SStefano Garzarella vrh->iotlb_lock = iotlb_lock; 13399ad9c49cSJason Wang } 13409ad9c49cSJason Wang EXPORT_SYMBOL(vringh_set_iotlb); 13419ad9c49cSJason Wang 13429ad9c49cSJason Wang /** 13439ad9c49cSJason Wang * vringh_getdesc_iotlb - get next available descriptor from ring with 13449ad9c49cSJason Wang * IOTLB. 13459ad9c49cSJason Wang * @vrh: the kernelspace vring. 13469ad9c49cSJason Wang * @riov: where to put the readable descriptors (or NULL) 13479ad9c49cSJason Wang * @wiov: where to put the writable descriptors (or NULL) 13489ad9c49cSJason Wang * @head: head index we received, for passing to vringh_complete_iotlb(). 13499ad9c49cSJason Wang * @gfp: flags for allocating larger riov/wiov. 13509ad9c49cSJason Wang * 13519ad9c49cSJason Wang * Returns 0 if there was no descriptor, 1 if there was, or -errno. 13529ad9c49cSJason Wang * 13539ad9c49cSJason Wang * Note that on error return, you can tell the difference between an 13549ad9c49cSJason Wang * invalid ring and a single invalid descriptor: in the former case, 13559ad9c49cSJason Wang * *head will be vrh->vring.num. You may be able to ignore an invalid 13569ad9c49cSJason Wang * descriptor, but there's not much you can do with an invalid ring. 13579ad9c49cSJason Wang * 135869c13c58SStefano Garzarella * Note that you can reuse riov and wiov with subsequent calls. Content is 135969c13c58SStefano Garzarella * overwritten and memory reallocated if more space is needed. 136069c13c58SStefano Garzarella * When you don't have to use riov and wiov anymore, you should clean up them 136169c13c58SStefano Garzarella * calling vringh_kiov_cleanup() to release the memory, even on error! 13629ad9c49cSJason Wang */ 13639ad9c49cSJason Wang int vringh_getdesc_iotlb(struct vringh *vrh, 13649ad9c49cSJason Wang struct vringh_kiov *riov, 13659ad9c49cSJason Wang struct vringh_kiov *wiov, 13669ad9c49cSJason Wang u16 *head, 13679ad9c49cSJason Wang gfp_t gfp) 13689ad9c49cSJason Wang { 13699ad9c49cSJason Wang int err; 13709ad9c49cSJason Wang 13719ad9c49cSJason Wang err = __vringh_get_head(vrh, getu16_iotlb, &vrh->last_avail_idx); 13729ad9c49cSJason Wang if (err < 0) 13739ad9c49cSJason Wang return err; 13749ad9c49cSJason Wang 13759ad9c49cSJason Wang /* Empty... */ 13769ad9c49cSJason Wang if (err == vrh->vring.num) 13779ad9c49cSJason Wang return 0; 13789ad9c49cSJason Wang 13799ad9c49cSJason Wang *head = err; 13809ad9c49cSJason Wang err = __vringh_iov(vrh, *head, riov, wiov, no_range_check, NULL, 13819ad9c49cSJason Wang gfp, copydesc_iotlb); 13829ad9c49cSJason Wang if (err) 13839ad9c49cSJason Wang return err; 13849ad9c49cSJason Wang 13859ad9c49cSJason Wang return 1; 13869ad9c49cSJason Wang } 13879ad9c49cSJason Wang EXPORT_SYMBOL(vringh_getdesc_iotlb); 13889ad9c49cSJason Wang 13899ad9c49cSJason Wang /** 13909ad9c49cSJason Wang * vringh_iov_pull_iotlb - copy bytes from vring_iov. 13919ad9c49cSJason Wang * @vrh: the vring. 13929ad9c49cSJason Wang * @riov: the riov as passed to vringh_getdesc_iotlb() (updated as we consume) 13939ad9c49cSJason Wang * @dst: the place to copy. 13949ad9c49cSJason Wang * @len: the maximum length to copy. 13959ad9c49cSJason Wang * 13969ad9c49cSJason Wang * Returns the bytes copied <= len or a negative errno. 13979ad9c49cSJason Wang */ 13989ad9c49cSJason Wang ssize_t vringh_iov_pull_iotlb(struct vringh *vrh, 13999ad9c49cSJason Wang struct vringh_kiov *riov, 14009ad9c49cSJason Wang void *dst, size_t len) 14019ad9c49cSJason Wang { 14029ad9c49cSJason Wang return vringh_iov_xfer(vrh, riov, dst, len, xfer_from_iotlb); 14039ad9c49cSJason Wang } 14049ad9c49cSJason Wang EXPORT_SYMBOL(vringh_iov_pull_iotlb); 14059ad9c49cSJason Wang 14069ad9c49cSJason Wang /** 14079ad9c49cSJason Wang * vringh_iov_push_iotlb - copy bytes into vring_iov. 14089ad9c49cSJason Wang * @vrh: the vring. 14099ad9c49cSJason Wang * @wiov: the wiov as passed to vringh_getdesc_iotlb() (updated as we consume) 14108009b0f4SStefano Garzarella * @src: the place to copy from. 14119ad9c49cSJason Wang * @len: the maximum length to copy. 14129ad9c49cSJason Wang * 14139ad9c49cSJason Wang * Returns the bytes copied <= len or a negative errno. 14149ad9c49cSJason Wang */ 14159ad9c49cSJason Wang ssize_t vringh_iov_push_iotlb(struct vringh *vrh, 14169ad9c49cSJason Wang struct vringh_kiov *wiov, 14179ad9c49cSJason Wang const void *src, size_t len) 14189ad9c49cSJason Wang { 14199ad9c49cSJason Wang return vringh_iov_xfer(vrh, wiov, (void *)src, len, xfer_to_iotlb); 14209ad9c49cSJason Wang } 14219ad9c49cSJason Wang EXPORT_SYMBOL(vringh_iov_push_iotlb); 14229ad9c49cSJason Wang 14239ad9c49cSJason Wang /** 14249ad9c49cSJason Wang * vringh_abandon_iotlb - we've decided not to handle the descriptor(s). 14259ad9c49cSJason Wang * @vrh: the vring. 14269ad9c49cSJason Wang * @num: the number of descriptors to put back (ie. num 14279ad9c49cSJason Wang * vringh_get_iotlb() to undo). 14289ad9c49cSJason Wang * 14299ad9c49cSJason Wang * The next vringh_get_iotlb() will return the old descriptor(s) again. 14309ad9c49cSJason Wang */ 14319ad9c49cSJason Wang void vringh_abandon_iotlb(struct vringh *vrh, unsigned int num) 14329ad9c49cSJason Wang { 14339ad9c49cSJason Wang /* We only update vring_avail_event(vr) when we want to be notified, 14349ad9c49cSJason Wang * so we haven't changed that yet. 14359ad9c49cSJason Wang */ 14369ad9c49cSJason Wang vrh->last_avail_idx -= num; 14379ad9c49cSJason Wang } 14389ad9c49cSJason Wang EXPORT_SYMBOL(vringh_abandon_iotlb); 14399ad9c49cSJason Wang 14409ad9c49cSJason Wang /** 14419ad9c49cSJason Wang * vringh_complete_iotlb - we've finished with descriptor, publish it. 14429ad9c49cSJason Wang * @vrh: the vring. 14439ad9c49cSJason Wang * @head: the head as filled in by vringh_getdesc_iotlb. 14449ad9c49cSJason Wang * @len: the length of data we have written. 14459ad9c49cSJason Wang * 14469ad9c49cSJason Wang * You should check vringh_need_notify_iotlb() after one or more calls 14479ad9c49cSJason Wang * to this function. 14489ad9c49cSJason Wang */ 14499ad9c49cSJason Wang int vringh_complete_iotlb(struct vringh *vrh, u16 head, u32 len) 14509ad9c49cSJason Wang { 14519ad9c49cSJason Wang struct vring_used_elem used; 14529ad9c49cSJason Wang 14539ad9c49cSJason Wang used.id = cpu_to_vringh32(vrh, head); 14549ad9c49cSJason Wang used.len = cpu_to_vringh32(vrh, len); 14559ad9c49cSJason Wang 14569ad9c49cSJason Wang return __vringh_complete(vrh, &used, 1, putu16_iotlb, putused_iotlb); 14579ad9c49cSJason Wang } 14589ad9c49cSJason Wang EXPORT_SYMBOL(vringh_complete_iotlb); 14599ad9c49cSJason Wang 14609ad9c49cSJason Wang /** 14619ad9c49cSJason Wang * vringh_notify_enable_iotlb - we want to know if something changes. 14629ad9c49cSJason Wang * @vrh: the vring. 14639ad9c49cSJason Wang * 14649ad9c49cSJason Wang * This always enables notifications, but returns false if there are 14659ad9c49cSJason Wang * now more buffers available in the vring. 14669ad9c49cSJason Wang */ 14679ad9c49cSJason Wang bool vringh_notify_enable_iotlb(struct vringh *vrh) 14689ad9c49cSJason Wang { 14699ad9c49cSJason Wang return __vringh_notify_enable(vrh, getu16_iotlb, putu16_iotlb); 14709ad9c49cSJason Wang } 14719ad9c49cSJason Wang EXPORT_SYMBOL(vringh_notify_enable_iotlb); 14729ad9c49cSJason Wang 14739ad9c49cSJason Wang /** 14749ad9c49cSJason Wang * vringh_notify_disable_iotlb - don't tell us if something changes. 14759ad9c49cSJason Wang * @vrh: the vring. 14769ad9c49cSJason Wang * 14779ad9c49cSJason Wang * This is our normal running state: we disable and then only enable when 14789ad9c49cSJason Wang * we're going to sleep. 14799ad9c49cSJason Wang */ 14809ad9c49cSJason Wang void vringh_notify_disable_iotlb(struct vringh *vrh) 14819ad9c49cSJason Wang { 14829ad9c49cSJason Wang __vringh_notify_disable(vrh, putu16_iotlb); 14839ad9c49cSJason Wang } 14849ad9c49cSJason Wang EXPORT_SYMBOL(vringh_notify_disable_iotlb); 14859ad9c49cSJason Wang 14869ad9c49cSJason Wang /** 14879ad9c49cSJason Wang * vringh_need_notify_iotlb - must we tell the other side about used buffers? 14889ad9c49cSJason Wang * @vrh: the vring we've called vringh_complete_iotlb() on. 14899ad9c49cSJason Wang * 14909ad9c49cSJason Wang * Returns -errno or 0 if we don't need to tell the other side, 1 if we do. 14919ad9c49cSJason Wang */ 14929ad9c49cSJason Wang int vringh_need_notify_iotlb(struct vringh *vrh) 14939ad9c49cSJason Wang { 14949ad9c49cSJason Wang return __vringh_need_notify(vrh, getu16_iotlb); 14959ad9c49cSJason Wang } 14969ad9c49cSJason Wang EXPORT_SYMBOL(vringh_need_notify_iotlb); 14979ad9c49cSJason Wang 14983302363aSMichael S. Tsirkin #endif 14999ad9c49cSJason Wang 1500f558a845SDave Jones MODULE_LICENSE("GPL"); 1501