10bbe3066SJason Wang // SPDX-License-Identifier: GPL-2.0-only
20bbe3066SJason Wang /* Copyright (C) 2020 Red Hat, Inc.
30bbe3066SJason Wang * Author: Jason Wang <jasowang@redhat.com>
40bbe3066SJason Wang *
50bbe3066SJason Wang * IOTLB implementation for vhost.
60bbe3066SJason Wang */
70bbe3066SJason Wang #include <linux/slab.h>
80bbe3066SJason Wang #include <linux/vhost_iotlb.h>
90bbe3066SJason Wang #include <linux/module.h>
100bbe3066SJason Wang
110bbe3066SJason Wang #define MOD_VERSION "0.1"
120bbe3066SJason Wang #define MOD_DESC "VHOST IOTLB"
130bbe3066SJason Wang #define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>"
140bbe3066SJason Wang #define MOD_LICENSE "GPL v2"
150bbe3066SJason Wang
160bbe3066SJason Wang #define START(map) ((map)->start)
170bbe3066SJason Wang #define LAST(map) ((map)->last)
180bbe3066SJason Wang
190bbe3066SJason Wang INTERVAL_TREE_DEFINE(struct vhost_iotlb_map,
200bbe3066SJason Wang rb, __u64, __subtree_last,
210bbe3066SJason Wang START, LAST, static inline, vhost_iotlb_itree);
220bbe3066SJason Wang
230bbe3066SJason Wang /**
240bbe3066SJason Wang * vhost_iotlb_map_free - remove a map node and free it
250bbe3066SJason Wang * @iotlb: the IOTLB
260bbe3066SJason Wang * @map: the map that want to be remove and freed
270bbe3066SJason Wang */
vhost_iotlb_map_free(struct vhost_iotlb * iotlb,struct vhost_iotlb_map * map)280bbe3066SJason Wang void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,
290bbe3066SJason Wang struct vhost_iotlb_map *map)
300bbe3066SJason Wang {
310bbe3066SJason Wang vhost_iotlb_itree_remove(map, &iotlb->root);
320bbe3066SJason Wang list_del(&map->link);
330bbe3066SJason Wang kfree(map);
340bbe3066SJason Wang iotlb->nmaps--;
350bbe3066SJason Wang }
360bbe3066SJason Wang EXPORT_SYMBOL_GPL(vhost_iotlb_map_free);
370bbe3066SJason Wang
380bbe3066SJason Wang /**
3959dfe4f1SXie Yongji * vhost_iotlb_add_range_ctx - add a new range to vhost IOTLB
400bbe3066SJason Wang * @iotlb: the IOTLB
410bbe3066SJason Wang * @start: start of the IOVA range
420bbe3066SJason Wang * @last: last of IOVA range
430bbe3066SJason Wang * @addr: the address that is mapped to @start
440bbe3066SJason Wang * @perm: access permission of this range
4559dfe4f1SXie Yongji * @opaque: the opaque pointer for the new mapping
460bbe3066SJason Wang *
470bbe3066SJason Wang * Returns an error last is smaller than start or memory allocation
480bbe3066SJason Wang * fails
490bbe3066SJason Wang */
vhost_iotlb_add_range_ctx(struct vhost_iotlb * iotlb,u64 start,u64 last,u64 addr,unsigned int perm,void * opaque)5059dfe4f1SXie Yongji int vhost_iotlb_add_range_ctx(struct vhost_iotlb *iotlb,
510bbe3066SJason Wang u64 start, u64 last,
5259dfe4f1SXie Yongji u64 addr, unsigned int perm,
5359dfe4f1SXie Yongji void *opaque)
540bbe3066SJason Wang {
550bbe3066SJason Wang struct vhost_iotlb_map *map;
560bbe3066SJason Wang
570bbe3066SJason Wang if (last < start)
580bbe3066SJason Wang return -EFAULT;
590bbe3066SJason Wang
60e2ae38cfSAnirudh Rayabharam /* If the range being mapped is [0, ULONG_MAX], split it into two entries
61e2ae38cfSAnirudh Rayabharam * otherwise its size would overflow u64.
62e2ae38cfSAnirudh Rayabharam */
63e2ae38cfSAnirudh Rayabharam if (start == 0 && last == ULONG_MAX) {
64e2ae38cfSAnirudh Rayabharam u64 mid = last / 2;
6503a91c9aSAnirudh Rayabharam int err = vhost_iotlb_add_range_ctx(iotlb, start, mid, addr,
6603a91c9aSAnirudh Rayabharam perm, opaque);
67e2ae38cfSAnirudh Rayabharam
6803a91c9aSAnirudh Rayabharam if (err)
6903a91c9aSAnirudh Rayabharam return err;
7003a91c9aSAnirudh Rayabharam
71e2ae38cfSAnirudh Rayabharam addr += mid + 1;
72e2ae38cfSAnirudh Rayabharam start = mid + 1;
73e2ae38cfSAnirudh Rayabharam }
74e2ae38cfSAnirudh Rayabharam
750bbe3066SJason Wang if (iotlb->limit &&
760bbe3066SJason Wang iotlb->nmaps == iotlb->limit &&
770bbe3066SJason Wang iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) {
780bbe3066SJason Wang map = list_first_entry(&iotlb->list, typeof(*map), link);
790bbe3066SJason Wang vhost_iotlb_map_free(iotlb, map);
800bbe3066SJason Wang }
810bbe3066SJason Wang
820bbe3066SJason Wang map = kmalloc(sizeof(*map), GFP_ATOMIC);
830bbe3066SJason Wang if (!map)
840bbe3066SJason Wang return -ENOMEM;
850bbe3066SJason Wang
860bbe3066SJason Wang map->start = start;
870bbe3066SJason Wang map->size = last - start + 1;
880bbe3066SJason Wang map->last = last;
890bbe3066SJason Wang map->addr = addr;
900bbe3066SJason Wang map->perm = perm;
9159dfe4f1SXie Yongji map->opaque = opaque;
920bbe3066SJason Wang
930bbe3066SJason Wang iotlb->nmaps++;
940bbe3066SJason Wang vhost_iotlb_itree_insert(map, &iotlb->root);
950bbe3066SJason Wang
960bbe3066SJason Wang INIT_LIST_HEAD(&map->link);
970bbe3066SJason Wang list_add_tail(&map->link, &iotlb->list);
980bbe3066SJason Wang
990bbe3066SJason Wang return 0;
1000bbe3066SJason Wang }
10159dfe4f1SXie Yongji EXPORT_SYMBOL_GPL(vhost_iotlb_add_range_ctx);
10259dfe4f1SXie Yongji
vhost_iotlb_add_range(struct vhost_iotlb * iotlb,u64 start,u64 last,u64 addr,unsigned int perm)10359dfe4f1SXie Yongji int vhost_iotlb_add_range(struct vhost_iotlb *iotlb,
10459dfe4f1SXie Yongji u64 start, u64 last,
10559dfe4f1SXie Yongji u64 addr, unsigned int perm)
10659dfe4f1SXie Yongji {
10759dfe4f1SXie Yongji return vhost_iotlb_add_range_ctx(iotlb, start, last,
10859dfe4f1SXie Yongji addr, perm, NULL);
10959dfe4f1SXie Yongji }
1100bbe3066SJason Wang EXPORT_SYMBOL_GPL(vhost_iotlb_add_range);
1110bbe3066SJason Wang
1120bbe3066SJason Wang /**
11386930592SStefano Garzarella * vhost_iotlb_del_range - delete overlapped ranges from vhost IOTLB
1140bbe3066SJason Wang * @iotlb: the IOTLB
1150bbe3066SJason Wang * @start: start of the IOVA range
1160bbe3066SJason Wang * @last: last of IOVA range
1170bbe3066SJason Wang */
vhost_iotlb_del_range(struct vhost_iotlb * iotlb,u64 start,u64 last)1180bbe3066SJason Wang void vhost_iotlb_del_range(struct vhost_iotlb *iotlb, u64 start, u64 last)
1190bbe3066SJason Wang {
1200bbe3066SJason Wang struct vhost_iotlb_map *map;
1210bbe3066SJason Wang
1220bbe3066SJason Wang while ((map = vhost_iotlb_itree_iter_first(&iotlb->root,
1230bbe3066SJason Wang start, last)))
1240bbe3066SJason Wang vhost_iotlb_map_free(iotlb, map);
1250bbe3066SJason Wang }
1260bbe3066SJason Wang EXPORT_SYMBOL_GPL(vhost_iotlb_del_range);
1270bbe3066SJason Wang
1280bbe3066SJason Wang /**
129*1cb10899SGautam Dawar * vhost_iotlb_init - initialize a vhost IOTLB
130*1cb10899SGautam Dawar * @iotlb: the IOTLB that needs to be initialized
131*1cb10899SGautam Dawar * @limit: maximum number of IOTLB entries
132*1cb10899SGautam Dawar * @flags: VHOST_IOTLB_FLAG_XXX
133*1cb10899SGautam Dawar */
vhost_iotlb_init(struct vhost_iotlb * iotlb,unsigned int limit,unsigned int flags)134*1cb10899SGautam Dawar void vhost_iotlb_init(struct vhost_iotlb *iotlb, unsigned int limit,
135*1cb10899SGautam Dawar unsigned int flags)
136*1cb10899SGautam Dawar {
137*1cb10899SGautam Dawar iotlb->root = RB_ROOT_CACHED;
138*1cb10899SGautam Dawar iotlb->limit = limit;
139*1cb10899SGautam Dawar iotlb->nmaps = 0;
140*1cb10899SGautam Dawar iotlb->flags = flags;
141*1cb10899SGautam Dawar INIT_LIST_HEAD(&iotlb->list);
142*1cb10899SGautam Dawar }
143*1cb10899SGautam Dawar EXPORT_SYMBOL_GPL(vhost_iotlb_init);
144*1cb10899SGautam Dawar
145*1cb10899SGautam Dawar /**
1460bbe3066SJason Wang * vhost_iotlb_alloc - add a new vhost IOTLB
1470bbe3066SJason Wang * @limit: maximum number of IOTLB entries
1480bbe3066SJason Wang * @flags: VHOST_IOTLB_FLAG_XXX
1490bbe3066SJason Wang *
1500bbe3066SJason Wang * Returns an error is memory allocation fails
1510bbe3066SJason Wang */
vhost_iotlb_alloc(unsigned int limit,unsigned int flags)1520bbe3066SJason Wang struct vhost_iotlb *vhost_iotlb_alloc(unsigned int limit, unsigned int flags)
1530bbe3066SJason Wang {
1540bbe3066SJason Wang struct vhost_iotlb *iotlb = kzalloc(sizeof(*iotlb), GFP_KERNEL);
1550bbe3066SJason Wang
1560bbe3066SJason Wang if (!iotlb)
1570bbe3066SJason Wang return NULL;
1580bbe3066SJason Wang
159*1cb10899SGautam Dawar vhost_iotlb_init(iotlb, limit, flags);
1600bbe3066SJason Wang
1610bbe3066SJason Wang return iotlb;
1620bbe3066SJason Wang }
1630bbe3066SJason Wang EXPORT_SYMBOL_GPL(vhost_iotlb_alloc);
1640bbe3066SJason Wang
1650bbe3066SJason Wang /**
1660bbe3066SJason Wang * vhost_iotlb_reset - reset vhost IOTLB (free all IOTLB entries)
1670bbe3066SJason Wang * @iotlb: the IOTLB to be reset
1680bbe3066SJason Wang */
vhost_iotlb_reset(struct vhost_iotlb * iotlb)1690bbe3066SJason Wang void vhost_iotlb_reset(struct vhost_iotlb *iotlb)
1700bbe3066SJason Wang {
1710bbe3066SJason Wang vhost_iotlb_del_range(iotlb, 0ULL, 0ULL - 1);
1720bbe3066SJason Wang }
1730bbe3066SJason Wang EXPORT_SYMBOL_GPL(vhost_iotlb_reset);
1740bbe3066SJason Wang
1750bbe3066SJason Wang /**
1760bbe3066SJason Wang * vhost_iotlb_free - reset and free vhost IOTLB
1770bbe3066SJason Wang * @iotlb: the IOTLB to be freed
1780bbe3066SJason Wang */
vhost_iotlb_free(struct vhost_iotlb * iotlb)1790bbe3066SJason Wang void vhost_iotlb_free(struct vhost_iotlb *iotlb)
1800bbe3066SJason Wang {
1810bbe3066SJason Wang if (iotlb) {
1820bbe3066SJason Wang vhost_iotlb_reset(iotlb);
1830bbe3066SJason Wang kfree(iotlb);
1840bbe3066SJason Wang }
1850bbe3066SJason Wang }
1860bbe3066SJason Wang EXPORT_SYMBOL_GPL(vhost_iotlb_free);
1870bbe3066SJason Wang
1880bbe3066SJason Wang /**
1890bbe3066SJason Wang * vhost_iotlb_itree_first - return the first overlapped range
1900bbe3066SJason Wang * @iotlb: the IOTLB
1910bbe3066SJason Wang * @start: start of IOVA range
19271c548c2SEli Cohen * @last: last byte in IOVA range
1930bbe3066SJason Wang */
1940bbe3066SJason Wang struct vhost_iotlb_map *
vhost_iotlb_itree_first(struct vhost_iotlb * iotlb,u64 start,u64 last)1950bbe3066SJason Wang vhost_iotlb_itree_first(struct vhost_iotlb *iotlb, u64 start, u64 last)
1960bbe3066SJason Wang {
1970bbe3066SJason Wang return vhost_iotlb_itree_iter_first(&iotlb->root, start, last);
1980bbe3066SJason Wang }
1990bbe3066SJason Wang EXPORT_SYMBOL_GPL(vhost_iotlb_itree_first);
2000bbe3066SJason Wang
2010bbe3066SJason Wang /**
202eb07d8f5SStefano Garzarella * vhost_iotlb_itree_next - return the next overlapped range
203eb07d8f5SStefano Garzarella * @map: the starting map node
2040bbe3066SJason Wang * @start: start of IOVA range
20571c548c2SEli Cohen * @last: last byte IOVA range
2060bbe3066SJason Wang */
2070bbe3066SJason Wang struct vhost_iotlb_map *
vhost_iotlb_itree_next(struct vhost_iotlb_map * map,u64 start,u64 last)2080bbe3066SJason Wang vhost_iotlb_itree_next(struct vhost_iotlb_map *map, u64 start, u64 last)
2090bbe3066SJason Wang {
2100bbe3066SJason Wang return vhost_iotlb_itree_iter_next(map, start, last);
2110bbe3066SJason Wang }
2120bbe3066SJason Wang EXPORT_SYMBOL_GPL(vhost_iotlb_itree_next);
2130bbe3066SJason Wang
2140bbe3066SJason Wang MODULE_VERSION(MOD_VERSION);
2150bbe3066SJason Wang MODULE_DESCRIPTION(MOD_DESC);
2160bbe3066SJason Wang MODULE_AUTHOR(MOD_AUTHOR);
2170bbe3066SJason Wang MODULE_LICENSE(MOD_LICENSE);
218