1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Red Hat, Inc.
3 * Author: Jason Wang <jasowang@redhat.com>
4 *
5 * IOTLB implementation for vhost.
6 */
7 #include <linux/slab.h>
8 #include <linux/vhost_iotlb.h>
9 #include <linux/module.h>
10
11 #define MOD_VERSION "0.1"
12 #define MOD_DESC "VHOST IOTLB"
13 #define MOD_AUTHOR "Jason Wang <jasowang@redhat.com>"
14 #define MOD_LICENSE "GPL v2"
15
16 #define START(map) ((map)->start)
17 #define LAST(map) ((map)->last)
18
19 INTERVAL_TREE_DEFINE(struct vhost_iotlb_map,
20 rb, __u64, __subtree_last,
21 START, LAST, static inline, vhost_iotlb_itree);
22
23 /**
24 * vhost_iotlb_map_free - remove a map node and free it
25 * @iotlb: the IOTLB
26 * @map: the map that want to be remove and freed
27 */
vhost_iotlb_map_free(struct vhost_iotlb * iotlb,struct vhost_iotlb_map * map)28 void vhost_iotlb_map_free(struct vhost_iotlb *iotlb,
29 struct vhost_iotlb_map *map)
30 {
31 vhost_iotlb_itree_remove(map, &iotlb->root);
32 list_del(&map->link);
33 kfree(map);
34 iotlb->nmaps--;
35 }
36 EXPORT_SYMBOL_GPL(vhost_iotlb_map_free);
37
38 /**
39 * vhost_iotlb_add_range_ctx - add a new range to vhost IOTLB
40 * @iotlb: the IOTLB
41 * @start: start of the IOVA range
42 * @last: last of IOVA range
43 * @addr: the address that is mapped to @start
44 * @perm: access permission of this range
45 * @opaque: the opaque pointer for the new mapping
46 *
47 * Returns an error last is smaller than start or memory allocation
48 * fails
49 */
vhost_iotlb_add_range_ctx(struct vhost_iotlb * iotlb,u64 start,u64 last,u64 addr,unsigned int perm,void * opaque)50 int vhost_iotlb_add_range_ctx(struct vhost_iotlb *iotlb,
51 u64 start, u64 last,
52 u64 addr, unsigned int perm,
53 void *opaque)
54 {
55 struct vhost_iotlb_map *map;
56
57 if (last < start)
58 return -EFAULT;
59
60 /* If the range being mapped is [0, ULONG_MAX], split it into two entries
61 * otherwise its size would overflow u64.
62 */
63 if (start == 0 && last == ULONG_MAX) {
64 u64 mid = last / 2;
65 int err = vhost_iotlb_add_range_ctx(iotlb, start, mid, addr,
66 perm, opaque);
67
68 if (err)
69 return err;
70
71 addr += mid + 1;
72 start = mid + 1;
73 }
74
75 if (iotlb->limit &&
76 iotlb->nmaps == iotlb->limit &&
77 iotlb->flags & VHOST_IOTLB_FLAG_RETIRE) {
78 map = list_first_entry(&iotlb->list, typeof(*map), link);
79 vhost_iotlb_map_free(iotlb, map);
80 }
81
82 map = kmalloc(sizeof(*map), GFP_ATOMIC);
83 if (!map)
84 return -ENOMEM;
85
86 map->start = start;
87 map->size = last - start + 1;
88 map->last = last;
89 map->addr = addr;
90 map->perm = perm;
91 map->opaque = opaque;
92
93 iotlb->nmaps++;
94 vhost_iotlb_itree_insert(map, &iotlb->root);
95
96 INIT_LIST_HEAD(&map->link);
97 list_add_tail(&map->link, &iotlb->list);
98
99 return 0;
100 }
101 EXPORT_SYMBOL_GPL(vhost_iotlb_add_range_ctx);
102
vhost_iotlb_add_range(struct vhost_iotlb * iotlb,u64 start,u64 last,u64 addr,unsigned int perm)103 int vhost_iotlb_add_range(struct vhost_iotlb *iotlb,
104 u64 start, u64 last,
105 u64 addr, unsigned int perm)
106 {
107 return vhost_iotlb_add_range_ctx(iotlb, start, last,
108 addr, perm, NULL);
109 }
110 EXPORT_SYMBOL_GPL(vhost_iotlb_add_range);
111
112 /**
113 * vhost_iotlb_del_range - delete overlapped ranges from vhost IOTLB
114 * @iotlb: the IOTLB
115 * @start: start of the IOVA range
116 * @last: last of IOVA range
117 */
vhost_iotlb_del_range(struct vhost_iotlb * iotlb,u64 start,u64 last)118 void vhost_iotlb_del_range(struct vhost_iotlb *iotlb, u64 start, u64 last)
119 {
120 struct vhost_iotlb_map *map;
121
122 while ((map = vhost_iotlb_itree_iter_first(&iotlb->root,
123 start, last)))
124 vhost_iotlb_map_free(iotlb, map);
125 }
126 EXPORT_SYMBOL_GPL(vhost_iotlb_del_range);
127
128 /**
129 * vhost_iotlb_init - initialize a vhost IOTLB
130 * @iotlb: the IOTLB that needs to be initialized
131 * @limit: maximum number of IOTLB entries
132 * @flags: VHOST_IOTLB_FLAG_XXX
133 */
vhost_iotlb_init(struct vhost_iotlb * iotlb,unsigned int limit,unsigned int flags)134 void vhost_iotlb_init(struct vhost_iotlb *iotlb, unsigned int limit,
135 unsigned int flags)
136 {
137 iotlb->root = RB_ROOT_CACHED;
138 iotlb->limit = limit;
139 iotlb->nmaps = 0;
140 iotlb->flags = flags;
141 INIT_LIST_HEAD(&iotlb->list);
142 }
143 EXPORT_SYMBOL_GPL(vhost_iotlb_init);
144
145 /**
146 * vhost_iotlb_alloc - add a new vhost IOTLB
147 * @limit: maximum number of IOTLB entries
148 * @flags: VHOST_IOTLB_FLAG_XXX
149 *
150 * Returns an error is memory allocation fails
151 */
vhost_iotlb_alloc(unsigned int limit,unsigned int flags)152 struct vhost_iotlb *vhost_iotlb_alloc(unsigned int limit, unsigned int flags)
153 {
154 struct vhost_iotlb *iotlb = kzalloc(sizeof(*iotlb), GFP_KERNEL);
155
156 if (!iotlb)
157 return NULL;
158
159 vhost_iotlb_init(iotlb, limit, flags);
160
161 return iotlb;
162 }
163 EXPORT_SYMBOL_GPL(vhost_iotlb_alloc);
164
165 /**
166 * vhost_iotlb_reset - reset vhost IOTLB (free all IOTLB entries)
167 * @iotlb: the IOTLB to be reset
168 */
vhost_iotlb_reset(struct vhost_iotlb * iotlb)169 void vhost_iotlb_reset(struct vhost_iotlb *iotlb)
170 {
171 vhost_iotlb_del_range(iotlb, 0ULL, 0ULL - 1);
172 }
173 EXPORT_SYMBOL_GPL(vhost_iotlb_reset);
174
175 /**
176 * vhost_iotlb_free - reset and free vhost IOTLB
177 * @iotlb: the IOTLB to be freed
178 */
vhost_iotlb_free(struct vhost_iotlb * iotlb)179 void vhost_iotlb_free(struct vhost_iotlb *iotlb)
180 {
181 if (iotlb) {
182 vhost_iotlb_reset(iotlb);
183 kfree(iotlb);
184 }
185 }
186 EXPORT_SYMBOL_GPL(vhost_iotlb_free);
187
188 /**
189 * vhost_iotlb_itree_first - return the first overlapped range
190 * @iotlb: the IOTLB
191 * @start: start of IOVA range
192 * @last: last byte in IOVA range
193 */
194 struct vhost_iotlb_map *
vhost_iotlb_itree_first(struct vhost_iotlb * iotlb,u64 start,u64 last)195 vhost_iotlb_itree_first(struct vhost_iotlb *iotlb, u64 start, u64 last)
196 {
197 return vhost_iotlb_itree_iter_first(&iotlb->root, start, last);
198 }
199 EXPORT_SYMBOL_GPL(vhost_iotlb_itree_first);
200
201 /**
202 * vhost_iotlb_itree_next - return the next overlapped range
203 * @map: the starting map node
204 * @start: start of IOVA range
205 * @last: last byte IOVA range
206 */
207 struct vhost_iotlb_map *
vhost_iotlb_itree_next(struct vhost_iotlb_map * map,u64 start,u64 last)208 vhost_iotlb_itree_next(struct vhost_iotlb_map *map, u64 start, u64 last)
209 {
210 return vhost_iotlb_itree_iter_next(map, start, last);
211 }
212 EXPORT_SYMBOL_GPL(vhost_iotlb_itree_next);
213
214 MODULE_VERSION(MOD_VERSION);
215 MODULE_DESCRIPTION(MOD_DESC);
216 MODULE_AUTHOR(MOD_AUTHOR);
217 MODULE_LICENSE(MOD_LICENSE);
218