1 /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
2 /*
3 * Copyright (c) 2016 Mellanox Technologies Ltd. All rights reserved.
4 * Copyright (c) 2015 System Fabric Works, Inc. All rights reserved.
5 */
6
7 #ifndef RXE_QUEUE_H
8 #define RXE_QUEUE_H
9
10 /* Implements a simple circular buffer that is shared between user
11 * and the driver and can be resized. The requested element size is
12 * rounded up to a power of 2 and the number of elements in the buffer
13 * is also rounded up to a power of 2. Since the queue is empty when
14 * the producer and consumer indices match the maximum capacity of the
15 * queue is one less than the number of element slots.
16 *
17 * Notes:
18 * - The driver indices are always masked off to q->index_mask
19 * before storing so do not need to be checked on reads.
20 * - The user whether user space or kernel is generally
21 * not trusted so its parameters are masked to make sure
22 * they do not access the queue out of bounds on reads.
23 * - The driver indices for queues must not be written
24 * by user so a local copy is used and a shared copy is
25 * stored when the local copy is changed.
26 * - By passing the type in the parameter list separate from q
27 * the compiler can eliminate the switch statement when the
28 * actual queue type is known when the function is called at
29 * compile time.
30 * - These queues are lock free. The user and driver must protect
31 * changes to their end of the queues with locks if more than one
32 * CPU can be accessing it at the same time.
33 */
34
35 /**
36 * enum queue_type - type of queue
37 * @QUEUE_TYPE_TO_CLIENT: Queue is written by rxe driver and
38 * read by client which may be a user space
39 * application or a kernel ulp.
40 * Used by rxe internals only.
41 * @QUEUE_TYPE_FROM_CLIENT: Queue is written by client and
42 * read by rxe driver.
43 * Used by rxe internals only.
44 * @QUEUE_TYPE_FROM_ULP: Queue is written by kernel ulp and
45 * read by rxe driver.
46 * Used by kernel verbs APIs only on
47 * behalf of ulps.
48 * @QUEUE_TYPE_TO_ULP: Queue is written by rxe driver and
49 * read by kernel ulp.
50 * Used by kernel verbs APIs only on
51 * behalf of ulps.
52 */
53 enum queue_type {
54 QUEUE_TYPE_TO_CLIENT,
55 QUEUE_TYPE_FROM_CLIENT,
56 QUEUE_TYPE_FROM_ULP,
57 QUEUE_TYPE_TO_ULP,
58 };
59
60 struct rxe_queue_buf;
61
62 struct rxe_queue {
63 struct rxe_dev *rxe;
64 struct rxe_queue_buf *buf;
65 struct rxe_mmap_info *ip;
66 size_t buf_size;
67 size_t elem_size;
68 unsigned int log2_elem_size;
69 u32 index_mask;
70 enum queue_type type;
71 /* private copy of index for shared queues between
72 * driver and clients. Driver reads and writes
73 * this copy and then replicates to rxe_queue_buf
74 * for read access by clients.
75 */
76 u32 index;
77 };
78
79 int do_mmap_info(struct rxe_dev *rxe, struct mminfo __user *outbuf,
80 struct ib_udata *udata, struct rxe_queue_buf *buf,
81 size_t buf_size, struct rxe_mmap_info **ip_p);
82
83 void rxe_queue_reset(struct rxe_queue *q);
84
85 struct rxe_queue *rxe_queue_init(struct rxe_dev *rxe, int *num_elem,
86 unsigned int elem_size, enum queue_type type);
87
88 int rxe_queue_resize(struct rxe_queue *q, unsigned int *num_elem_p,
89 unsigned int elem_size, struct ib_udata *udata,
90 struct mminfo __user *outbuf,
91 spinlock_t *producer_lock, spinlock_t *consumer_lock);
92
93 void rxe_queue_cleanup(struct rxe_queue *queue);
94
queue_next_index(struct rxe_queue * q,int index)95 static inline u32 queue_next_index(struct rxe_queue *q, int index)
96 {
97 return (index + 1) & q->index_mask;
98 }
99
queue_get_producer(const struct rxe_queue * q,enum queue_type type)100 static inline u32 queue_get_producer(const struct rxe_queue *q,
101 enum queue_type type)
102 {
103 u32 prod;
104
105 switch (type) {
106 case QUEUE_TYPE_FROM_CLIENT:
107 /* used by rxe, client owns the index */
108 prod = smp_load_acquire(&q->buf->producer_index);
109 break;
110 case QUEUE_TYPE_TO_CLIENT:
111 /* used by rxe which owns the index */
112 prod = q->index;
113 break;
114 case QUEUE_TYPE_FROM_ULP:
115 /* used by ulp which owns the index */
116 prod = q->buf->producer_index;
117 break;
118 case QUEUE_TYPE_TO_ULP:
119 /* used by ulp, rxe owns the index */
120 prod = smp_load_acquire(&q->buf->producer_index);
121 break;
122 }
123
124 return prod;
125 }
126
queue_get_consumer(const struct rxe_queue * q,enum queue_type type)127 static inline u32 queue_get_consumer(const struct rxe_queue *q,
128 enum queue_type type)
129 {
130 u32 cons;
131
132 switch (type) {
133 case QUEUE_TYPE_FROM_CLIENT:
134 /* used by rxe which owns the index */
135 cons = q->index;
136 break;
137 case QUEUE_TYPE_TO_CLIENT:
138 /* used by rxe, client owns the index */
139 cons = smp_load_acquire(&q->buf->consumer_index);
140 break;
141 case QUEUE_TYPE_FROM_ULP:
142 /* used by ulp, rxe owns the index */
143 cons = smp_load_acquire(&q->buf->consumer_index);
144 break;
145 case QUEUE_TYPE_TO_ULP:
146 /* used by ulp which owns the index */
147 cons = q->buf->consumer_index;
148 break;
149 }
150
151 return cons;
152 }
153
queue_empty(struct rxe_queue * q,enum queue_type type)154 static inline int queue_empty(struct rxe_queue *q, enum queue_type type)
155 {
156 u32 prod = queue_get_producer(q, type);
157 u32 cons = queue_get_consumer(q, type);
158
159 return ((prod - cons) & q->index_mask) == 0;
160 }
161
queue_full(struct rxe_queue * q,enum queue_type type)162 static inline int queue_full(struct rxe_queue *q, enum queue_type type)
163 {
164 u32 prod = queue_get_producer(q, type);
165 u32 cons = queue_get_consumer(q, type);
166
167 return ((prod + 1 - cons) & q->index_mask) == 0;
168 }
169
queue_count(const struct rxe_queue * q,enum queue_type type)170 static inline u32 queue_count(const struct rxe_queue *q,
171 enum queue_type type)
172 {
173 u32 prod = queue_get_producer(q, type);
174 u32 cons = queue_get_consumer(q, type);
175
176 return (prod - cons) & q->index_mask;
177 }
178
queue_advance_producer(struct rxe_queue * q,enum queue_type type)179 static inline void queue_advance_producer(struct rxe_queue *q,
180 enum queue_type type)
181 {
182 u32 prod;
183
184 switch (type) {
185 case QUEUE_TYPE_FROM_CLIENT:
186 /* used by rxe, client owns the index */
187 if (WARN_ON(1))
188 pr_warn("%s: attempt to advance client index\n",
189 __func__);
190 break;
191 case QUEUE_TYPE_TO_CLIENT:
192 /* used by rxe which owns the index */
193 prod = q->index;
194 prod = (prod + 1) & q->index_mask;
195 q->index = prod;
196 /* release so client can read it safely */
197 smp_store_release(&q->buf->producer_index, prod);
198 break;
199 case QUEUE_TYPE_FROM_ULP:
200 /* used by ulp which owns the index */
201 prod = q->buf->producer_index;
202 prod = (prod + 1) & q->index_mask;
203 /* release so rxe can read it safely */
204 smp_store_release(&q->buf->producer_index, prod);
205 break;
206 case QUEUE_TYPE_TO_ULP:
207 /* used by ulp, rxe owns the index */
208 if (WARN_ON(1))
209 pr_warn("%s: attempt to advance driver index\n",
210 __func__);
211 break;
212 }
213 }
214
queue_advance_consumer(struct rxe_queue * q,enum queue_type type)215 static inline void queue_advance_consumer(struct rxe_queue *q,
216 enum queue_type type)
217 {
218 u32 cons;
219
220 switch (type) {
221 case QUEUE_TYPE_FROM_CLIENT:
222 /* used by rxe which owns the index */
223 cons = (q->index + 1) & q->index_mask;
224 q->index = cons;
225 /* release so client can read it safely */
226 smp_store_release(&q->buf->consumer_index, cons);
227 break;
228 case QUEUE_TYPE_TO_CLIENT:
229 /* used by rxe, client owns the index */
230 if (WARN_ON(1))
231 pr_warn("%s: attempt to advance client index\n",
232 __func__);
233 break;
234 case QUEUE_TYPE_FROM_ULP:
235 /* used by ulp, rxe owns the index */
236 if (WARN_ON(1))
237 pr_warn("%s: attempt to advance driver index\n",
238 __func__);
239 break;
240 case QUEUE_TYPE_TO_ULP:
241 /* used by ulp which owns the index */
242 cons = q->buf->consumer_index;
243 cons = (cons + 1) & q->index_mask;
244 /* release so rxe can read it safely */
245 smp_store_release(&q->buf->consumer_index, cons);
246 break;
247 }
248 }
249
queue_producer_addr(struct rxe_queue * q,enum queue_type type)250 static inline void *queue_producer_addr(struct rxe_queue *q,
251 enum queue_type type)
252 {
253 u32 prod = queue_get_producer(q, type);
254
255 return q->buf->data + (prod << q->log2_elem_size);
256 }
257
queue_consumer_addr(struct rxe_queue * q,enum queue_type type)258 static inline void *queue_consumer_addr(struct rxe_queue *q,
259 enum queue_type type)
260 {
261 u32 cons = queue_get_consumer(q, type);
262
263 return q->buf->data + (cons << q->log2_elem_size);
264 }
265
queue_addr_from_index(struct rxe_queue * q,u32 index)266 static inline void *queue_addr_from_index(struct rxe_queue *q, u32 index)
267 {
268 return q->buf->data + ((index & q->index_mask)
269 << q->log2_elem_size);
270 }
271
queue_index_from_addr(const struct rxe_queue * q,const void * addr)272 static inline u32 queue_index_from_addr(const struct rxe_queue *q,
273 const void *addr)
274 {
275 return (((u8 *)addr - q->buf->data) >> q->log2_elem_size)
276 & q->index_mask;
277 }
278
queue_head(struct rxe_queue * q,enum queue_type type)279 static inline void *queue_head(struct rxe_queue *q, enum queue_type type)
280 {
281 return queue_empty(q, type) ? NULL : queue_consumer_addr(q, type);
282 }
283
284 #endif /* RXE_QUEUE_H */
285