1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause)
2
3 #include <linux/dma-mapping.h>
4 #include <linux/interrupt.h>
5 #include <linux/log2.h>
6 #include <linux/mm.h>
7 #include <linux/netdevice.h>
8 #include <linux/pci.h>
9 #include <linux/slab.h>
10
11 #include "fun_dev.h"
12 #include "fun_queue.h"
13
14 /* Allocate memory for a queue. This includes the memory for the HW descriptor
15 * ring, an optional 64b HW write-back area, and an optional SW state ring.
16 * Returns the virtual and DMA addresses of the HW ring, the VA of the SW ring,
17 * and the VA of the write-back area.
18 */
fun_alloc_ring_mem(struct device * dma_dev,size_t depth,size_t hw_desc_sz,size_t sw_desc_sz,bool wb,int numa_node,dma_addr_t * dma_addr,void ** sw_va,volatile __be64 ** wb_va)19 void *fun_alloc_ring_mem(struct device *dma_dev, size_t depth,
20 size_t hw_desc_sz, size_t sw_desc_sz, bool wb,
21 int numa_node, dma_addr_t *dma_addr, void **sw_va,
22 volatile __be64 **wb_va)
23 {
24 int dev_node = dev_to_node(dma_dev);
25 size_t dma_sz;
26 void *va;
27
28 if (numa_node == NUMA_NO_NODE)
29 numa_node = dev_node;
30
31 /* Place optional write-back area at end of descriptor ring. */
32 dma_sz = hw_desc_sz * depth;
33 if (wb)
34 dma_sz += sizeof(u64);
35
36 set_dev_node(dma_dev, numa_node);
37 va = dma_alloc_coherent(dma_dev, dma_sz, dma_addr, GFP_KERNEL);
38 set_dev_node(dma_dev, dev_node);
39 if (!va)
40 return NULL;
41
42 if (sw_desc_sz) {
43 *sw_va = kvzalloc_node(sw_desc_sz * depth, GFP_KERNEL,
44 numa_node);
45 if (!*sw_va) {
46 dma_free_coherent(dma_dev, dma_sz, va, *dma_addr);
47 return NULL;
48 }
49 }
50
51 if (wb)
52 *wb_va = va + dma_sz - sizeof(u64);
53 return va;
54 }
55 EXPORT_SYMBOL_GPL(fun_alloc_ring_mem);
56
fun_free_ring_mem(struct device * dma_dev,size_t depth,size_t hw_desc_sz,bool wb,void * hw_va,dma_addr_t dma_addr,void * sw_va)57 void fun_free_ring_mem(struct device *dma_dev, size_t depth, size_t hw_desc_sz,
58 bool wb, void *hw_va, dma_addr_t dma_addr, void *sw_va)
59 {
60 if (hw_va) {
61 size_t sz = depth * hw_desc_sz;
62
63 if (wb)
64 sz += sizeof(u64);
65 dma_free_coherent(dma_dev, sz, hw_va, dma_addr);
66 }
67 kvfree(sw_va);
68 }
69 EXPORT_SYMBOL_GPL(fun_free_ring_mem);
70
71 /* Prepare and issue an admin command to create an SQ on the device with the
72 * provided parameters. If the queue ID is auto-allocated by the device it is
73 * returned in *sqidp.
74 */
fun_sq_create(struct fun_dev * fdev,u16 flags,u32 sqid,u32 cqid,u8 sqe_size_log2,u32 sq_depth,dma_addr_t dma_addr,u8 coal_nentries,u8 coal_usec,u32 irq_num,u32 scan_start_id,u32 scan_end_id,u32 rq_buf_size_log2,u32 * sqidp,u32 __iomem ** dbp)75 int fun_sq_create(struct fun_dev *fdev, u16 flags, u32 sqid, u32 cqid,
76 u8 sqe_size_log2, u32 sq_depth, dma_addr_t dma_addr,
77 u8 coal_nentries, u8 coal_usec, u32 irq_num,
78 u32 scan_start_id, u32 scan_end_id,
79 u32 rq_buf_size_log2, u32 *sqidp, u32 __iomem **dbp)
80 {
81 union {
82 struct fun_admin_epsq_req req;
83 struct fun_admin_generic_create_rsp rsp;
84 } cmd;
85 dma_addr_t wb_addr;
86 u32 hw_qid;
87 int rc;
88
89 if (sq_depth > fdev->q_depth)
90 return -EINVAL;
91 if (flags & FUN_ADMIN_EPSQ_CREATE_FLAG_RQ)
92 sqe_size_log2 = ilog2(sizeof(struct fun_eprq_rqbuf));
93
94 wb_addr = dma_addr + (sq_depth << sqe_size_log2);
95
96 cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_EPSQ,
97 sizeof(cmd.req));
98 cmd.req.u.create =
99 FUN_ADMIN_EPSQ_CREATE_REQ_INIT(FUN_ADMIN_SUBOP_CREATE, flags,
100 sqid, cqid, sqe_size_log2,
101 sq_depth - 1, dma_addr, 0,
102 coal_nentries, coal_usec,
103 irq_num, scan_start_id,
104 scan_end_id, 0,
105 rq_buf_size_log2,
106 ilog2(sizeof(u64)), wb_addr);
107
108 rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common,
109 &cmd.rsp, sizeof(cmd.rsp), 0);
110 if (rc)
111 return rc;
112
113 hw_qid = be32_to_cpu(cmd.rsp.id);
114 *dbp = fun_sq_db_addr(fdev, hw_qid);
115 if (flags & FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR)
116 *sqidp = hw_qid;
117 return rc;
118 }
119 EXPORT_SYMBOL_GPL(fun_sq_create);
120
121 /* Prepare and issue an admin command to create a CQ on the device with the
122 * provided parameters. If the queue ID is auto-allocated by the device it is
123 * returned in *cqidp.
124 */
fun_cq_create(struct fun_dev * fdev,u16 flags,u32 cqid,u32 rqid,u8 cqe_size_log2,u32 cq_depth,dma_addr_t dma_addr,u16 headroom,u16 tailroom,u8 coal_nentries,u8 coal_usec,u32 irq_num,u32 scan_start_id,u32 scan_end_id,u32 * cqidp,u32 __iomem ** dbp)125 int fun_cq_create(struct fun_dev *fdev, u16 flags, u32 cqid, u32 rqid,
126 u8 cqe_size_log2, u32 cq_depth, dma_addr_t dma_addr,
127 u16 headroom, u16 tailroom, u8 coal_nentries, u8 coal_usec,
128 u32 irq_num, u32 scan_start_id, u32 scan_end_id, u32 *cqidp,
129 u32 __iomem **dbp)
130 {
131 union {
132 struct fun_admin_epcq_req req;
133 struct fun_admin_generic_create_rsp rsp;
134 } cmd;
135 u32 hw_qid;
136 int rc;
137
138 if (cq_depth > fdev->q_depth)
139 return -EINVAL;
140
141 cmd.req.common = FUN_ADMIN_REQ_COMMON_INIT2(FUN_ADMIN_OP_EPCQ,
142 sizeof(cmd.req));
143 cmd.req.u.create =
144 FUN_ADMIN_EPCQ_CREATE_REQ_INIT(FUN_ADMIN_SUBOP_CREATE, flags,
145 cqid, rqid, cqe_size_log2,
146 cq_depth - 1, dma_addr, tailroom,
147 headroom / 2, 0, coal_nentries,
148 coal_usec, irq_num,
149 scan_start_id, scan_end_id, 0);
150
151 rc = fun_submit_admin_sync_cmd(fdev, &cmd.req.common,
152 &cmd.rsp, sizeof(cmd.rsp), 0);
153 if (rc)
154 return rc;
155
156 hw_qid = be32_to_cpu(cmd.rsp.id);
157 *dbp = fun_cq_db_addr(fdev, hw_qid);
158 if (flags & FUN_ADMIN_RES_CREATE_FLAG_ALLOCATOR)
159 *cqidp = hw_qid;
160 return rc;
161 }
162 EXPORT_SYMBOL_GPL(fun_cq_create);
163
fun_sq_is_head_wb(const struct fun_queue * funq)164 static bool fun_sq_is_head_wb(const struct fun_queue *funq)
165 {
166 return funq->sq_flags & FUN_ADMIN_EPSQ_CREATE_FLAG_HEAD_WB_ADDRESS;
167 }
168
fun_clean_rq(struct fun_queue * funq)169 static void fun_clean_rq(struct fun_queue *funq)
170 {
171 struct fun_dev *fdev = funq->fdev;
172 struct fun_rq_info *rqinfo;
173 unsigned int i;
174
175 for (i = 0; i < funq->rq_depth; i++) {
176 rqinfo = &funq->rq_info[i];
177 if (rqinfo->page) {
178 dma_unmap_page(fdev->dev, rqinfo->dma, PAGE_SIZE,
179 DMA_FROM_DEVICE);
180 put_page(rqinfo->page);
181 rqinfo->page = NULL;
182 }
183 }
184 }
185
fun_fill_rq(struct fun_queue * funq)186 static int fun_fill_rq(struct fun_queue *funq)
187 {
188 struct device *dev = funq->fdev->dev;
189 int i, node = dev_to_node(dev);
190 struct fun_rq_info *rqinfo;
191
192 for (i = 0; i < funq->rq_depth; i++) {
193 rqinfo = &funq->rq_info[i];
194 rqinfo->page = alloc_pages_node(node, GFP_KERNEL, 0);
195 if (unlikely(!rqinfo->page))
196 return -ENOMEM;
197
198 rqinfo->dma = dma_map_page(dev, rqinfo->page, 0,
199 PAGE_SIZE, DMA_FROM_DEVICE);
200 if (unlikely(dma_mapping_error(dev, rqinfo->dma))) {
201 put_page(rqinfo->page);
202 rqinfo->page = NULL;
203 return -ENOMEM;
204 }
205
206 funq->rqes[i] = FUN_EPRQ_RQBUF_INIT(rqinfo->dma);
207 }
208
209 funq->rq_tail = funq->rq_depth - 1;
210 return 0;
211 }
212
fun_rq_update_pos(struct fun_queue * funq,int buf_offset)213 static void fun_rq_update_pos(struct fun_queue *funq, int buf_offset)
214 {
215 if (buf_offset <= funq->rq_buf_offset) {
216 struct fun_rq_info *rqinfo = &funq->rq_info[funq->rq_buf_idx];
217 struct device *dev = funq->fdev->dev;
218
219 dma_sync_single_for_device(dev, rqinfo->dma, PAGE_SIZE,
220 DMA_FROM_DEVICE);
221 funq->num_rqe_to_fill++;
222 if (++funq->rq_buf_idx == funq->rq_depth)
223 funq->rq_buf_idx = 0;
224 }
225 funq->rq_buf_offset = buf_offset;
226 }
227
228 /* Given a command response with data scattered across >= 1 RQ buffers return
229 * a pointer to a contiguous buffer containing all the data. If the data is in
230 * one RQ buffer the start address within that buffer is returned, otherwise a
231 * new buffer is allocated and the data is gathered into it.
232 */
fun_data_from_rq(struct fun_queue * funq,const struct fun_rsp_common * rsp,bool * need_free)233 static void *fun_data_from_rq(struct fun_queue *funq,
234 const struct fun_rsp_common *rsp, bool *need_free)
235 {
236 u32 bufoff, total_len, remaining, fragsize, dataoff;
237 struct device *dma_dev = funq->fdev->dev;
238 const struct fun_dataop_rqbuf *databuf;
239 const struct fun_dataop_hdr *dataop;
240 const struct fun_rq_info *rqinfo;
241 void *data;
242
243 dataop = (void *)rsp + rsp->suboff8 * 8;
244 total_len = be32_to_cpu(dataop->total_len);
245
246 if (likely(dataop->nsgl == 1)) {
247 databuf = (struct fun_dataop_rqbuf *)dataop->imm;
248 bufoff = be32_to_cpu(databuf->bufoff);
249 fun_rq_update_pos(funq, bufoff);
250 rqinfo = &funq->rq_info[funq->rq_buf_idx];
251 dma_sync_single_for_cpu(dma_dev, rqinfo->dma + bufoff,
252 total_len, DMA_FROM_DEVICE);
253 *need_free = false;
254 return page_address(rqinfo->page) + bufoff;
255 }
256
257 /* For scattered completions gather the fragments into one buffer. */
258
259 data = kmalloc(total_len, GFP_ATOMIC);
260 /* NULL is OK here. In case of failure we still need to consume the data
261 * for proper buffer accounting but indicate an error in the response.
262 */
263 if (likely(data))
264 *need_free = true;
265
266 dataoff = 0;
267 for (remaining = total_len; remaining; remaining -= fragsize) {
268 fun_rq_update_pos(funq, 0);
269 fragsize = min_t(unsigned int, PAGE_SIZE, remaining);
270 if (data) {
271 rqinfo = &funq->rq_info[funq->rq_buf_idx];
272 dma_sync_single_for_cpu(dma_dev, rqinfo->dma, fragsize,
273 DMA_FROM_DEVICE);
274 memcpy(data + dataoff, page_address(rqinfo->page),
275 fragsize);
276 dataoff += fragsize;
277 }
278 }
279 return data;
280 }
281
__fun_process_cq(struct fun_queue * funq,unsigned int max)282 unsigned int __fun_process_cq(struct fun_queue *funq, unsigned int max)
283 {
284 const struct fun_cqe_info *info;
285 struct fun_rsp_common *rsp;
286 unsigned int new_cqes;
287 u16 sf_p, flags;
288 bool need_free;
289 void *cqe;
290
291 if (!max)
292 max = funq->cq_depth - 1;
293
294 for (new_cqes = 0; new_cqes < max; new_cqes++) {
295 cqe = funq->cqes + (funq->cq_head << funq->cqe_size_log2);
296 info = funq_cqe_info(funq, cqe);
297 sf_p = be16_to_cpu(info->sf_p);
298
299 if ((sf_p & 1) != funq->cq_phase)
300 break;
301
302 /* ensure the phase tag is read before other CQE fields */
303 dma_rmb();
304
305 if (++funq->cq_head == funq->cq_depth) {
306 funq->cq_head = 0;
307 funq->cq_phase = !funq->cq_phase;
308 }
309
310 rsp = cqe;
311 flags = be16_to_cpu(rsp->flags);
312
313 need_free = false;
314 if (unlikely(flags & FUN_REQ_COMMON_FLAG_CQE_IN_RQBUF)) {
315 rsp = fun_data_from_rq(funq, rsp, &need_free);
316 if (!rsp) {
317 rsp = cqe;
318 rsp->len8 = 1;
319 if (rsp->ret == 0)
320 rsp->ret = ENOMEM;
321 }
322 }
323
324 if (funq->cq_cb)
325 funq->cq_cb(funq, funq->cb_data, rsp, info);
326 if (need_free)
327 kfree(rsp);
328 }
329
330 dev_dbg(funq->fdev->dev, "CQ %u, new CQEs %u/%u, head %u, phase %u\n",
331 funq->cqid, new_cqes, max, funq->cq_head, funq->cq_phase);
332 return new_cqes;
333 }
334
fun_process_cq(struct fun_queue * funq,unsigned int max)335 unsigned int fun_process_cq(struct fun_queue *funq, unsigned int max)
336 {
337 unsigned int processed;
338 u32 db;
339
340 processed = __fun_process_cq(funq, max);
341
342 if (funq->num_rqe_to_fill) {
343 funq->rq_tail = (funq->rq_tail + funq->num_rqe_to_fill) %
344 funq->rq_depth;
345 funq->num_rqe_to_fill = 0;
346 writel(funq->rq_tail, funq->rq_db);
347 }
348
349 db = funq->cq_head | FUN_DB_IRQ_ARM_F;
350 writel(db, funq->cq_db);
351 return processed;
352 }
353
fun_alloc_sqes(struct fun_queue * funq)354 static int fun_alloc_sqes(struct fun_queue *funq)
355 {
356 funq->sq_cmds = fun_alloc_ring_mem(funq->fdev->dev, funq->sq_depth,
357 1 << funq->sqe_size_log2, 0,
358 fun_sq_is_head_wb(funq),
359 NUMA_NO_NODE, &funq->sq_dma_addr,
360 NULL, &funq->sq_head);
361 return funq->sq_cmds ? 0 : -ENOMEM;
362 }
363
fun_alloc_cqes(struct fun_queue * funq)364 static int fun_alloc_cqes(struct fun_queue *funq)
365 {
366 funq->cqes = fun_alloc_ring_mem(funq->fdev->dev, funq->cq_depth,
367 1 << funq->cqe_size_log2, 0, false,
368 NUMA_NO_NODE, &funq->cq_dma_addr, NULL,
369 NULL);
370 return funq->cqes ? 0 : -ENOMEM;
371 }
372
fun_alloc_rqes(struct fun_queue * funq)373 static int fun_alloc_rqes(struct fun_queue *funq)
374 {
375 funq->rqes = fun_alloc_ring_mem(funq->fdev->dev, funq->rq_depth,
376 sizeof(*funq->rqes),
377 sizeof(*funq->rq_info), false,
378 NUMA_NO_NODE, &funq->rq_dma_addr,
379 (void **)&funq->rq_info, NULL);
380 return funq->rqes ? 0 : -ENOMEM;
381 }
382
383 /* Free a queue's structures. */
fun_free_queue(struct fun_queue * funq)384 void fun_free_queue(struct fun_queue *funq)
385 {
386 struct device *dev = funq->fdev->dev;
387
388 fun_free_ring_mem(dev, funq->cq_depth, 1 << funq->cqe_size_log2, false,
389 funq->cqes, funq->cq_dma_addr, NULL);
390 fun_free_ring_mem(dev, funq->sq_depth, 1 << funq->sqe_size_log2,
391 fun_sq_is_head_wb(funq), funq->sq_cmds,
392 funq->sq_dma_addr, NULL);
393
394 if (funq->rqes) {
395 fun_clean_rq(funq);
396 fun_free_ring_mem(dev, funq->rq_depth, sizeof(*funq->rqes),
397 false, funq->rqes, funq->rq_dma_addr,
398 funq->rq_info);
399 }
400
401 kfree(funq);
402 }
403
404 /* Allocate and initialize a funq's structures. */
fun_alloc_queue(struct fun_dev * fdev,int qid,const struct fun_queue_alloc_req * req)405 struct fun_queue *fun_alloc_queue(struct fun_dev *fdev, int qid,
406 const struct fun_queue_alloc_req *req)
407 {
408 struct fun_queue *funq = kzalloc(sizeof(*funq), GFP_KERNEL);
409
410 if (!funq)
411 return NULL;
412
413 funq->fdev = fdev;
414 spin_lock_init(&funq->sq_lock);
415
416 funq->qid = qid;
417
418 /* Initial CQ/SQ/RQ ids */
419 if (req->rq_depth) {
420 funq->cqid = 2 * qid;
421 if (funq->qid) {
422 /* I/O Q: use rqid = cqid, sqid = +1 */
423 funq->rqid = funq->cqid;
424 funq->sqid = funq->rqid + 1;
425 } else {
426 /* Admin Q: sqid is always 0, use ID 1 for RQ */
427 funq->sqid = 0;
428 funq->rqid = 1;
429 }
430 } else {
431 funq->cqid = qid;
432 funq->sqid = qid;
433 }
434
435 funq->cq_flags = req->cq_flags;
436 funq->sq_flags = req->sq_flags;
437
438 funq->cqe_size_log2 = req->cqe_size_log2;
439 funq->sqe_size_log2 = req->sqe_size_log2;
440
441 funq->cq_depth = req->cq_depth;
442 funq->sq_depth = req->sq_depth;
443
444 funq->cq_intcoal_nentries = req->cq_intcoal_nentries;
445 funq->cq_intcoal_usec = req->cq_intcoal_usec;
446
447 funq->sq_intcoal_nentries = req->sq_intcoal_nentries;
448 funq->sq_intcoal_usec = req->sq_intcoal_usec;
449
450 if (fun_alloc_cqes(funq))
451 goto free_funq;
452
453 funq->cq_phase = 1;
454
455 if (fun_alloc_sqes(funq))
456 goto free_funq;
457
458 if (req->rq_depth) {
459 funq->rq_flags = req->rq_flags | FUN_ADMIN_EPSQ_CREATE_FLAG_RQ;
460 funq->rq_depth = req->rq_depth;
461 funq->rq_buf_offset = -1;
462
463 if (fun_alloc_rqes(funq) || fun_fill_rq(funq))
464 goto free_funq;
465 }
466
467 funq->cq_vector = -1;
468 funq->cqe_info_offset = (1 << funq->cqe_size_log2) - sizeof(struct fun_cqe_info);
469
470 /* SQ/CQ 0 are implicitly created, assign their doorbells now.
471 * Other queues are assigned doorbells at their explicit creation.
472 */
473 if (funq->sqid == 0)
474 funq->sq_db = fun_sq_db_addr(fdev, 0);
475 if (funq->cqid == 0)
476 funq->cq_db = fun_cq_db_addr(fdev, 0);
477
478 return funq;
479
480 free_funq:
481 fun_free_queue(funq);
482 return NULL;
483 }
484
485 /* Create a funq's CQ on the device. */
fun_create_cq(struct fun_queue * funq)486 static int fun_create_cq(struct fun_queue *funq)
487 {
488 struct fun_dev *fdev = funq->fdev;
489 unsigned int rqid;
490 int rc;
491
492 rqid = funq->cq_flags & FUN_ADMIN_EPCQ_CREATE_FLAG_RQ ?
493 funq->rqid : FUN_HCI_ID_INVALID;
494 rc = fun_cq_create(fdev, funq->cq_flags, funq->cqid, rqid,
495 funq->cqe_size_log2, funq->cq_depth,
496 funq->cq_dma_addr, 0, 0, funq->cq_intcoal_nentries,
497 funq->cq_intcoal_usec, funq->cq_vector, 0, 0,
498 &funq->cqid, &funq->cq_db);
499 if (!rc)
500 dev_dbg(fdev->dev, "created CQ %u\n", funq->cqid);
501
502 return rc;
503 }
504
505 /* Create a funq's SQ on the device. */
fun_create_sq(struct fun_queue * funq)506 static int fun_create_sq(struct fun_queue *funq)
507 {
508 struct fun_dev *fdev = funq->fdev;
509 int rc;
510
511 rc = fun_sq_create(fdev, funq->sq_flags, funq->sqid, funq->cqid,
512 funq->sqe_size_log2, funq->sq_depth,
513 funq->sq_dma_addr, funq->sq_intcoal_nentries,
514 funq->sq_intcoal_usec, funq->cq_vector, 0, 0,
515 0, &funq->sqid, &funq->sq_db);
516 if (!rc)
517 dev_dbg(fdev->dev, "created SQ %u\n", funq->sqid);
518
519 return rc;
520 }
521
522 /* Create a funq's RQ on the device. */
fun_create_rq(struct fun_queue * funq)523 int fun_create_rq(struct fun_queue *funq)
524 {
525 struct fun_dev *fdev = funq->fdev;
526 int rc;
527
528 rc = fun_sq_create(fdev, funq->rq_flags, funq->rqid, funq->cqid, 0,
529 funq->rq_depth, funq->rq_dma_addr, 0, 0,
530 funq->cq_vector, 0, 0, PAGE_SHIFT, &funq->rqid,
531 &funq->rq_db);
532 if (!rc)
533 dev_dbg(fdev->dev, "created RQ %u\n", funq->rqid);
534
535 return rc;
536 }
537
funq_irq(struct fun_queue * funq)538 static unsigned int funq_irq(struct fun_queue *funq)
539 {
540 return pci_irq_vector(to_pci_dev(funq->fdev->dev), funq->cq_vector);
541 }
542
fun_request_irq(struct fun_queue * funq,const char * devname,irq_handler_t handler,void * data)543 int fun_request_irq(struct fun_queue *funq, const char *devname,
544 irq_handler_t handler, void *data)
545 {
546 int rc;
547
548 if (funq->cq_vector < 0)
549 return -EINVAL;
550
551 funq->irq_handler = handler;
552 funq->irq_data = data;
553
554 snprintf(funq->irqname, sizeof(funq->irqname),
555 funq->qid ? "%s-q[%d]" : "%s-adminq", devname, funq->qid);
556
557 rc = request_irq(funq_irq(funq), handler, 0, funq->irqname, data);
558 if (rc)
559 funq->irq_handler = NULL;
560
561 return rc;
562 }
563
564 /* Create all component queues of a funq on the device. */
fun_create_queue(struct fun_queue * funq)565 int fun_create_queue(struct fun_queue *funq)
566 {
567 int rc;
568
569 rc = fun_create_cq(funq);
570 if (rc)
571 return rc;
572
573 if (funq->rq_depth) {
574 rc = fun_create_rq(funq);
575 if (rc)
576 goto release_cq;
577 }
578
579 rc = fun_create_sq(funq);
580 if (rc)
581 goto release_rq;
582
583 return 0;
584
585 release_rq:
586 fun_destroy_sq(funq->fdev, funq->rqid);
587 release_cq:
588 fun_destroy_cq(funq->fdev, funq->cqid);
589 return rc;
590 }
591
fun_free_irq(struct fun_queue * funq)592 void fun_free_irq(struct fun_queue *funq)
593 {
594 if (funq->irq_handler) {
595 unsigned int vector = funq_irq(funq);
596
597 free_irq(vector, funq->irq_data);
598 funq->irq_handler = NULL;
599 funq->irq_data = NULL;
600 }
601 }
602