1 /*
2  * Copyright (c) 2004, 2005, 2006 Voltaire, Inc. All rights reserved.
3  * Copyright (c) 2013 Mellanox Technologies. All rights reserved.
4  *
5  * This software is available to you under a choice of one of two
6  * licenses.  You may choose to be licensed under the terms of the GNU
7  * General Public License (GPL) Version 2, available from the file
8  * COPYING in the main directory of this source tree, or the
9  * OpenIB.org BSD license below:
10  *
11  *     Redistribution and use in source and binary forms, with or
12  *     without modification, are permitted provided that the following
13  *     conditions are met:
14  *
15  *	- Redistributions of source code must retain the above
16  *	  copyright notice, this list of conditions and the following
17  *	  disclaimer.
18  *
19  *	- Redistributions in binary form must reproduce the above
20  *	  copyright notice, this list of conditions and the following
21  *	  disclaimer in the documentation and/or other materials
22  *	  provided with the distribution.
23  *
24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31  * SOFTWARE.
32  */
33 #include <linux/module.h>
34 #include <linux/kernel.h>
35 #include <linux/slab.h>
36 #include <linux/mm.h>
37 #include <linux/highmem.h>
38 #include <linux/scatterlist.h>
39 
40 #include "iscsi_iser.h"
41 
42 #define ISER_KMALLOC_THRESHOLD 0x20000 /* 128K - kmalloc limit */
43 
44 /**
45  * iser_start_rdma_unaligned_sg
46  */
47 static int iser_start_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
48 					enum iser_data_dir cmd_dir)
49 {
50 	int dma_nents;
51 	struct ib_device *dev;
52 	char *mem = NULL;
53 	struct iser_data_buf *data = &iser_task->data[cmd_dir];
54 	unsigned long  cmd_data_len = data->data_len;
55 
56 	if (cmd_data_len > ISER_KMALLOC_THRESHOLD)
57 		mem = (void *)__get_free_pages(GFP_ATOMIC,
58 		      ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT);
59 	else
60 		mem = kmalloc(cmd_data_len, GFP_ATOMIC);
61 
62 	if (mem == NULL) {
63 		iser_err("Failed to allocate mem size %d %d for copying sglist\n",
64 			 data->size,(int)cmd_data_len);
65 		return -ENOMEM;
66 	}
67 
68 	if (cmd_dir == ISER_DIR_OUT) {
69 		/* copy the unaligned sg the buffer which is used for RDMA */
70 		struct scatterlist *sgl = (struct scatterlist *)data->buf;
71 		struct scatterlist *sg;
72 		int i;
73 		char *p, *from;
74 
75 		p = mem;
76 		for_each_sg(sgl, sg, data->size, i) {
77 			from = kmap_atomic(sg_page(sg));
78 			memcpy(p,
79 			       from + sg->offset,
80 			       sg->length);
81 			kunmap_atomic(from);
82 			p += sg->length;
83 		}
84 	}
85 
86 	sg_init_one(&iser_task->data_copy[cmd_dir].sg_single, mem, cmd_data_len);
87 	iser_task->data_copy[cmd_dir].buf  =
88 		&iser_task->data_copy[cmd_dir].sg_single;
89 	iser_task->data_copy[cmd_dir].size = 1;
90 
91 	iser_task->data_copy[cmd_dir].copy_buf  = mem;
92 
93 	dev = iser_task->iser_conn->ib_conn->device->ib_device;
94 	dma_nents = ib_dma_map_sg(dev,
95 				  &iser_task->data_copy[cmd_dir].sg_single,
96 				  1,
97 				  (cmd_dir == ISER_DIR_OUT) ?
98 				  DMA_TO_DEVICE : DMA_FROM_DEVICE);
99 	BUG_ON(dma_nents == 0);
100 
101 	iser_task->data_copy[cmd_dir].dma_nents = dma_nents;
102 	return 0;
103 }
104 
105 /**
106  * iser_finalize_rdma_unaligned_sg
107  */
108 void iser_finalize_rdma_unaligned_sg(struct iscsi_iser_task *iser_task,
109 				     enum iser_data_dir         cmd_dir)
110 {
111 	struct ib_device *dev;
112 	struct iser_data_buf *mem_copy;
113 	unsigned long  cmd_data_len;
114 
115 	dev = iser_task->iser_conn->ib_conn->device->ib_device;
116 	mem_copy = &iser_task->data_copy[cmd_dir];
117 
118 	ib_dma_unmap_sg(dev, &mem_copy->sg_single, 1,
119 			(cmd_dir == ISER_DIR_OUT) ?
120 			DMA_TO_DEVICE : DMA_FROM_DEVICE);
121 
122 	if (cmd_dir == ISER_DIR_IN) {
123 		char *mem;
124 		struct scatterlist *sgl, *sg;
125 		unsigned char *p, *to;
126 		unsigned int sg_size;
127 		int i;
128 
129 		/* copy back read RDMA to unaligned sg */
130 		mem	= mem_copy->copy_buf;
131 
132 		sgl	= (struct scatterlist *)iser_task->data[ISER_DIR_IN].buf;
133 		sg_size = iser_task->data[ISER_DIR_IN].size;
134 
135 		p = mem;
136 		for_each_sg(sgl, sg, sg_size, i) {
137 			to = kmap_atomic(sg_page(sg));
138 			memcpy(to + sg->offset,
139 			       p,
140 			       sg->length);
141 			kunmap_atomic(to);
142 			p += sg->length;
143 		}
144 	}
145 
146 	cmd_data_len = iser_task->data[cmd_dir].data_len;
147 
148 	if (cmd_data_len > ISER_KMALLOC_THRESHOLD)
149 		free_pages((unsigned long)mem_copy->copy_buf,
150 			   ilog2(roundup_pow_of_two(cmd_data_len)) - PAGE_SHIFT);
151 	else
152 		kfree(mem_copy->copy_buf);
153 
154 	mem_copy->copy_buf = NULL;
155 }
156 
157 #define IS_4K_ALIGNED(addr)	((((unsigned long)addr) & ~MASK_4K) == 0)
158 
159 /**
160  * iser_sg_to_page_vec - Translates scatterlist entries to physical addresses
161  * and returns the length of resulting physical address array (may be less than
162  * the original due to possible compaction).
163  *
164  * we build a "page vec" under the assumption that the SG meets the RDMA
165  * alignment requirements. Other then the first and last SG elements, all
166  * the "internal" elements can be compacted into a list whose elements are
167  * dma addresses of physical pages. The code supports also the weird case
168  * where --few fragments of the same page-- are present in the SG as
169  * consecutive elements. Also, it handles one entry SG.
170  */
171 
172 static int iser_sg_to_page_vec(struct iser_data_buf *data,
173 			       struct ib_device *ibdev, u64 *pages,
174 			       int *offset, int *data_size)
175 {
176 	struct scatterlist *sg, *sgl = (struct scatterlist *)data->buf;
177 	u64 start_addr, end_addr, page, chunk_start = 0;
178 	unsigned long total_sz = 0;
179 	unsigned int dma_len;
180 	int i, new_chunk, cur_page, last_ent = data->dma_nents - 1;
181 
182 	/* compute the offset of first element */
183 	*offset = (u64) sgl[0].offset & ~MASK_4K;
184 
185 	new_chunk = 1;
186 	cur_page  = 0;
187 	for_each_sg(sgl, sg, data->dma_nents, i) {
188 		start_addr = ib_sg_dma_address(ibdev, sg);
189 		if (new_chunk)
190 			chunk_start = start_addr;
191 		dma_len = ib_sg_dma_len(ibdev, sg);
192 		end_addr = start_addr + dma_len;
193 		total_sz += dma_len;
194 
195 		/* collect page fragments until aligned or end of SG list */
196 		if (!IS_4K_ALIGNED(end_addr) && i < last_ent) {
197 			new_chunk = 0;
198 			continue;
199 		}
200 		new_chunk = 1;
201 
202 		/* address of the first page in the contiguous chunk;
203 		   masking relevant for the very first SG entry,
204 		   which might be unaligned */
205 		page = chunk_start & MASK_4K;
206 		do {
207 			pages[cur_page++] = page;
208 			page += SIZE_4K;
209 		} while (page < end_addr);
210 	}
211 
212 	*data_size = total_sz;
213 	iser_dbg("page_vec->data_size:%d cur_page %d\n",
214 		 *data_size, cur_page);
215 	return cur_page;
216 }
217 
218 
219 /**
220  * iser_data_buf_aligned_len - Tries to determine the maximal correctly aligned
221  * for RDMA sub-list of a scatter-gather list of memory buffers, and  returns
222  * the number of entries which are aligned correctly. Supports the case where
223  * consecutive SG elements are actually fragments of the same physcial page.
224  */
225 static int iser_data_buf_aligned_len(struct iser_data_buf *data,
226 				      struct ib_device *ibdev)
227 {
228 	struct scatterlist *sgl, *sg, *next_sg = NULL;
229 	u64 start_addr, end_addr;
230 	int i, ret_len, start_check = 0;
231 
232 	if (data->dma_nents == 1)
233 		return 1;
234 
235 	sgl = (struct scatterlist *)data->buf;
236 	start_addr  = ib_sg_dma_address(ibdev, sgl);
237 
238 	for_each_sg(sgl, sg, data->dma_nents, i) {
239 		if (start_check && !IS_4K_ALIGNED(start_addr))
240 			break;
241 
242 		next_sg = sg_next(sg);
243 		if (!next_sg)
244 			break;
245 
246 		end_addr    = start_addr + ib_sg_dma_len(ibdev, sg);
247 		start_addr  = ib_sg_dma_address(ibdev, next_sg);
248 
249 		if (end_addr == start_addr) {
250 			start_check = 0;
251 			continue;
252 		} else
253 			start_check = 1;
254 
255 		if (!IS_4K_ALIGNED(end_addr))
256 			break;
257 	}
258 	ret_len = (next_sg) ? i : i+1;
259 	iser_dbg("Found %d aligned entries out of %d in sg:0x%p\n",
260 		 ret_len, data->dma_nents, data);
261 	return ret_len;
262 }
263 
264 static void iser_data_buf_dump(struct iser_data_buf *data,
265 			       struct ib_device *ibdev)
266 {
267 	struct scatterlist *sgl = (struct scatterlist *)data->buf;
268 	struct scatterlist *sg;
269 	int i;
270 
271 	for_each_sg(sgl, sg, data->dma_nents, i)
272 		iser_dbg("sg[%d] dma_addr:0x%lX page:0x%p "
273 			 "off:0x%x sz:0x%x dma_len:0x%x\n",
274 			 i, (unsigned long)ib_sg_dma_address(ibdev, sg),
275 			 sg_page(sg), sg->offset,
276 			 sg->length, ib_sg_dma_len(ibdev, sg));
277 }
278 
279 static void iser_dump_page_vec(struct iser_page_vec *page_vec)
280 {
281 	int i;
282 
283 	iser_err("page vec length %d data size %d\n",
284 		 page_vec->length, page_vec->data_size);
285 	for (i = 0; i < page_vec->length; i++)
286 		iser_err("%d %lx\n",i,(unsigned long)page_vec->pages[i]);
287 }
288 
289 static void iser_page_vec_build(struct iser_data_buf *data,
290 				struct iser_page_vec *page_vec,
291 				struct ib_device *ibdev)
292 {
293 	int page_vec_len = 0;
294 
295 	page_vec->length = 0;
296 	page_vec->offset = 0;
297 
298 	iser_dbg("Translating sg sz: %d\n", data->dma_nents);
299 	page_vec_len = iser_sg_to_page_vec(data, ibdev, page_vec->pages,
300 					   &page_vec->offset,
301 					   &page_vec->data_size);
302 	iser_dbg("sg len %d page_vec_len %d\n", data->dma_nents, page_vec_len);
303 
304 	page_vec->length = page_vec_len;
305 
306 	if (page_vec_len * SIZE_4K < page_vec->data_size) {
307 		iser_err("page_vec too short to hold this SG\n");
308 		iser_data_buf_dump(data, ibdev);
309 		iser_dump_page_vec(page_vec);
310 		BUG();
311 	}
312 }
313 
314 int iser_dma_map_task_data(struct iscsi_iser_task *iser_task,
315 			    struct iser_data_buf *data,
316 			    enum iser_data_dir iser_dir,
317 			    enum dma_data_direction dma_dir)
318 {
319 	struct ib_device *dev;
320 
321 	iser_task->dir[iser_dir] = 1;
322 	dev = iser_task->iser_conn->ib_conn->device->ib_device;
323 
324 	data->dma_nents = ib_dma_map_sg(dev, data->buf, data->size, dma_dir);
325 	if (data->dma_nents == 0) {
326 		iser_err("dma_map_sg failed!!!\n");
327 		return -EINVAL;
328 	}
329 	return 0;
330 }
331 
332 void iser_dma_unmap_task_data(struct iscsi_iser_task *iser_task)
333 {
334 	struct ib_device *dev;
335 	struct iser_data_buf *data;
336 
337 	dev = iser_task->iser_conn->ib_conn->device->ib_device;
338 
339 	if (iser_task->dir[ISER_DIR_IN]) {
340 		data = &iser_task->data[ISER_DIR_IN];
341 		ib_dma_unmap_sg(dev, data->buf, data->size, DMA_FROM_DEVICE);
342 	}
343 
344 	if (iser_task->dir[ISER_DIR_OUT]) {
345 		data = &iser_task->data[ISER_DIR_OUT];
346 		ib_dma_unmap_sg(dev, data->buf, data->size, DMA_TO_DEVICE);
347 	}
348 }
349 
350 static int fall_to_bounce_buf(struct iscsi_iser_task *iser_task,
351 			      struct ib_device *ibdev,
352 			      enum iser_data_dir cmd_dir,
353 			      int aligned_len)
354 {
355 	struct iscsi_conn    *iscsi_conn = iser_task->iser_conn->iscsi_conn;
356 	struct iser_data_buf *mem = &iser_task->data[cmd_dir];
357 
358 	iscsi_conn->fmr_unalign_cnt++;
359 	iser_warn("rdma alignment violation (%d/%d aligned) or FMR not supported\n",
360 		  aligned_len, mem->size);
361 
362 	if (iser_debug_level > 0)
363 		iser_data_buf_dump(mem, ibdev);
364 
365 	/* unmap the command data before accessing it */
366 	iser_dma_unmap_task_data(iser_task);
367 
368 	/* allocate copy buf, if we are writing, copy the */
369 	/* unaligned scatterlist, dma map the copy        */
370 	if (iser_start_rdma_unaligned_sg(iser_task, cmd_dir) != 0)
371 			return -ENOMEM;
372 
373 	return 0;
374 }
375 
376 /**
377  * iser_reg_rdma_mem_fmr - Registers memory intended for RDMA,
378  * using FMR (if possible) obtaining rkey and va
379  *
380  * returns 0 on success, errno code on failure
381  */
382 int iser_reg_rdma_mem_fmr(struct iscsi_iser_task *iser_task,
383 			  enum iser_data_dir cmd_dir)
384 {
385 	struct iser_conn     *ib_conn = iser_task->iser_conn->ib_conn;
386 	struct iser_device   *device = ib_conn->device;
387 	struct ib_device     *ibdev = device->ib_device;
388 	struct iser_data_buf *mem = &iser_task->data[cmd_dir];
389 	struct iser_regd_buf *regd_buf;
390 	int aligned_len;
391 	int err;
392 	int i;
393 	struct scatterlist *sg;
394 
395 	regd_buf = &iser_task->rdma_regd[cmd_dir];
396 
397 	aligned_len = iser_data_buf_aligned_len(mem, ibdev);
398 	if (aligned_len != mem->dma_nents) {
399 		err = fall_to_bounce_buf(iser_task, ibdev,
400 					 cmd_dir, aligned_len);
401 		if (err) {
402 			iser_err("failed to allocate bounce buffer\n");
403 			return err;
404 		}
405 		mem = &iser_task->data_copy[cmd_dir];
406 	}
407 
408 	/* if there a single dma entry, FMR is not needed */
409 	if (mem->dma_nents == 1) {
410 		sg = (struct scatterlist *)mem->buf;
411 
412 		regd_buf->reg.lkey = device->mr->lkey;
413 		regd_buf->reg.rkey = device->mr->rkey;
414 		regd_buf->reg.len  = ib_sg_dma_len(ibdev, &sg[0]);
415 		regd_buf->reg.va   = ib_sg_dma_address(ibdev, &sg[0]);
416 		regd_buf->reg.is_mr = 0;
417 
418 		iser_dbg("PHYSICAL Mem.register: lkey: 0x%08X rkey: 0x%08X  "
419 			 "va: 0x%08lX sz: %ld]\n",
420 			 (unsigned int)regd_buf->reg.lkey,
421 			 (unsigned int)regd_buf->reg.rkey,
422 			 (unsigned long)regd_buf->reg.va,
423 			 (unsigned long)regd_buf->reg.len);
424 	} else { /* use FMR for multiple dma entries */
425 		iser_page_vec_build(mem, ib_conn->fastreg.fmr.page_vec, ibdev);
426 		err = iser_reg_page_vec(ib_conn, ib_conn->fastreg.fmr.page_vec,
427 					&regd_buf->reg);
428 		if (err && err != -EAGAIN) {
429 			iser_data_buf_dump(mem, ibdev);
430 			iser_err("mem->dma_nents = %d (dlength = 0x%x)\n",
431 				 mem->dma_nents,
432 				 ntoh24(iser_task->desc.iscsi_header.dlength));
433 			iser_err("page_vec: data_size = 0x%x, length = %d, offset = 0x%x\n",
434 				 ib_conn->fastreg.fmr.page_vec->data_size,
435 				 ib_conn->fastreg.fmr.page_vec->length,
436 				 ib_conn->fastreg.fmr.page_vec->offset);
437 			for (i = 0; i < ib_conn->fastreg.fmr.page_vec->length; i++)
438 				iser_err("page_vec[%d] = 0x%llx\n", i,
439 					 (unsigned long long) ib_conn->fastreg.fmr.page_vec->pages[i]);
440 		}
441 		if (err)
442 			return err;
443 	}
444 	return 0;
445 }
446 
447 static int iser_fast_reg_mr(struct fast_reg_descriptor *desc,
448 			    struct iser_conn *ib_conn,
449 			    struct iser_regd_buf *regd_buf,
450 			    u32 offset, unsigned int data_size,
451 			    unsigned int page_list_len)
452 {
453 	struct ib_send_wr fastreg_wr, inv_wr;
454 	struct ib_send_wr *bad_wr, *wr = NULL;
455 	u8 key;
456 	int ret;
457 
458 	if (!desc->valid) {
459 		memset(&inv_wr, 0, sizeof(inv_wr));
460 		inv_wr.opcode = IB_WR_LOCAL_INV;
461 		inv_wr.send_flags = IB_SEND_SIGNALED;
462 		inv_wr.ex.invalidate_rkey = desc->data_mr->rkey;
463 		wr = &inv_wr;
464 		/* Bump the key */
465 		key = (u8)(desc->data_mr->rkey & 0x000000FF);
466 		ib_update_fast_reg_key(desc->data_mr, ++key);
467 	}
468 
469 	/* Prepare FASTREG WR */
470 	memset(&fastreg_wr, 0, sizeof(fastreg_wr));
471 	fastreg_wr.opcode = IB_WR_FAST_REG_MR;
472 	fastreg_wr.send_flags = IB_SEND_SIGNALED;
473 	fastreg_wr.wr.fast_reg.iova_start = desc->data_frpl->page_list[0] + offset;
474 	fastreg_wr.wr.fast_reg.page_list = desc->data_frpl;
475 	fastreg_wr.wr.fast_reg.page_list_len = page_list_len;
476 	fastreg_wr.wr.fast_reg.page_shift = SHIFT_4K;
477 	fastreg_wr.wr.fast_reg.length = data_size;
478 	fastreg_wr.wr.fast_reg.rkey = desc->data_mr->rkey;
479 	fastreg_wr.wr.fast_reg.access_flags = (IB_ACCESS_LOCAL_WRITE  |
480 					       IB_ACCESS_REMOTE_WRITE |
481 					       IB_ACCESS_REMOTE_READ);
482 
483 	if (!wr) {
484 		wr = &fastreg_wr;
485 		atomic_inc(&ib_conn->post_send_buf_count);
486 	} else {
487 		wr->next = &fastreg_wr;
488 		atomic_add(2, &ib_conn->post_send_buf_count);
489 	}
490 
491 	ret = ib_post_send(ib_conn->qp, wr, &bad_wr);
492 	if (ret) {
493 		if (bad_wr->next)
494 			atomic_sub(2, &ib_conn->post_send_buf_count);
495 		else
496 			atomic_dec(&ib_conn->post_send_buf_count);
497 		iser_err("fast registration failed, ret:%d\n", ret);
498 		return ret;
499 	}
500 	desc->valid = false;
501 
502 	regd_buf->reg.mem_h = desc;
503 	regd_buf->reg.lkey = desc->data_mr->lkey;
504 	regd_buf->reg.rkey = desc->data_mr->rkey;
505 	regd_buf->reg.va = desc->data_frpl->page_list[0] + offset;
506 	regd_buf->reg.len = data_size;
507 	regd_buf->reg.is_mr = 1;
508 
509 	return ret;
510 }
511 
512 /**
513  * iser_reg_rdma_mem_frwr - Registers memory intended for RDMA,
514  * using Fast Registration WR (if possible) obtaining rkey and va
515  *
516  * returns 0 on success, errno code on failure
517  */
518 int iser_reg_rdma_mem_frwr(struct iscsi_iser_task *iser_task,
519 			   enum iser_data_dir cmd_dir)
520 {
521 	struct iser_conn *ib_conn = iser_task->iser_conn->ib_conn;
522 	struct iser_device *device = ib_conn->device;
523 	struct ib_device *ibdev = device->ib_device;
524 	struct iser_data_buf *mem = &iser_task->data[cmd_dir];
525 	struct iser_regd_buf *regd_buf = &iser_task->rdma_regd[cmd_dir];
526 	struct fast_reg_descriptor *desc;
527 	unsigned int data_size, page_list_len;
528 	int err, aligned_len;
529 	unsigned long flags;
530 	u32 offset;
531 
532 	aligned_len = iser_data_buf_aligned_len(mem, ibdev);
533 	if (aligned_len != mem->dma_nents) {
534 		err = fall_to_bounce_buf(iser_task, ibdev,
535 					 cmd_dir, aligned_len);
536 		if (err) {
537 			iser_err("failed to allocate bounce buffer\n");
538 			return err;
539 		}
540 		mem = &iser_task->data_copy[cmd_dir];
541 	}
542 
543 	/* if there a single dma entry, dma mr suffices */
544 	if (mem->dma_nents == 1) {
545 		struct scatterlist *sg = (struct scatterlist *)mem->buf;
546 
547 		regd_buf->reg.lkey = device->mr->lkey;
548 		regd_buf->reg.rkey = device->mr->rkey;
549 		regd_buf->reg.len  = ib_sg_dma_len(ibdev, &sg[0]);
550 		regd_buf->reg.va   = ib_sg_dma_address(ibdev, &sg[0]);
551 		regd_buf->reg.is_mr = 0;
552 	} else {
553 		spin_lock_irqsave(&ib_conn->lock, flags);
554 		desc = list_first_entry(&ib_conn->fastreg.frwr.pool,
555 					struct fast_reg_descriptor, list);
556 		list_del(&desc->list);
557 		spin_unlock_irqrestore(&ib_conn->lock, flags);
558 		page_list_len = iser_sg_to_page_vec(mem, device->ib_device,
559 						    desc->data_frpl->page_list,
560 						    &offset, &data_size);
561 
562 		if (page_list_len * SIZE_4K < data_size) {
563 			iser_err("fast reg page_list too short to hold this SG\n");
564 			err = -EINVAL;
565 			goto err_reg;
566 		}
567 
568 		err = iser_fast_reg_mr(desc, ib_conn, regd_buf,
569 				       offset, data_size, page_list_len);
570 		if (err)
571 			goto err_reg;
572 	}
573 
574 	return 0;
575 err_reg:
576 	spin_lock_irqsave(&ib_conn->lock, flags);
577 	list_add_tail(&desc->list, &ib_conn->fastreg.frwr.pool);
578 	spin_unlock_irqrestore(&ib_conn->lock, flags);
579 	return err;
580 }
581