xref: /openbmc/linux/drivers/infiniband/hw/cxgb4/mem.c (revision 92a76f6d)
1 /*
2  * Copyright (c) 2009-2010 Chelsio, Inc. All rights reserved.
3  *
4  * This software is available to you under a choice of one of two
5  * licenses.  You may choose to be licensed under the terms of the GNU
6  * General Public License (GPL) Version 2, available from the file
7  * COPYING in the main directory of this source tree, or the
8  * OpenIB.org BSD license below:
9  *
10  *     Redistribution and use in source and binary forms, with or
11  *     without modification, are permitted provided that the following
12  *     conditions are met:
13  *
14  *      - Redistributions of source code must retain the above
15  *        copyright notice, this list of conditions and the following
16  *        disclaimer.
17  *
18  *      - Redistributions in binary form must reproduce the above
19  *        copyright notice, this list of conditions and the following
20  *        disclaimer in the documentation and/or other materials
21  *        provided with the distribution.
22  *
23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30  * SOFTWARE.
31  */
32 
33 #include <linux/module.h>
34 #include <linux/moduleparam.h>
35 #include <rdma/ib_umem.h>
36 #include <linux/atomic.h>
37 #include <rdma/ib_user_verbs.h>
38 
39 #include "iw_cxgb4.h"
40 
41 int use_dsgl = 0;
42 module_param(use_dsgl, int, 0644);
43 MODULE_PARM_DESC(use_dsgl, "Use DSGL for PBL/FastReg (default=0)");
44 
45 #define T4_ULPTX_MIN_IO 32
46 #define C4IW_MAX_INLINE_SIZE 96
47 #define T4_ULPTX_MAX_DMA 1024
48 #define C4IW_INLINE_THRESHOLD 128
49 
50 static int inline_threshold = C4IW_INLINE_THRESHOLD;
51 module_param(inline_threshold, int, 0644);
52 MODULE_PARM_DESC(inline_threshold, "inline vs dsgl threshold (default=128)");
53 
54 static int mr_exceeds_hw_limits(struct c4iw_dev *dev, u64 length)
55 {
56 	return (is_t4(dev->rdev.lldi.adapter_type) ||
57 		is_t5(dev->rdev.lldi.adapter_type)) &&
58 		length >= 8*1024*1024*1024ULL;
59 }
60 
61 static int _c4iw_write_mem_dma_aligned(struct c4iw_rdev *rdev, u32 addr,
62 				       u32 len, dma_addr_t data, int wait)
63 {
64 	struct sk_buff *skb;
65 	struct ulp_mem_io *req;
66 	struct ulptx_sgl *sgl;
67 	u8 wr_len;
68 	int ret = 0;
69 	struct c4iw_wr_wait wr_wait;
70 
71 	addr &= 0x7FFFFFF;
72 
73 	if (wait)
74 		c4iw_init_wr_wait(&wr_wait);
75 	wr_len = roundup(sizeof(*req) + sizeof(*sgl), 16);
76 
77 	skb = alloc_skb(wr_len, GFP_KERNEL);
78 	if (!skb)
79 		return -ENOMEM;
80 	set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0);
81 
82 	req = (struct ulp_mem_io *)__skb_put(skb, wr_len);
83 	memset(req, 0, wr_len);
84 	INIT_ULPTX_WR(req, wr_len, 0, 0);
85 	req->wr.wr_hi = cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR) |
86 			(wait ? FW_WR_COMPL_F : 0));
87 	req->wr.wr_lo = wait ? (__force __be64)(unsigned long) &wr_wait : 0L;
88 	req->wr.wr_mid = cpu_to_be32(FW_WR_LEN16_V(DIV_ROUND_UP(wr_len, 16)));
89 	req->cmd = cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE));
90 	req->cmd |= cpu_to_be32(T5_ULP_MEMIO_ORDER_V(1));
91 	req->dlen = cpu_to_be32(ULP_MEMIO_DATA_LEN_V(len>>5));
92 	req->len16 = cpu_to_be32(DIV_ROUND_UP(wr_len-sizeof(req->wr), 16));
93 	req->lock_addr = cpu_to_be32(ULP_MEMIO_ADDR_V(addr));
94 
95 	sgl = (struct ulptx_sgl *)(req + 1);
96 	sgl->cmd_nsge = cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_DSGL) |
97 				    ULPTX_NSGE_V(1));
98 	sgl->len0 = cpu_to_be32(len);
99 	sgl->addr0 = cpu_to_be64(data);
100 
101 	ret = c4iw_ofld_send(rdev, skb);
102 	if (ret)
103 		return ret;
104 	if (wait)
105 		ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__);
106 	return ret;
107 }
108 
109 static int _c4iw_write_mem_inline(struct c4iw_rdev *rdev, u32 addr, u32 len,
110 				  void *data)
111 {
112 	struct sk_buff *skb;
113 	struct ulp_mem_io *req;
114 	struct ulptx_idata *sc;
115 	u8 wr_len, *to_dp, *from_dp;
116 	int copy_len, num_wqe, i, ret = 0;
117 	struct c4iw_wr_wait wr_wait;
118 	__be32 cmd = cpu_to_be32(ULPTX_CMD_V(ULP_TX_MEM_WRITE));
119 
120 	if (is_t4(rdev->lldi.adapter_type))
121 		cmd |= cpu_to_be32(ULP_MEMIO_ORDER_F);
122 	else
123 		cmd |= cpu_to_be32(T5_ULP_MEMIO_IMM_F);
124 
125 	addr &= 0x7FFFFFF;
126 	PDBG("%s addr 0x%x len %u\n", __func__, addr, len);
127 	num_wqe = DIV_ROUND_UP(len, C4IW_MAX_INLINE_SIZE);
128 	c4iw_init_wr_wait(&wr_wait);
129 	for (i = 0; i < num_wqe; i++) {
130 
131 		copy_len = len > C4IW_MAX_INLINE_SIZE ? C4IW_MAX_INLINE_SIZE :
132 			   len;
133 		wr_len = roundup(sizeof *req + sizeof *sc +
134 				 roundup(copy_len, T4_ULPTX_MIN_IO), 16);
135 
136 		skb = alloc_skb(wr_len, GFP_KERNEL);
137 		if (!skb)
138 			return -ENOMEM;
139 		set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0);
140 
141 		req = (struct ulp_mem_io *)__skb_put(skb, wr_len);
142 		memset(req, 0, wr_len);
143 		INIT_ULPTX_WR(req, wr_len, 0, 0);
144 
145 		if (i == (num_wqe-1)) {
146 			req->wr.wr_hi = cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR) |
147 						    FW_WR_COMPL_F);
148 			req->wr.wr_lo = (__force __be64)(unsigned long)&wr_wait;
149 		} else
150 			req->wr.wr_hi = cpu_to_be32(FW_WR_OP_V(FW_ULPTX_WR));
151 		req->wr.wr_mid = cpu_to_be32(
152 				       FW_WR_LEN16_V(DIV_ROUND_UP(wr_len, 16)));
153 
154 		req->cmd = cmd;
155 		req->dlen = cpu_to_be32(ULP_MEMIO_DATA_LEN_V(
156 				DIV_ROUND_UP(copy_len, T4_ULPTX_MIN_IO)));
157 		req->len16 = cpu_to_be32(DIV_ROUND_UP(wr_len-sizeof(req->wr),
158 						      16));
159 		req->lock_addr = cpu_to_be32(ULP_MEMIO_ADDR_V(addr + i * 3));
160 
161 		sc = (struct ulptx_idata *)(req + 1);
162 		sc->cmd_more = cpu_to_be32(ULPTX_CMD_V(ULP_TX_SC_IMM));
163 		sc->len = cpu_to_be32(roundup(copy_len, T4_ULPTX_MIN_IO));
164 
165 		to_dp = (u8 *)(sc + 1);
166 		from_dp = (u8 *)data + i * C4IW_MAX_INLINE_SIZE;
167 		if (data)
168 			memcpy(to_dp, from_dp, copy_len);
169 		else
170 			memset(to_dp, 0, copy_len);
171 		if (copy_len % T4_ULPTX_MIN_IO)
172 			memset(to_dp + copy_len, 0, T4_ULPTX_MIN_IO -
173 			       (copy_len % T4_ULPTX_MIN_IO));
174 		ret = c4iw_ofld_send(rdev, skb);
175 		if (ret)
176 			return ret;
177 		len -= C4IW_MAX_INLINE_SIZE;
178 	}
179 
180 	ret = c4iw_wait_for_reply(rdev, &wr_wait, 0, 0, __func__);
181 	return ret;
182 }
183 
184 static int _c4iw_write_mem_dma(struct c4iw_rdev *rdev, u32 addr, u32 len, void *data)
185 {
186 	u32 remain = len;
187 	u32 dmalen;
188 	int ret = 0;
189 	dma_addr_t daddr;
190 	dma_addr_t save;
191 
192 	daddr = dma_map_single(&rdev->lldi.pdev->dev, data, len, DMA_TO_DEVICE);
193 	if (dma_mapping_error(&rdev->lldi.pdev->dev, daddr))
194 		return -1;
195 	save = daddr;
196 
197 	while (remain > inline_threshold) {
198 		if (remain < T4_ULPTX_MAX_DMA) {
199 			if (remain & ~T4_ULPTX_MIN_IO)
200 				dmalen = remain & ~(T4_ULPTX_MIN_IO-1);
201 			else
202 				dmalen = remain;
203 		} else
204 			dmalen = T4_ULPTX_MAX_DMA;
205 		remain -= dmalen;
206 		ret = _c4iw_write_mem_dma_aligned(rdev, addr, dmalen, daddr,
207 						 !remain);
208 		if (ret)
209 			goto out;
210 		addr += dmalen >> 5;
211 		data += dmalen;
212 		daddr += dmalen;
213 	}
214 	if (remain)
215 		ret = _c4iw_write_mem_inline(rdev, addr, remain, data);
216 out:
217 	dma_unmap_single(&rdev->lldi.pdev->dev, save, len, DMA_TO_DEVICE);
218 	return ret;
219 }
220 
221 /*
222  * write len bytes of data into addr (32B aligned address)
223  * If data is NULL, clear len byte of memory to zero.
224  */
225 static int write_adapter_mem(struct c4iw_rdev *rdev, u32 addr, u32 len,
226 			     void *data)
227 {
228 	if (is_t5(rdev->lldi.adapter_type) && use_dsgl) {
229 		if (len > inline_threshold) {
230 			if (_c4iw_write_mem_dma(rdev, addr, len, data)) {
231 				printk_ratelimited(KERN_WARNING
232 						   "%s: dma map"
233 						   " failure (non fatal)\n",
234 						   pci_name(rdev->lldi.pdev));
235 				return _c4iw_write_mem_inline(rdev, addr, len,
236 							      data);
237 			} else
238 				return 0;
239 		} else
240 			return _c4iw_write_mem_inline(rdev, addr, len, data);
241 	} else
242 		return _c4iw_write_mem_inline(rdev, addr, len, data);
243 }
244 
245 /*
246  * Build and write a TPT entry.
247  * IN: stag key, pdid, perm, bind_enabled, zbva, to, len, page_size,
248  *     pbl_size and pbl_addr
249  * OUT: stag index
250  */
251 static int write_tpt_entry(struct c4iw_rdev *rdev, u32 reset_tpt_entry,
252 			   u32 *stag, u8 stag_state, u32 pdid,
253 			   enum fw_ri_stag_type type, enum fw_ri_mem_perms perm,
254 			   int bind_enabled, u32 zbva, u64 to,
255 			   u64 len, u8 page_size, u32 pbl_size, u32 pbl_addr)
256 {
257 	int err;
258 	struct fw_ri_tpte tpt;
259 	u32 stag_idx;
260 	static atomic_t key;
261 
262 	if (c4iw_fatal_error(rdev))
263 		return -EIO;
264 
265 	stag_state = stag_state > 0;
266 	stag_idx = (*stag) >> 8;
267 
268 	if ((!reset_tpt_entry) && (*stag == T4_STAG_UNSET)) {
269 		stag_idx = c4iw_get_resource(&rdev->resource.tpt_table);
270 		if (!stag_idx) {
271 			mutex_lock(&rdev->stats.lock);
272 			rdev->stats.stag.fail++;
273 			mutex_unlock(&rdev->stats.lock);
274 			return -ENOMEM;
275 		}
276 		mutex_lock(&rdev->stats.lock);
277 		rdev->stats.stag.cur += 32;
278 		if (rdev->stats.stag.cur > rdev->stats.stag.max)
279 			rdev->stats.stag.max = rdev->stats.stag.cur;
280 		mutex_unlock(&rdev->stats.lock);
281 		*stag = (stag_idx << 8) | (atomic_inc_return(&key) & 0xff);
282 	}
283 	PDBG("%s stag_state 0x%0x type 0x%0x pdid 0x%0x, stag_idx 0x%x\n",
284 	     __func__, stag_state, type, pdid, stag_idx);
285 
286 	/* write TPT entry */
287 	if (reset_tpt_entry)
288 		memset(&tpt, 0, sizeof(tpt));
289 	else {
290 		tpt.valid_to_pdid = cpu_to_be32(FW_RI_TPTE_VALID_F |
291 			FW_RI_TPTE_STAGKEY_V((*stag & FW_RI_TPTE_STAGKEY_M)) |
292 			FW_RI_TPTE_STAGSTATE_V(stag_state) |
293 			FW_RI_TPTE_STAGTYPE_V(type) | FW_RI_TPTE_PDID_V(pdid));
294 		tpt.locread_to_qpid = cpu_to_be32(FW_RI_TPTE_PERM_V(perm) |
295 			(bind_enabled ? FW_RI_TPTE_MWBINDEN_F : 0) |
296 			FW_RI_TPTE_ADDRTYPE_V((zbva ? FW_RI_ZERO_BASED_TO :
297 						      FW_RI_VA_BASED_TO))|
298 			FW_RI_TPTE_PS_V(page_size));
299 		tpt.nosnoop_pbladdr = !pbl_size ? 0 : cpu_to_be32(
300 			FW_RI_TPTE_PBLADDR_V(PBL_OFF(rdev, pbl_addr)>>3));
301 		tpt.len_lo = cpu_to_be32((u32)(len & 0xffffffffUL));
302 		tpt.va_hi = cpu_to_be32((u32)(to >> 32));
303 		tpt.va_lo_fbo = cpu_to_be32((u32)(to & 0xffffffffUL));
304 		tpt.dca_mwbcnt_pstag = cpu_to_be32(0);
305 		tpt.len_hi = cpu_to_be32((u32)(len >> 32));
306 	}
307 	err = write_adapter_mem(rdev, stag_idx +
308 				(rdev->lldi.vr->stag.start >> 5),
309 				sizeof(tpt), &tpt);
310 
311 	if (reset_tpt_entry) {
312 		c4iw_put_resource(&rdev->resource.tpt_table, stag_idx);
313 		mutex_lock(&rdev->stats.lock);
314 		rdev->stats.stag.cur -= 32;
315 		mutex_unlock(&rdev->stats.lock);
316 	}
317 	return err;
318 }
319 
320 static int write_pbl(struct c4iw_rdev *rdev, __be64 *pbl,
321 		     u32 pbl_addr, u32 pbl_size)
322 {
323 	int err;
324 
325 	PDBG("%s *pdb_addr 0x%x, pbl_base 0x%x, pbl_size %d\n",
326 	     __func__, pbl_addr, rdev->lldi.vr->pbl.start,
327 	     pbl_size);
328 
329 	err = write_adapter_mem(rdev, pbl_addr >> 5, pbl_size << 3, pbl);
330 	return err;
331 }
332 
333 static int dereg_mem(struct c4iw_rdev *rdev, u32 stag, u32 pbl_size,
334 		     u32 pbl_addr)
335 {
336 	return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0,
337 			       pbl_size, pbl_addr);
338 }
339 
340 static int allocate_window(struct c4iw_rdev *rdev, u32 * stag, u32 pdid)
341 {
342 	*stag = T4_STAG_UNSET;
343 	return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_MW, 0, 0, 0,
344 			       0UL, 0, 0, 0, 0);
345 }
346 
347 static int deallocate_window(struct c4iw_rdev *rdev, u32 stag)
348 {
349 	return write_tpt_entry(rdev, 1, &stag, 0, 0, 0, 0, 0, 0, 0UL, 0, 0, 0,
350 			       0);
351 }
352 
353 static int allocate_stag(struct c4iw_rdev *rdev, u32 *stag, u32 pdid,
354 			 u32 pbl_size, u32 pbl_addr)
355 {
356 	*stag = T4_STAG_UNSET;
357 	return write_tpt_entry(rdev, 0, stag, 0, pdid, FW_RI_STAG_NSMR, 0, 0, 0,
358 			       0UL, 0, 0, pbl_size, pbl_addr);
359 }
360 
361 static int finish_mem_reg(struct c4iw_mr *mhp, u32 stag)
362 {
363 	u32 mmid;
364 
365 	mhp->attr.state = 1;
366 	mhp->attr.stag = stag;
367 	mmid = stag >> 8;
368 	mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
369 	PDBG("%s mmid 0x%x mhp %p\n", __func__, mmid, mhp);
370 	return insert_handle(mhp->rhp, &mhp->rhp->mmidr, mhp, mmid);
371 }
372 
373 static int register_mem(struct c4iw_dev *rhp, struct c4iw_pd *php,
374 		      struct c4iw_mr *mhp, int shift)
375 {
376 	u32 stag = T4_STAG_UNSET;
377 	int ret;
378 
379 	ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, mhp->attr.pdid,
380 			      FW_RI_STAG_NSMR, mhp->attr.len ?
381 			      mhp->attr.perms : 0,
382 			      mhp->attr.mw_bind_enable, mhp->attr.zbva,
383 			      mhp->attr.va_fbo, mhp->attr.len ?
384 			      mhp->attr.len : -1, shift - 12,
385 			      mhp->attr.pbl_size, mhp->attr.pbl_addr);
386 	if (ret)
387 		return ret;
388 
389 	ret = finish_mem_reg(mhp, stag);
390 	if (ret)
391 		dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
392 		       mhp->attr.pbl_addr);
393 	return ret;
394 }
395 
396 static int alloc_pbl(struct c4iw_mr *mhp, int npages)
397 {
398 	mhp->attr.pbl_addr = c4iw_pblpool_alloc(&mhp->rhp->rdev,
399 						    npages << 3);
400 
401 	if (!mhp->attr.pbl_addr)
402 		return -ENOMEM;
403 
404 	mhp->attr.pbl_size = npages;
405 
406 	return 0;
407 }
408 
409 struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc)
410 {
411 	struct c4iw_dev *rhp;
412 	struct c4iw_pd *php;
413 	struct c4iw_mr *mhp;
414 	int ret;
415 	u32 stag = T4_STAG_UNSET;
416 
417 	PDBG("%s ib_pd %p\n", __func__, pd);
418 	php = to_c4iw_pd(pd);
419 	rhp = php->rhp;
420 
421 	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
422 	if (!mhp)
423 		return ERR_PTR(-ENOMEM);
424 
425 	mhp->rhp = rhp;
426 	mhp->attr.pdid = php->pdid;
427 	mhp->attr.perms = c4iw_ib_to_tpt_access(acc);
428 	mhp->attr.mw_bind_enable = (acc&IB_ACCESS_MW_BIND) == IB_ACCESS_MW_BIND;
429 	mhp->attr.zbva = 0;
430 	mhp->attr.va_fbo = 0;
431 	mhp->attr.page_size = 0;
432 	mhp->attr.len = ~0ULL;
433 	mhp->attr.pbl_size = 0;
434 
435 	ret = write_tpt_entry(&rhp->rdev, 0, &stag, 1, php->pdid,
436 			      FW_RI_STAG_NSMR, mhp->attr.perms,
437 			      mhp->attr.mw_bind_enable, 0, 0, ~0ULL, 0, 0, 0);
438 	if (ret)
439 		goto err1;
440 
441 	ret = finish_mem_reg(mhp, stag);
442 	if (ret)
443 		goto err2;
444 	return &mhp->ibmr;
445 err2:
446 	dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
447 		  mhp->attr.pbl_addr);
448 err1:
449 	kfree(mhp);
450 	return ERR_PTR(ret);
451 }
452 
453 struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
454 			       u64 virt, int acc, struct ib_udata *udata)
455 {
456 	__be64 *pages;
457 	int shift, n, len;
458 	int i, k, entry;
459 	int err = 0;
460 	struct scatterlist *sg;
461 	struct c4iw_dev *rhp;
462 	struct c4iw_pd *php;
463 	struct c4iw_mr *mhp;
464 
465 	PDBG("%s ib_pd %p\n", __func__, pd);
466 
467 	if (length == ~0ULL)
468 		return ERR_PTR(-EINVAL);
469 
470 	if ((length + start) < start)
471 		return ERR_PTR(-EINVAL);
472 
473 	php = to_c4iw_pd(pd);
474 	rhp = php->rhp;
475 
476 	if (mr_exceeds_hw_limits(rhp, length))
477 		return ERR_PTR(-EINVAL);
478 
479 	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
480 	if (!mhp)
481 		return ERR_PTR(-ENOMEM);
482 
483 	mhp->rhp = rhp;
484 
485 	mhp->umem = ib_umem_get(pd->uobject->context, start, length, acc, 0);
486 	if (IS_ERR(mhp->umem)) {
487 		err = PTR_ERR(mhp->umem);
488 		kfree(mhp);
489 		return ERR_PTR(err);
490 	}
491 
492 	shift = ffs(mhp->umem->page_size) - 1;
493 
494 	n = mhp->umem->nmap;
495 	err = alloc_pbl(mhp, n);
496 	if (err)
497 		goto err;
498 
499 	pages = (__be64 *) __get_free_page(GFP_KERNEL);
500 	if (!pages) {
501 		err = -ENOMEM;
502 		goto err_pbl;
503 	}
504 
505 	i = n = 0;
506 
507 	for_each_sg(mhp->umem->sg_head.sgl, sg, mhp->umem->nmap, entry) {
508 		len = sg_dma_len(sg) >> shift;
509 		for (k = 0; k < len; ++k) {
510 			pages[i++] = cpu_to_be64(sg_dma_address(sg) +
511 				mhp->umem->page_size * k);
512 			if (i == PAGE_SIZE / sizeof *pages) {
513 				err = write_pbl(&mhp->rhp->rdev,
514 				      pages,
515 				      mhp->attr.pbl_addr + (n << 3), i);
516 				if (err)
517 					goto pbl_done;
518 				n += i;
519 				i = 0;
520 			}
521 		}
522 	}
523 
524 	if (i)
525 		err = write_pbl(&mhp->rhp->rdev, pages,
526 				     mhp->attr.pbl_addr + (n << 3), i);
527 
528 pbl_done:
529 	free_page((unsigned long) pages);
530 	if (err)
531 		goto err_pbl;
532 
533 	mhp->attr.pdid = php->pdid;
534 	mhp->attr.zbva = 0;
535 	mhp->attr.perms = c4iw_ib_to_tpt_access(acc);
536 	mhp->attr.va_fbo = virt;
537 	mhp->attr.page_size = shift - 12;
538 	mhp->attr.len = length;
539 
540 	err = register_mem(rhp, php, mhp, shift);
541 	if (err)
542 		goto err_pbl;
543 
544 	return &mhp->ibmr;
545 
546 err_pbl:
547 	c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr,
548 			      mhp->attr.pbl_size << 3);
549 
550 err:
551 	ib_umem_release(mhp->umem);
552 	kfree(mhp);
553 	return ERR_PTR(err);
554 }
555 
556 struct ib_mw *c4iw_alloc_mw(struct ib_pd *pd, enum ib_mw_type type,
557 			    struct ib_udata *udata)
558 {
559 	struct c4iw_dev *rhp;
560 	struct c4iw_pd *php;
561 	struct c4iw_mw *mhp;
562 	u32 mmid;
563 	u32 stag = 0;
564 	int ret;
565 
566 	if (type != IB_MW_TYPE_1)
567 		return ERR_PTR(-EINVAL);
568 
569 	php = to_c4iw_pd(pd);
570 	rhp = php->rhp;
571 	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
572 	if (!mhp)
573 		return ERR_PTR(-ENOMEM);
574 	ret = allocate_window(&rhp->rdev, &stag, php->pdid);
575 	if (ret) {
576 		kfree(mhp);
577 		return ERR_PTR(ret);
578 	}
579 	mhp->rhp = rhp;
580 	mhp->attr.pdid = php->pdid;
581 	mhp->attr.type = FW_RI_STAG_MW;
582 	mhp->attr.stag = stag;
583 	mmid = (stag) >> 8;
584 	mhp->ibmw.rkey = stag;
585 	if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {
586 		deallocate_window(&rhp->rdev, mhp->attr.stag);
587 		kfree(mhp);
588 		return ERR_PTR(-ENOMEM);
589 	}
590 	PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
591 	return &(mhp->ibmw);
592 }
593 
594 int c4iw_dealloc_mw(struct ib_mw *mw)
595 {
596 	struct c4iw_dev *rhp;
597 	struct c4iw_mw *mhp;
598 	u32 mmid;
599 
600 	mhp = to_c4iw_mw(mw);
601 	rhp = mhp->rhp;
602 	mmid = (mw->rkey) >> 8;
603 	remove_handle(rhp, &rhp->mmidr, mmid);
604 	deallocate_window(&rhp->rdev, mhp->attr.stag);
605 	kfree(mhp);
606 	PDBG("%s ib_mw %p mmid 0x%x ptr %p\n", __func__, mw, mmid, mhp);
607 	return 0;
608 }
609 
610 struct ib_mr *c4iw_alloc_mr(struct ib_pd *pd,
611 			    enum ib_mr_type mr_type,
612 			    u32 max_num_sg)
613 {
614 	struct c4iw_dev *rhp;
615 	struct c4iw_pd *php;
616 	struct c4iw_mr *mhp;
617 	u32 mmid;
618 	u32 stag = 0;
619 	int ret = 0;
620 	int length = roundup(max_num_sg * sizeof(u64), 32);
621 
622 	php = to_c4iw_pd(pd);
623 	rhp = php->rhp;
624 
625 	if (mr_type != IB_MR_TYPE_MEM_REG ||
626 	    max_num_sg > t4_max_fr_depth(&rhp->rdev.lldi.ulptx_memwrite_dsgl &&
627 					 use_dsgl))
628 		return ERR_PTR(-EINVAL);
629 
630 	mhp = kzalloc(sizeof(*mhp), GFP_KERNEL);
631 	if (!mhp) {
632 		ret = -ENOMEM;
633 		goto err;
634 	}
635 
636 	mhp->mpl = dma_alloc_coherent(&rhp->rdev.lldi.pdev->dev,
637 				      length, &mhp->mpl_addr, GFP_KERNEL);
638 	if (!mhp->mpl) {
639 		ret = -ENOMEM;
640 		goto err_mpl;
641 	}
642 	mhp->max_mpl_len = length;
643 
644 	mhp->rhp = rhp;
645 	ret = alloc_pbl(mhp, max_num_sg);
646 	if (ret)
647 		goto err1;
648 	mhp->attr.pbl_size = max_num_sg;
649 	ret = allocate_stag(&rhp->rdev, &stag, php->pdid,
650 				 mhp->attr.pbl_size, mhp->attr.pbl_addr);
651 	if (ret)
652 		goto err2;
653 	mhp->attr.pdid = php->pdid;
654 	mhp->attr.type = FW_RI_STAG_NSMR;
655 	mhp->attr.stag = stag;
656 	mhp->attr.state = 1;
657 	mmid = (stag) >> 8;
658 	mhp->ibmr.rkey = mhp->ibmr.lkey = stag;
659 	if (insert_handle(rhp, &rhp->mmidr, mhp, mmid)) {
660 		ret = -ENOMEM;
661 		goto err3;
662 	}
663 
664 	PDBG("%s mmid 0x%x mhp %p stag 0x%x\n", __func__, mmid, mhp, stag);
665 	return &(mhp->ibmr);
666 err3:
667 	dereg_mem(&rhp->rdev, stag, mhp->attr.pbl_size,
668 		       mhp->attr.pbl_addr);
669 err2:
670 	c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr,
671 			      mhp->attr.pbl_size << 3);
672 err1:
673 	dma_free_coherent(&mhp->rhp->rdev.lldi.pdev->dev,
674 			  mhp->max_mpl_len, mhp->mpl, mhp->mpl_addr);
675 err_mpl:
676 	kfree(mhp);
677 err:
678 	return ERR_PTR(ret);
679 }
680 
681 static int c4iw_set_page(struct ib_mr *ibmr, u64 addr)
682 {
683 	struct c4iw_mr *mhp = to_c4iw_mr(ibmr);
684 
685 	if (unlikely(mhp->mpl_len == mhp->max_mpl_len))
686 		return -ENOMEM;
687 
688 	mhp->mpl[mhp->mpl_len++] = addr;
689 
690 	return 0;
691 }
692 
693 int c4iw_map_mr_sg(struct ib_mr *ibmr,
694 		   struct scatterlist *sg,
695 		   int sg_nents)
696 {
697 	struct c4iw_mr *mhp = to_c4iw_mr(ibmr);
698 
699 	mhp->mpl_len = 0;
700 
701 	return ib_sg_to_pages(ibmr, sg, sg_nents, c4iw_set_page);
702 }
703 
704 int c4iw_dereg_mr(struct ib_mr *ib_mr)
705 {
706 	struct c4iw_dev *rhp;
707 	struct c4iw_mr *mhp;
708 	u32 mmid;
709 
710 	PDBG("%s ib_mr %p\n", __func__, ib_mr);
711 
712 	mhp = to_c4iw_mr(ib_mr);
713 	rhp = mhp->rhp;
714 	mmid = mhp->attr.stag >> 8;
715 	remove_handle(rhp, &rhp->mmidr, mmid);
716 	if (mhp->mpl)
717 		dma_free_coherent(&mhp->rhp->rdev.lldi.pdev->dev,
718 				  mhp->max_mpl_len, mhp->mpl, mhp->mpl_addr);
719 	dereg_mem(&rhp->rdev, mhp->attr.stag, mhp->attr.pbl_size,
720 		       mhp->attr.pbl_addr);
721 	if (mhp->attr.pbl_size)
722 		c4iw_pblpool_free(&mhp->rhp->rdev, mhp->attr.pbl_addr,
723 				  mhp->attr.pbl_size << 3);
724 	if (mhp->kva)
725 		kfree((void *) (unsigned long) mhp->kva);
726 	if (mhp->umem)
727 		ib_umem_release(mhp->umem);
728 	PDBG("%s mmid 0x%x ptr %p\n", __func__, mmid, mhp);
729 	kfree(mhp);
730 	return 0;
731 }
732