51 static int alloc_mr_key(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)  in alloc_mr_key()  argument
66 	mr->key = hw_index_to_key(id); /* MR key */  in alloc_mr_key()
81 static void free_mr_key(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)  in free_mr_key()  argument
83 	unsigned long obj = key_to_hw_index(mr->key);  in free_mr_key()
89 static int alloc_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr,  in alloc_mr_pbl()  argument
93 	bool is_fast = mr->type == MR_TYPE_FRMR;  in alloc_mr_pbl()
97 	mr->pbl_hop_num = is_fast ? 1 : hr_dev->caps.pbl_hop_num;  in alloc_mr_pbl()
100 	buf_attr.region[0].size = mr->size;  in alloc_mr_pbl()
101 	buf_attr.region[0].hopnum = mr->pbl_hop_num;  in alloc_mr_pbl()
103 	buf_attr.user_access = mr->access;  in alloc_mr_pbl()
107 	err = hns_roce_mtr_create(hr_dev, &mr->pbl_mtr, &buf_attr,  in alloc_mr_pbl()
113 		mr->npages = mr->pbl_mtr.hem_cfg.buf_pg_count;  in alloc_mr_pbl()
118 static void free_mr_pbl(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)  in free_mr_pbl()  argument
120 	hns_roce_mtr_destroy(hr_dev, &mr->pbl_mtr);  in free_mr_pbl()
123 static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr)  in hns_roce_mr_free()  argument
128 	if (mr->enabled) {  in hns_roce_mr_free()
130 					      key_to_hw_index(mr->key) &  in hns_roce_mr_free()
137 	free_mr_pbl(hr_dev, mr);  in hns_roce_mr_free()
138 	free_mr_key(hr_dev, mr);  in hns_roce_mr_free()
142 			      struct hns_roce_mr *mr)  in hns_roce_mr_enable()  argument
144 	unsigned long mtpt_idx = key_to_hw_index(mr->key);  in hns_roce_mr_enable()
154 	if (mr->type != MR_TYPE_FRMR)  in hns_roce_mr_enable()
155 		ret = hr_dev->hw->write_mtpt(hr_dev, mailbox->buf, mr);  in hns_roce_mr_enable()
157 		ret = hr_dev->hw->frmr_write_mtpt(mailbox->buf, mr);  in hns_roce_mr_enable()
170 	mr->enabled = 1;  in hns_roce_mr_enable()
190 	struct hns_roce_mr *mr;  in hns_roce_get_dma_mr()  local
193 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);  in hns_roce_get_dma_mr()
194 	if (!mr)  in hns_roce_get_dma_mr()
197 	mr->type = MR_TYPE_DMA;  in hns_roce_get_dma_mr()
198 	mr->pd = to_hr_pd(pd)->pdn;  in hns_roce_get_dma_mr()
199 	mr->access = acc;  in hns_roce_get_dma_mr()
202 	hns_roce_hem_list_init(&mr->pbl_mtr.hem_list);  in hns_roce_get_dma_mr()
203 	ret = alloc_mr_key(hr_dev, mr);  in hns_roce_get_dma_mr()
207 	ret = hns_roce_mr_enable(hr_dev, mr);  in hns_roce_get_dma_mr()
211 	mr->ibmr.rkey = mr->ibmr.lkey = mr->key;  in hns_roce_get_dma_mr()
213 	return &mr->ibmr;  in hns_roce_get_dma_mr()
215 	free_mr_key(hr_dev, mr);  in hns_roce_get_dma_mr()
218 	kfree(mr);  in hns_roce_get_dma_mr()
227 	struct hns_roce_mr *mr;  in hns_roce_reg_user_mr()  local
230 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);  in hns_roce_reg_user_mr()
231 	if (!mr)  in hns_roce_reg_user_mr()
234 	mr->iova = virt_addr;  in hns_roce_reg_user_mr()
235 	mr->size = length;  in hns_roce_reg_user_mr()
236 	mr->pd = to_hr_pd(pd)->pdn;  in hns_roce_reg_user_mr()
237 	mr->access = access_flags;  in hns_roce_reg_user_mr()
238 	mr->type = MR_TYPE_MR;  in hns_roce_reg_user_mr()
240 	ret = alloc_mr_key(hr_dev, mr);  in hns_roce_reg_user_mr()
244 	ret = alloc_mr_pbl(hr_dev, mr, udata, start);  in hns_roce_reg_user_mr()
248 	ret = hns_roce_mr_enable(hr_dev, mr);  in hns_roce_reg_user_mr()
252 	mr->ibmr.rkey = mr->ibmr.lkey = mr->key;  in hns_roce_reg_user_mr()
254 	return &mr->ibmr;  in hns_roce_reg_user_mr()
257 	free_mr_pbl(hr_dev, mr);  in hns_roce_reg_user_mr()
259 	free_mr_key(hr_dev, mr);  in hns_roce_reg_user_mr()
261 	kfree(mr);  in hns_roce_reg_user_mr()
272 	struct hns_roce_mr *mr = to_hr_mr(ibmr);  in hns_roce_rereg_user_mr()  local
277 	if (!mr->enabled)  in hns_roce_rereg_user_mr()
284 	mtpt_idx = key_to_hw_index(mr->key) & (hr_dev->caps.num_mtpts - 1);  in hns_roce_rereg_user_mr()
296 	mr->enabled = 0;  in hns_roce_rereg_user_mr()
297 	mr->iova = virt_addr;  in hns_roce_rereg_user_mr()
298 	mr->size = length;  in hns_roce_rereg_user_mr()
301 		mr->pd = to_hr_pd(pd)->pdn;  in hns_roce_rereg_user_mr()
304 		mr->access = mr_access_flags;  in hns_roce_rereg_user_mr()
307 		free_mr_pbl(hr_dev, mr);  in hns_roce_rereg_user_mr()
308 		ret = alloc_mr_pbl(hr_dev, mr, udata, start);  in hns_roce_rereg_user_mr()
316 	ret = hr_dev->hw->rereg_write_mtpt(hr_dev, mr, flags, mailbox->buf);  in hns_roce_rereg_user_mr()
329 	mr->enabled = 1;  in hns_roce_rereg_user_mr()
342 	struct hns_roce_mr *mr = to_hr_mr(ibmr);  in hns_roce_dereg_mr()  local
347 	hns_roce_mr_free(hr_dev, mr);  in hns_roce_dereg_mr()
348 	kfree(mr);  in hns_roce_dereg_mr()
358 	struct hns_roce_mr *mr;  in hns_roce_alloc_mr()  local
370 	mr = kzalloc(sizeof(*mr), GFP_KERNEL);  in hns_roce_alloc_mr()
371 	if (!mr)  in hns_roce_alloc_mr()
374 	mr->type = MR_TYPE_FRMR;  in hns_roce_alloc_mr()
375 	mr->pd = to_hr_pd(pd)->pdn;  in hns_roce_alloc_mr()
376 	mr->size = max_num_sg * (1 << PAGE_SHIFT);  in hns_roce_alloc_mr()
379 	ret = alloc_mr_key(hr_dev, mr);  in hns_roce_alloc_mr()
383 	ret = alloc_mr_pbl(hr_dev, mr, NULL, 0);  in hns_roce_alloc_mr()
387 	ret = hns_roce_mr_enable(hr_dev, mr);  in hns_roce_alloc_mr()
391 	mr->ibmr.rkey = mr->ibmr.lkey = mr->key;  in hns_roce_alloc_mr()
392 	mr->ibmr.length = mr->size;  in hns_roce_alloc_mr()
394 	return &mr->ibmr;  in hns_roce_alloc_mr()
397 	free_mr_pbl(hr_dev, mr);  in hns_roce_alloc_mr()
399 	free_mr_key(hr_dev, mr);  in hns_roce_alloc_mr()
401 	kfree(mr);  in hns_roce_alloc_mr()
407 	struct hns_roce_mr *mr = to_hr_mr(ibmr);  in hns_roce_set_page()  local
409 	if (likely(mr->npages < mr->pbl_mtr.hem_cfg.buf_pg_count)) {  in hns_roce_set_page()
410 		mr->page_list[mr->npages++] = addr;  in hns_roce_set_page()
423 	struct hns_roce_mr *mr = to_hr_mr(ibmr);  in hns_roce_map_mr_sg()  local
424 	struct hns_roce_mtr *mtr = &mr->pbl_mtr;  in hns_roce_map_mr_sg()
432 	mr->npages = 0;  in hns_roce_map_mr_sg()
433 	mr->page_list = kvcalloc(mr->pbl_mtr.hem_cfg.buf_pg_count,  in hns_roce_map_mr_sg()
435 	if (!mr->page_list)  in hns_roce_map_mr_sg()
441 			  mr->npages, mr->pbl_mtr.hem_cfg.buf_pg_count, sg_num);  in hns_roce_map_mr_sg()
446 	mtr->hem_cfg.region[0].count = mr->npages;  in hns_roce_map_mr_sg()
447 	mtr->hem_cfg.region[0].hopnum = mr->pbl_hop_num;  in hns_roce_map_mr_sg()
449 	ret = hns_roce_mtr_map(hr_dev, mtr, mr->page_list, mr->npages);  in hns_roce_map_mr_sg()
454 		mr->pbl_mtr.hem_cfg.buf_pg_shift = (u32)ilog2(ibmr->page_size);  in hns_roce_map_mr_sg()
458 	kvfree(mr->page_list);  in hns_roce_map_mr_sg()
459 	mr->page_list = NULL;  in hns_roce_map_mr_sg()