xref: /openbmc/linux/drivers/infiniband/hw/irdma/hmc.c (revision 2c4b14ea)
1 // SPDX-License-Identifier: GPL-2.0 or Linux-OpenIB
2 /* Copyright (c) 2015 - 2021 Intel Corporation */
3 #include "osdep.h"
4 #include "hmc.h"
5 #include "defs.h"
6 #include "type.h"
7 #include "protos.h"
8 
9 /**
10  * irdma_find_sd_index_limit - finds segment descriptor index limit
11  * @hmc_info: pointer to the HMC configuration information structure
12  * @type: type of HMC resources we're searching
13  * @idx: starting index for the object
14  * @cnt: number of objects we're trying to create
15  * @sd_idx: pointer to return index of the segment descriptor in question
16  * @sd_limit: pointer to return the maximum number of segment descriptors
17  *
18  * This function calculates the segment descriptor index and index limit
19  * for the resource defined by irdma_hmc_rsrc_type.
20  */
21 
irdma_find_sd_index_limit(struct irdma_hmc_info * hmc_info,u32 type,u32 idx,u32 cnt,u32 * sd_idx,u32 * sd_limit)22 static void irdma_find_sd_index_limit(struct irdma_hmc_info *hmc_info, u32 type,
23 				      u32 idx, u32 cnt, u32 *sd_idx,
24 				      u32 *sd_limit)
25 {
26 	u64 fpm_addr, fpm_limit;
27 
28 	fpm_addr = hmc_info->hmc_obj[(type)].base +
29 		   hmc_info->hmc_obj[type].size * idx;
30 	fpm_limit = fpm_addr + hmc_info->hmc_obj[type].size * cnt;
31 	*sd_idx = (u32)(fpm_addr / IRDMA_HMC_DIRECT_BP_SIZE);
32 	*sd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_DIRECT_BP_SIZE);
33 	*sd_limit += 1;
34 }
35 
36 /**
37  * irdma_find_pd_index_limit - finds page descriptor index limit
38  * @hmc_info: pointer to the HMC configuration information struct
39  * @type: HMC resource type we're examining
40  * @idx: starting index for the object
41  * @cnt: number of objects we're trying to create
42  * @pd_idx: pointer to return page descriptor index
43  * @pd_limit: pointer to return page descriptor index limit
44  *
45  * Calculates the page descriptor index and index limit for the resource
46  * defined by irdma_hmc_rsrc_type.
47  */
48 
irdma_find_pd_index_limit(struct irdma_hmc_info * hmc_info,u32 type,u32 idx,u32 cnt,u32 * pd_idx,u32 * pd_limit)49 static void irdma_find_pd_index_limit(struct irdma_hmc_info *hmc_info, u32 type,
50 				      u32 idx, u32 cnt, u32 *pd_idx,
51 				      u32 *pd_limit)
52 {
53 	u64 fpm_adr, fpm_limit;
54 
55 	fpm_adr = hmc_info->hmc_obj[type].base +
56 		  hmc_info->hmc_obj[type].size * idx;
57 	fpm_limit = fpm_adr + (hmc_info)->hmc_obj[(type)].size * (cnt);
58 	*pd_idx = (u32)(fpm_adr / IRDMA_HMC_PAGED_BP_SIZE);
59 	*pd_limit = (u32)((fpm_limit - 1) / IRDMA_HMC_PAGED_BP_SIZE);
60 	*pd_limit += 1;
61 }
62 
63 /**
64  * irdma_set_sd_entry - setup entry for sd programming
65  * @pa: physical addr
66  * @idx: sd index
67  * @type: paged or direct sd
68  * @entry: sd entry ptr
69  */
irdma_set_sd_entry(u64 pa,u32 idx,enum irdma_sd_entry_type type,struct irdma_update_sd_entry * entry)70 static void irdma_set_sd_entry(u64 pa, u32 idx, enum irdma_sd_entry_type type,
71 			       struct irdma_update_sd_entry *entry)
72 {
73 	entry->data = pa |
74 		      FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) |
75 		      FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE,
76 				 type == IRDMA_SD_TYPE_PAGED ? 0 : 1) |
77 		      FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDVALID, 1);
78 
79 	entry->cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) | BIT(15);
80 }
81 
82 /**
83  * irdma_clr_sd_entry - setup entry for sd clear
84  * @idx: sd index
85  * @type: paged or direct sd
86  * @entry: sd entry ptr
87  */
irdma_clr_sd_entry(u32 idx,enum irdma_sd_entry_type type,struct irdma_update_sd_entry * entry)88 static void irdma_clr_sd_entry(u32 idx, enum irdma_sd_entry_type type,
89 			       struct irdma_update_sd_entry *entry)
90 {
91 	entry->data = FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDBPCOUNT, IRDMA_HMC_MAX_BP_COUNT) |
92 		      FIELD_PREP(IRDMA_PFHMC_SDDATALOW_PMSDTYPE,
93 				 type == IRDMA_SD_TYPE_PAGED ? 0 : 1);
94 
95 	entry->cmd = idx | FIELD_PREP(IRDMA_PFHMC_SDCMD_PMSDWR, 1) | BIT(15);
96 }
97 
98 /**
99  * irdma_invalidate_pf_hmc_pd - Invalidates the pd cache in the hardware for PF
100  * @dev: pointer to our device struct
101  * @sd_idx: segment descriptor index
102  * @pd_idx: page descriptor index
103  */
irdma_invalidate_pf_hmc_pd(struct irdma_sc_dev * dev,u32 sd_idx,u32 pd_idx)104 static inline void irdma_invalidate_pf_hmc_pd(struct irdma_sc_dev *dev, u32 sd_idx,
105 					      u32 pd_idx)
106 {
107 	u32 val = FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDIDX, sd_idx) |
108 		  FIELD_PREP(IRDMA_PFHMC_PDINV_PMSDPARTSEL, 1) |
109 		  FIELD_PREP(IRDMA_PFHMC_PDINV_PMPDIDX, pd_idx);
110 
111 	writel(val, dev->hw_regs[IRDMA_PFHMC_PDINV]);
112 }
113 
114 /**
115  * irdma_hmc_sd_one - setup 1 sd entry for cqp
116  * @dev: pointer to the device structure
117  * @hmc_fn_id: hmc's function id
118  * @pa: physical addr
119  * @sd_idx: sd index
120  * @type: paged or direct sd
121  * @setsd: flag to set or clear sd
122  */
irdma_hmc_sd_one(struct irdma_sc_dev * dev,u8 hmc_fn_id,u64 pa,u32 sd_idx,enum irdma_sd_entry_type type,bool setsd)123 int irdma_hmc_sd_one(struct irdma_sc_dev *dev, u8 hmc_fn_id, u64 pa, u32 sd_idx,
124 		     enum irdma_sd_entry_type type, bool setsd)
125 {
126 	struct irdma_update_sds_info sdinfo;
127 
128 	sdinfo.cnt = 1;
129 	sdinfo.hmc_fn_id = hmc_fn_id;
130 	if (setsd)
131 		irdma_set_sd_entry(pa, sd_idx, type, sdinfo.entry);
132 	else
133 		irdma_clr_sd_entry(sd_idx, type, sdinfo.entry);
134 	return dev->cqp->process_cqp_sds(dev, &sdinfo);
135 }
136 
137 /**
138  * irdma_hmc_sd_grp - setup group of sd entries for cqp
139  * @dev: pointer to the device structure
140  * @hmc_info: pointer to the HMC configuration information struct
141  * @sd_index: sd index
142  * @sd_cnt: number of sd entries
143  * @setsd: flag to set or clear sd
144  */
irdma_hmc_sd_grp(struct irdma_sc_dev * dev,struct irdma_hmc_info * hmc_info,u32 sd_index,u32 sd_cnt,bool setsd)145 static int irdma_hmc_sd_grp(struct irdma_sc_dev *dev,
146 			    struct irdma_hmc_info *hmc_info, u32 sd_index,
147 			    u32 sd_cnt, bool setsd)
148 {
149 	struct irdma_hmc_sd_entry *sd_entry;
150 	struct irdma_update_sds_info sdinfo = {};
151 	u64 pa;
152 	u32 i;
153 	int ret_code = 0;
154 
155 	sdinfo.hmc_fn_id = hmc_info->hmc_fn_id;
156 	for (i = sd_index; i < sd_index + sd_cnt; i++) {
157 		sd_entry = &hmc_info->sd_table.sd_entry[i];
158 		if (!sd_entry || (!sd_entry->valid && setsd) ||
159 		    (sd_entry->valid && !setsd))
160 			continue;
161 		if (setsd) {
162 			pa = (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED) ?
163 				     sd_entry->u.pd_table.pd_page_addr.pa :
164 				     sd_entry->u.bp.addr.pa;
165 			irdma_set_sd_entry(pa, i, sd_entry->entry_type,
166 					   &sdinfo.entry[sdinfo.cnt]);
167 		} else {
168 			irdma_clr_sd_entry(i, sd_entry->entry_type,
169 					   &sdinfo.entry[sdinfo.cnt]);
170 		}
171 		sdinfo.cnt++;
172 		if (sdinfo.cnt == IRDMA_MAX_SD_ENTRIES) {
173 			ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo);
174 			if (ret_code) {
175 				ibdev_dbg(to_ibdev(dev),
176 					  "HMC: sd_programming failed err=%d\n",
177 					  ret_code);
178 				return ret_code;
179 			}
180 
181 			sdinfo.cnt = 0;
182 		}
183 	}
184 	if (sdinfo.cnt)
185 		ret_code = dev->cqp->process_cqp_sds(dev, &sdinfo);
186 
187 	return ret_code;
188 }
189 
190 /**
191  * irdma_hmc_finish_add_sd_reg - program sd entries for objects
192  * @dev: pointer to the device structure
193  * @info: create obj info
194  */
irdma_hmc_finish_add_sd_reg(struct irdma_sc_dev * dev,struct irdma_hmc_create_obj_info * info)195 static int irdma_hmc_finish_add_sd_reg(struct irdma_sc_dev *dev,
196 				       struct irdma_hmc_create_obj_info *info)
197 {
198 	if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
199 		return -EINVAL;
200 
201 	if ((info->start_idx + info->count) >
202 	    info->hmc_info->hmc_obj[info->rsrc_type].cnt)
203 		return -EINVAL;
204 
205 	if (!info->add_sd_cnt)
206 		return 0;
207 	return irdma_hmc_sd_grp(dev, info->hmc_info,
208 				info->hmc_info->sd_indexes[0], info->add_sd_cnt,
209 				true);
210 }
211 
212 /**
213  * irdma_sc_create_hmc_obj - allocate backing store for hmc objects
214  * @dev: pointer to the device structure
215  * @info: pointer to irdma_hmc_create_obj_info struct
216  *
217  * This will allocate memory for PDs and backing pages and populate
218  * the sd and pd entries.
219  */
irdma_sc_create_hmc_obj(struct irdma_sc_dev * dev,struct irdma_hmc_create_obj_info * info)220 int irdma_sc_create_hmc_obj(struct irdma_sc_dev *dev,
221 			    struct irdma_hmc_create_obj_info *info)
222 {
223 	struct irdma_hmc_sd_entry *sd_entry;
224 	u32 sd_idx, sd_lmt;
225 	u32 pd_idx = 0, pd_lmt = 0;
226 	u32 pd_idx1 = 0, pd_lmt1 = 0;
227 	u32 i, j;
228 	bool pd_error = false;
229 	int ret_code = 0;
230 
231 	if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt)
232 		return -EINVAL;
233 
234 	if ((info->start_idx + info->count) >
235 	    info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
236 		ibdev_dbg(to_ibdev(dev),
237 			  "HMC: error type %u, start = %u, req cnt %u, cnt = %u\n",
238 			  info->rsrc_type, info->start_idx, info->count,
239 			  info->hmc_info->hmc_obj[info->rsrc_type].cnt);
240 		return -EINVAL;
241 	}
242 
243 	irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type,
244 				  info->start_idx, info->count, &sd_idx,
245 				  &sd_lmt);
246 	if (sd_idx >= info->hmc_info->sd_table.sd_cnt ||
247 	    sd_lmt > info->hmc_info->sd_table.sd_cnt) {
248 		return -EINVAL;
249 	}
250 
251 	irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type,
252 				  info->start_idx, info->count, &pd_idx,
253 				  &pd_lmt);
254 
255 	for (j = sd_idx; j < sd_lmt; j++) {
256 		ret_code = irdma_add_sd_table_entry(dev->hw, info->hmc_info, j,
257 						    info->entry_type,
258 						    IRDMA_HMC_DIRECT_BP_SIZE);
259 		if (ret_code)
260 			goto exit_sd_error;
261 
262 		sd_entry = &info->hmc_info->sd_table.sd_entry[j];
263 		if (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED &&
264 		    (dev->hmc_info == info->hmc_info &&
265 		     info->rsrc_type != IRDMA_HMC_IW_PBLE)) {
266 			pd_idx1 = max(pd_idx, (j * IRDMA_HMC_MAX_BP_COUNT));
267 			pd_lmt1 = min(pd_lmt, (j + 1) * IRDMA_HMC_MAX_BP_COUNT);
268 			for (i = pd_idx1; i < pd_lmt1; i++) {
269 				/* update the pd table entry */
270 				ret_code = irdma_add_pd_table_entry(dev,
271 								    info->hmc_info,
272 								    i, NULL);
273 				if (ret_code) {
274 					pd_error = true;
275 					break;
276 				}
277 			}
278 			if (pd_error) {
279 				while (i && (i > pd_idx1)) {
280 					irdma_remove_pd_bp(dev, info->hmc_info,
281 							   i - 1);
282 					i--;
283 				}
284 			}
285 		}
286 		if (sd_entry->valid)
287 			continue;
288 
289 		info->hmc_info->sd_indexes[info->add_sd_cnt] = (u16)j;
290 		info->add_sd_cnt++;
291 		sd_entry->valid = true;
292 	}
293 	return irdma_hmc_finish_add_sd_reg(dev, info);
294 
295 exit_sd_error:
296 	while (j && (j > sd_idx)) {
297 		sd_entry = &info->hmc_info->sd_table.sd_entry[j - 1];
298 		switch (sd_entry->entry_type) {
299 		case IRDMA_SD_TYPE_PAGED:
300 			pd_idx1 = max(pd_idx, (j - 1) * IRDMA_HMC_MAX_BP_COUNT);
301 			pd_lmt1 = min(pd_lmt, (j * IRDMA_HMC_MAX_BP_COUNT));
302 			for (i = pd_idx1; i < pd_lmt1; i++)
303 				irdma_prep_remove_pd_page(info->hmc_info, i);
304 			break;
305 		case IRDMA_SD_TYPE_DIRECT:
306 			irdma_prep_remove_pd_page(info->hmc_info, (j - 1));
307 			break;
308 		default:
309 			ret_code = -EINVAL;
310 			break;
311 		}
312 		j--;
313 	}
314 
315 	return ret_code;
316 }
317 
318 /**
319  * irdma_finish_del_sd_reg - delete sd entries for objects
320  * @dev: pointer to the device structure
321  * @info: dele obj info
322  * @reset: true if called before reset
323  */
irdma_finish_del_sd_reg(struct irdma_sc_dev * dev,struct irdma_hmc_del_obj_info * info,bool reset)324 static int irdma_finish_del_sd_reg(struct irdma_sc_dev *dev,
325 				   struct irdma_hmc_del_obj_info *info,
326 				   bool reset)
327 {
328 	struct irdma_hmc_sd_entry *sd_entry;
329 	int ret_code = 0;
330 	u32 i, sd_idx;
331 	struct irdma_dma_mem *mem;
332 
333 	if (!reset)
334 		ret_code = irdma_hmc_sd_grp(dev, info->hmc_info,
335 					    info->hmc_info->sd_indexes[0],
336 					    info->del_sd_cnt, false);
337 
338 	if (ret_code)
339 		ibdev_dbg(to_ibdev(dev), "HMC: error cqp sd sd_grp\n");
340 	for (i = 0; i < info->del_sd_cnt; i++) {
341 		sd_idx = info->hmc_info->sd_indexes[i];
342 		sd_entry = &info->hmc_info->sd_table.sd_entry[sd_idx];
343 		mem = (sd_entry->entry_type == IRDMA_SD_TYPE_PAGED) ?
344 			      &sd_entry->u.pd_table.pd_page_addr :
345 			      &sd_entry->u.bp.addr;
346 
347 		if (!mem || !mem->va) {
348 			ibdev_dbg(to_ibdev(dev), "HMC: error cqp sd mem\n");
349 		} else {
350 			dma_free_coherent(dev->hw->device, mem->size, mem->va,
351 					  mem->pa);
352 			mem->va = NULL;
353 		}
354 	}
355 
356 	return ret_code;
357 }
358 
359 /**
360  * irdma_sc_del_hmc_obj - remove pe hmc objects
361  * @dev: pointer to the device structure
362  * @info: pointer to irdma_hmc_del_obj_info struct
363  * @reset: true if called before reset
364  *
365  * This will de-populate the SDs and PDs.  It frees
366  * the memory for PDS and backing storage.  After this function is returned,
367  * caller should deallocate memory allocated previously for
368  * book-keeping information about PDs and backing storage.
369  */
irdma_sc_del_hmc_obj(struct irdma_sc_dev * dev,struct irdma_hmc_del_obj_info * info,bool reset)370 int irdma_sc_del_hmc_obj(struct irdma_sc_dev *dev,
371 			 struct irdma_hmc_del_obj_info *info, bool reset)
372 {
373 	struct irdma_hmc_pd_table *pd_table;
374 	u32 sd_idx, sd_lmt;
375 	u32 pd_idx, pd_lmt, rel_pd_idx;
376 	u32 i, j;
377 	int ret_code = 0;
378 
379 	if (info->start_idx >= info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
380 		ibdev_dbg(to_ibdev(dev),
381 			  "HMC: error start_idx[%04d]  >= [type %04d].cnt[%04d]\n",
382 			  info->start_idx, info->rsrc_type,
383 			  info->hmc_info->hmc_obj[info->rsrc_type].cnt);
384 		return -EINVAL;
385 	}
386 
387 	if ((info->start_idx + info->count) >
388 	    info->hmc_info->hmc_obj[info->rsrc_type].cnt) {
389 		ibdev_dbg(to_ibdev(dev),
390 			  "HMC: error start_idx[%04d] + count %04d  >= [type %04d].cnt[%04d]\n",
391 			  info->start_idx, info->count, info->rsrc_type,
392 			  info->hmc_info->hmc_obj[info->rsrc_type].cnt);
393 		return -EINVAL;
394 	}
395 
396 	irdma_find_pd_index_limit(info->hmc_info, info->rsrc_type,
397 				  info->start_idx, info->count, &pd_idx,
398 				  &pd_lmt);
399 
400 	for (j = pd_idx; j < pd_lmt; j++) {
401 		sd_idx = j / IRDMA_HMC_PD_CNT_IN_SD;
402 
403 		if (!info->hmc_info->sd_table.sd_entry[sd_idx].valid)
404 			continue;
405 
406 		if (info->hmc_info->sd_table.sd_entry[sd_idx].entry_type !=
407 		    IRDMA_SD_TYPE_PAGED)
408 			continue;
409 
410 		rel_pd_idx = j % IRDMA_HMC_PD_CNT_IN_SD;
411 		pd_table = &info->hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
412 		if (pd_table->pd_entry &&
413 		    pd_table->pd_entry[rel_pd_idx].valid) {
414 			ret_code = irdma_remove_pd_bp(dev, info->hmc_info, j);
415 			if (ret_code) {
416 				ibdev_dbg(to_ibdev(dev),
417 					  "HMC: remove_pd_bp error\n");
418 				return ret_code;
419 			}
420 		}
421 	}
422 
423 	irdma_find_sd_index_limit(info->hmc_info, info->rsrc_type,
424 				  info->start_idx, info->count, &sd_idx,
425 				  &sd_lmt);
426 	if (sd_idx >= info->hmc_info->sd_table.sd_cnt ||
427 	    sd_lmt > info->hmc_info->sd_table.sd_cnt) {
428 		ibdev_dbg(to_ibdev(dev), "HMC: invalid sd_idx\n");
429 		return -EINVAL;
430 	}
431 
432 	for (i = sd_idx; i < sd_lmt; i++) {
433 		pd_table = &info->hmc_info->sd_table.sd_entry[i].u.pd_table;
434 		if (!info->hmc_info->sd_table.sd_entry[i].valid)
435 			continue;
436 		switch (info->hmc_info->sd_table.sd_entry[i].entry_type) {
437 		case IRDMA_SD_TYPE_DIRECT:
438 			ret_code = irdma_prep_remove_sd_bp(info->hmc_info, i);
439 			if (!ret_code) {
440 				info->hmc_info->sd_indexes[info->del_sd_cnt] =
441 					(u16)i;
442 				info->del_sd_cnt++;
443 			}
444 			break;
445 		case IRDMA_SD_TYPE_PAGED:
446 			ret_code = irdma_prep_remove_pd_page(info->hmc_info, i);
447 			if (ret_code)
448 				break;
449 			if (dev->hmc_info != info->hmc_info &&
450 			    info->rsrc_type == IRDMA_HMC_IW_PBLE &&
451 			    pd_table->pd_entry) {
452 				kfree(pd_table->pd_entry_virt_mem.va);
453 				pd_table->pd_entry = NULL;
454 			}
455 			info->hmc_info->sd_indexes[info->del_sd_cnt] = (u16)i;
456 			info->del_sd_cnt++;
457 			break;
458 		default:
459 			break;
460 		}
461 	}
462 	return irdma_finish_del_sd_reg(dev, info, reset);
463 }
464 
465 /**
466  * irdma_add_sd_table_entry - Adds a segment descriptor to the table
467  * @hw: pointer to our hw struct
468  * @hmc_info: pointer to the HMC configuration information struct
469  * @sd_index: segment descriptor index to manipulate
470  * @type: what type of segment descriptor we're manipulating
471  * @direct_mode_sz: size to alloc in direct mode
472  */
irdma_add_sd_table_entry(struct irdma_hw * hw,struct irdma_hmc_info * hmc_info,u32 sd_index,enum irdma_sd_entry_type type,u64 direct_mode_sz)473 int irdma_add_sd_table_entry(struct irdma_hw *hw,
474 			     struct irdma_hmc_info *hmc_info, u32 sd_index,
475 			     enum irdma_sd_entry_type type, u64 direct_mode_sz)
476 {
477 	struct irdma_hmc_sd_entry *sd_entry;
478 	struct irdma_dma_mem dma_mem;
479 	u64 alloc_len;
480 
481 	sd_entry = &hmc_info->sd_table.sd_entry[sd_index];
482 	if (!sd_entry->valid) {
483 		if (type == IRDMA_SD_TYPE_PAGED)
484 			alloc_len = IRDMA_HMC_PAGED_BP_SIZE;
485 		else
486 			alloc_len = direct_mode_sz;
487 
488 		/* allocate a 4K pd page or 2M backing page */
489 		dma_mem.size = ALIGN(alloc_len, IRDMA_HMC_PD_BP_BUF_ALIGNMENT);
490 		dma_mem.va = dma_alloc_coherent(hw->device, dma_mem.size,
491 						&dma_mem.pa, GFP_KERNEL);
492 		if (!dma_mem.va)
493 			return -ENOMEM;
494 		if (type == IRDMA_SD_TYPE_PAGED) {
495 			struct irdma_virt_mem *vmem =
496 				&sd_entry->u.pd_table.pd_entry_virt_mem;
497 
498 			vmem->size = sizeof(struct irdma_hmc_pd_entry) * 512;
499 			vmem->va = kzalloc(vmem->size, GFP_KERNEL);
500 			if (!vmem->va) {
501 				dma_free_coherent(hw->device, dma_mem.size,
502 						  dma_mem.va, dma_mem.pa);
503 				dma_mem.va = NULL;
504 				return -ENOMEM;
505 			}
506 			sd_entry->u.pd_table.pd_entry = vmem->va;
507 
508 			memcpy(&sd_entry->u.pd_table.pd_page_addr, &dma_mem,
509 			       sizeof(sd_entry->u.pd_table.pd_page_addr));
510 		} else {
511 			memcpy(&sd_entry->u.bp.addr, &dma_mem,
512 			       sizeof(sd_entry->u.bp.addr));
513 
514 			sd_entry->u.bp.sd_pd_index = sd_index;
515 		}
516 
517 		hmc_info->sd_table.sd_entry[sd_index].entry_type = type;
518 		hmc_info->sd_table.use_cnt++;
519 	}
520 	if (sd_entry->entry_type == IRDMA_SD_TYPE_DIRECT)
521 		sd_entry->u.bp.use_cnt++;
522 
523 	return 0;
524 }
525 
526 /**
527  * irdma_add_pd_table_entry - Adds page descriptor to the specified table
528  * @dev: pointer to our device structure
529  * @hmc_info: pointer to the HMC configuration information structure
530  * @pd_index: which page descriptor index to manipulate
531  * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one.
532  *
533  * This function:
534  *	1. Initializes the pd entry
535  *	2. Adds pd_entry in the pd_table
536  *	3. Mark the entry valid in irdma_hmc_pd_entry structure
537  *	4. Initializes the pd_entry's ref count to 1
538  * assumptions:
539  *	1. The memory for pd should be pinned down, physically contiguous and
540  *	   aligned on 4K boundary and zeroed memory.
541  *	2. It should be 4K in size.
542  */
irdma_add_pd_table_entry(struct irdma_sc_dev * dev,struct irdma_hmc_info * hmc_info,u32 pd_index,struct irdma_dma_mem * rsrc_pg)543 int irdma_add_pd_table_entry(struct irdma_sc_dev *dev,
544 			     struct irdma_hmc_info *hmc_info, u32 pd_index,
545 			     struct irdma_dma_mem *rsrc_pg)
546 {
547 	struct irdma_hmc_pd_table *pd_table;
548 	struct irdma_hmc_pd_entry *pd_entry;
549 	struct irdma_dma_mem mem;
550 	struct irdma_dma_mem *page = &mem;
551 	u32 sd_idx, rel_pd_idx;
552 	u64 *pd_addr;
553 	u64 page_desc;
554 
555 	if (pd_index / IRDMA_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt)
556 		return -EINVAL;
557 
558 	sd_idx = (pd_index / IRDMA_HMC_PD_CNT_IN_SD);
559 	if (hmc_info->sd_table.sd_entry[sd_idx].entry_type !=
560 	    IRDMA_SD_TYPE_PAGED)
561 		return 0;
562 
563 	rel_pd_idx = (pd_index % IRDMA_HMC_PD_CNT_IN_SD);
564 	pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
565 	pd_entry = &pd_table->pd_entry[rel_pd_idx];
566 	if (!pd_entry->valid) {
567 		if (rsrc_pg) {
568 			pd_entry->rsrc_pg = true;
569 			page = rsrc_pg;
570 		} else {
571 			page->size = ALIGN(IRDMA_HMC_PAGED_BP_SIZE,
572 					   IRDMA_HMC_PD_BP_BUF_ALIGNMENT);
573 			page->va = dma_alloc_coherent(dev->hw->device,
574 						      page->size, &page->pa,
575 						      GFP_KERNEL);
576 			if (!page->va)
577 				return -ENOMEM;
578 
579 			pd_entry->rsrc_pg = false;
580 		}
581 
582 		memcpy(&pd_entry->bp.addr, page, sizeof(pd_entry->bp.addr));
583 		pd_entry->bp.sd_pd_index = pd_index;
584 		pd_entry->bp.entry_type = IRDMA_SD_TYPE_PAGED;
585 		page_desc = page->pa | 0x1;
586 		pd_addr = pd_table->pd_page_addr.va;
587 		pd_addr += rel_pd_idx;
588 		memcpy(pd_addr, &page_desc, sizeof(*pd_addr));
589 		pd_entry->sd_index = sd_idx;
590 		pd_entry->valid = true;
591 		pd_table->use_cnt++;
592 		irdma_invalidate_pf_hmc_pd(dev, sd_idx, rel_pd_idx);
593 	}
594 	pd_entry->bp.use_cnt++;
595 
596 	return 0;
597 }
598 
599 /**
600  * irdma_remove_pd_bp - remove a backing page from a page descriptor
601  * @dev: pointer to our HW structure
602  * @hmc_info: pointer to the HMC configuration information structure
603  * @idx: the page index
604  *
605  * This function:
606  *	1. Marks the entry in pd table (for paged address mode) or in sd table
607  *	   (for direct address mode) invalid.
608  *	2. Write to register PMPDINV to invalidate the backing page in FV cache
609  *	3. Decrement the ref count for the pd _entry
610  * assumptions:
611  *	1. Caller can deallocate the memory used by backing storage after this
612  *	   function returns.
613  */
irdma_remove_pd_bp(struct irdma_sc_dev * dev,struct irdma_hmc_info * hmc_info,u32 idx)614 int irdma_remove_pd_bp(struct irdma_sc_dev *dev,
615 		       struct irdma_hmc_info *hmc_info, u32 idx)
616 {
617 	struct irdma_hmc_pd_entry *pd_entry;
618 	struct irdma_hmc_pd_table *pd_table;
619 	struct irdma_hmc_sd_entry *sd_entry;
620 	u32 sd_idx, rel_pd_idx;
621 	struct irdma_dma_mem *mem;
622 	u64 *pd_addr;
623 
624 	sd_idx = idx / IRDMA_HMC_PD_CNT_IN_SD;
625 	rel_pd_idx = idx % IRDMA_HMC_PD_CNT_IN_SD;
626 	if (sd_idx >= hmc_info->sd_table.sd_cnt)
627 		return -EINVAL;
628 
629 	sd_entry = &hmc_info->sd_table.sd_entry[sd_idx];
630 	if (sd_entry->entry_type != IRDMA_SD_TYPE_PAGED)
631 		return -EINVAL;
632 
633 	pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
634 	pd_entry = &pd_table->pd_entry[rel_pd_idx];
635 	if (--pd_entry->bp.use_cnt)
636 		return 0;
637 
638 	pd_entry->valid = false;
639 	pd_table->use_cnt--;
640 	pd_addr = pd_table->pd_page_addr.va;
641 	pd_addr += rel_pd_idx;
642 	memset(pd_addr, 0, sizeof(u64));
643 	irdma_invalidate_pf_hmc_pd(dev, sd_idx, idx);
644 
645 	if (!pd_entry->rsrc_pg) {
646 		mem = &pd_entry->bp.addr;
647 		if (!mem || !mem->va)
648 			return -EINVAL;
649 
650 		dma_free_coherent(dev->hw->device, mem->size, mem->va,
651 				  mem->pa);
652 		mem->va = NULL;
653 	}
654 	if (!pd_table->use_cnt)
655 		kfree(pd_table->pd_entry_virt_mem.va);
656 
657 	return 0;
658 }
659 
660 /**
661  * irdma_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry
662  * @hmc_info: pointer to the HMC configuration information structure
663  * @idx: the page index
664  */
irdma_prep_remove_sd_bp(struct irdma_hmc_info * hmc_info,u32 idx)665 int irdma_prep_remove_sd_bp(struct irdma_hmc_info *hmc_info, u32 idx)
666 {
667 	struct irdma_hmc_sd_entry *sd_entry;
668 
669 	sd_entry = &hmc_info->sd_table.sd_entry[idx];
670 	if (--sd_entry->u.bp.use_cnt)
671 		return -EBUSY;
672 
673 	hmc_info->sd_table.use_cnt--;
674 	sd_entry->valid = false;
675 
676 	return 0;
677 }
678 
679 /**
680  * irdma_prep_remove_pd_page - Prepares to remove a PD page from sd entry.
681  * @hmc_info: pointer to the HMC configuration information structure
682  * @idx: segment descriptor index to find the relevant page descriptor
683  */
irdma_prep_remove_pd_page(struct irdma_hmc_info * hmc_info,u32 idx)684 int irdma_prep_remove_pd_page(struct irdma_hmc_info *hmc_info, u32 idx)
685 {
686 	struct irdma_hmc_sd_entry *sd_entry;
687 
688 	sd_entry = &hmc_info->sd_table.sd_entry[idx];
689 
690 	if (sd_entry->u.pd_table.use_cnt)
691 		return -EBUSY;
692 
693 	sd_entry->valid = false;
694 	hmc_info->sd_table.use_cnt--;
695 
696 	return 0;
697 }
698