1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright(c) 2013 - 2018 Intel Corporation. */
3 
4 #include "i40e_alloc.h"
5 #include "i40e_debug.h"
6 #include "i40e_hmc.h"
7 #include "i40e_type.h"
8 
9 /**
10  * i40e_add_sd_table_entry - Adds a segment descriptor to the table
11  * @hw: pointer to our hw struct
12  * @hmc_info: pointer to the HMC configuration information struct
13  * @sd_index: segment descriptor index to manipulate
14  * @type: what type of segment descriptor we're manipulating
15  * @direct_mode_sz: size to alloc in direct mode
16  **/
i40e_add_sd_table_entry(struct i40e_hw * hw,struct i40e_hmc_info * hmc_info,u32 sd_index,enum i40e_sd_entry_type type,u64 direct_mode_sz)17 int i40e_add_sd_table_entry(struct i40e_hw *hw,
18 			    struct i40e_hmc_info *hmc_info,
19 			    u32 sd_index,
20 			    enum i40e_sd_entry_type type,
21 			    u64 direct_mode_sz)
22 {
23 	struct i40e_hmc_sd_entry *sd_entry;
24 	bool dma_mem_alloc_done = false;
25 	struct i40e_dma_mem mem;
26 	int ret_code = 0;
27 	u64 alloc_len;
28 
29 	if (NULL == hmc_info->sd_table.sd_entry) {
30 		ret_code = -EINVAL;
31 		hw_dbg(hw, "i40e_add_sd_table_entry: bad sd_entry\n");
32 		goto exit;
33 	}
34 
35 	if (sd_index >= hmc_info->sd_table.sd_cnt) {
36 		ret_code = -EINVAL;
37 		hw_dbg(hw, "i40e_add_sd_table_entry: bad sd_index\n");
38 		goto exit;
39 	}
40 
41 	sd_entry = &hmc_info->sd_table.sd_entry[sd_index];
42 	if (!sd_entry->valid) {
43 		if (type == I40E_SD_TYPE_PAGED)
44 			alloc_len = I40E_HMC_PAGED_BP_SIZE;
45 		else
46 			alloc_len = direct_mode_sz;
47 
48 		/* allocate a 4K pd page or 2M backing page */
49 		ret_code = i40e_allocate_dma_mem(hw, &mem, alloc_len,
50 						 I40E_HMC_PD_BP_BUF_ALIGNMENT);
51 		if (ret_code)
52 			goto exit;
53 		dma_mem_alloc_done = true;
54 		if (I40E_SD_TYPE_PAGED == type) {
55 			ret_code = i40e_allocate_virt_mem(hw,
56 					&sd_entry->u.pd_table.pd_entry_virt_mem,
57 					sizeof(struct i40e_hmc_pd_entry) * 512);
58 			if (ret_code)
59 				goto exit;
60 			sd_entry->u.pd_table.pd_entry =
61 				(struct i40e_hmc_pd_entry *)
62 				sd_entry->u.pd_table.pd_entry_virt_mem.va;
63 			sd_entry->u.pd_table.pd_page_addr = mem;
64 		} else {
65 			sd_entry->u.bp.addr = mem;
66 			sd_entry->u.bp.sd_pd_index = sd_index;
67 		}
68 		/* initialize the sd entry */
69 		hmc_info->sd_table.sd_entry[sd_index].entry_type = type;
70 
71 		/* increment the ref count */
72 		I40E_INC_SD_REFCNT(&hmc_info->sd_table);
73 	}
74 	/* Increment backing page reference count */
75 	if (I40E_SD_TYPE_DIRECT == sd_entry->entry_type)
76 		I40E_INC_BP_REFCNT(&sd_entry->u.bp);
77 exit:
78 	if (ret_code)
79 		if (dma_mem_alloc_done)
80 			i40e_free_dma_mem(hw, &mem);
81 
82 	return ret_code;
83 }
84 
85 /**
86  * i40e_add_pd_table_entry - Adds page descriptor to the specified table
87  * @hw: pointer to our HW structure
88  * @hmc_info: pointer to the HMC configuration information structure
89  * @pd_index: which page descriptor index to manipulate
90  * @rsrc_pg: if not NULL, use preallocated page instead of allocating new one.
91  *
92  * This function:
93  *	1. Initializes the pd entry
94  *	2. Adds pd_entry in the pd_table
95  *	3. Mark the entry valid in i40e_hmc_pd_entry structure
96  *	4. Initializes the pd_entry's ref count to 1
97  * assumptions:
98  *	1. The memory for pd should be pinned down, physically contiguous and
99  *	   aligned on 4K boundary and zeroed memory.
100  *	2. It should be 4K in size.
101  **/
i40e_add_pd_table_entry(struct i40e_hw * hw,struct i40e_hmc_info * hmc_info,u32 pd_index,struct i40e_dma_mem * rsrc_pg)102 int i40e_add_pd_table_entry(struct i40e_hw *hw,
103 			    struct i40e_hmc_info *hmc_info,
104 			    u32 pd_index,
105 			    struct i40e_dma_mem *rsrc_pg)
106 {
107 	struct i40e_hmc_pd_table *pd_table;
108 	struct i40e_hmc_pd_entry *pd_entry;
109 	struct i40e_dma_mem mem;
110 	struct i40e_dma_mem *page = &mem;
111 	u32 sd_idx, rel_pd_idx;
112 	int ret_code = 0;
113 	u64 page_desc;
114 	u64 *pd_addr;
115 
116 	if (pd_index / I40E_HMC_PD_CNT_IN_SD >= hmc_info->sd_table.sd_cnt) {
117 		ret_code = -EINVAL;
118 		hw_dbg(hw, "i40e_add_pd_table_entry: bad pd_index\n");
119 		goto exit;
120 	}
121 
122 	/* find corresponding sd */
123 	sd_idx = (pd_index / I40E_HMC_PD_CNT_IN_SD);
124 	if (I40E_SD_TYPE_PAGED !=
125 	    hmc_info->sd_table.sd_entry[sd_idx].entry_type)
126 		goto exit;
127 
128 	rel_pd_idx = (pd_index % I40E_HMC_PD_CNT_IN_SD);
129 	pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
130 	pd_entry = &pd_table->pd_entry[rel_pd_idx];
131 	if (!pd_entry->valid) {
132 		if (rsrc_pg) {
133 			pd_entry->rsrc_pg = true;
134 			page = rsrc_pg;
135 		} else {
136 			/* allocate a 4K backing page */
137 			ret_code = i40e_allocate_dma_mem(hw, page,
138 						I40E_HMC_PAGED_BP_SIZE,
139 						I40E_HMC_PD_BP_BUF_ALIGNMENT);
140 			if (ret_code)
141 				goto exit;
142 			pd_entry->rsrc_pg = false;
143 		}
144 
145 		pd_entry->bp.addr = *page;
146 		pd_entry->bp.sd_pd_index = pd_index;
147 		pd_entry->bp.entry_type = I40E_SD_TYPE_PAGED;
148 		/* Set page address and valid bit */
149 		page_desc = page->pa | 0x1;
150 
151 		pd_addr = (u64 *)pd_table->pd_page_addr.va;
152 		pd_addr += rel_pd_idx;
153 
154 		/* Add the backing page physical address in the pd entry */
155 		memcpy(pd_addr, &page_desc, sizeof(u64));
156 
157 		pd_entry->sd_index = sd_idx;
158 		pd_entry->valid = true;
159 		I40E_INC_PD_REFCNT(pd_table);
160 	}
161 	I40E_INC_BP_REFCNT(&pd_entry->bp);
162 exit:
163 	return ret_code;
164 }
165 
166 /**
167  * i40e_remove_pd_bp - remove a backing page from a page descriptor
168  * @hw: pointer to our HW structure
169  * @hmc_info: pointer to the HMC configuration information structure
170  * @idx: the page index
171  *
172  * This function:
173  *	1. Marks the entry in pd tabe (for paged address mode) or in sd table
174  *	   (for direct address mode) invalid.
175  *	2. Write to register PMPDINV to invalidate the backing page in FV cache
176  *	3. Decrement the ref count for the pd _entry
177  * assumptions:
178  *	1. Caller can deallocate the memory used by backing storage after this
179  *	   function returns.
180  **/
i40e_remove_pd_bp(struct i40e_hw * hw,struct i40e_hmc_info * hmc_info,u32 idx)181 int i40e_remove_pd_bp(struct i40e_hw *hw,
182 		      struct i40e_hmc_info *hmc_info,
183 		      u32 idx)
184 {
185 	struct i40e_hmc_pd_entry *pd_entry;
186 	struct i40e_hmc_pd_table *pd_table;
187 	struct i40e_hmc_sd_entry *sd_entry;
188 	u32 sd_idx, rel_pd_idx;
189 	int ret_code = 0;
190 	u64 *pd_addr;
191 
192 	/* calculate index */
193 	sd_idx = idx / I40E_HMC_PD_CNT_IN_SD;
194 	rel_pd_idx = idx % I40E_HMC_PD_CNT_IN_SD;
195 	if (sd_idx >= hmc_info->sd_table.sd_cnt) {
196 		ret_code = -EINVAL;
197 		hw_dbg(hw, "i40e_remove_pd_bp: bad idx\n");
198 		goto exit;
199 	}
200 	sd_entry = &hmc_info->sd_table.sd_entry[sd_idx];
201 	if (I40E_SD_TYPE_PAGED != sd_entry->entry_type) {
202 		ret_code = -EINVAL;
203 		hw_dbg(hw, "i40e_remove_pd_bp: wrong sd_entry type\n");
204 		goto exit;
205 	}
206 	/* get the entry and decrease its ref counter */
207 	pd_table = &hmc_info->sd_table.sd_entry[sd_idx].u.pd_table;
208 	pd_entry = &pd_table->pd_entry[rel_pd_idx];
209 	I40E_DEC_BP_REFCNT(&pd_entry->bp);
210 	if (pd_entry->bp.ref_cnt)
211 		goto exit;
212 
213 	/* mark the entry invalid */
214 	pd_entry->valid = false;
215 	I40E_DEC_PD_REFCNT(pd_table);
216 	pd_addr = (u64 *)pd_table->pd_page_addr.va;
217 	pd_addr += rel_pd_idx;
218 	memset(pd_addr, 0, sizeof(u64));
219 	I40E_INVALIDATE_PF_HMC_PD(hw, sd_idx, idx);
220 
221 	/* free memory here */
222 	if (!pd_entry->rsrc_pg)
223 		ret_code = i40e_free_dma_mem(hw, &pd_entry->bp.addr);
224 	if (ret_code)
225 		goto exit;
226 	if (!pd_table->ref_cnt)
227 		i40e_free_virt_mem(hw, &pd_table->pd_entry_virt_mem);
228 exit:
229 	return ret_code;
230 }
231 
232 /**
233  * i40e_prep_remove_sd_bp - Prepares to remove a backing page from a sd entry
234  * @hmc_info: pointer to the HMC configuration information structure
235  * @idx: the page index
236  **/
i40e_prep_remove_sd_bp(struct i40e_hmc_info * hmc_info,u32 idx)237 int i40e_prep_remove_sd_bp(struct i40e_hmc_info *hmc_info,
238 			   u32 idx)
239 {
240 	struct i40e_hmc_sd_entry *sd_entry;
241 	int ret_code = 0;
242 
243 	/* get the entry and decrease its ref counter */
244 	sd_entry = &hmc_info->sd_table.sd_entry[idx];
245 	I40E_DEC_BP_REFCNT(&sd_entry->u.bp);
246 	if (sd_entry->u.bp.ref_cnt) {
247 		ret_code = -EBUSY;
248 		goto exit;
249 	}
250 	I40E_DEC_SD_REFCNT(&hmc_info->sd_table);
251 
252 	/* mark the entry invalid */
253 	sd_entry->valid = false;
254 exit:
255 	return ret_code;
256 }
257 
258 /**
259  * i40e_remove_sd_bp_new - Removes a backing page from a segment descriptor
260  * @hw: pointer to our hw struct
261  * @hmc_info: pointer to the HMC configuration information structure
262  * @idx: the page index
263  * @is_pf: used to distinguish between VF and PF
264  **/
i40e_remove_sd_bp_new(struct i40e_hw * hw,struct i40e_hmc_info * hmc_info,u32 idx,bool is_pf)265 int i40e_remove_sd_bp_new(struct i40e_hw *hw,
266 			  struct i40e_hmc_info *hmc_info,
267 			  u32 idx, bool is_pf)
268 {
269 	struct i40e_hmc_sd_entry *sd_entry;
270 
271 	if (!is_pf)
272 		return -EOPNOTSUPP;
273 
274 	/* get the entry and decrease its ref counter */
275 	sd_entry = &hmc_info->sd_table.sd_entry[idx];
276 	I40E_CLEAR_PF_SD_ENTRY(hw, idx, I40E_SD_TYPE_DIRECT);
277 
278 	return i40e_free_dma_mem(hw, &sd_entry->u.bp.addr);
279 }
280 
281 /**
282  * i40e_prep_remove_pd_page - Prepares to remove a PD page from sd entry.
283  * @hmc_info: pointer to the HMC configuration information structure
284  * @idx: segment descriptor index to find the relevant page descriptor
285  **/
i40e_prep_remove_pd_page(struct i40e_hmc_info * hmc_info,u32 idx)286 int i40e_prep_remove_pd_page(struct i40e_hmc_info *hmc_info,
287 			     u32 idx)
288 {
289 	struct i40e_hmc_sd_entry *sd_entry;
290 	int ret_code = 0;
291 
292 	sd_entry = &hmc_info->sd_table.sd_entry[idx];
293 
294 	if (sd_entry->u.pd_table.ref_cnt) {
295 		ret_code = -EBUSY;
296 		goto exit;
297 	}
298 
299 	/* mark the entry invalid */
300 	sd_entry->valid = false;
301 
302 	I40E_DEC_SD_REFCNT(&hmc_info->sd_table);
303 exit:
304 	return ret_code;
305 }
306 
307 /**
308  * i40e_remove_pd_page_new - Removes a PD page from sd entry.
309  * @hw: pointer to our hw struct
310  * @hmc_info: pointer to the HMC configuration information structure
311  * @idx: segment descriptor index to find the relevant page descriptor
312  * @is_pf: used to distinguish between VF and PF
313  **/
i40e_remove_pd_page_new(struct i40e_hw * hw,struct i40e_hmc_info * hmc_info,u32 idx,bool is_pf)314 int i40e_remove_pd_page_new(struct i40e_hw *hw,
315 			    struct i40e_hmc_info *hmc_info,
316 			    u32 idx, bool is_pf)
317 {
318 	struct i40e_hmc_sd_entry *sd_entry;
319 
320 	if (!is_pf)
321 		return -EOPNOTSUPP;
322 
323 	sd_entry = &hmc_info->sd_table.sd_entry[idx];
324 	I40E_CLEAR_PF_SD_ENTRY(hw, idx, I40E_SD_TYPE_PAGED);
325 
326 	return  i40e_free_dma_mem(hw, &sd_entry->u.pd_table.pd_page_addr);
327 }
328