1 // SPDX-License-Identifier: GPL-2.0
2 
3 /*
4  * Copyright 2016-2022 HabanaLabs, Ltd.
5  * All Rights Reserved.
6  */
7 
8 #include <linux/slab.h>
9 
10 #include "../habanalabs.h"
11 
12 #include <trace/events/habanalabs.h>
13 
14 /**
15  * hl_mmu_get_funcs() - get MMU functions structure
16  * @hdev: habanalabs device structure.
17  * @pgt_residency: page table residency.
18  * @is_dram_addr: true if we need HMMU functions
19  *
20  * @return appropriate MMU functions structure
21  */
hl_mmu_get_funcs(struct hl_device * hdev,int pgt_residency,bool is_dram_addr)22 static struct hl_mmu_funcs *hl_mmu_get_funcs(struct hl_device *hdev, int pgt_residency,
23 									bool is_dram_addr)
24 {
25 	return &hdev->mmu_func[pgt_residency];
26 }
27 
hl_is_dram_va(struct hl_device * hdev,u64 virt_addr)28 bool hl_is_dram_va(struct hl_device *hdev, u64 virt_addr)
29 {
30 	struct asic_fixed_properties *prop = &hdev->asic_prop;
31 
32 	return hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size,
33 					prop->dmmu.start_addr,
34 					prop->dmmu.end_addr);
35 }
36 
37 /**
38  * hl_mmu_init() - initialize the MMU module.
39  * @hdev: habanalabs device structure.
40  *
41  * Return: 0 for success, non-zero for failure.
42  */
hl_mmu_init(struct hl_device * hdev)43 int hl_mmu_init(struct hl_device *hdev)
44 {
45 	int rc = -EOPNOTSUPP;
46 
47 	if (hdev->mmu_disable)
48 		return 0;
49 
50 	mutex_init(&hdev->mmu_lock);
51 
52 	if (hdev->mmu_func[MMU_DR_PGT].init != NULL) {
53 		rc = hdev->mmu_func[MMU_DR_PGT].init(hdev);
54 		if (rc)
55 			return rc;
56 	}
57 
58 	if (hdev->mmu_func[MMU_HR_PGT].init != NULL) {
59 		rc = hdev->mmu_func[MMU_HR_PGT].init(hdev);
60 		if (rc)
61 			goto fini_dr_mmu;
62 	}
63 
64 	return 0;
65 
66 fini_dr_mmu:
67 	if (hdev->mmu_func[MMU_DR_PGT].fini != NULL)
68 		hdev->mmu_func[MMU_DR_PGT].fini(hdev);
69 
70 	return rc;
71 }
72 
73 /**
74  * hl_mmu_fini() - release the MMU module.
75  * @hdev: habanalabs device structure.
76  *
77  * This function does the following:
78  * - Disable MMU in H/W.
79  * - Free the pgt_infos pool.
80  *
81  * All contexts should be freed before calling this function.
82  */
hl_mmu_fini(struct hl_device * hdev)83 void hl_mmu_fini(struct hl_device *hdev)
84 {
85 	if (hdev->mmu_disable)
86 		return;
87 
88 	if (hdev->mmu_func[MMU_DR_PGT].fini != NULL)
89 		hdev->mmu_func[MMU_DR_PGT].fini(hdev);
90 
91 	if (hdev->mmu_func[MMU_HR_PGT].fini != NULL)
92 		hdev->mmu_func[MMU_HR_PGT].fini(hdev);
93 
94 	mutex_destroy(&hdev->mmu_lock);
95 }
96 
97 /**
98  * hl_mmu_ctx_init() - initialize a context for using the MMU module.
99  * @ctx: pointer to the context structure to initialize.
100  *
101  * Initialize a mutex to protect the concurrent mapping flow, a hash to hold all
102  * page tables hops related to this context.
103  * Return: 0 on success, non-zero otherwise.
104  */
hl_mmu_ctx_init(struct hl_ctx * ctx)105 int hl_mmu_ctx_init(struct hl_ctx *ctx)
106 {
107 	struct hl_device *hdev = ctx->hdev;
108 	int rc = -EOPNOTSUPP;
109 
110 	if (hdev->mmu_disable)
111 		return 0;
112 
113 	if (hdev->mmu_func[MMU_DR_PGT].ctx_init != NULL) {
114 		rc = hdev->mmu_func[MMU_DR_PGT].ctx_init(ctx);
115 		if (rc)
116 			return rc;
117 	}
118 
119 	if (hdev->mmu_func[MMU_HR_PGT].ctx_init != NULL) {
120 		rc = hdev->mmu_func[MMU_HR_PGT].ctx_init(ctx);
121 		if (rc)
122 			goto fini_dr_ctx;
123 	}
124 
125 	return 0;
126 
127 fini_dr_ctx:
128 	if (hdev->mmu_func[MMU_DR_PGT].fini != NULL)
129 		hdev->mmu_func[MMU_DR_PGT].fini(hdev);
130 
131 	return rc;
132 }
133 
134 /*
135  * hl_mmu_ctx_fini - disable a ctx from using the mmu module
136  *
137  * @ctx: pointer to the context structure
138  *
139  * This function does the following:
140  * - Free any pgts which were not freed yet
141  * - Free the mutex
142  * - Free DRAM default page mapping hops
143  */
hl_mmu_ctx_fini(struct hl_ctx * ctx)144 void hl_mmu_ctx_fini(struct hl_ctx *ctx)
145 {
146 	struct hl_device *hdev = ctx->hdev;
147 
148 	if (hdev->mmu_disable)
149 		return;
150 
151 	if (hdev->mmu_func[MMU_DR_PGT].ctx_fini != NULL)
152 		hdev->mmu_func[MMU_DR_PGT].ctx_fini(ctx);
153 
154 	if (hdev->mmu_func[MMU_HR_PGT].ctx_fini != NULL)
155 		hdev->mmu_func[MMU_HR_PGT].ctx_fini(ctx);
156 }
157 
158 /*
159  * hl_mmu_get_real_page_size - get real page size to use in map/unmap operation
160  *
161  * @hdev: pointer to device data.
162  * @mmu_prop: MMU properties.
163  * @page_size: page size
164  * @real_page_size: set here the actual page size to use for the operation
165  * @is_dram_addr: true if DRAM address, otherwise false.
166  *
167  * @return 0 on success, otherwise non 0 error code
168  *
169  * note that this is general implementation that can fit most MMU arch. but as this is used as an
170  * MMU function:
171  * 1. it shall not be called directly- only from mmu_func structure instance
172  * 2. each MMU may modify the implementation internally
173  */
hl_mmu_get_real_page_size(struct hl_device * hdev,struct hl_mmu_properties * mmu_prop,u32 page_size,u32 * real_page_size,bool is_dram_addr)174 int hl_mmu_get_real_page_size(struct hl_device *hdev, struct hl_mmu_properties *mmu_prop,
175 				u32 page_size, u32 *real_page_size, bool is_dram_addr)
176 {
177 	/*
178 	 * The H/W handles mapping of specific page sizes. Hence if the page
179 	 * size is bigger, we break it to sub-pages and map them separately.
180 	 */
181 	if ((page_size % mmu_prop->page_size) == 0) {
182 		*real_page_size = mmu_prop->page_size;
183 		return 0;
184 	}
185 
186 	dev_err(hdev->dev, "page size of %u is not %uKB aligned, can't map\n",
187 						page_size, mmu_prop->page_size >> 10);
188 
189 	return -EFAULT;
190 }
191 
hl_mmu_get_prop(struct hl_device * hdev,u32 page_size,bool is_dram_addr)192 static struct hl_mmu_properties *hl_mmu_get_prop(struct hl_device *hdev, u32 page_size,
193 							bool is_dram_addr)
194 {
195 	struct asic_fixed_properties *prop = &hdev->asic_prop;
196 
197 	if (is_dram_addr)
198 		return &prop->dmmu;
199 	else if ((page_size % prop->pmmu_huge.page_size) == 0)
200 		return &prop->pmmu_huge;
201 
202 	return &prop->pmmu;
203 }
204 
205 /*
206  * hl_mmu_unmap_page - unmaps a virtual addr
207  *
208  * @ctx: pointer to the context structure
209  * @virt_addr: virt addr to map from
210  * @page_size: size of the page to unmap
211  * @flush_pte: whether to do a PCI flush
212  *
213  * This function does the following:
214  * - Check that the virt addr is mapped
215  * - Unmap the virt addr and frees pgts if possible
216  * - Returns 0 on success, -EINVAL if the given addr is not mapped
217  *
218  * Because this function changes the page tables in the device and because it
219  * changes the MMU hash, it must be protected by a lock.
220  * However, because it maps only a single page, the lock should be implemented
221  * in a higher level in order to protect the entire mapping of the memory area
222  *
223  * For optimization reasons PCI flush may be requested once after unmapping of
224  * large area.
225  */
hl_mmu_unmap_page(struct hl_ctx * ctx,u64 virt_addr,u32 page_size,bool flush_pte)226 int hl_mmu_unmap_page(struct hl_ctx *ctx, u64 virt_addr, u32 page_size, bool flush_pte)
227 {
228 	struct hl_device *hdev = ctx->hdev;
229 	struct hl_mmu_properties *mmu_prop;
230 	struct hl_mmu_funcs *mmu_funcs;
231 	int i, pgt_residency, rc = 0;
232 	u32 real_page_size, npages;
233 	u64 real_virt_addr;
234 	bool is_dram_addr;
235 
236 	if (hdev->mmu_disable)
237 		return 0;
238 
239 	is_dram_addr = hl_is_dram_va(hdev, virt_addr);
240 	mmu_prop = hl_mmu_get_prop(hdev, page_size, is_dram_addr);
241 
242 	pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT;
243 	mmu_funcs = hl_mmu_get_funcs(hdev, pgt_residency, is_dram_addr);
244 
245 	rc = hdev->asic_funcs->mmu_get_real_page_size(hdev, mmu_prop, page_size, &real_page_size,
246 							is_dram_addr);
247 	if (rc)
248 		return rc;
249 
250 	npages = page_size / real_page_size;
251 	real_virt_addr = virt_addr;
252 
253 	for (i = 0 ; i < npages ; i++) {
254 		rc = mmu_funcs->unmap(ctx, real_virt_addr, is_dram_addr);
255 		if (rc)
256 			break;
257 
258 		real_virt_addr += real_page_size;
259 	}
260 
261 	if (flush_pte)
262 		mmu_funcs->flush(ctx);
263 
264 	if (trace_habanalabs_mmu_unmap_enabled() && !rc)
265 		trace_habanalabs_mmu_unmap(hdev->dev, virt_addr, 0, page_size, flush_pte);
266 
267 	return rc;
268 }
269 
270 /*
271  * hl_mmu_map_page - maps a virtual addr to physical addr
272  *
273  * @ctx: pointer to the context structure
274  * @virt_addr: virt addr to map from
275  * @phys_addr: phys addr to map to
276  * @page_size: physical page size
277  * @flush_pte: whether to do a PCI flush
278  *
279  * This function does the following:
280  * - Check that the virt addr is not mapped
281  * - Allocate pgts as necessary in order to map the virt addr to the phys
282  * - Returns 0 on success, -EINVAL if addr is already mapped, or -ENOMEM.
283  *
284  * Because this function changes the page tables in the device and because it
285  * changes the MMU hash, it must be protected by a lock.
286  * However, because it maps only a single page, the lock should be implemented
287  * in a higher level in order to protect the entire mapping of the memory area
288  *
289  * For optimization reasons PCI flush may be requested once after mapping of
290  * large area.
291  */
hl_mmu_map_page(struct hl_ctx * ctx,u64 virt_addr,u64 phys_addr,u32 page_size,bool flush_pte)292 int hl_mmu_map_page(struct hl_ctx *ctx, u64 virt_addr, u64 phys_addr, u32 page_size,
293 			bool flush_pte)
294 {
295 	int i, rc, pgt_residency, mapped_cnt = 0;
296 	struct hl_device *hdev = ctx->hdev;
297 	struct hl_mmu_properties *mmu_prop;
298 	u64 real_virt_addr, real_phys_addr;
299 	struct hl_mmu_funcs *mmu_funcs;
300 	u32 real_page_size, npages;
301 	bool is_dram_addr;
302 
303 
304 	if (hdev->mmu_disable)
305 		return 0;
306 
307 	is_dram_addr = hl_is_dram_va(hdev, virt_addr);
308 	mmu_prop = hl_mmu_get_prop(hdev, page_size, is_dram_addr);
309 
310 	pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT;
311 	mmu_funcs = hl_mmu_get_funcs(hdev, pgt_residency, is_dram_addr);
312 
313 	rc = hdev->asic_funcs->mmu_get_real_page_size(hdev, mmu_prop, page_size, &real_page_size,
314 							is_dram_addr);
315 	if (rc)
316 		return rc;
317 
318 	/*
319 	 * Verify that the phys and virt addresses are aligned with the
320 	 * MMU page size (in dram this means checking the address and MMU
321 	 * after scrambling)
322 	 */
323 	if ((is_dram_addr &&
324 			((hdev->asic_funcs->scramble_addr(hdev, phys_addr) &
325 				(mmu_prop->page_size - 1)) ||
326 			(hdev->asic_funcs->scramble_addr(hdev, virt_addr) &
327 				(mmu_prop->page_size - 1)))) ||
328 		(!is_dram_addr && ((phys_addr & (real_page_size - 1)) ||
329 				(virt_addr & (real_page_size - 1)))))
330 		dev_crit(hdev->dev,
331 			"Mapping address 0x%llx with virtual address 0x%llx and page size of 0x%x is erroneous! Addresses must be divisible by page size",
332 			phys_addr, virt_addr, real_page_size);
333 
334 	npages = page_size / real_page_size;
335 	real_virt_addr = virt_addr;
336 	real_phys_addr = phys_addr;
337 
338 	for (i = 0 ; i < npages ; i++) {
339 		rc = mmu_funcs->map(ctx, real_virt_addr, real_phys_addr, real_page_size,
340 										is_dram_addr);
341 		if (rc)
342 			goto err;
343 
344 		real_virt_addr += real_page_size;
345 		real_phys_addr += real_page_size;
346 		mapped_cnt++;
347 	}
348 
349 	if (flush_pte)
350 		mmu_funcs->flush(ctx);
351 
352 	trace_habanalabs_mmu_map(hdev->dev, virt_addr, phys_addr, page_size, flush_pte);
353 
354 	return 0;
355 
356 err:
357 	real_virt_addr = virt_addr;
358 	for (i = 0 ; i < mapped_cnt ; i++) {
359 		if (mmu_funcs->unmap(ctx, real_virt_addr, is_dram_addr))
360 			dev_warn_ratelimited(hdev->dev,
361 				"failed to unmap va: 0x%llx\n", real_virt_addr);
362 
363 		real_virt_addr += real_page_size;
364 	}
365 
366 	mmu_funcs->flush(ctx);
367 
368 	return rc;
369 }
370 
371 /*
372  * hl_mmu_map_contiguous - implements a wrapper for hl_mmu_map_page
373  *                         for mapping contiguous physical memory
374  *
375  * @ctx: pointer to the context structure
376  * @virt_addr: virt addr to map from
377  * @phys_addr: phys addr to map to
378  * @size: size to map
379  *
380  */
hl_mmu_map_contiguous(struct hl_ctx * ctx,u64 virt_addr,u64 phys_addr,u32 size)381 int hl_mmu_map_contiguous(struct hl_ctx *ctx, u64 virt_addr,
382 					u64 phys_addr, u32 size)
383 {
384 	struct hl_device *hdev = ctx->hdev;
385 	struct asic_fixed_properties *prop = &hdev->asic_prop;
386 	u64 curr_va, curr_pa;
387 	u32 page_size;
388 	bool flush_pte;
389 	int rc = 0, off;
390 
391 	if (hl_mem_area_inside_range(virt_addr, size,
392 			prop->dmmu.start_addr, prop->dmmu.end_addr))
393 		page_size = prop->dmmu.page_size;
394 	else if (hl_mem_area_inside_range(virt_addr, size,
395 			prop->pmmu.start_addr, prop->pmmu.end_addr))
396 		page_size = prop->pmmu.page_size;
397 	else if (hl_mem_area_inside_range(virt_addr, size,
398 			prop->pmmu_huge.start_addr, prop->pmmu_huge.end_addr))
399 		page_size = prop->pmmu_huge.page_size;
400 	else
401 		return -EINVAL;
402 
403 	for (off = 0 ; off < size ; off += page_size) {
404 		curr_va = virt_addr + off;
405 		curr_pa = phys_addr + off;
406 		flush_pte = (off + page_size) >= size;
407 		rc = hl_mmu_map_page(ctx, curr_va, curr_pa, page_size,
408 								flush_pte);
409 		if (rc) {
410 			dev_err(hdev->dev,
411 				"Map failed for va 0x%llx to pa 0x%llx\n",
412 				curr_va, curr_pa);
413 			/* last mapping failed so don't try to unmap it - reduce off by page_size */
414 			off -= page_size;
415 			goto unmap;
416 		}
417 	}
418 
419 	return rc;
420 
421 unmap:
422 	for (; off >= 0 ; off -= page_size) {
423 		curr_va = virt_addr + off;
424 		flush_pte = (off - (s32) page_size) < 0;
425 		if (hl_mmu_unmap_page(ctx, curr_va, page_size, flush_pte))
426 			dev_warn_ratelimited(hdev->dev,
427 				"failed to unmap va 0x%llx\n", curr_va);
428 	}
429 
430 	return rc;
431 }
432 
433 /*
434  * hl_mmu_unmap_contiguous - implements a wrapper for hl_mmu_unmap_page
435  *                           for unmapping contiguous physical memory
436  *
437  * @ctx: pointer to the context structure
438  * @virt_addr: virt addr to unmap
439  * @size: size to unmap
440  *
441  */
hl_mmu_unmap_contiguous(struct hl_ctx * ctx,u64 virt_addr,u32 size)442 int hl_mmu_unmap_contiguous(struct hl_ctx *ctx, u64 virt_addr, u32 size)
443 {
444 	struct hl_device *hdev = ctx->hdev;
445 	struct asic_fixed_properties *prop = &hdev->asic_prop;
446 	u64 curr_va;
447 	u32 page_size;
448 	bool flush_pte;
449 	int rc = 0, off;
450 
451 	if (hl_mem_area_inside_range(virt_addr, size,
452 			prop->dmmu.start_addr, prop->dmmu.end_addr))
453 		page_size = prop->dmmu.page_size;
454 	else if (hl_mem_area_inside_range(virt_addr, size,
455 			prop->pmmu.start_addr, prop->pmmu.end_addr))
456 		page_size = prop->pmmu.page_size;
457 	else if (hl_mem_area_inside_range(virt_addr, size,
458 			prop->pmmu_huge.start_addr, prop->pmmu_huge.end_addr))
459 		page_size = prop->pmmu_huge.page_size;
460 	else
461 		return -EINVAL;
462 
463 	for (off = 0 ; off < size ; off += page_size) {
464 		curr_va = virt_addr + off;
465 		flush_pte = (off + page_size) >= size;
466 		rc = hl_mmu_unmap_page(ctx, curr_va, page_size, flush_pte);
467 		if (rc)
468 			dev_warn_ratelimited(hdev->dev,
469 				"Unmap failed for va 0x%llx\n", curr_va);
470 	}
471 
472 	return rc;
473 }
474 
hl_mmu_pa_page_with_offset(struct hl_ctx * ctx,u64 virt_addr,struct hl_mmu_hop_info * hops,u64 * phys_addr)475 static void hl_mmu_pa_page_with_offset(struct hl_ctx *ctx, u64 virt_addr,
476 						struct hl_mmu_hop_info *hops,
477 						u64 *phys_addr)
478 {
479 	struct asic_fixed_properties *prop = &ctx->hdev->asic_prop;
480 	u64 offset_mask, addr_mask, hop_shift, tmp_phys_addr;
481 	struct hl_mmu_properties *mmu_prop;
482 
483 	/* last hop holds the phys address and flags */
484 	if (hops->unscrambled_paddr)
485 		tmp_phys_addr = hops->unscrambled_paddr;
486 	else
487 		tmp_phys_addr = hops->hop_info[hops->used_hops - 1].hop_pte_val;
488 
489 	if (hops->range_type == HL_VA_RANGE_TYPE_HOST_HUGE)
490 		mmu_prop = &prop->pmmu_huge;
491 	else if (hops->range_type == HL_VA_RANGE_TYPE_HOST)
492 		mmu_prop = &prop->pmmu;
493 	else /* HL_VA_RANGE_TYPE_DRAM */
494 		mmu_prop = &prop->dmmu;
495 
496 	if ((hops->range_type == HL_VA_RANGE_TYPE_DRAM) &&
497 			!is_power_of_2(prop->dram_page_size)) {
498 		u64 dram_page_size, dram_base, abs_phys_addr, abs_virt_addr,
499 			page_id, page_start;
500 		u32 page_off;
501 
502 		/*
503 		 * Bit arithmetic cannot be used for non power of two page
504 		 * sizes. In addition, since bit arithmetic is not used,
505 		 * we cannot ignore dram base. All that shall be considered.
506 		 */
507 
508 		dram_page_size = prop->dram_page_size;
509 		dram_base = prop->dram_base_address;
510 		abs_phys_addr = tmp_phys_addr - dram_base;
511 		abs_virt_addr = virt_addr - dram_base;
512 		page_id = DIV_ROUND_DOWN_ULL(abs_phys_addr, dram_page_size);
513 		page_start = page_id * dram_page_size;
514 		div_u64_rem(abs_virt_addr, dram_page_size, &page_off);
515 
516 		*phys_addr = page_start + page_off + dram_base;
517 	} else {
518 		/*
519 		 * find the correct hop shift field in hl_mmu_properties
520 		 * structure in order to determine the right masks
521 		 * for the page offset.
522 		 */
523 		hop_shift = mmu_prop->hop_shifts[hops->used_hops - 1];
524 		offset_mask = (1ull << hop_shift) - 1;
525 		addr_mask = ~(offset_mask);
526 		*phys_addr = (tmp_phys_addr & addr_mask) |
527 				(virt_addr & offset_mask);
528 	}
529 }
530 
hl_mmu_va_to_pa(struct hl_ctx * ctx,u64 virt_addr,u64 * phys_addr)531 int hl_mmu_va_to_pa(struct hl_ctx *ctx, u64 virt_addr, u64 *phys_addr)
532 {
533 	struct hl_mmu_hop_info hops;
534 	int rc;
535 
536 	memset(&hops, 0, sizeof(hops));
537 
538 	rc = hl_mmu_get_tlb_info(ctx, virt_addr, &hops);
539 	if (rc)
540 		return rc;
541 
542 	hl_mmu_pa_page_with_offset(ctx, virt_addr, &hops,  phys_addr);
543 
544 	return 0;
545 }
546 
hl_mmu_get_tlb_info(struct hl_ctx * ctx,u64 virt_addr,struct hl_mmu_hop_info * hops)547 int hl_mmu_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr,
548 			struct hl_mmu_hop_info *hops)
549 {
550 	struct hl_device *hdev = ctx->hdev;
551 	struct asic_fixed_properties *prop;
552 	struct hl_mmu_properties *mmu_prop;
553 	struct hl_mmu_funcs *mmu_funcs;
554 	int pgt_residency, rc;
555 	bool is_dram_addr;
556 
557 	if (hdev->mmu_disable)
558 		return -EOPNOTSUPP;
559 
560 	prop = &hdev->asic_prop;
561 	hops->scrambled_vaddr = virt_addr;      /* assume no scrambling */
562 
563 	is_dram_addr = hl_mem_area_inside_range(virt_addr, prop->dmmu.page_size,
564 								prop->dmmu.start_addr,
565 								prop->dmmu.end_addr);
566 
567 	/* host-residency is the same in PMMU and PMMU huge, no need to distinguish here */
568 	mmu_prop = is_dram_addr ? &prop->dmmu : &prop->pmmu;
569 	pgt_residency = mmu_prop->host_resident ? MMU_HR_PGT : MMU_DR_PGT;
570 	mmu_funcs = hl_mmu_get_funcs(hdev, pgt_residency, is_dram_addr);
571 
572 	mutex_lock(&hdev->mmu_lock);
573 	rc = mmu_funcs->get_tlb_info(ctx, virt_addr, hops);
574 	mutex_unlock(&hdev->mmu_lock);
575 
576 	if (rc)
577 		return rc;
578 
579 	/* add page offset to physical address */
580 	if (hops->unscrambled_paddr)
581 		hl_mmu_pa_page_with_offset(ctx, virt_addr, hops, &hops->unscrambled_paddr);
582 
583 	return 0;
584 }
585 
hl_mmu_if_set_funcs(struct hl_device * hdev)586 int hl_mmu_if_set_funcs(struct hl_device *hdev)
587 {
588 	if (hdev->mmu_disable)
589 		return 0;
590 
591 	switch (hdev->asic_type) {
592 	case ASIC_GOYA:
593 	case ASIC_GAUDI:
594 	case ASIC_GAUDI_SEC:
595 		hl_mmu_v1_set_funcs(hdev, &hdev->mmu_func[MMU_DR_PGT]);
596 		break;
597 	case ASIC_GAUDI2:
598 	case ASIC_GAUDI2B:
599 	case ASIC_GAUDI2C:
600 		/* MMUs in Gaudi2 are always host resident */
601 		hl_mmu_v2_hr_set_funcs(hdev, &hdev->mmu_func[MMU_HR_PGT]);
602 		break;
603 	default:
604 		dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
605 			hdev->asic_type);
606 		return -EOPNOTSUPP;
607 	}
608 
609 	return 0;
610 }
611 
612 /**
613  * hl_mmu_scramble_addr() - The generic mmu address scrambling routine.
614  * @hdev: pointer to device data.
615  * @addr: The address to scramble.
616  *
617  * Return: The scrambled address.
618  */
hl_mmu_scramble_addr(struct hl_device * hdev,u64 addr)619 u64 hl_mmu_scramble_addr(struct hl_device *hdev, u64 addr)
620 {
621 	return addr;
622 }
623 
624 /**
625  * hl_mmu_descramble_addr() - The generic mmu address descrambling
626  * routine.
627  * @hdev: pointer to device data.
628  * @addr: The address to descramble.
629  *
630  * Return: The un-scrambled address.
631  */
hl_mmu_descramble_addr(struct hl_device * hdev,u64 addr)632 u64 hl_mmu_descramble_addr(struct hl_device *hdev, u64 addr)
633 {
634 	return addr;
635 }
636 
hl_mmu_invalidate_cache(struct hl_device * hdev,bool is_hard,u32 flags)637 int hl_mmu_invalidate_cache(struct hl_device *hdev, bool is_hard, u32 flags)
638 {
639 	int rc;
640 
641 	rc = hdev->asic_funcs->mmu_invalidate_cache(hdev, is_hard, flags);
642 	if (rc)
643 		dev_err_ratelimited(hdev->dev,
644 				"%s cache invalidation failed, rc=%d\n",
645 				flags == VM_TYPE_USERPTR ? "PMMU" : "HMMU", rc);
646 
647 	return rc;
648 }
649 
hl_mmu_invalidate_cache_range(struct hl_device * hdev,bool is_hard,u32 flags,u32 asid,u64 va,u64 size)650 int hl_mmu_invalidate_cache_range(struct hl_device *hdev, bool is_hard,
651 					u32 flags, u32 asid, u64 va, u64 size)
652 {
653 	int rc;
654 
655 	rc = hdev->asic_funcs->mmu_invalidate_cache_range(hdev, is_hard, flags,
656 								asid, va, size);
657 	if (rc)
658 		dev_err_ratelimited(hdev->dev,
659 				"%s cache range invalidation failed: va=%#llx, size=%llu, rc=%d",
660 				flags == VM_TYPE_USERPTR ? "PMMU" : "HMMU", va, size, rc);
661 
662 	return rc;
663 }
664 
hl_mmu_prefetch_work_function(struct work_struct * work)665 static void hl_mmu_prefetch_work_function(struct work_struct *work)
666 {
667 	struct hl_prefetch_work *pfw = container_of(work, struct hl_prefetch_work, prefetch_work);
668 	struct hl_ctx *ctx = pfw->ctx;
669 	struct hl_device *hdev = ctx->hdev;
670 
671 	if (!hl_device_operational(hdev, NULL))
672 		goto put_ctx;
673 
674 	mutex_lock(&hdev->mmu_lock);
675 
676 	hdev->asic_funcs->mmu_prefetch_cache_range(ctx, pfw->flags, pfw->asid, pfw->va, pfw->size);
677 
678 	mutex_unlock(&hdev->mmu_lock);
679 
680 put_ctx:
681 	/*
682 	 * context was taken in the common mmu prefetch function- see comment there about
683 	 * context handling.
684 	 */
685 	hl_ctx_put(ctx);
686 	kfree(pfw);
687 }
688 
hl_mmu_prefetch_cache_range(struct hl_ctx * ctx,u32 flags,u32 asid,u64 va,u64 size)689 int hl_mmu_prefetch_cache_range(struct hl_ctx *ctx, u32 flags, u32 asid, u64 va, u64 size)
690 {
691 	struct hl_prefetch_work *handle_prefetch_work;
692 
693 	handle_prefetch_work = kmalloc(sizeof(*handle_prefetch_work), GFP_KERNEL);
694 	if (!handle_prefetch_work)
695 		return -ENOMEM;
696 
697 	INIT_WORK(&handle_prefetch_work->prefetch_work, hl_mmu_prefetch_work_function);
698 	handle_prefetch_work->ctx = ctx;
699 	handle_prefetch_work->va = va;
700 	handle_prefetch_work->size = size;
701 	handle_prefetch_work->flags = flags;
702 	handle_prefetch_work->asid = asid;
703 
704 	/*
705 	 * as actual prefetch is done in a WQ we must get the context (and put it
706 	 * at the end of the work function)
707 	 */
708 	hl_ctx_get(ctx);
709 	queue_work(ctx->hdev->prefetch_wq, &handle_prefetch_work->prefetch_work);
710 
711 	return 0;
712 }
713 
hl_mmu_get_next_hop_addr(struct hl_ctx * ctx,u64 curr_pte)714 u64 hl_mmu_get_next_hop_addr(struct hl_ctx *ctx, u64 curr_pte)
715 {
716 	return (curr_pte & PAGE_PRESENT_MASK) ? (curr_pte & HOP_PHYS_ADDR_MASK) : ULLONG_MAX;
717 }
718 
719 /**
720  * hl_mmu_get_hop_pte_phys_addr() - extract PTE address from HOP
721  * @ctx: pointer to the context structure to initialize.
722  * @mmu_prop: MMU properties.
723  * @hop_idx: HOP index.
724  * @hop_addr: HOP address.
725  * @virt_addr: virtual address for the translation.
726  *
727  * @return the matching PTE value on success, otherwise U64_MAX.
728  */
hl_mmu_get_hop_pte_phys_addr(struct hl_ctx * ctx,struct hl_mmu_properties * mmu_prop,u8 hop_idx,u64 hop_addr,u64 virt_addr)729 u64 hl_mmu_get_hop_pte_phys_addr(struct hl_ctx *ctx, struct hl_mmu_properties *mmu_prop,
730 					u8 hop_idx, u64 hop_addr, u64 virt_addr)
731 {
732 	u64 mask, shift;
733 
734 	if (hop_idx >= mmu_prop->num_hops) {
735 		dev_err_ratelimited(ctx->hdev->dev, "Invalid hop index %d\n", hop_idx);
736 		return U64_MAX;
737 	}
738 
739 	shift = mmu_prop->hop_shifts[hop_idx];
740 	mask = mmu_prop->hop_masks[hop_idx];
741 
742 	return hop_addr + ctx->hdev->asic_prop.mmu_pte_size * ((virt_addr & mask) >> shift);
743 }
744 
mmu_dma_mem_free_from_chunk(struct gen_pool * pool,struct gen_pool_chunk * chunk,void * data)745 static void mmu_dma_mem_free_from_chunk(struct gen_pool *pool,
746 					struct gen_pool_chunk *chunk,
747 					void *data)
748 {
749 	struct hl_device *hdev = data;
750 
751 	hl_asic_dma_free_coherent(hdev, (chunk->end_addr - chunk->start_addr) + 1,
752 					(void *)chunk->start_addr, chunk->phys_addr);
753 }
754 
hl_mmu_hr_flush(struct hl_ctx * ctx)755 void hl_mmu_hr_flush(struct hl_ctx *ctx)
756 {
757 	/* a flush operation requires memory barrier */
758 	mb();
759 }
760 
761 /**
762  * hl_mmu_hr_pool_destroy() - destroy genpool
763  * @hdev: habanalabs device structure.
764  * @hr_priv: MMU HR private data.
765  * @hop_table_size: HOP table size.
766  *
767  * This function does the following:
768  * - free entries allocated for shadow HOP0
769  * - free pool chunks
770  * - free pool
771  */
hl_mmu_hr_pool_destroy(struct hl_device * hdev,struct hl_mmu_hr_priv * hr_priv,u32 hop_table_size)772 static void hl_mmu_hr_pool_destroy(struct hl_device *hdev, struct hl_mmu_hr_priv *hr_priv,
773 					u32 hop_table_size)
774 {
775 	struct asic_fixed_properties *prop = &hdev->asic_prop;
776 	struct gen_pool **pool = &hr_priv->mmu_pgt_pool;
777 	struct pgt_info *hop0_pgt;
778 	int asid;
779 
780 	if (ZERO_OR_NULL_PTR(*pool))
781 		return;
782 
783 	/* Free the Fixed allocation of HOPs0 */
784 	if (hr_priv->mmu_asid_hop0) {
785 		for (asid = 0 ; asid < prop->max_asid ; asid++) {
786 			hop0_pgt = &hr_priv->mmu_asid_hop0[asid];
787 			if (ZERO_OR_NULL_PTR(hop0_pgt->virt_addr))
788 				continue;
789 
790 			gen_pool_free(*pool, (uintptr_t) hop0_pgt->virt_addr, hop_table_size);
791 		}
792 	}
793 
794 	gen_pool_for_each_chunk(*pool, mmu_dma_mem_free_from_chunk, hdev);
795 	gen_pool_destroy(*pool);
796 
797 	/* Make sure that if we arrive here again without init was called we
798 	 * won't cause kernel panic. This can happen for example if we fail
799 	 * during hard reset code at certain points
800 	 */
801 	*pool = NULL;
802 }
803 
804 /**
805  * hl_mmu_hr_init() - initialize the MMU module.
806  * @hdev: habanalabs device structure.
807  * @hr_priv: MMU HR private data.
808  * @hop_table_size: HOP table size.
809  * @pgt_size: memory size allocated for the page table
810  *
811  * @return 0 on success otherwise non-zero error code
812  *
813  * This function does the following:
814  * - Create a pool of pages for pgt_infos.
815  * - Create a shadow table for pgt
816  */
hl_mmu_hr_init(struct hl_device * hdev,struct hl_mmu_hr_priv * hr_priv,u32 hop_table_size,u64 pgt_size)817 int hl_mmu_hr_init(struct hl_device *hdev, struct hl_mmu_hr_priv *hr_priv, u32 hop_table_size,
818 			u64 pgt_size)
819 {
820 	struct asic_fixed_properties *prop = &hdev->asic_prop;
821 	size_t pool_chunk_size = SZ_4M;
822 	struct pgt_info *hop0_pgt;
823 	dma_addr_t dma_addr;
824 	u64 virt_addr;
825 	int i, rc;
826 
827 	/*
828 	 * we set alloc size as PAGE_SIZE (sine dma_alloc_coherent allocation order/size is
829 	 * PAGE_SHIFT/PAGE_SIZE) in order to be able to control the allocations alignment.
830 	 * This way we can call "DMA alloc align" according to dma_alloc granularity and supply
831 	 * allocations with higher-order alignment restrictions
832 	 */
833 	hr_priv->mmu_pgt_pool = gen_pool_create(PAGE_SHIFT, -1);
834 	if (ZERO_OR_NULL_PTR(hr_priv->mmu_pgt_pool)) {
835 		dev_err(hdev->dev, "Failed to create hr page pool\n");
836 		return -ENOMEM;
837 	}
838 
839 	hr_priv->mmu_asid_hop0 = kvcalloc(prop->max_asid, sizeof(struct pgt_info), GFP_KERNEL);
840 	if (ZERO_OR_NULL_PTR(hr_priv->mmu_asid_hop0)) {
841 		dev_err(hdev->dev, "Failed to allocate hr-mmu hop0 table\n");
842 		rc = -ENOMEM;
843 		goto destroy_mmu_pgt_pool;
844 	}
845 
846 	for (i = 0 ; i < pgt_size ; i += pool_chunk_size) {
847 		virt_addr = (uintptr_t) hl_asic_dma_alloc_coherent(hdev, pool_chunk_size,
848 									&dma_addr,
849 									GFP_KERNEL | __GFP_ZERO);
850 		if (ZERO_OR_NULL_PTR(virt_addr)) {
851 			dev_err(hdev->dev,
852 				"Failed to allocate memory for host-resident page pool\n");
853 			rc = -ENOMEM;
854 			goto destroy_mmu_pgt_pool;
855 		}
856 
857 		rc = gen_pool_add_virt(hr_priv->mmu_pgt_pool, virt_addr, (phys_addr_t) dma_addr,
858 						pool_chunk_size, -1);
859 		if (rc) {
860 			dev_err(hdev->dev, "Failed to fill host-resident page pool\n");
861 			goto destroy_mmu_pgt_pool;
862 		}
863 	}
864 
865 	for (i = 0 ; i < prop->max_asid ; i++) {
866 		hop0_pgt = &hr_priv->mmu_asid_hop0[i];
867 		hop0_pgt->virt_addr = (uintptr_t)
868 					gen_pool_dma_zalloc_align(hr_priv->mmu_pgt_pool,
869 								hop_table_size,
870 								(dma_addr_t *) &hop0_pgt->phys_addr,
871 								hop_table_size);
872 		if (!hop0_pgt->virt_addr) {
873 			dev_err(hdev->dev, "Failed to allocate HOP from pgt pool\n");
874 			rc = -ENOMEM;
875 			goto destroy_mmu_pgt_pool;
876 		}
877 	}
878 
879 	/* MMU H/W init will be done in device hw_init() */
880 
881 	return 0;
882 
883 destroy_mmu_pgt_pool:
884 	hl_mmu_hr_pool_destroy(hdev, hr_priv, hop_table_size);
885 	if (!ZERO_OR_NULL_PTR(hr_priv->mmu_asid_hop0))
886 		kvfree(hr_priv->mmu_asid_hop0);
887 
888 	return rc;
889 }
890 
891 /**
892  * hl_mmu_hr_fini() - release the MMU module.
893  * @hdev: habanalabs device structure.
894  * @hr_priv: MMU host resident private info.
895  * @hop_table_size: HOP table size
896  *
897  * This function does the following:
898  * - Disable MMU in H/W.
899  * - Free the pgt_infos pool.
900  *
901  * All contexts should be freed before calling this function.
902  */
hl_mmu_hr_fini(struct hl_device * hdev,struct hl_mmu_hr_priv * hr_priv,u32 hop_table_size)903 void hl_mmu_hr_fini(struct hl_device *hdev, struct hl_mmu_hr_priv *hr_priv, u32 hop_table_size)
904 {
905 	/* MMU H/W fini was already done in device hw_fini() */
906 
907 	hl_mmu_hr_pool_destroy(hdev, hr_priv, hop_table_size);
908 
909 	if (!ZERO_OR_NULL_PTR(hr_priv->mmu_asid_hop0)) {
910 		kvfree(hr_priv->mmu_asid_hop0);
911 
912 		/* Make sure that if we arrive here again without init was
913 		 * called we won't cause kernel panic. This can happen for
914 		 * example if we fail during hard reset code at certain points
915 		 */
916 		hr_priv->mmu_asid_hop0 = NULL;
917 	}
918 }
919 
920 /**
921  * hl_mmu_hr_free_hop_remove_pgt() - free HOP and remove PGT from hash
922  * @pgt_info: page table info structure.
923  * @hr_priv: MMU HR private data.
924  * @hop_table_size: HOP table size.
925  */
hl_mmu_hr_free_hop_remove_pgt(struct pgt_info * pgt_info,struct hl_mmu_hr_priv * hr_priv,u32 hop_table_size)926 void hl_mmu_hr_free_hop_remove_pgt(struct pgt_info *pgt_info, struct hl_mmu_hr_priv *hr_priv,
927 					u32 hop_table_size)
928 {
929 	gen_pool_free(hr_priv->mmu_pgt_pool, pgt_info->virt_addr, hop_table_size);
930 	hash_del(&pgt_info->node);
931 	kfree(pgt_info);
932 }
933 
934 /**
935  * hl_mmu_hr_pte_phys_to_virt() - translate PTE phys addr to virt addr
936  * @ctx: pointer to the context structure
937  * @pgt: pgt_info for the HOP hosting the PTE
938  * @phys_pte_addr: phys address of the PTE
939  * @hop_table_size: HOP table size
940  *
941  * @return PTE virtual address
942  *
943  * The function use the pgt_info to get HOP base virt addr and obtain the PTE's virt addr
944  * by adding the PTE offset.
945  */
hl_mmu_hr_pte_phys_to_virt(struct hl_ctx * ctx,struct pgt_info * pgt,u64 phys_pte_addr,u32 hop_table_size)946 u64 hl_mmu_hr_pte_phys_to_virt(struct hl_ctx *ctx, struct pgt_info *pgt,
947 							u64 phys_pte_addr, u32 hop_table_size)
948 {
949 	u64 page_mask = (hop_table_size - 1);
950 	u64 pte_offset = phys_pte_addr & page_mask;
951 
952 	return pgt->virt_addr + pte_offset;
953 }
954 
955 /**
956  * hl_mmu_hr_write_pte() - write HR PTE
957  * @ctx: pointer to the context structure
958  * @pgt_info: HOP's page table info structure
959  * @phys_pte_addr: phys PTE address
960  * @val: raw PTE data
961  * @hop_table_size: HOP table size
962  */
hl_mmu_hr_write_pte(struct hl_ctx * ctx,struct pgt_info * pgt_info,u64 phys_pte_addr,u64 val,u32 hop_table_size)963 void hl_mmu_hr_write_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, u64 phys_pte_addr,
964 								u64 val, u32 hop_table_size)
965 {
966 	/*
967 	 * The value to write is the phys address of the next hop +
968 	 * flags at the 12 LSBs.
969 	 */
970 	u64 virt_addr = hl_mmu_hr_pte_phys_to_virt(ctx, pgt_info, phys_pte_addr, hop_table_size);
971 
972 	*((u64 *) (uintptr_t) virt_addr) = val;
973 }
974 
975 /**
976  * hl_mmu_hr_clear_pte() - clear HR PTE
977  * @ctx: pointer to the context structure
978  * @pgt_info: HOP's page table info structure
979  * @phys_pte_addr: phys PTE address
980  * @hop_table_size: HOP table size
981  */
hl_mmu_hr_clear_pte(struct hl_ctx * ctx,struct pgt_info * pgt_info,u64 phys_pte_addr,u32 hop_table_size)982 void hl_mmu_hr_clear_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info, u64 phys_pte_addr,
983 						u32 hop_table_size)
984 {
985 	/* no need to transform the value to physical address */
986 	hl_mmu_hr_write_pte(ctx, pgt_info, phys_pte_addr, 0, hop_table_size);
987 }
988 
989 /**
990  * hl_mmu_hr_put_pte() - put HR PTE and remove it if necessary (no more PTEs)
991  * @ctx: pointer to the context structure
992  * @pgt_info: HOP's page table info structure
993  * @hr_priv: HR MMU private info
994  * @hop_table_size: HOP table size
995  *
996  * @return number of PTEs still in the HOP
997  */
hl_mmu_hr_put_pte(struct hl_ctx * ctx,struct pgt_info * pgt_info,struct hl_mmu_hr_priv * hr_priv,u32 hop_table_size)998 int hl_mmu_hr_put_pte(struct hl_ctx *ctx, struct pgt_info *pgt_info,
999 						struct hl_mmu_hr_priv *hr_priv,
1000 						u32 hop_table_size)
1001 {
1002 	int num_of_ptes_left;
1003 
1004 	pgt_info->num_of_ptes--;
1005 
1006 	/*
1007 	 * Need to save the number of ptes left because free_hop might free
1008 	 * the pgt_info
1009 	 */
1010 	num_of_ptes_left = pgt_info->num_of_ptes;
1011 	if (!num_of_ptes_left)
1012 		hl_mmu_hr_free_hop_remove_pgt(pgt_info, hr_priv, hop_table_size);
1013 
1014 	return num_of_ptes_left;
1015 }
1016 
1017 /**
1018  * hl_mmu_hr_get_pte() - increase PGT PTE count
1019  * @ctx: pointer to the context structure
1020  * @hr_func: host resident functions
1021  * @phys_hop_addr: HOP phys address
1022  */
hl_mmu_hr_get_pte(struct hl_ctx * ctx,struct hl_hr_mmu_funcs * hr_func,u64 phys_hop_addr)1023 void hl_mmu_hr_get_pte(struct hl_ctx *ctx, struct hl_hr_mmu_funcs *hr_func, u64 phys_hop_addr)
1024 {
1025 	hr_func->get_pgt_info(ctx, phys_hop_addr)->num_of_ptes++;
1026 }
1027 
1028 /**
1029  * hl_mmu_hr_get_next_hop_pgt_info() - get pgt_info structure for the next HOP
1030  * @ctx: pointer to the context structure.
1031  * @hr_func: host resident functions.
1032  * @curr_pte: current PTE value.
1033  *
1034  * @return pgt_info structure on success, otherwise NULL.
1035  */
hl_mmu_hr_get_next_hop_pgt_info(struct hl_ctx * ctx,struct hl_hr_mmu_funcs * hr_func,u64 curr_pte)1036 struct pgt_info *hl_mmu_hr_get_next_hop_pgt_info(struct hl_ctx *ctx,
1037 							struct hl_hr_mmu_funcs *hr_func,
1038 							u64 curr_pte)
1039 {
1040 	u64 next_hop_phys_addr = hl_mmu_get_next_hop_addr(ctx, curr_pte);
1041 
1042 	if (next_hop_phys_addr == ULLONG_MAX)
1043 		return NULL;
1044 
1045 	return hr_func->get_pgt_info(ctx, next_hop_phys_addr);
1046 }
1047 
1048 /**
1049  * hl_mmu_hr_alloc_hop() - allocate HOP
1050  * @ctx: pointer to the context structure.
1051  * @hr_priv: host resident private info structure.
1052  * @hr_func: host resident functions.
1053  * @mmu_prop: MMU properties.
1054  *
1055  * @return pgt_info structure associated with the allocated HOP on success, otherwise NULL.
1056  */
hl_mmu_hr_alloc_hop(struct hl_ctx * ctx,struct hl_mmu_hr_priv * hr_priv,struct hl_hr_mmu_funcs * hr_func,struct hl_mmu_properties * mmu_prop)1057 struct pgt_info *hl_mmu_hr_alloc_hop(struct hl_ctx *ctx, struct hl_mmu_hr_priv *hr_priv,
1058 							struct hl_hr_mmu_funcs *hr_func,
1059 							struct hl_mmu_properties *mmu_prop)
1060 {
1061 	struct hl_device *hdev = ctx->hdev;
1062 	struct pgt_info *pgt_info;
1063 	dma_addr_t phys_addr;
1064 	void *virt_addr;
1065 	int i, retry = 1;
1066 
1067 	pgt_info = kmalloc(sizeof(*pgt_info), GFP_KERNEL);
1068 	if (!pgt_info)
1069 		return NULL;
1070 
1071 	for (i = 0; i <= retry; i++) {
1072 		virt_addr = gen_pool_dma_zalloc_align(hr_priv->mmu_pgt_pool,
1073 							mmu_prop->hop_table_size,
1074 							&phys_addr,
1075 							mmu_prop->hop_table_size);
1076 		if (virt_addr)
1077 			break;
1078 
1079 		/* No memory in pool - get some and try again */
1080 		virt_addr = hl_asic_dma_alloc_coherent(hdev, SZ_2M, &phys_addr,
1081 							GFP_KERNEL | __GFP_ZERO);
1082 		if (ZERO_OR_NULL_PTR(virt_addr))
1083 			break;
1084 
1085 		if (gen_pool_add_virt(hr_priv->mmu_pgt_pool, (unsigned long)virt_addr,
1086 								phys_addr, SZ_2M, -1)) {
1087 			hl_asic_dma_free_coherent(hdev, SZ_2M, virt_addr, phys_addr);
1088 			virt_addr = NULL;
1089 			break;
1090 		}
1091 	}
1092 
1093 	if (ZERO_OR_NULL_PTR(virt_addr)) {
1094 		dev_err(hdev->dev, "failed to allocate page\n");
1095 		goto pool_alloc_err;
1096 	}
1097 
1098 	pgt_info->phys_addr = phys_addr;
1099 	pgt_info->shadow_addr = (unsigned long) NULL;
1100 	pgt_info->virt_addr = (unsigned long)virt_addr;
1101 	pgt_info->ctx = ctx;
1102 	pgt_info->num_of_ptes = 0;
1103 	hr_func->add_pgt_info(ctx, pgt_info, phys_addr);
1104 
1105 	return pgt_info;
1106 
1107 pool_alloc_err:
1108 	kfree(pgt_info);
1109 
1110 	return NULL;
1111 }
1112 
1113 /**
1114  * hl_mmu_hr_get_alloc_next_hop() - get the next HOP, allocate it if it does not exist
1115  * @ctx: pointer to the context structure.
1116  * @hr_priv: host resident private info structure.
1117  * @hr_func: host resident functions.
1118  * @mmu_prop: MMU properties.
1119  * @curr_pte: current PTE value.
1120  * @is_new_hop: set to true if HOP is new (caller responsibility to set it to false).
1121  *
1122  * @return pgt_info structure associated with the allocated HOP on success, otherwise NULL.
1123  */
hl_mmu_hr_get_alloc_next_hop(struct hl_ctx * ctx,struct hl_mmu_hr_priv * hr_priv,struct hl_hr_mmu_funcs * hr_func,struct hl_mmu_properties * mmu_prop,u64 curr_pte,bool * is_new_hop)1124 struct pgt_info *hl_mmu_hr_get_alloc_next_hop(struct hl_ctx *ctx,
1125 							struct hl_mmu_hr_priv *hr_priv,
1126 							struct hl_hr_mmu_funcs *hr_func,
1127 							struct hl_mmu_properties *mmu_prop,
1128 							u64 curr_pte, bool *is_new_hop)
1129 {
1130 	u64 hop_addr = hl_mmu_get_next_hop_addr(ctx, curr_pte);
1131 
1132 	if (hop_addr != ULLONG_MAX)
1133 		return hr_func->get_pgt_info(ctx, hop_addr);
1134 
1135 	*is_new_hop = true;
1136 	return hl_mmu_hr_alloc_hop(ctx, hr_priv, hr_func, mmu_prop);
1137 }
1138 
1139 /**
1140  * hl_mmu_hr_get_tlb_info() - get the TLB info (info for a specific mapping)
1141  * @ctx: pointer to the context structure.
1142  * @virt_addr: the virt address for which to get info.
1143  * @hops: HOPs info structure.
1144  * @hr_func: host resident functions.
1145  *
1146  * @return 0 on success, otherwise non 0 error code..
1147  */
hl_mmu_hr_get_tlb_info(struct hl_ctx * ctx,u64 virt_addr,struct hl_mmu_hop_info * hops,struct hl_hr_mmu_funcs * hr_func)1148 int hl_mmu_hr_get_tlb_info(struct hl_ctx *ctx, u64 virt_addr, struct hl_mmu_hop_info *hops,
1149 								struct hl_hr_mmu_funcs *hr_func)
1150 {
1151 	/* using 6 HOPs as this is the maximum number of HOPs */
1152 	struct pgt_info *hops_pgt_info[MMU_ARCH_6_HOPS] = { NULL };
1153 	struct hl_device *hdev = ctx->hdev;
1154 	struct hl_mmu_properties *mmu_prop;
1155 	int rc, i, used_hops;
1156 	bool is_huge;
1157 
1158 	rc = hr_func->get_tlb_mapping_params(hdev, &mmu_prop, hops, virt_addr, &is_huge);
1159 	if (rc)
1160 		return rc;
1161 
1162 	used_hops = mmu_prop->num_hops;
1163 
1164 	/* huge pages use one less hop */
1165 	if (is_huge)
1166 		used_hops--;
1167 
1168 	hops->scrambled_vaddr = hdev->asic_funcs->scramble_addr(hdev, virt_addr);
1169 
1170 	for (i = 0 ; i < used_hops ; i++) {
1171 		if (i == 0)
1172 			hops_pgt_info[i] = hr_func->get_hop0_pgt_info(ctx);
1173 		else
1174 			hops_pgt_info[i] = hl_mmu_hr_get_next_hop_pgt_info(ctx, hr_func,
1175 								hops->hop_info[i - 1].hop_pte_val);
1176 
1177 		if (!hops_pgt_info[i])
1178 			return -EFAULT;
1179 
1180 		hops->hop_info[i].hop_addr = hops_pgt_info[i]->phys_addr;
1181 		hops->hop_info[i].hop_pte_addr =
1182 				hl_mmu_get_hop_pte_phys_addr(ctx, mmu_prop, i,
1183 								hops->hop_info[i].hop_addr,
1184 								hops->scrambled_vaddr);
1185 		hops->hop_info[i].hop_pte_val = *(u64 *) (uintptr_t)
1186 						hl_mmu_hr_pte_phys_to_virt(ctx, hops_pgt_info[i],
1187 								hops->hop_info[i].hop_pte_addr,
1188 								mmu_prop->hop_table_size);
1189 
1190 		if (!(hops->hop_info[i].hop_pte_val & PAGE_PRESENT_MASK))
1191 			return -EFAULT;
1192 
1193 		if (hops->hop_info[i].hop_pte_val & mmu_prop->last_mask)
1194 			break;
1195 	}
1196 
1197 	/* if passed over all hops then no last hop was found */
1198 	if (i == mmu_prop->num_hops)
1199 		return -EFAULT;
1200 
1201 	if (hops->scrambled_vaddr != virt_addr)
1202 		hops->unscrambled_paddr = hdev->asic_funcs->descramble_addr
1203 				(hdev, hops->hop_info[i].hop_pte_val);
1204 	else
1205 		hops->unscrambled_paddr = hops->hop_info[i].hop_pte_val;
1206 
1207 	hops->used_hops = i + 1;
1208 
1209 	return 0;
1210 }
1211 
1212