xref: /openbmc/linux/drivers/gpu/drm/i915/gvt/gtt.c (revision 6aa7de05)
1 /*
2  * GTT virtualization
3  *
4  * Copyright(c) 2011-2016 Intel Corporation. All rights reserved.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the "Software"),
8  * to deal in the Software without restriction, including without limitation
9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10  * and/or sell copies of the Software, and to permit persons to whom the
11  * Software is furnished to do so, subject to the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the next
14  * paragraph) shall be included in all copies or substantial portions of the
15  * Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
20  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23  * SOFTWARE.
24  *
25  * Authors:
26  *    Zhi Wang <zhi.a.wang@intel.com>
27  *    Zhenyu Wang <zhenyuw@linux.intel.com>
28  *    Xiao Zheng <xiao.zheng@intel.com>
29  *
30  * Contributors:
31  *    Min He <min.he@intel.com>
32  *    Bing Niu <bing.niu@intel.com>
33  *
34  */
35 
36 #include "i915_drv.h"
37 #include "gvt.h"
38 #include "i915_pvinfo.h"
39 #include "trace.h"
40 
41 static bool enable_out_of_sync = false;
42 static int preallocated_oos_pages = 8192;
43 
44 /*
45  * validate a gm address and related range size,
46  * translate it to host gm address
47  */
48 bool intel_gvt_ggtt_validate_range(struct intel_vgpu *vgpu, u64 addr, u32 size)
49 {
50 	if ((!vgpu_gmadr_is_valid(vgpu, addr)) || (size
51 			&& !vgpu_gmadr_is_valid(vgpu, addr + size - 1))) {
52 		gvt_vgpu_err("invalid range gmadr 0x%llx size 0x%x\n",
53 				addr, size);
54 		return false;
55 	}
56 	return true;
57 }
58 
59 /* translate a guest gmadr to host gmadr */
60 int intel_gvt_ggtt_gmadr_g2h(struct intel_vgpu *vgpu, u64 g_addr, u64 *h_addr)
61 {
62 	if (WARN(!vgpu_gmadr_is_valid(vgpu, g_addr),
63 		 "invalid guest gmadr %llx\n", g_addr))
64 		return -EACCES;
65 
66 	if (vgpu_gmadr_is_aperture(vgpu, g_addr))
67 		*h_addr = vgpu_aperture_gmadr_base(vgpu)
68 			  + (g_addr - vgpu_aperture_offset(vgpu));
69 	else
70 		*h_addr = vgpu_hidden_gmadr_base(vgpu)
71 			  + (g_addr - vgpu_hidden_offset(vgpu));
72 	return 0;
73 }
74 
75 /* translate a host gmadr to guest gmadr */
76 int intel_gvt_ggtt_gmadr_h2g(struct intel_vgpu *vgpu, u64 h_addr, u64 *g_addr)
77 {
78 	if (WARN(!gvt_gmadr_is_valid(vgpu->gvt, h_addr),
79 		 "invalid host gmadr %llx\n", h_addr))
80 		return -EACCES;
81 
82 	if (gvt_gmadr_is_aperture(vgpu->gvt, h_addr))
83 		*g_addr = vgpu_aperture_gmadr_base(vgpu)
84 			+ (h_addr - gvt_aperture_gmadr_base(vgpu->gvt));
85 	else
86 		*g_addr = vgpu_hidden_gmadr_base(vgpu)
87 			+ (h_addr - gvt_hidden_gmadr_base(vgpu->gvt));
88 	return 0;
89 }
90 
91 int intel_gvt_ggtt_index_g2h(struct intel_vgpu *vgpu, unsigned long g_index,
92 			     unsigned long *h_index)
93 {
94 	u64 h_addr;
95 	int ret;
96 
97 	ret = intel_gvt_ggtt_gmadr_g2h(vgpu, g_index << GTT_PAGE_SHIFT,
98 				       &h_addr);
99 	if (ret)
100 		return ret;
101 
102 	*h_index = h_addr >> GTT_PAGE_SHIFT;
103 	return 0;
104 }
105 
106 int intel_gvt_ggtt_h2g_index(struct intel_vgpu *vgpu, unsigned long h_index,
107 			     unsigned long *g_index)
108 {
109 	u64 g_addr;
110 	int ret;
111 
112 	ret = intel_gvt_ggtt_gmadr_h2g(vgpu, h_index << GTT_PAGE_SHIFT,
113 				       &g_addr);
114 	if (ret)
115 		return ret;
116 
117 	*g_index = g_addr >> GTT_PAGE_SHIFT;
118 	return 0;
119 }
120 
121 #define gtt_type_is_entry(type) \
122 	(type > GTT_TYPE_INVALID && type < GTT_TYPE_PPGTT_ENTRY \
123 	 && type != GTT_TYPE_PPGTT_PTE_ENTRY \
124 	 && type != GTT_TYPE_PPGTT_ROOT_ENTRY)
125 
126 #define gtt_type_is_pt(type) \
127 	(type >= GTT_TYPE_PPGTT_PTE_PT && type < GTT_TYPE_MAX)
128 
129 #define gtt_type_is_pte_pt(type) \
130 	(type == GTT_TYPE_PPGTT_PTE_PT)
131 
132 #define gtt_type_is_root_pointer(type) \
133 	(gtt_type_is_entry(type) && type > GTT_TYPE_PPGTT_ROOT_ENTRY)
134 
135 #define gtt_init_entry(e, t, p, v) do { \
136 	(e)->type = t; \
137 	(e)->pdev = p; \
138 	memcpy(&(e)->val64, &v, sizeof(v)); \
139 } while (0)
140 
141 /*
142  * Mappings between GTT_TYPE* enumerations.
143  * Following information can be found according to the given type:
144  * - type of next level page table
145  * - type of entry inside this level page table
146  * - type of entry with PSE set
147  *
148  * If the given type doesn't have such a kind of information,
149  * e.g. give a l4 root entry type, then request to get its PSE type,
150  * give a PTE page table type, then request to get its next level page
151  * table type, as we know l4 root entry doesn't have a PSE bit,
152  * and a PTE page table doesn't have a next level page table type,
153  * GTT_TYPE_INVALID will be returned. This is useful when traversing a
154  * page table.
155  */
156 
157 struct gtt_type_table_entry {
158 	int entry_type;
159 	int next_pt_type;
160 	int pse_entry_type;
161 };
162 
163 #define GTT_TYPE_TABLE_ENTRY(type, e_type, npt_type, pse_type) \
164 	[type] = { \
165 		.entry_type = e_type, \
166 		.next_pt_type = npt_type, \
167 		.pse_entry_type = pse_type, \
168 	}
169 
170 static struct gtt_type_table_entry gtt_type_table[] = {
171 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_ROOT_L4_ENTRY,
172 			GTT_TYPE_PPGTT_ROOT_L4_ENTRY,
173 			GTT_TYPE_PPGTT_PML4_PT,
174 			GTT_TYPE_INVALID),
175 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PML4_PT,
176 			GTT_TYPE_PPGTT_PML4_ENTRY,
177 			GTT_TYPE_PPGTT_PDP_PT,
178 			GTT_TYPE_INVALID),
179 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PML4_ENTRY,
180 			GTT_TYPE_PPGTT_PML4_ENTRY,
181 			GTT_TYPE_PPGTT_PDP_PT,
182 			GTT_TYPE_INVALID),
183 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDP_PT,
184 			GTT_TYPE_PPGTT_PDP_ENTRY,
185 			GTT_TYPE_PPGTT_PDE_PT,
186 			GTT_TYPE_PPGTT_PTE_1G_ENTRY),
187 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_ROOT_L3_ENTRY,
188 			GTT_TYPE_PPGTT_ROOT_L3_ENTRY,
189 			GTT_TYPE_PPGTT_PDE_PT,
190 			GTT_TYPE_PPGTT_PTE_1G_ENTRY),
191 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDP_ENTRY,
192 			GTT_TYPE_PPGTT_PDP_ENTRY,
193 			GTT_TYPE_PPGTT_PDE_PT,
194 			GTT_TYPE_PPGTT_PTE_1G_ENTRY),
195 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDE_PT,
196 			GTT_TYPE_PPGTT_PDE_ENTRY,
197 			GTT_TYPE_PPGTT_PTE_PT,
198 			GTT_TYPE_PPGTT_PTE_2M_ENTRY),
199 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PDE_ENTRY,
200 			GTT_TYPE_PPGTT_PDE_ENTRY,
201 			GTT_TYPE_PPGTT_PTE_PT,
202 			GTT_TYPE_PPGTT_PTE_2M_ENTRY),
203 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_PT,
204 			GTT_TYPE_PPGTT_PTE_4K_ENTRY,
205 			GTT_TYPE_INVALID,
206 			GTT_TYPE_INVALID),
207 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_4K_ENTRY,
208 			GTT_TYPE_PPGTT_PTE_4K_ENTRY,
209 			GTT_TYPE_INVALID,
210 			GTT_TYPE_INVALID),
211 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_2M_ENTRY,
212 			GTT_TYPE_PPGTT_PDE_ENTRY,
213 			GTT_TYPE_INVALID,
214 			GTT_TYPE_PPGTT_PTE_2M_ENTRY),
215 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_PPGTT_PTE_1G_ENTRY,
216 			GTT_TYPE_PPGTT_PDP_ENTRY,
217 			GTT_TYPE_INVALID,
218 			GTT_TYPE_PPGTT_PTE_1G_ENTRY),
219 	GTT_TYPE_TABLE_ENTRY(GTT_TYPE_GGTT_PTE,
220 			GTT_TYPE_GGTT_PTE,
221 			GTT_TYPE_INVALID,
222 			GTT_TYPE_INVALID),
223 };
224 
225 static inline int get_next_pt_type(int type)
226 {
227 	return gtt_type_table[type].next_pt_type;
228 }
229 
230 static inline int get_entry_type(int type)
231 {
232 	return gtt_type_table[type].entry_type;
233 }
234 
235 static inline int get_pse_type(int type)
236 {
237 	return gtt_type_table[type].pse_entry_type;
238 }
239 
240 static u64 read_pte64(struct drm_i915_private *dev_priv, unsigned long index)
241 {
242 	void __iomem *addr = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + index;
243 
244 	return readq(addr);
245 }
246 
247 static void gtt_invalidate(struct drm_i915_private *dev_priv)
248 {
249 	mmio_hw_access_pre(dev_priv);
250 	I915_WRITE(GFX_FLSH_CNTL_GEN6, GFX_FLSH_CNTL_EN);
251 	mmio_hw_access_post(dev_priv);
252 }
253 
254 static void write_pte64(struct drm_i915_private *dev_priv,
255 		unsigned long index, u64 pte)
256 {
257 	void __iomem *addr = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + index;
258 
259 	writeq(pte, addr);
260 }
261 
262 static inline int gtt_get_entry64(void *pt,
263 		struct intel_gvt_gtt_entry *e,
264 		unsigned long index, bool hypervisor_access, unsigned long gpa,
265 		struct intel_vgpu *vgpu)
266 {
267 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
268 	int ret;
269 
270 	if (WARN_ON(info->gtt_entry_size != 8))
271 		return -EINVAL;
272 
273 	if (hypervisor_access) {
274 		ret = intel_gvt_hypervisor_read_gpa(vgpu, gpa +
275 				(index << info->gtt_entry_size_shift),
276 				&e->val64, 8);
277 		if (WARN_ON(ret))
278 			return ret;
279 	} else if (!pt) {
280 		e->val64 = read_pte64(vgpu->gvt->dev_priv, index);
281 	} else {
282 		e->val64 = *((u64 *)pt + index);
283 	}
284 	return 0;
285 }
286 
287 static inline int gtt_set_entry64(void *pt,
288 		struct intel_gvt_gtt_entry *e,
289 		unsigned long index, bool hypervisor_access, unsigned long gpa,
290 		struct intel_vgpu *vgpu)
291 {
292 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
293 	int ret;
294 
295 	if (WARN_ON(info->gtt_entry_size != 8))
296 		return -EINVAL;
297 
298 	if (hypervisor_access) {
299 		ret = intel_gvt_hypervisor_write_gpa(vgpu, gpa +
300 				(index << info->gtt_entry_size_shift),
301 				&e->val64, 8);
302 		if (WARN_ON(ret))
303 			return ret;
304 	} else if (!pt) {
305 		write_pte64(vgpu->gvt->dev_priv, index, e->val64);
306 	} else {
307 		*((u64 *)pt + index) = e->val64;
308 	}
309 	return 0;
310 }
311 
312 #define GTT_HAW 46
313 
314 #define ADDR_1G_MASK (((1UL << (GTT_HAW - 30 + 1)) - 1) << 30)
315 #define ADDR_2M_MASK (((1UL << (GTT_HAW - 21 + 1)) - 1) << 21)
316 #define ADDR_4K_MASK (((1UL << (GTT_HAW - 12 + 1)) - 1) << 12)
317 
318 static unsigned long gen8_gtt_get_pfn(struct intel_gvt_gtt_entry *e)
319 {
320 	unsigned long pfn;
321 
322 	if (e->type == GTT_TYPE_PPGTT_PTE_1G_ENTRY)
323 		pfn = (e->val64 & ADDR_1G_MASK) >> 12;
324 	else if (e->type == GTT_TYPE_PPGTT_PTE_2M_ENTRY)
325 		pfn = (e->val64 & ADDR_2M_MASK) >> 12;
326 	else
327 		pfn = (e->val64 & ADDR_4K_MASK) >> 12;
328 	return pfn;
329 }
330 
331 static void gen8_gtt_set_pfn(struct intel_gvt_gtt_entry *e, unsigned long pfn)
332 {
333 	if (e->type == GTT_TYPE_PPGTT_PTE_1G_ENTRY) {
334 		e->val64 &= ~ADDR_1G_MASK;
335 		pfn &= (ADDR_1G_MASK >> 12);
336 	} else if (e->type == GTT_TYPE_PPGTT_PTE_2M_ENTRY) {
337 		e->val64 &= ~ADDR_2M_MASK;
338 		pfn &= (ADDR_2M_MASK >> 12);
339 	} else {
340 		e->val64 &= ~ADDR_4K_MASK;
341 		pfn &= (ADDR_4K_MASK >> 12);
342 	}
343 
344 	e->val64 |= (pfn << 12);
345 }
346 
347 static bool gen8_gtt_test_pse(struct intel_gvt_gtt_entry *e)
348 {
349 	/* Entry doesn't have PSE bit. */
350 	if (get_pse_type(e->type) == GTT_TYPE_INVALID)
351 		return false;
352 
353 	e->type = get_entry_type(e->type);
354 	if (!(e->val64 & (1 << 7)))
355 		return false;
356 
357 	e->type = get_pse_type(e->type);
358 	return true;
359 }
360 
361 static bool gen8_gtt_test_present(struct intel_gvt_gtt_entry *e)
362 {
363 	/*
364 	 * i915 writes PDP root pointer registers without present bit,
365 	 * it also works, so we need to treat root pointer entry
366 	 * specifically.
367 	 */
368 	if (e->type == GTT_TYPE_PPGTT_ROOT_L3_ENTRY
369 			|| e->type == GTT_TYPE_PPGTT_ROOT_L4_ENTRY)
370 		return (e->val64 != 0);
371 	else
372 		return (e->val64 & (1 << 0));
373 }
374 
375 static void gtt_entry_clear_present(struct intel_gvt_gtt_entry *e)
376 {
377 	e->val64 &= ~(1 << 0);
378 }
379 
380 /*
381  * Per-platform GMA routines.
382  */
383 static unsigned long gma_to_ggtt_pte_index(unsigned long gma)
384 {
385 	unsigned long x = (gma >> GTT_PAGE_SHIFT);
386 
387 	trace_gma_index(__func__, gma, x);
388 	return x;
389 }
390 
391 #define DEFINE_PPGTT_GMA_TO_INDEX(prefix, ename, exp) \
392 static unsigned long prefix##_gma_to_##ename##_index(unsigned long gma) \
393 { \
394 	unsigned long x = (exp); \
395 	trace_gma_index(__func__, gma, x); \
396 	return x; \
397 }
398 
399 DEFINE_PPGTT_GMA_TO_INDEX(gen8, pte, (gma >> 12 & 0x1ff));
400 DEFINE_PPGTT_GMA_TO_INDEX(gen8, pde, (gma >> 21 & 0x1ff));
401 DEFINE_PPGTT_GMA_TO_INDEX(gen8, l3_pdp, (gma >> 30 & 0x3));
402 DEFINE_PPGTT_GMA_TO_INDEX(gen8, l4_pdp, (gma >> 30 & 0x1ff));
403 DEFINE_PPGTT_GMA_TO_INDEX(gen8, pml4, (gma >> 39 & 0x1ff));
404 
405 static struct intel_gvt_gtt_pte_ops gen8_gtt_pte_ops = {
406 	.get_entry = gtt_get_entry64,
407 	.set_entry = gtt_set_entry64,
408 	.clear_present = gtt_entry_clear_present,
409 	.test_present = gen8_gtt_test_present,
410 	.test_pse = gen8_gtt_test_pse,
411 	.get_pfn = gen8_gtt_get_pfn,
412 	.set_pfn = gen8_gtt_set_pfn,
413 };
414 
415 static struct intel_gvt_gtt_gma_ops gen8_gtt_gma_ops = {
416 	.gma_to_ggtt_pte_index = gma_to_ggtt_pte_index,
417 	.gma_to_pte_index = gen8_gma_to_pte_index,
418 	.gma_to_pde_index = gen8_gma_to_pde_index,
419 	.gma_to_l3_pdp_index = gen8_gma_to_l3_pdp_index,
420 	.gma_to_l4_pdp_index = gen8_gma_to_l4_pdp_index,
421 	.gma_to_pml4_index = gen8_gma_to_pml4_index,
422 };
423 
424 static int gtt_entry_p2m(struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *p,
425 		struct intel_gvt_gtt_entry *m)
426 {
427 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
428 	unsigned long gfn, mfn;
429 
430 	*m = *p;
431 
432 	if (!ops->test_present(p))
433 		return 0;
434 
435 	gfn = ops->get_pfn(p);
436 
437 	mfn = intel_gvt_hypervisor_gfn_to_mfn(vgpu, gfn);
438 	if (mfn == INTEL_GVT_INVALID_ADDR) {
439 		gvt_vgpu_err("fail to translate gfn: 0x%lx\n", gfn);
440 		return -ENXIO;
441 	}
442 
443 	ops->set_pfn(m, mfn);
444 	return 0;
445 }
446 
447 /*
448  * MM helpers.
449  */
450 int intel_vgpu_mm_get_entry(struct intel_vgpu_mm *mm,
451 		void *page_table, struct intel_gvt_gtt_entry *e,
452 		unsigned long index)
453 {
454 	struct intel_gvt *gvt = mm->vgpu->gvt;
455 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
456 	int ret;
457 
458 	e->type = mm->page_table_entry_type;
459 
460 	ret = ops->get_entry(page_table, e, index, false, 0, mm->vgpu);
461 	if (ret)
462 		return ret;
463 
464 	ops->test_pse(e);
465 	return 0;
466 }
467 
468 int intel_vgpu_mm_set_entry(struct intel_vgpu_mm *mm,
469 		void *page_table, struct intel_gvt_gtt_entry *e,
470 		unsigned long index)
471 {
472 	struct intel_gvt *gvt = mm->vgpu->gvt;
473 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
474 
475 	return ops->set_entry(page_table, e, index, false, 0, mm->vgpu);
476 }
477 
478 /*
479  * PPGTT shadow page table helpers.
480  */
481 static inline int ppgtt_spt_get_entry(
482 		struct intel_vgpu_ppgtt_spt *spt,
483 		void *page_table, int type,
484 		struct intel_gvt_gtt_entry *e, unsigned long index,
485 		bool guest)
486 {
487 	struct intel_gvt *gvt = spt->vgpu->gvt;
488 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
489 	int ret;
490 
491 	e->type = get_entry_type(type);
492 
493 	if (WARN(!gtt_type_is_entry(e->type), "invalid entry type\n"))
494 		return -EINVAL;
495 
496 	ret = ops->get_entry(page_table, e, index, guest,
497 			spt->guest_page.gfn << GTT_PAGE_SHIFT,
498 			spt->vgpu);
499 	if (ret)
500 		return ret;
501 
502 	ops->test_pse(e);
503 	return 0;
504 }
505 
506 static inline int ppgtt_spt_set_entry(
507 		struct intel_vgpu_ppgtt_spt *spt,
508 		void *page_table, int type,
509 		struct intel_gvt_gtt_entry *e, unsigned long index,
510 		bool guest)
511 {
512 	struct intel_gvt *gvt = spt->vgpu->gvt;
513 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
514 
515 	if (WARN(!gtt_type_is_entry(e->type), "invalid entry type\n"))
516 		return -EINVAL;
517 
518 	return ops->set_entry(page_table, e, index, guest,
519 			spt->guest_page.gfn << GTT_PAGE_SHIFT,
520 			spt->vgpu);
521 }
522 
523 #define ppgtt_get_guest_entry(spt, e, index) \
524 	ppgtt_spt_get_entry(spt, NULL, \
525 		spt->guest_page_type, e, index, true)
526 
527 #define ppgtt_set_guest_entry(spt, e, index) \
528 	ppgtt_spt_set_entry(spt, NULL, \
529 		spt->guest_page_type, e, index, true)
530 
531 #define ppgtt_get_shadow_entry(spt, e, index) \
532 	ppgtt_spt_get_entry(spt, spt->shadow_page.vaddr, \
533 		spt->shadow_page.type, e, index, false)
534 
535 #define ppgtt_set_shadow_entry(spt, e, index) \
536 	ppgtt_spt_set_entry(spt, spt->shadow_page.vaddr, \
537 		spt->shadow_page.type, e, index, false)
538 
539 /**
540  * intel_vgpu_init_guest_page - init a guest page data structure
541  * @vgpu: a vGPU
542  * @p: a guest page data structure
543  * @gfn: guest memory page frame number
544  * @handler: function will be called when target guest memory page has
545  * been modified.
546  *
547  * This function is called when user wants to track a guest memory page.
548  *
549  * Returns:
550  * Zero on success, negative error code if failed.
551  */
552 int intel_vgpu_init_guest_page(struct intel_vgpu *vgpu,
553 		struct intel_vgpu_guest_page *p,
554 		unsigned long gfn,
555 		int (*handler)(void *, u64, void *, int),
556 		void *data)
557 {
558 	INIT_HLIST_NODE(&p->node);
559 
560 	p->writeprotection = false;
561 	p->gfn = gfn;
562 	p->handler = handler;
563 	p->data = data;
564 	p->oos_page = NULL;
565 	p->write_cnt = 0;
566 
567 	hash_add(vgpu->gtt.guest_page_hash_table, &p->node, p->gfn);
568 	return 0;
569 }
570 
571 static int detach_oos_page(struct intel_vgpu *vgpu,
572 		struct intel_vgpu_oos_page *oos_page);
573 
574 /**
575  * intel_vgpu_clean_guest_page - release the resource owned by guest page data
576  * structure
577  * @vgpu: a vGPU
578  * @p: a tracked guest page
579  *
580  * This function is called when user tries to stop tracking a guest memory
581  * page.
582  */
583 void intel_vgpu_clean_guest_page(struct intel_vgpu *vgpu,
584 		struct intel_vgpu_guest_page *p)
585 {
586 	if (!hlist_unhashed(&p->node))
587 		hash_del(&p->node);
588 
589 	if (p->oos_page)
590 		detach_oos_page(vgpu, p->oos_page);
591 
592 	if (p->writeprotection)
593 		intel_gvt_hypervisor_unset_wp_page(vgpu, p);
594 }
595 
596 /**
597  * intel_vgpu_find_guest_page - find a guest page data structure by GFN.
598  * @vgpu: a vGPU
599  * @gfn: guest memory page frame number
600  *
601  * This function is called when emulation logic wants to know if a trapped GFN
602  * is a tracked guest page.
603  *
604  * Returns:
605  * Pointer to guest page data structure, NULL if failed.
606  */
607 struct intel_vgpu_guest_page *intel_vgpu_find_guest_page(
608 		struct intel_vgpu *vgpu, unsigned long gfn)
609 {
610 	struct intel_vgpu_guest_page *p;
611 
612 	hash_for_each_possible(vgpu->gtt.guest_page_hash_table,
613 		p, node, gfn) {
614 		if (p->gfn == gfn)
615 			return p;
616 	}
617 	return NULL;
618 }
619 
620 static inline int init_shadow_page(struct intel_vgpu *vgpu,
621 		struct intel_vgpu_shadow_page *p, int type)
622 {
623 	struct device *kdev = &vgpu->gvt->dev_priv->drm.pdev->dev;
624 	dma_addr_t daddr;
625 
626 	daddr = dma_map_page(kdev, p->page, 0, 4096, PCI_DMA_BIDIRECTIONAL);
627 	if (dma_mapping_error(kdev, daddr)) {
628 		gvt_vgpu_err("fail to map dma addr\n");
629 		return -EINVAL;
630 	}
631 
632 	p->vaddr = page_address(p->page);
633 	p->type = type;
634 
635 	INIT_HLIST_NODE(&p->node);
636 
637 	p->mfn = daddr >> GTT_PAGE_SHIFT;
638 	hash_add(vgpu->gtt.shadow_page_hash_table, &p->node, p->mfn);
639 	return 0;
640 }
641 
642 static inline void clean_shadow_page(struct intel_vgpu *vgpu,
643 		struct intel_vgpu_shadow_page *p)
644 {
645 	struct device *kdev = &vgpu->gvt->dev_priv->drm.pdev->dev;
646 
647 	dma_unmap_page(kdev, p->mfn << GTT_PAGE_SHIFT, 4096,
648 			PCI_DMA_BIDIRECTIONAL);
649 
650 	if (!hlist_unhashed(&p->node))
651 		hash_del(&p->node);
652 }
653 
654 static inline struct intel_vgpu_shadow_page *find_shadow_page(
655 		struct intel_vgpu *vgpu, unsigned long mfn)
656 {
657 	struct intel_vgpu_shadow_page *p;
658 
659 	hash_for_each_possible(vgpu->gtt.shadow_page_hash_table,
660 		p, node, mfn) {
661 		if (p->mfn == mfn)
662 			return p;
663 	}
664 	return NULL;
665 }
666 
667 #define guest_page_to_ppgtt_spt(ptr) \
668 	container_of(ptr, struct intel_vgpu_ppgtt_spt, guest_page)
669 
670 #define shadow_page_to_ppgtt_spt(ptr) \
671 	container_of(ptr, struct intel_vgpu_ppgtt_spt, shadow_page)
672 
673 static void *alloc_spt(gfp_t gfp_mask)
674 {
675 	struct intel_vgpu_ppgtt_spt *spt;
676 
677 	spt = kzalloc(sizeof(*spt), gfp_mask);
678 	if (!spt)
679 		return NULL;
680 
681 	spt->shadow_page.page = alloc_page(gfp_mask);
682 	if (!spt->shadow_page.page) {
683 		kfree(spt);
684 		return NULL;
685 	}
686 	return spt;
687 }
688 
689 static void free_spt(struct intel_vgpu_ppgtt_spt *spt)
690 {
691 	__free_page(spt->shadow_page.page);
692 	kfree(spt);
693 }
694 
695 static void ppgtt_free_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
696 {
697 	trace_spt_free(spt->vgpu->id, spt, spt->shadow_page.type);
698 
699 	clean_shadow_page(spt->vgpu, &spt->shadow_page);
700 	intel_vgpu_clean_guest_page(spt->vgpu, &spt->guest_page);
701 	list_del_init(&spt->post_shadow_list);
702 
703 	free_spt(spt);
704 }
705 
706 static void ppgtt_free_all_shadow_page(struct intel_vgpu *vgpu)
707 {
708 	struct hlist_node *n;
709 	struct intel_vgpu_shadow_page *sp;
710 	int i;
711 
712 	hash_for_each_safe(vgpu->gtt.shadow_page_hash_table, i, n, sp, node)
713 		ppgtt_free_shadow_page(shadow_page_to_ppgtt_spt(sp));
714 }
715 
716 static int ppgtt_handle_guest_write_page_table_bytes(void *gp,
717 		u64 pa, void *p_data, int bytes);
718 
719 static int ppgtt_write_protection_handler(void *gp, u64 pa,
720 		void *p_data, int bytes)
721 {
722 	struct intel_vgpu_guest_page *gpt = (struct intel_vgpu_guest_page *)gp;
723 	int ret;
724 
725 	if (bytes != 4 && bytes != 8)
726 		return -EINVAL;
727 
728 	if (!gpt->writeprotection)
729 		return -EINVAL;
730 
731 	ret = ppgtt_handle_guest_write_page_table_bytes(gp,
732 		pa, p_data, bytes);
733 	if (ret)
734 		return ret;
735 	return ret;
736 }
737 
738 static int reclaim_one_mm(struct intel_gvt *gvt);
739 
740 static struct intel_vgpu_ppgtt_spt *ppgtt_alloc_shadow_page(
741 		struct intel_vgpu *vgpu, int type, unsigned long gfn)
742 {
743 	struct intel_vgpu_ppgtt_spt *spt = NULL;
744 	int ret;
745 
746 retry:
747 	spt = alloc_spt(GFP_KERNEL | __GFP_ZERO);
748 	if (!spt) {
749 		if (reclaim_one_mm(vgpu->gvt))
750 			goto retry;
751 
752 		gvt_vgpu_err("fail to allocate ppgtt shadow page\n");
753 		return ERR_PTR(-ENOMEM);
754 	}
755 
756 	spt->vgpu = vgpu;
757 	spt->guest_page_type = type;
758 	atomic_set(&spt->refcount, 1);
759 	INIT_LIST_HEAD(&spt->post_shadow_list);
760 
761 	/*
762 	 * TODO: guest page type may be different with shadow page type,
763 	 *	 when we support PSE page in future.
764 	 */
765 	ret = init_shadow_page(vgpu, &spt->shadow_page, type);
766 	if (ret) {
767 		gvt_vgpu_err("fail to initialize shadow page for spt\n");
768 		goto err;
769 	}
770 
771 	ret = intel_vgpu_init_guest_page(vgpu, &spt->guest_page,
772 			gfn, ppgtt_write_protection_handler, NULL);
773 	if (ret) {
774 		gvt_vgpu_err("fail to initialize guest page for spt\n");
775 		goto err;
776 	}
777 
778 	trace_spt_alloc(vgpu->id, spt, type, spt->shadow_page.mfn, gfn);
779 	return spt;
780 err:
781 	ppgtt_free_shadow_page(spt);
782 	return ERR_PTR(ret);
783 }
784 
785 static struct intel_vgpu_ppgtt_spt *ppgtt_find_shadow_page(
786 		struct intel_vgpu *vgpu, unsigned long mfn)
787 {
788 	struct intel_vgpu_shadow_page *p = find_shadow_page(vgpu, mfn);
789 
790 	if (p)
791 		return shadow_page_to_ppgtt_spt(p);
792 
793 	gvt_vgpu_err("fail to find ppgtt shadow page: 0x%lx\n", mfn);
794 	return NULL;
795 }
796 
797 #define pt_entry_size_shift(spt) \
798 	((spt)->vgpu->gvt->device_info.gtt_entry_size_shift)
799 
800 #define pt_entries(spt) \
801 	(GTT_PAGE_SIZE >> pt_entry_size_shift(spt))
802 
803 #define for_each_present_guest_entry(spt, e, i) \
804 	for (i = 0; i < pt_entries(spt); i++) \
805 		if (!ppgtt_get_guest_entry(spt, e, i) && \
806 		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
807 
808 #define for_each_present_shadow_entry(spt, e, i) \
809 	for (i = 0; i < pt_entries(spt); i++) \
810 		if (!ppgtt_get_shadow_entry(spt, e, i) && \
811 		    spt->vgpu->gvt->gtt.pte_ops->test_present(e))
812 
813 static void ppgtt_get_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
814 {
815 	int v = atomic_read(&spt->refcount);
816 
817 	trace_spt_refcount(spt->vgpu->id, "inc", spt, v, (v + 1));
818 
819 	atomic_inc(&spt->refcount);
820 }
821 
822 static int ppgtt_invalidate_shadow_page(struct intel_vgpu_ppgtt_spt *spt);
823 
824 static int ppgtt_invalidate_shadow_page_by_shadow_entry(struct intel_vgpu *vgpu,
825 		struct intel_gvt_gtt_entry *e)
826 {
827 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
828 	struct intel_vgpu_ppgtt_spt *s;
829 	intel_gvt_gtt_type_t cur_pt_type;
830 
831 	if (WARN_ON(!gtt_type_is_pt(get_next_pt_type(e->type))))
832 		return -EINVAL;
833 
834 	if (e->type != GTT_TYPE_PPGTT_ROOT_L3_ENTRY
835 		&& e->type != GTT_TYPE_PPGTT_ROOT_L4_ENTRY) {
836 		cur_pt_type = get_next_pt_type(e->type) + 1;
837 		if (ops->get_pfn(e) ==
838 			vgpu->gtt.scratch_pt[cur_pt_type].page_mfn)
839 			return 0;
840 	}
841 	s = ppgtt_find_shadow_page(vgpu, ops->get_pfn(e));
842 	if (!s) {
843 		gvt_vgpu_err("fail to find shadow page: mfn: 0x%lx\n",
844 				ops->get_pfn(e));
845 		return -ENXIO;
846 	}
847 	return ppgtt_invalidate_shadow_page(s);
848 }
849 
850 static int ppgtt_invalidate_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
851 {
852 	struct intel_vgpu *vgpu = spt->vgpu;
853 	struct intel_gvt_gtt_entry e;
854 	unsigned long index;
855 	int ret;
856 	int v = atomic_read(&spt->refcount);
857 
858 	trace_spt_change(spt->vgpu->id, "die", spt,
859 			spt->guest_page.gfn, spt->shadow_page.type);
860 
861 	trace_spt_refcount(spt->vgpu->id, "dec", spt, v, (v - 1));
862 
863 	if (atomic_dec_return(&spt->refcount) > 0)
864 		return 0;
865 
866 	if (gtt_type_is_pte_pt(spt->shadow_page.type))
867 		goto release;
868 
869 	for_each_present_shadow_entry(spt, &e, index) {
870 		if (!gtt_type_is_pt(get_next_pt_type(e.type))) {
871 			gvt_vgpu_err("GVT doesn't support pse bit for now\n");
872 			return -EINVAL;
873 		}
874 		ret = ppgtt_invalidate_shadow_page_by_shadow_entry(
875 				spt->vgpu, &e);
876 		if (ret)
877 			goto fail;
878 	}
879 release:
880 	trace_spt_change(spt->vgpu->id, "release", spt,
881 			spt->guest_page.gfn, spt->shadow_page.type);
882 	ppgtt_free_shadow_page(spt);
883 	return 0;
884 fail:
885 	gvt_vgpu_err("fail: shadow page %p shadow entry 0x%llx type %d\n",
886 			spt, e.val64, e.type);
887 	return ret;
888 }
889 
890 static int ppgtt_populate_shadow_page(struct intel_vgpu_ppgtt_spt *spt);
891 
892 static struct intel_vgpu_ppgtt_spt *ppgtt_populate_shadow_page_by_guest_entry(
893 		struct intel_vgpu *vgpu, struct intel_gvt_gtt_entry *we)
894 {
895 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
896 	struct intel_vgpu_ppgtt_spt *s = NULL;
897 	struct intel_vgpu_guest_page *g;
898 	int ret;
899 
900 	if (WARN_ON(!gtt_type_is_pt(get_next_pt_type(we->type)))) {
901 		ret = -EINVAL;
902 		goto fail;
903 	}
904 
905 	g = intel_vgpu_find_guest_page(vgpu, ops->get_pfn(we));
906 	if (g) {
907 		s = guest_page_to_ppgtt_spt(g);
908 		ppgtt_get_shadow_page(s);
909 	} else {
910 		int type = get_next_pt_type(we->type);
911 
912 		s = ppgtt_alloc_shadow_page(vgpu, type, ops->get_pfn(we));
913 		if (IS_ERR(s)) {
914 			ret = PTR_ERR(s);
915 			goto fail;
916 		}
917 
918 		ret = intel_gvt_hypervisor_set_wp_page(vgpu, &s->guest_page);
919 		if (ret)
920 			goto fail;
921 
922 		ret = ppgtt_populate_shadow_page(s);
923 		if (ret)
924 			goto fail;
925 
926 		trace_spt_change(vgpu->id, "new", s, s->guest_page.gfn,
927 			s->shadow_page.type);
928 	}
929 	return s;
930 fail:
931 	gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d\n",
932 			s, we->val64, we->type);
933 	return ERR_PTR(ret);
934 }
935 
936 static inline void ppgtt_generate_shadow_entry(struct intel_gvt_gtt_entry *se,
937 		struct intel_vgpu_ppgtt_spt *s, struct intel_gvt_gtt_entry *ge)
938 {
939 	struct intel_gvt_gtt_pte_ops *ops = s->vgpu->gvt->gtt.pte_ops;
940 
941 	se->type = ge->type;
942 	se->val64 = ge->val64;
943 
944 	ops->set_pfn(se, s->shadow_page.mfn);
945 }
946 
947 static int ppgtt_populate_shadow_page(struct intel_vgpu_ppgtt_spt *spt)
948 {
949 	struct intel_vgpu *vgpu = spt->vgpu;
950 	struct intel_vgpu_ppgtt_spt *s;
951 	struct intel_gvt_gtt_entry se, ge;
952 	unsigned long i;
953 	int ret;
954 
955 	trace_spt_change(spt->vgpu->id, "born", spt,
956 			spt->guest_page.gfn, spt->shadow_page.type);
957 
958 	if (gtt_type_is_pte_pt(spt->shadow_page.type)) {
959 		for_each_present_guest_entry(spt, &ge, i) {
960 			ret = gtt_entry_p2m(vgpu, &ge, &se);
961 			if (ret)
962 				goto fail;
963 			ppgtt_set_shadow_entry(spt, &se, i);
964 		}
965 		return 0;
966 	}
967 
968 	for_each_present_guest_entry(spt, &ge, i) {
969 		if (!gtt_type_is_pt(get_next_pt_type(ge.type))) {
970 			gvt_vgpu_err("GVT doesn't support pse bit now\n");
971 			ret = -EINVAL;
972 			goto fail;
973 		}
974 
975 		s = ppgtt_populate_shadow_page_by_guest_entry(vgpu, &ge);
976 		if (IS_ERR(s)) {
977 			ret = PTR_ERR(s);
978 			goto fail;
979 		}
980 		ppgtt_get_shadow_entry(spt, &se, i);
981 		ppgtt_generate_shadow_entry(&se, s, &ge);
982 		ppgtt_set_shadow_entry(spt, &se, i);
983 	}
984 	return 0;
985 fail:
986 	gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d\n",
987 			spt, ge.val64, ge.type);
988 	return ret;
989 }
990 
991 static int ppgtt_handle_guest_entry_removal(struct intel_vgpu_guest_page *gpt,
992 		struct intel_gvt_gtt_entry *se, unsigned long index)
993 {
994 	struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
995 	struct intel_vgpu_shadow_page *sp = &spt->shadow_page;
996 	struct intel_vgpu *vgpu = spt->vgpu;
997 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
998 	int ret;
999 
1000 	trace_gpt_change(spt->vgpu->id, "remove", spt, sp->type, se->val64,
1001 			 index);
1002 
1003 	if (!ops->test_present(se))
1004 		return 0;
1005 
1006 	if (ops->get_pfn(se) == vgpu->gtt.scratch_pt[sp->type].page_mfn)
1007 		return 0;
1008 
1009 	if (gtt_type_is_pt(get_next_pt_type(se->type))) {
1010 		struct intel_vgpu_ppgtt_spt *s =
1011 			ppgtt_find_shadow_page(vgpu, ops->get_pfn(se));
1012 		if (!s) {
1013 			gvt_vgpu_err("fail to find guest page\n");
1014 			ret = -ENXIO;
1015 			goto fail;
1016 		}
1017 		ret = ppgtt_invalidate_shadow_page(s);
1018 		if (ret)
1019 			goto fail;
1020 	}
1021 	return 0;
1022 fail:
1023 	gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d\n",
1024 			spt, se->val64, se->type);
1025 	return ret;
1026 }
1027 
1028 static int ppgtt_handle_guest_entry_add(struct intel_vgpu_guest_page *gpt,
1029 		struct intel_gvt_gtt_entry *we, unsigned long index)
1030 {
1031 	struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
1032 	struct intel_vgpu_shadow_page *sp = &spt->shadow_page;
1033 	struct intel_vgpu *vgpu = spt->vgpu;
1034 	struct intel_gvt_gtt_entry m;
1035 	struct intel_vgpu_ppgtt_spt *s;
1036 	int ret;
1037 
1038 	trace_gpt_change(spt->vgpu->id, "add", spt, sp->type,
1039 		we->val64, index);
1040 
1041 	if (gtt_type_is_pt(get_next_pt_type(we->type))) {
1042 		s = ppgtt_populate_shadow_page_by_guest_entry(vgpu, we);
1043 		if (IS_ERR(s)) {
1044 			ret = PTR_ERR(s);
1045 			goto fail;
1046 		}
1047 		ppgtt_get_shadow_entry(spt, &m, index);
1048 		ppgtt_generate_shadow_entry(&m, s, we);
1049 		ppgtt_set_shadow_entry(spt, &m, index);
1050 	} else {
1051 		ret = gtt_entry_p2m(vgpu, we, &m);
1052 		if (ret)
1053 			goto fail;
1054 		ppgtt_set_shadow_entry(spt, &m, index);
1055 	}
1056 	return 0;
1057 fail:
1058 	gvt_vgpu_err("fail: spt %p guest entry 0x%llx type %d\n",
1059 		spt, we->val64, we->type);
1060 	return ret;
1061 }
1062 
1063 static int sync_oos_page(struct intel_vgpu *vgpu,
1064 		struct intel_vgpu_oos_page *oos_page)
1065 {
1066 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
1067 	struct intel_gvt *gvt = vgpu->gvt;
1068 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
1069 	struct intel_vgpu_ppgtt_spt *spt =
1070 		guest_page_to_ppgtt_spt(oos_page->guest_page);
1071 	struct intel_gvt_gtt_entry old, new, m;
1072 	int index;
1073 	int ret;
1074 
1075 	trace_oos_change(vgpu->id, "sync", oos_page->id,
1076 			oos_page->guest_page, spt->guest_page_type);
1077 
1078 	old.type = new.type = get_entry_type(spt->guest_page_type);
1079 	old.val64 = new.val64 = 0;
1080 
1081 	for (index = 0; index < (GTT_PAGE_SIZE >> info->gtt_entry_size_shift);
1082 		index++) {
1083 		ops->get_entry(oos_page->mem, &old, index, false, 0, vgpu);
1084 		ops->get_entry(NULL, &new, index, true,
1085 			oos_page->guest_page->gfn << PAGE_SHIFT, vgpu);
1086 
1087 		if (old.val64 == new.val64
1088 			&& !test_and_clear_bit(index, spt->post_shadow_bitmap))
1089 			continue;
1090 
1091 		trace_oos_sync(vgpu->id, oos_page->id,
1092 				oos_page->guest_page, spt->guest_page_type,
1093 				new.val64, index);
1094 
1095 		ret = gtt_entry_p2m(vgpu, &new, &m);
1096 		if (ret)
1097 			return ret;
1098 
1099 		ops->set_entry(oos_page->mem, &new, index, false, 0, vgpu);
1100 		ppgtt_set_shadow_entry(spt, &m, index);
1101 	}
1102 
1103 	oos_page->guest_page->write_cnt = 0;
1104 	list_del_init(&spt->post_shadow_list);
1105 	return 0;
1106 }
1107 
1108 static int detach_oos_page(struct intel_vgpu *vgpu,
1109 		struct intel_vgpu_oos_page *oos_page)
1110 {
1111 	struct intel_gvt *gvt = vgpu->gvt;
1112 	struct intel_vgpu_ppgtt_spt *spt =
1113 		guest_page_to_ppgtt_spt(oos_page->guest_page);
1114 
1115 	trace_oos_change(vgpu->id, "detach", oos_page->id,
1116 			oos_page->guest_page, spt->guest_page_type);
1117 
1118 	oos_page->guest_page->write_cnt = 0;
1119 	oos_page->guest_page->oos_page = NULL;
1120 	oos_page->guest_page = NULL;
1121 
1122 	list_del_init(&oos_page->vm_list);
1123 	list_move_tail(&oos_page->list, &gvt->gtt.oos_page_free_list_head);
1124 
1125 	return 0;
1126 }
1127 
1128 static int attach_oos_page(struct intel_vgpu *vgpu,
1129 		struct intel_vgpu_oos_page *oos_page,
1130 		struct intel_vgpu_guest_page *gpt)
1131 {
1132 	struct intel_gvt *gvt = vgpu->gvt;
1133 	int ret;
1134 
1135 	ret = intel_gvt_hypervisor_read_gpa(vgpu, gpt->gfn << GTT_PAGE_SHIFT,
1136 		oos_page->mem, GTT_PAGE_SIZE);
1137 	if (ret)
1138 		return ret;
1139 
1140 	oos_page->guest_page = gpt;
1141 	gpt->oos_page = oos_page;
1142 
1143 	list_move_tail(&oos_page->list, &gvt->gtt.oos_page_use_list_head);
1144 
1145 	trace_oos_change(vgpu->id, "attach", gpt->oos_page->id,
1146 			gpt, guest_page_to_ppgtt_spt(gpt)->guest_page_type);
1147 	return 0;
1148 }
1149 
1150 static int ppgtt_set_guest_page_sync(struct intel_vgpu *vgpu,
1151 		struct intel_vgpu_guest_page *gpt)
1152 {
1153 	int ret;
1154 
1155 	ret = intel_gvt_hypervisor_set_wp_page(vgpu, gpt);
1156 	if (ret)
1157 		return ret;
1158 
1159 	trace_oos_change(vgpu->id, "set page sync", gpt->oos_page->id,
1160 			gpt, guest_page_to_ppgtt_spt(gpt)->guest_page_type);
1161 
1162 	list_del_init(&gpt->oos_page->vm_list);
1163 	return sync_oos_page(vgpu, gpt->oos_page);
1164 }
1165 
1166 static int ppgtt_allocate_oos_page(struct intel_vgpu *vgpu,
1167 		struct intel_vgpu_guest_page *gpt)
1168 {
1169 	struct intel_gvt *gvt = vgpu->gvt;
1170 	struct intel_gvt_gtt *gtt = &gvt->gtt;
1171 	struct intel_vgpu_oos_page *oos_page = gpt->oos_page;
1172 	int ret;
1173 
1174 	WARN(oos_page, "shadow PPGTT page has already has a oos page\n");
1175 
1176 	if (list_empty(&gtt->oos_page_free_list_head)) {
1177 		oos_page = container_of(gtt->oos_page_use_list_head.next,
1178 			struct intel_vgpu_oos_page, list);
1179 		ret = ppgtt_set_guest_page_sync(vgpu, oos_page->guest_page);
1180 		if (ret)
1181 			return ret;
1182 		ret = detach_oos_page(vgpu, oos_page);
1183 		if (ret)
1184 			return ret;
1185 	} else
1186 		oos_page = container_of(gtt->oos_page_free_list_head.next,
1187 			struct intel_vgpu_oos_page, list);
1188 	return attach_oos_page(vgpu, oos_page, gpt);
1189 }
1190 
1191 static int ppgtt_set_guest_page_oos(struct intel_vgpu *vgpu,
1192 		struct intel_vgpu_guest_page *gpt)
1193 {
1194 	struct intel_vgpu_oos_page *oos_page = gpt->oos_page;
1195 
1196 	if (WARN(!oos_page, "shadow PPGTT page should have a oos page\n"))
1197 		return -EINVAL;
1198 
1199 	trace_oos_change(vgpu->id, "set page out of sync", gpt->oos_page->id,
1200 			gpt, guest_page_to_ppgtt_spt(gpt)->guest_page_type);
1201 
1202 	list_add_tail(&oos_page->vm_list, &vgpu->gtt.oos_page_list_head);
1203 	return intel_gvt_hypervisor_unset_wp_page(vgpu, gpt);
1204 }
1205 
1206 /**
1207  * intel_vgpu_sync_oos_pages - sync all the out-of-synced shadow for vGPU
1208  * @vgpu: a vGPU
1209  *
1210  * This function is called before submitting a guest workload to host,
1211  * to sync all the out-of-synced shadow for vGPU
1212  *
1213  * Returns:
1214  * Zero on success, negative error code if failed.
1215  */
1216 int intel_vgpu_sync_oos_pages(struct intel_vgpu *vgpu)
1217 {
1218 	struct list_head *pos, *n;
1219 	struct intel_vgpu_oos_page *oos_page;
1220 	int ret;
1221 
1222 	if (!enable_out_of_sync)
1223 		return 0;
1224 
1225 	list_for_each_safe(pos, n, &vgpu->gtt.oos_page_list_head) {
1226 		oos_page = container_of(pos,
1227 				struct intel_vgpu_oos_page, vm_list);
1228 		ret = ppgtt_set_guest_page_sync(vgpu, oos_page->guest_page);
1229 		if (ret)
1230 			return ret;
1231 	}
1232 	return 0;
1233 }
1234 
1235 /*
1236  * The heart of PPGTT shadow page table.
1237  */
1238 static int ppgtt_handle_guest_write_page_table(
1239 		struct intel_vgpu_guest_page *gpt,
1240 		struct intel_gvt_gtt_entry *we, unsigned long index)
1241 {
1242 	struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
1243 	struct intel_vgpu *vgpu = spt->vgpu;
1244 	int type = spt->shadow_page.type;
1245 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
1246 	struct intel_gvt_gtt_entry se;
1247 
1248 	int ret;
1249 	int new_present;
1250 
1251 	new_present = ops->test_present(we);
1252 
1253 	/*
1254 	 * Adding the new entry first and then removing the old one, that can
1255 	 * guarantee the ppgtt table is validated during the window between
1256 	 * adding and removal.
1257 	 */
1258 	ppgtt_get_shadow_entry(spt, &se, index);
1259 
1260 	if (new_present) {
1261 		ret = ppgtt_handle_guest_entry_add(gpt, we, index);
1262 		if (ret)
1263 			goto fail;
1264 	}
1265 
1266 	ret = ppgtt_handle_guest_entry_removal(gpt, &se, index);
1267 	if (ret)
1268 		goto fail;
1269 
1270 	if (!new_present) {
1271 		ops->set_pfn(&se, vgpu->gtt.scratch_pt[type].page_mfn);
1272 		ppgtt_set_shadow_entry(spt, &se, index);
1273 	}
1274 
1275 	return 0;
1276 fail:
1277 	gvt_vgpu_err("fail: shadow page %p guest entry 0x%llx type %d.\n",
1278 			spt, we->val64, we->type);
1279 	return ret;
1280 }
1281 
1282 static inline bool can_do_out_of_sync(struct intel_vgpu_guest_page *gpt)
1283 {
1284 	return enable_out_of_sync
1285 		&& gtt_type_is_pte_pt(
1286 			guest_page_to_ppgtt_spt(gpt)->guest_page_type)
1287 		&& gpt->write_cnt >= 2;
1288 }
1289 
1290 static void ppgtt_set_post_shadow(struct intel_vgpu_ppgtt_spt *spt,
1291 		unsigned long index)
1292 {
1293 	set_bit(index, spt->post_shadow_bitmap);
1294 	if (!list_empty(&spt->post_shadow_list))
1295 		return;
1296 
1297 	list_add_tail(&spt->post_shadow_list,
1298 			&spt->vgpu->gtt.post_shadow_list_head);
1299 }
1300 
1301 /**
1302  * intel_vgpu_flush_post_shadow - flush the post shadow transactions
1303  * @vgpu: a vGPU
1304  *
1305  * This function is called before submitting a guest workload to host,
1306  * to flush all the post shadows for a vGPU.
1307  *
1308  * Returns:
1309  * Zero on success, negative error code if failed.
1310  */
1311 int intel_vgpu_flush_post_shadow(struct intel_vgpu *vgpu)
1312 {
1313 	struct list_head *pos, *n;
1314 	struct intel_vgpu_ppgtt_spt *spt;
1315 	struct intel_gvt_gtt_entry ge;
1316 	unsigned long index;
1317 	int ret;
1318 
1319 	list_for_each_safe(pos, n, &vgpu->gtt.post_shadow_list_head) {
1320 		spt = container_of(pos, struct intel_vgpu_ppgtt_spt,
1321 				post_shadow_list);
1322 
1323 		for_each_set_bit(index, spt->post_shadow_bitmap,
1324 				GTT_ENTRY_NUM_IN_ONE_PAGE) {
1325 			ppgtt_get_guest_entry(spt, &ge, index);
1326 
1327 			ret = ppgtt_handle_guest_write_page_table(
1328 					&spt->guest_page, &ge, index);
1329 			if (ret)
1330 				return ret;
1331 			clear_bit(index, spt->post_shadow_bitmap);
1332 		}
1333 		list_del_init(&spt->post_shadow_list);
1334 	}
1335 	return 0;
1336 }
1337 
1338 static int ppgtt_handle_guest_write_page_table_bytes(void *gp,
1339 		u64 pa, void *p_data, int bytes)
1340 {
1341 	struct intel_vgpu_guest_page *gpt = (struct intel_vgpu_guest_page *)gp;
1342 	struct intel_vgpu_ppgtt_spt *spt = guest_page_to_ppgtt_spt(gpt);
1343 	struct intel_vgpu *vgpu = spt->vgpu;
1344 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
1345 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
1346 	struct intel_gvt_gtt_entry we, se;
1347 	unsigned long index;
1348 	int ret;
1349 
1350 	index = (pa & (PAGE_SIZE - 1)) >> info->gtt_entry_size_shift;
1351 
1352 	ppgtt_get_guest_entry(spt, &we, index);
1353 
1354 	ops->test_pse(&we);
1355 
1356 	if (bytes == info->gtt_entry_size) {
1357 		ret = ppgtt_handle_guest_write_page_table(gpt, &we, index);
1358 		if (ret)
1359 			return ret;
1360 	} else {
1361 		if (!test_bit(index, spt->post_shadow_bitmap)) {
1362 			ppgtt_get_shadow_entry(spt, &se, index);
1363 			ret = ppgtt_handle_guest_entry_removal(gpt, &se, index);
1364 			if (ret)
1365 				return ret;
1366 		}
1367 
1368 		ppgtt_set_post_shadow(spt, index);
1369 	}
1370 
1371 	if (!enable_out_of_sync)
1372 		return 0;
1373 
1374 	gpt->write_cnt++;
1375 
1376 	if (gpt->oos_page)
1377 		ops->set_entry(gpt->oos_page->mem, &we, index,
1378 				false, 0, vgpu);
1379 
1380 	if (can_do_out_of_sync(gpt)) {
1381 		if (!gpt->oos_page)
1382 			ppgtt_allocate_oos_page(vgpu, gpt);
1383 
1384 		ret = ppgtt_set_guest_page_oos(vgpu, gpt);
1385 		if (ret < 0)
1386 			return ret;
1387 	}
1388 	return 0;
1389 }
1390 
1391 /*
1392  * mm page table allocation policy for bdw+
1393  *  - for ggtt, only virtual page table will be allocated.
1394  *  - for ppgtt, dedicated virtual/shadow page table will be allocated.
1395  */
1396 static int gen8_mm_alloc_page_table(struct intel_vgpu_mm *mm)
1397 {
1398 	struct intel_vgpu *vgpu = mm->vgpu;
1399 	struct intel_gvt *gvt = vgpu->gvt;
1400 	const struct intel_gvt_device_info *info = &gvt->device_info;
1401 	void *mem;
1402 
1403 	if (mm->type == INTEL_GVT_MM_PPGTT) {
1404 		mm->page_table_entry_cnt = 4;
1405 		mm->page_table_entry_size = mm->page_table_entry_cnt *
1406 			info->gtt_entry_size;
1407 		mem = kzalloc(mm->has_shadow_page_table ?
1408 			mm->page_table_entry_size * 2
1409 				: mm->page_table_entry_size, GFP_KERNEL);
1410 		if (!mem)
1411 			return -ENOMEM;
1412 		mm->virtual_page_table = mem;
1413 		if (!mm->has_shadow_page_table)
1414 			return 0;
1415 		mm->shadow_page_table = mem + mm->page_table_entry_size;
1416 	} else if (mm->type == INTEL_GVT_MM_GGTT) {
1417 		mm->page_table_entry_cnt =
1418 			(gvt_ggtt_gm_sz(gvt) >> GTT_PAGE_SHIFT);
1419 		mm->page_table_entry_size = mm->page_table_entry_cnt *
1420 			info->gtt_entry_size;
1421 		mem = vzalloc(mm->page_table_entry_size);
1422 		if (!mem)
1423 			return -ENOMEM;
1424 		mm->virtual_page_table = mem;
1425 	}
1426 	return 0;
1427 }
1428 
1429 static void gen8_mm_free_page_table(struct intel_vgpu_mm *mm)
1430 {
1431 	if (mm->type == INTEL_GVT_MM_PPGTT) {
1432 		kfree(mm->virtual_page_table);
1433 	} else if (mm->type == INTEL_GVT_MM_GGTT) {
1434 		if (mm->virtual_page_table)
1435 			vfree(mm->virtual_page_table);
1436 	}
1437 	mm->virtual_page_table = mm->shadow_page_table = NULL;
1438 }
1439 
1440 static void invalidate_mm(struct intel_vgpu_mm *mm)
1441 {
1442 	struct intel_vgpu *vgpu = mm->vgpu;
1443 	struct intel_gvt *gvt = vgpu->gvt;
1444 	struct intel_gvt_gtt *gtt = &gvt->gtt;
1445 	struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;
1446 	struct intel_gvt_gtt_entry se;
1447 	int i;
1448 
1449 	if (WARN_ON(!mm->has_shadow_page_table || !mm->shadowed))
1450 		return;
1451 
1452 	for (i = 0; i < mm->page_table_entry_cnt; i++) {
1453 		ppgtt_get_shadow_root_entry(mm, &se, i);
1454 		if (!ops->test_present(&se))
1455 			continue;
1456 		ppgtt_invalidate_shadow_page_by_shadow_entry(
1457 				vgpu, &se);
1458 		se.val64 = 0;
1459 		ppgtt_set_shadow_root_entry(mm, &se, i);
1460 
1461 		trace_gpt_change(vgpu->id, "destroy root pointer",
1462 				NULL, se.type, se.val64, i);
1463 	}
1464 	mm->shadowed = false;
1465 }
1466 
1467 /**
1468  * intel_vgpu_destroy_mm - destroy a mm object
1469  * @mm: a kref object
1470  *
1471  * This function is used to destroy a mm object for vGPU
1472  *
1473  */
1474 void intel_vgpu_destroy_mm(struct kref *mm_ref)
1475 {
1476 	struct intel_vgpu_mm *mm = container_of(mm_ref, typeof(*mm), ref);
1477 	struct intel_vgpu *vgpu = mm->vgpu;
1478 	struct intel_gvt *gvt = vgpu->gvt;
1479 	struct intel_gvt_gtt *gtt = &gvt->gtt;
1480 
1481 	if (!mm->initialized)
1482 		goto out;
1483 
1484 	list_del(&mm->list);
1485 	list_del(&mm->lru_list);
1486 
1487 	if (mm->has_shadow_page_table)
1488 		invalidate_mm(mm);
1489 
1490 	gtt->mm_free_page_table(mm);
1491 out:
1492 	kfree(mm);
1493 }
1494 
1495 static int shadow_mm(struct intel_vgpu_mm *mm)
1496 {
1497 	struct intel_vgpu *vgpu = mm->vgpu;
1498 	struct intel_gvt *gvt = vgpu->gvt;
1499 	struct intel_gvt_gtt *gtt = &gvt->gtt;
1500 	struct intel_gvt_gtt_pte_ops *ops = gtt->pte_ops;
1501 	struct intel_vgpu_ppgtt_spt *spt;
1502 	struct intel_gvt_gtt_entry ge, se;
1503 	int i;
1504 	int ret;
1505 
1506 	if (WARN_ON(!mm->has_shadow_page_table || mm->shadowed))
1507 		return 0;
1508 
1509 	mm->shadowed = true;
1510 
1511 	for (i = 0; i < mm->page_table_entry_cnt; i++) {
1512 		ppgtt_get_guest_root_entry(mm, &ge, i);
1513 		if (!ops->test_present(&ge))
1514 			continue;
1515 
1516 		trace_gpt_change(vgpu->id, __func__, NULL,
1517 				ge.type, ge.val64, i);
1518 
1519 		spt = ppgtt_populate_shadow_page_by_guest_entry(vgpu, &ge);
1520 		if (IS_ERR(spt)) {
1521 			gvt_vgpu_err("fail to populate guest root pointer\n");
1522 			ret = PTR_ERR(spt);
1523 			goto fail;
1524 		}
1525 		ppgtt_generate_shadow_entry(&se, spt, &ge);
1526 		ppgtt_set_shadow_root_entry(mm, &se, i);
1527 
1528 		trace_gpt_change(vgpu->id, "populate root pointer",
1529 				NULL, se.type, se.val64, i);
1530 	}
1531 	return 0;
1532 fail:
1533 	invalidate_mm(mm);
1534 	return ret;
1535 }
1536 
1537 /**
1538  * intel_vgpu_create_mm - create a mm object for a vGPU
1539  * @vgpu: a vGPU
1540  * @mm_type: mm object type, should be PPGTT or GGTT
1541  * @virtual_page_table: page table root pointers. Could be NULL if user wants
1542  *	to populate shadow later.
1543  * @page_table_level: describe the page table level of the mm object
1544  * @pde_base_index: pde root pointer base in GGTT MMIO.
1545  *
1546  * This function is used to create a mm object for a vGPU.
1547  *
1548  * Returns:
1549  * Zero on success, negative error code in pointer if failed.
1550  */
1551 struct intel_vgpu_mm *intel_vgpu_create_mm(struct intel_vgpu *vgpu,
1552 		int mm_type, void *virtual_page_table, int page_table_level,
1553 		u32 pde_base_index)
1554 {
1555 	struct intel_gvt *gvt = vgpu->gvt;
1556 	struct intel_gvt_gtt *gtt = &gvt->gtt;
1557 	struct intel_vgpu_mm *mm;
1558 	int ret;
1559 
1560 	mm = kzalloc(sizeof(*mm), GFP_KERNEL);
1561 	if (!mm) {
1562 		ret = -ENOMEM;
1563 		goto fail;
1564 	}
1565 
1566 	mm->type = mm_type;
1567 
1568 	if (page_table_level == 1)
1569 		mm->page_table_entry_type = GTT_TYPE_GGTT_PTE;
1570 	else if (page_table_level == 3)
1571 		mm->page_table_entry_type = GTT_TYPE_PPGTT_ROOT_L3_ENTRY;
1572 	else if (page_table_level == 4)
1573 		mm->page_table_entry_type = GTT_TYPE_PPGTT_ROOT_L4_ENTRY;
1574 	else {
1575 		WARN_ON(1);
1576 		ret = -EINVAL;
1577 		goto fail;
1578 	}
1579 
1580 	mm->page_table_level = page_table_level;
1581 	mm->pde_base_index = pde_base_index;
1582 
1583 	mm->vgpu = vgpu;
1584 	mm->has_shadow_page_table = !!(mm_type == INTEL_GVT_MM_PPGTT);
1585 
1586 	kref_init(&mm->ref);
1587 	atomic_set(&mm->pincount, 0);
1588 	INIT_LIST_HEAD(&mm->list);
1589 	INIT_LIST_HEAD(&mm->lru_list);
1590 	list_add_tail(&mm->list, &vgpu->gtt.mm_list_head);
1591 
1592 	ret = gtt->mm_alloc_page_table(mm);
1593 	if (ret) {
1594 		gvt_vgpu_err("fail to allocate page table for mm\n");
1595 		goto fail;
1596 	}
1597 
1598 	mm->initialized = true;
1599 
1600 	if (virtual_page_table)
1601 		memcpy(mm->virtual_page_table, virtual_page_table,
1602 				mm->page_table_entry_size);
1603 
1604 	if (mm->has_shadow_page_table) {
1605 		ret = shadow_mm(mm);
1606 		if (ret)
1607 			goto fail;
1608 		list_add_tail(&mm->lru_list, &gvt->gtt.mm_lru_list_head);
1609 	}
1610 	return mm;
1611 fail:
1612 	gvt_vgpu_err("fail to create mm\n");
1613 	if (mm)
1614 		intel_gvt_mm_unreference(mm);
1615 	return ERR_PTR(ret);
1616 }
1617 
1618 /**
1619  * intel_vgpu_unpin_mm - decrease the pin count of a vGPU mm object
1620  * @mm: a vGPU mm object
1621  *
1622  * This function is called when user doesn't want to use a vGPU mm object
1623  */
1624 void intel_vgpu_unpin_mm(struct intel_vgpu_mm *mm)
1625 {
1626 	if (WARN_ON(mm->type != INTEL_GVT_MM_PPGTT))
1627 		return;
1628 
1629 	atomic_dec(&mm->pincount);
1630 }
1631 
1632 /**
1633  * intel_vgpu_pin_mm - increase the pin count of a vGPU mm object
1634  * @vgpu: a vGPU
1635  *
1636  * This function is called when user wants to use a vGPU mm object. If this
1637  * mm object hasn't been shadowed yet, the shadow will be populated at this
1638  * time.
1639  *
1640  * Returns:
1641  * Zero on success, negative error code if failed.
1642  */
1643 int intel_vgpu_pin_mm(struct intel_vgpu_mm *mm)
1644 {
1645 	int ret;
1646 
1647 	if (WARN_ON(mm->type != INTEL_GVT_MM_PPGTT))
1648 		return 0;
1649 
1650 	atomic_inc(&mm->pincount);
1651 
1652 	if (!mm->shadowed) {
1653 		ret = shadow_mm(mm);
1654 		if (ret)
1655 			return ret;
1656 	}
1657 
1658 	list_del_init(&mm->lru_list);
1659 	list_add_tail(&mm->lru_list, &mm->vgpu->gvt->gtt.mm_lru_list_head);
1660 	return 0;
1661 }
1662 
1663 static int reclaim_one_mm(struct intel_gvt *gvt)
1664 {
1665 	struct intel_vgpu_mm *mm;
1666 	struct list_head *pos, *n;
1667 
1668 	list_for_each_safe(pos, n, &gvt->gtt.mm_lru_list_head) {
1669 		mm = container_of(pos, struct intel_vgpu_mm, lru_list);
1670 
1671 		if (mm->type != INTEL_GVT_MM_PPGTT)
1672 			continue;
1673 		if (atomic_read(&mm->pincount))
1674 			continue;
1675 
1676 		list_del_init(&mm->lru_list);
1677 		invalidate_mm(mm);
1678 		return 1;
1679 	}
1680 	return 0;
1681 }
1682 
1683 /*
1684  * GMA translation APIs.
1685  */
1686 static inline int ppgtt_get_next_level_entry(struct intel_vgpu_mm *mm,
1687 		struct intel_gvt_gtt_entry *e, unsigned long index, bool guest)
1688 {
1689 	struct intel_vgpu *vgpu = mm->vgpu;
1690 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
1691 	struct intel_vgpu_ppgtt_spt *s;
1692 
1693 	if (WARN_ON(!mm->has_shadow_page_table))
1694 		return -EINVAL;
1695 
1696 	s = ppgtt_find_shadow_page(vgpu, ops->get_pfn(e));
1697 	if (!s)
1698 		return -ENXIO;
1699 
1700 	if (!guest)
1701 		ppgtt_get_shadow_entry(s, e, index);
1702 	else
1703 		ppgtt_get_guest_entry(s, e, index);
1704 	return 0;
1705 }
1706 
1707 /**
1708  * intel_vgpu_gma_to_gpa - translate a gma to GPA
1709  * @mm: mm object. could be a PPGTT or GGTT mm object
1710  * @gma: graphics memory address in this mm object
1711  *
1712  * This function is used to translate a graphics memory address in specific
1713  * graphics memory space to guest physical address.
1714  *
1715  * Returns:
1716  * Guest physical address on success, INTEL_GVT_INVALID_ADDR if failed.
1717  */
1718 unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma)
1719 {
1720 	struct intel_vgpu *vgpu = mm->vgpu;
1721 	struct intel_gvt *gvt = vgpu->gvt;
1722 	struct intel_gvt_gtt_pte_ops *pte_ops = gvt->gtt.pte_ops;
1723 	struct intel_gvt_gtt_gma_ops *gma_ops = gvt->gtt.gma_ops;
1724 	unsigned long gpa = INTEL_GVT_INVALID_ADDR;
1725 	unsigned long gma_index[4];
1726 	struct intel_gvt_gtt_entry e;
1727 	int i, index;
1728 	int ret;
1729 
1730 	if (mm->type != INTEL_GVT_MM_GGTT && mm->type != INTEL_GVT_MM_PPGTT)
1731 		return INTEL_GVT_INVALID_ADDR;
1732 
1733 	if (mm->type == INTEL_GVT_MM_GGTT) {
1734 		if (!vgpu_gmadr_is_valid(vgpu, gma))
1735 			goto err;
1736 
1737 		ret = ggtt_get_guest_entry(mm, &e,
1738 				gma_ops->gma_to_ggtt_pte_index(gma));
1739 		if (ret)
1740 			goto err;
1741 		gpa = (pte_ops->get_pfn(&e) << GTT_PAGE_SHIFT)
1742 			+ (gma & ~GTT_PAGE_MASK);
1743 
1744 		trace_gma_translate(vgpu->id, "ggtt", 0, 0, gma, gpa);
1745 		return gpa;
1746 	}
1747 
1748 	switch (mm->page_table_level) {
1749 	case 4:
1750 		ret = ppgtt_get_shadow_root_entry(mm, &e, 0);
1751 		if (ret)
1752 			goto err;
1753 		gma_index[0] = gma_ops->gma_to_pml4_index(gma);
1754 		gma_index[1] = gma_ops->gma_to_l4_pdp_index(gma);
1755 		gma_index[2] = gma_ops->gma_to_pde_index(gma);
1756 		gma_index[3] = gma_ops->gma_to_pte_index(gma);
1757 		index = 4;
1758 		break;
1759 	case 3:
1760 		ret = ppgtt_get_shadow_root_entry(mm, &e,
1761 				gma_ops->gma_to_l3_pdp_index(gma));
1762 		if (ret)
1763 			goto err;
1764 		gma_index[0] = gma_ops->gma_to_pde_index(gma);
1765 		gma_index[1] = gma_ops->gma_to_pte_index(gma);
1766 		index = 2;
1767 		break;
1768 	case 2:
1769 		ret = ppgtt_get_shadow_root_entry(mm, &e,
1770 				gma_ops->gma_to_pde_index(gma));
1771 		if (ret)
1772 			goto err;
1773 		gma_index[0] = gma_ops->gma_to_pte_index(gma);
1774 		index = 1;
1775 		break;
1776 	default:
1777 		WARN_ON(1);
1778 		goto err;
1779 	}
1780 
1781 	/* walk into the shadow page table and get gpa from guest entry */
1782 	for (i = 0; i < index; i++) {
1783 		ret = ppgtt_get_next_level_entry(mm, &e, gma_index[i],
1784 			(i == index - 1));
1785 		if (ret)
1786 			goto err;
1787 
1788 		if (!pte_ops->test_present(&e)) {
1789 			gvt_dbg_core("GMA 0x%lx is not present\n", gma);
1790 			goto err;
1791 		}
1792 	}
1793 
1794 	gpa = (pte_ops->get_pfn(&e) << GTT_PAGE_SHIFT)
1795 		+ (gma & ~GTT_PAGE_MASK);
1796 
1797 	trace_gma_translate(vgpu->id, "ppgtt", 0,
1798 			mm->page_table_level, gma, gpa);
1799 	return gpa;
1800 err:
1801 	gvt_vgpu_err("invalid mm type: %d gma %lx\n", mm->type, gma);
1802 	return INTEL_GVT_INVALID_ADDR;
1803 }
1804 
1805 static int emulate_gtt_mmio_read(struct intel_vgpu *vgpu,
1806 	unsigned int off, void *p_data, unsigned int bytes)
1807 {
1808 	struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;
1809 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
1810 	unsigned long index = off >> info->gtt_entry_size_shift;
1811 	struct intel_gvt_gtt_entry e;
1812 
1813 	if (bytes != 4 && bytes != 8)
1814 		return -EINVAL;
1815 
1816 	ggtt_get_guest_entry(ggtt_mm, &e, index);
1817 	memcpy(p_data, (void *)&e.val64 + (off & (info->gtt_entry_size - 1)),
1818 			bytes);
1819 	return 0;
1820 }
1821 
1822 /**
1823  * intel_vgpu_emulate_gtt_mmio_read - emulate GTT MMIO register read
1824  * @vgpu: a vGPU
1825  * @off: register offset
1826  * @p_data: data will be returned to guest
1827  * @bytes: data length
1828  *
1829  * This function is used to emulate the GTT MMIO register read
1830  *
1831  * Returns:
1832  * Zero on success, error code if failed.
1833  */
1834 int intel_vgpu_emulate_gtt_mmio_read(struct intel_vgpu *vgpu, unsigned int off,
1835 	void *p_data, unsigned int bytes)
1836 {
1837 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
1838 	int ret;
1839 
1840 	if (bytes != 4 && bytes != 8)
1841 		return -EINVAL;
1842 
1843 	off -= info->gtt_start_offset;
1844 	ret = emulate_gtt_mmio_read(vgpu, off, p_data, bytes);
1845 	return ret;
1846 }
1847 
1848 static int emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
1849 	void *p_data, unsigned int bytes)
1850 {
1851 	struct intel_gvt *gvt = vgpu->gvt;
1852 	const struct intel_gvt_device_info *info = &gvt->device_info;
1853 	struct intel_vgpu_mm *ggtt_mm = vgpu->gtt.ggtt_mm;
1854 	struct intel_gvt_gtt_pte_ops *ops = gvt->gtt.pte_ops;
1855 	unsigned long g_gtt_index = off >> info->gtt_entry_size_shift;
1856 	unsigned long gma;
1857 	struct intel_gvt_gtt_entry e, m;
1858 	int ret;
1859 
1860 	if (bytes != 4 && bytes != 8)
1861 		return -EINVAL;
1862 
1863 	gma = g_gtt_index << GTT_PAGE_SHIFT;
1864 
1865 	/* the VM may configure the whole GM space when ballooning is used */
1866 	if (!vgpu_gmadr_is_valid(vgpu, gma))
1867 		return 0;
1868 
1869 	ggtt_get_guest_entry(ggtt_mm, &e, g_gtt_index);
1870 
1871 	memcpy((void *)&e.val64 + (off & (info->gtt_entry_size - 1)), p_data,
1872 			bytes);
1873 
1874 	if (ops->test_present(&e)) {
1875 		ret = gtt_entry_p2m(vgpu, &e, &m);
1876 		if (ret) {
1877 			gvt_vgpu_err("fail to translate guest gtt entry\n");
1878 			/* guest driver may read/write the entry when partial
1879 			 * update the entry in this situation p2m will fail
1880 			 * settting the shadow entry to point to a scratch page
1881 			 */
1882 			ops->set_pfn(&m, gvt->gtt.scratch_ggtt_mfn);
1883 		}
1884 	} else {
1885 		m = e;
1886 		ops->set_pfn(&m, gvt->gtt.scratch_ggtt_mfn);
1887 	}
1888 
1889 	ggtt_set_shadow_entry(ggtt_mm, &m, g_gtt_index);
1890 	gtt_invalidate(gvt->dev_priv);
1891 	ggtt_set_guest_entry(ggtt_mm, &e, g_gtt_index);
1892 	return 0;
1893 }
1894 
1895 /*
1896  * intel_vgpu_emulate_gtt_mmio_write - emulate GTT MMIO register write
1897  * @vgpu: a vGPU
1898  * @off: register offset
1899  * @p_data: data from guest write
1900  * @bytes: data length
1901  *
1902  * This function is used to emulate the GTT MMIO register write
1903  *
1904  * Returns:
1905  * Zero on success, error code if failed.
1906  */
1907 int intel_vgpu_emulate_gtt_mmio_write(struct intel_vgpu *vgpu, unsigned int off,
1908 	void *p_data, unsigned int bytes)
1909 {
1910 	const struct intel_gvt_device_info *info = &vgpu->gvt->device_info;
1911 	int ret;
1912 
1913 	if (bytes != 4 && bytes != 8)
1914 		return -EINVAL;
1915 
1916 	off -= info->gtt_start_offset;
1917 	ret = emulate_gtt_mmio_write(vgpu, off, p_data, bytes);
1918 	return ret;
1919 }
1920 
1921 static int alloc_scratch_pages(struct intel_vgpu *vgpu,
1922 		intel_gvt_gtt_type_t type)
1923 {
1924 	struct intel_vgpu_gtt *gtt = &vgpu->gtt;
1925 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
1926 	int page_entry_num = GTT_PAGE_SIZE >>
1927 				vgpu->gvt->device_info.gtt_entry_size_shift;
1928 	void *scratch_pt;
1929 	int i;
1930 	struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
1931 	dma_addr_t daddr;
1932 
1933 	if (WARN_ON(type < GTT_TYPE_PPGTT_PTE_PT || type >= GTT_TYPE_MAX))
1934 		return -EINVAL;
1935 
1936 	scratch_pt = (void *)get_zeroed_page(GFP_KERNEL);
1937 	if (!scratch_pt) {
1938 		gvt_vgpu_err("fail to allocate scratch page\n");
1939 		return -ENOMEM;
1940 	}
1941 
1942 	daddr = dma_map_page(dev, virt_to_page(scratch_pt), 0,
1943 			4096, PCI_DMA_BIDIRECTIONAL);
1944 	if (dma_mapping_error(dev, daddr)) {
1945 		gvt_vgpu_err("fail to dmamap scratch_pt\n");
1946 		__free_page(virt_to_page(scratch_pt));
1947 		return -ENOMEM;
1948 	}
1949 	gtt->scratch_pt[type].page_mfn =
1950 		(unsigned long)(daddr >> GTT_PAGE_SHIFT);
1951 	gtt->scratch_pt[type].page = virt_to_page(scratch_pt);
1952 	gvt_dbg_mm("vgpu%d create scratch_pt: type %d mfn=0x%lx\n",
1953 			vgpu->id, type, gtt->scratch_pt[type].page_mfn);
1954 
1955 	/* Build the tree by full filled the scratch pt with the entries which
1956 	 * point to the next level scratch pt or scratch page. The
1957 	 * scratch_pt[type] indicate the scratch pt/scratch page used by the
1958 	 * 'type' pt.
1959 	 * e.g. scratch_pt[GTT_TYPE_PPGTT_PDE_PT] is used by
1960 	 * GTT_TYPE_PPGTT_PDE_PT level pt, that means this scratch_pt it self
1961 	 * is GTT_TYPE_PPGTT_PTE_PT, and full filled by scratch page mfn.
1962 	 */
1963 	if (type > GTT_TYPE_PPGTT_PTE_PT && type < GTT_TYPE_MAX) {
1964 		struct intel_gvt_gtt_entry se;
1965 
1966 		memset(&se, 0, sizeof(struct intel_gvt_gtt_entry));
1967 		se.type = get_entry_type(type - 1);
1968 		ops->set_pfn(&se, gtt->scratch_pt[type - 1].page_mfn);
1969 
1970 		/* The entry parameters like present/writeable/cache type
1971 		 * set to the same as i915's scratch page tree.
1972 		 */
1973 		se.val64 |= _PAGE_PRESENT | _PAGE_RW;
1974 		if (type == GTT_TYPE_PPGTT_PDE_PT)
1975 			se.val64 |= PPAT_CACHED_INDEX;
1976 
1977 		for (i = 0; i < page_entry_num; i++)
1978 			ops->set_entry(scratch_pt, &se, i, false, 0, vgpu);
1979 	}
1980 
1981 	return 0;
1982 }
1983 
1984 static int release_scratch_page_tree(struct intel_vgpu *vgpu)
1985 {
1986 	int i;
1987 	struct device *dev = &vgpu->gvt->dev_priv->drm.pdev->dev;
1988 	dma_addr_t daddr;
1989 
1990 	for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) {
1991 		if (vgpu->gtt.scratch_pt[i].page != NULL) {
1992 			daddr = (dma_addr_t)(vgpu->gtt.scratch_pt[i].page_mfn <<
1993 					GTT_PAGE_SHIFT);
1994 			dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL);
1995 			__free_page(vgpu->gtt.scratch_pt[i].page);
1996 			vgpu->gtt.scratch_pt[i].page = NULL;
1997 			vgpu->gtt.scratch_pt[i].page_mfn = 0;
1998 		}
1999 	}
2000 
2001 	return 0;
2002 }
2003 
2004 static int create_scratch_page_tree(struct intel_vgpu *vgpu)
2005 {
2006 	int i, ret;
2007 
2008 	for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) {
2009 		ret = alloc_scratch_pages(vgpu, i);
2010 		if (ret)
2011 			goto err;
2012 	}
2013 
2014 	return 0;
2015 
2016 err:
2017 	release_scratch_page_tree(vgpu);
2018 	return ret;
2019 }
2020 
2021 /**
2022  * intel_vgpu_init_gtt - initialize per-vGPU graphics memory virulization
2023  * @vgpu: a vGPU
2024  *
2025  * This function is used to initialize per-vGPU graphics memory virtualization
2026  * components.
2027  *
2028  * Returns:
2029  * Zero on success, error code if failed.
2030  */
2031 int intel_vgpu_init_gtt(struct intel_vgpu *vgpu)
2032 {
2033 	struct intel_vgpu_gtt *gtt = &vgpu->gtt;
2034 	struct intel_vgpu_mm *ggtt_mm;
2035 
2036 	hash_init(gtt->guest_page_hash_table);
2037 	hash_init(gtt->shadow_page_hash_table);
2038 
2039 	INIT_LIST_HEAD(&gtt->mm_list_head);
2040 	INIT_LIST_HEAD(&gtt->oos_page_list_head);
2041 	INIT_LIST_HEAD(&gtt->post_shadow_list_head);
2042 
2043 	intel_vgpu_reset_ggtt(vgpu);
2044 
2045 	ggtt_mm = intel_vgpu_create_mm(vgpu, INTEL_GVT_MM_GGTT,
2046 			NULL, 1, 0);
2047 	if (IS_ERR(ggtt_mm)) {
2048 		gvt_vgpu_err("fail to create mm for ggtt.\n");
2049 		return PTR_ERR(ggtt_mm);
2050 	}
2051 
2052 	gtt->ggtt_mm = ggtt_mm;
2053 
2054 	return create_scratch_page_tree(vgpu);
2055 }
2056 
2057 static void intel_vgpu_free_mm(struct intel_vgpu *vgpu, int type)
2058 {
2059 	struct list_head *pos, *n;
2060 	struct intel_vgpu_mm *mm;
2061 
2062 	list_for_each_safe(pos, n, &vgpu->gtt.mm_list_head) {
2063 		mm = container_of(pos, struct intel_vgpu_mm, list);
2064 		if (mm->type == type) {
2065 			vgpu->gvt->gtt.mm_free_page_table(mm);
2066 			list_del(&mm->list);
2067 			list_del(&mm->lru_list);
2068 			kfree(mm);
2069 		}
2070 	}
2071 }
2072 
2073 /**
2074  * intel_vgpu_clean_gtt - clean up per-vGPU graphics memory virulization
2075  * @vgpu: a vGPU
2076  *
2077  * This function is used to clean up per-vGPU graphics memory virtualization
2078  * components.
2079  *
2080  * Returns:
2081  * Zero on success, error code if failed.
2082  */
2083 void intel_vgpu_clean_gtt(struct intel_vgpu *vgpu)
2084 {
2085 	ppgtt_free_all_shadow_page(vgpu);
2086 	release_scratch_page_tree(vgpu);
2087 
2088 	intel_vgpu_free_mm(vgpu, INTEL_GVT_MM_PPGTT);
2089 	intel_vgpu_free_mm(vgpu, INTEL_GVT_MM_GGTT);
2090 }
2091 
2092 static void clean_spt_oos(struct intel_gvt *gvt)
2093 {
2094 	struct intel_gvt_gtt *gtt = &gvt->gtt;
2095 	struct list_head *pos, *n;
2096 	struct intel_vgpu_oos_page *oos_page;
2097 
2098 	WARN(!list_empty(&gtt->oos_page_use_list_head),
2099 		"someone is still using oos page\n");
2100 
2101 	list_for_each_safe(pos, n, &gtt->oos_page_free_list_head) {
2102 		oos_page = container_of(pos, struct intel_vgpu_oos_page, list);
2103 		list_del(&oos_page->list);
2104 		kfree(oos_page);
2105 	}
2106 }
2107 
2108 static int setup_spt_oos(struct intel_gvt *gvt)
2109 {
2110 	struct intel_gvt_gtt *gtt = &gvt->gtt;
2111 	struct intel_vgpu_oos_page *oos_page;
2112 	int i;
2113 	int ret;
2114 
2115 	INIT_LIST_HEAD(&gtt->oos_page_free_list_head);
2116 	INIT_LIST_HEAD(&gtt->oos_page_use_list_head);
2117 
2118 	for (i = 0; i < preallocated_oos_pages; i++) {
2119 		oos_page = kzalloc(sizeof(*oos_page), GFP_KERNEL);
2120 		if (!oos_page) {
2121 			ret = -ENOMEM;
2122 			goto fail;
2123 		}
2124 
2125 		INIT_LIST_HEAD(&oos_page->list);
2126 		INIT_LIST_HEAD(&oos_page->vm_list);
2127 		oos_page->id = i;
2128 		list_add_tail(&oos_page->list, &gtt->oos_page_free_list_head);
2129 	}
2130 
2131 	gvt_dbg_mm("%d oos pages preallocated\n", i);
2132 
2133 	return 0;
2134 fail:
2135 	clean_spt_oos(gvt);
2136 	return ret;
2137 }
2138 
2139 /**
2140  * intel_vgpu_find_ppgtt_mm - find a PPGTT mm object
2141  * @vgpu: a vGPU
2142  * @page_table_level: PPGTT page table level
2143  * @root_entry: PPGTT page table root pointers
2144  *
2145  * This function is used to find a PPGTT mm object from mm object pool
2146  *
2147  * Returns:
2148  * pointer to mm object on success, NULL if failed.
2149  */
2150 struct intel_vgpu_mm *intel_vgpu_find_ppgtt_mm(struct intel_vgpu *vgpu,
2151 		int page_table_level, void *root_entry)
2152 {
2153 	struct list_head *pos;
2154 	struct intel_vgpu_mm *mm;
2155 	u64 *src, *dst;
2156 
2157 	list_for_each(pos, &vgpu->gtt.mm_list_head) {
2158 		mm = container_of(pos, struct intel_vgpu_mm, list);
2159 		if (mm->type != INTEL_GVT_MM_PPGTT)
2160 			continue;
2161 
2162 		if (mm->page_table_level != page_table_level)
2163 			continue;
2164 
2165 		src = root_entry;
2166 		dst = mm->virtual_page_table;
2167 
2168 		if (page_table_level == 3) {
2169 			if (src[0] == dst[0]
2170 					&& src[1] == dst[1]
2171 					&& src[2] == dst[2]
2172 					&& src[3] == dst[3])
2173 				return mm;
2174 		} else {
2175 			if (src[0] == dst[0])
2176 				return mm;
2177 		}
2178 	}
2179 	return NULL;
2180 }
2181 
2182 /**
2183  * intel_vgpu_g2v_create_ppgtt_mm - create a PPGTT mm object from
2184  * g2v notification
2185  * @vgpu: a vGPU
2186  * @page_table_level: PPGTT page table level
2187  *
2188  * This function is used to create a PPGTT mm object from a guest to GVT-g
2189  * notification.
2190  *
2191  * Returns:
2192  * Zero on success, negative error code if failed.
2193  */
2194 int intel_vgpu_g2v_create_ppgtt_mm(struct intel_vgpu *vgpu,
2195 		int page_table_level)
2196 {
2197 	u64 *pdp = (u64 *)&vgpu_vreg64(vgpu, vgtif_reg(pdp[0]));
2198 	struct intel_vgpu_mm *mm;
2199 
2200 	if (WARN_ON((page_table_level != 4) && (page_table_level != 3)))
2201 		return -EINVAL;
2202 
2203 	mm = intel_vgpu_find_ppgtt_mm(vgpu, page_table_level, pdp);
2204 	if (mm) {
2205 		intel_gvt_mm_reference(mm);
2206 	} else {
2207 		mm = intel_vgpu_create_mm(vgpu, INTEL_GVT_MM_PPGTT,
2208 				pdp, page_table_level, 0);
2209 		if (IS_ERR(mm)) {
2210 			gvt_vgpu_err("fail to create mm\n");
2211 			return PTR_ERR(mm);
2212 		}
2213 	}
2214 	return 0;
2215 }
2216 
2217 /**
2218  * intel_vgpu_g2v_destroy_ppgtt_mm - destroy a PPGTT mm object from
2219  * g2v notification
2220  * @vgpu: a vGPU
2221  * @page_table_level: PPGTT page table level
2222  *
2223  * This function is used to create a PPGTT mm object from a guest to GVT-g
2224  * notification.
2225  *
2226  * Returns:
2227  * Zero on success, negative error code if failed.
2228  */
2229 int intel_vgpu_g2v_destroy_ppgtt_mm(struct intel_vgpu *vgpu,
2230 		int page_table_level)
2231 {
2232 	u64 *pdp = (u64 *)&vgpu_vreg64(vgpu, vgtif_reg(pdp[0]));
2233 	struct intel_vgpu_mm *mm;
2234 
2235 	if (WARN_ON((page_table_level != 4) && (page_table_level != 3)))
2236 		return -EINVAL;
2237 
2238 	mm = intel_vgpu_find_ppgtt_mm(vgpu, page_table_level, pdp);
2239 	if (!mm) {
2240 		gvt_vgpu_err("fail to find ppgtt instance.\n");
2241 		return -EINVAL;
2242 	}
2243 	intel_gvt_mm_unreference(mm);
2244 	return 0;
2245 }
2246 
2247 /**
2248  * intel_gvt_init_gtt - initialize mm components of a GVT device
2249  * @gvt: GVT device
2250  *
2251  * This function is called at the initialization stage, to initialize
2252  * the mm components of a GVT device.
2253  *
2254  * Returns:
2255  * zero on success, negative error code if failed.
2256  */
2257 int intel_gvt_init_gtt(struct intel_gvt *gvt)
2258 {
2259 	int ret;
2260 	void *page;
2261 	struct device *dev = &gvt->dev_priv->drm.pdev->dev;
2262 	dma_addr_t daddr;
2263 
2264 	gvt_dbg_core("init gtt\n");
2265 
2266 	if (IS_BROADWELL(gvt->dev_priv) || IS_SKYLAKE(gvt->dev_priv)
2267 		|| IS_KABYLAKE(gvt->dev_priv)) {
2268 		gvt->gtt.pte_ops = &gen8_gtt_pte_ops;
2269 		gvt->gtt.gma_ops = &gen8_gtt_gma_ops;
2270 		gvt->gtt.mm_alloc_page_table = gen8_mm_alloc_page_table;
2271 		gvt->gtt.mm_free_page_table = gen8_mm_free_page_table;
2272 	} else {
2273 		return -ENODEV;
2274 	}
2275 
2276 	page = (void *)get_zeroed_page(GFP_KERNEL);
2277 	if (!page) {
2278 		gvt_err("fail to allocate scratch ggtt page\n");
2279 		return -ENOMEM;
2280 	}
2281 
2282 	daddr = dma_map_page(dev, virt_to_page(page), 0,
2283 			4096, PCI_DMA_BIDIRECTIONAL);
2284 	if (dma_mapping_error(dev, daddr)) {
2285 		gvt_err("fail to dmamap scratch ggtt page\n");
2286 		__free_page(virt_to_page(page));
2287 		return -ENOMEM;
2288 	}
2289 	gvt->gtt.scratch_ggtt_page = virt_to_page(page);
2290 	gvt->gtt.scratch_ggtt_mfn = (unsigned long)(daddr >> GTT_PAGE_SHIFT);
2291 
2292 	if (enable_out_of_sync) {
2293 		ret = setup_spt_oos(gvt);
2294 		if (ret) {
2295 			gvt_err("fail to initialize SPT oos\n");
2296 			dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL);
2297 			__free_page(gvt->gtt.scratch_ggtt_page);
2298 			return ret;
2299 		}
2300 	}
2301 	INIT_LIST_HEAD(&gvt->gtt.mm_lru_list_head);
2302 	return 0;
2303 }
2304 
2305 /**
2306  * intel_gvt_clean_gtt - clean up mm components of a GVT device
2307  * @gvt: GVT device
2308  *
2309  * This function is called at the driver unloading stage, to clean up the
2310  * the mm components of a GVT device.
2311  *
2312  */
2313 void intel_gvt_clean_gtt(struct intel_gvt *gvt)
2314 {
2315 	struct device *dev = &gvt->dev_priv->drm.pdev->dev;
2316 	dma_addr_t daddr = (dma_addr_t)(gvt->gtt.scratch_ggtt_mfn <<
2317 					GTT_PAGE_SHIFT);
2318 
2319 	dma_unmap_page(dev, daddr, 4096, PCI_DMA_BIDIRECTIONAL);
2320 
2321 	__free_page(gvt->gtt.scratch_ggtt_page);
2322 
2323 	if (enable_out_of_sync)
2324 		clean_spt_oos(gvt);
2325 }
2326 
2327 /**
2328  * intel_vgpu_reset_ggtt - reset the GGTT entry
2329  * @vgpu: a vGPU
2330  *
2331  * This function is called at the vGPU create stage
2332  * to reset all the GGTT entries.
2333  *
2334  */
2335 void intel_vgpu_reset_ggtt(struct intel_vgpu *vgpu)
2336 {
2337 	struct intel_gvt *gvt = vgpu->gvt;
2338 	struct drm_i915_private *dev_priv = gvt->dev_priv;
2339 	struct intel_gvt_gtt_pte_ops *ops = vgpu->gvt->gtt.pte_ops;
2340 	u32 index;
2341 	u32 offset;
2342 	u32 num_entries;
2343 	struct intel_gvt_gtt_entry e;
2344 
2345 	memset(&e, 0, sizeof(struct intel_gvt_gtt_entry));
2346 	e.type = GTT_TYPE_GGTT_PTE;
2347 	ops->set_pfn(&e, gvt->gtt.scratch_ggtt_mfn);
2348 	e.val64 |= _PAGE_PRESENT;
2349 
2350 	index = vgpu_aperture_gmadr_base(vgpu) >> PAGE_SHIFT;
2351 	num_entries = vgpu_aperture_sz(vgpu) >> PAGE_SHIFT;
2352 	for (offset = 0; offset < num_entries; offset++)
2353 		ops->set_entry(NULL, &e, index + offset, false, 0, vgpu);
2354 
2355 	index = vgpu_hidden_gmadr_base(vgpu) >> PAGE_SHIFT;
2356 	num_entries = vgpu_hidden_sz(vgpu) >> PAGE_SHIFT;
2357 	for (offset = 0; offset < num_entries; offset++)
2358 		ops->set_entry(NULL, &e, index + offset, false, 0, vgpu);
2359 
2360 	gtt_invalidate(dev_priv);
2361 }
2362 
2363 /**
2364  * intel_vgpu_reset_gtt - reset the all GTT related status
2365  * @vgpu: a vGPU
2366  *
2367  * This function is called from vfio core to reset reset all
2368  * GTT related status, including GGTT, PPGTT, scratch page.
2369  *
2370  */
2371 void intel_vgpu_reset_gtt(struct intel_vgpu *vgpu)
2372 {
2373 	int i;
2374 
2375 	ppgtt_free_all_shadow_page(vgpu);
2376 
2377 	/* Shadow pages are only created when there is no page
2378 	 * table tracking data, so remove page tracking data after
2379 	 * removing the shadow pages.
2380 	 */
2381 	intel_vgpu_free_mm(vgpu, INTEL_GVT_MM_PPGTT);
2382 
2383 	intel_vgpu_reset_ggtt(vgpu);
2384 
2385 	/* clear scratch page for security */
2386 	for (i = GTT_TYPE_PPGTT_PTE_PT; i < GTT_TYPE_MAX; i++) {
2387 		if (vgpu->gtt.scratch_pt[i].page != NULL)
2388 			memset(page_address(vgpu->gtt.scratch_pt[i].page),
2389 				0, PAGE_SIZE);
2390 	}
2391 }
2392