1 /*
2  * Copyright © 2016 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  */
24 
25 #include <linux/prime_numbers.h>
26 
27 #include "gem/i915_gem_context.h"
28 #include "gem/selftests/mock_context.h"
29 
30 #include "i915_scatterlist.h"
31 #include "i915_selftest.h"
32 
33 #include "mock_gem_device.h"
34 #include "mock_gtt.h"
35 
36 static bool assert_vma(struct i915_vma *vma,
37 		       struct drm_i915_gem_object *obj,
38 		       struct i915_gem_context *ctx)
39 {
40 	bool ok = true;
41 
42 	if (vma->vm != rcu_access_pointer(ctx->vm)) {
43 		pr_err("VMA created with wrong VM\n");
44 		ok = false;
45 	}
46 
47 	if (vma->size != obj->base.size) {
48 		pr_err("VMA created with wrong size, found %llu, expected %zu\n",
49 		       vma->size, obj->base.size);
50 		ok = false;
51 	}
52 
53 	if (vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) {
54 		pr_err("VMA created with wrong type [%d]\n",
55 		       vma->ggtt_view.type);
56 		ok = false;
57 	}
58 
59 	return ok;
60 }
61 
62 static struct i915_vma *
63 checked_vma_instance(struct drm_i915_gem_object *obj,
64 		     struct i915_address_space *vm,
65 		     const struct i915_ggtt_view *view)
66 {
67 	struct i915_vma *vma;
68 	bool ok = true;
69 
70 	vma = i915_vma_instance(obj, vm, view);
71 	if (IS_ERR(vma))
72 		return vma;
73 
74 	/* Manual checks, will be reinforced by i915_vma_compare! */
75 	if (vma->vm != vm) {
76 		pr_err("VMA's vm [%p] does not match request [%p]\n",
77 		       vma->vm, vm);
78 		ok = false;
79 	}
80 
81 	if (i915_is_ggtt(vm) != i915_vma_is_ggtt(vma)) {
82 		pr_err("VMA ggtt status [%d] does not match parent [%d]\n",
83 		       i915_vma_is_ggtt(vma), i915_is_ggtt(vm));
84 		ok = false;
85 	}
86 
87 	if (i915_vma_compare(vma, vm, view)) {
88 		pr_err("i915_vma_compare failed with create parameters!\n");
89 		return ERR_PTR(-EINVAL);
90 	}
91 
92 	if (i915_vma_compare(vma, vma->vm,
93 			     i915_vma_is_ggtt(vma) ? &vma->ggtt_view : NULL)) {
94 		pr_err("i915_vma_compare failed with itself\n");
95 		return ERR_PTR(-EINVAL);
96 	}
97 
98 	if (!ok) {
99 		pr_err("i915_vma_compare failed to detect the difference!\n");
100 		return ERR_PTR(-EINVAL);
101 	}
102 
103 	return vma;
104 }
105 
106 static int create_vmas(struct drm_i915_private *i915,
107 		       struct list_head *objects,
108 		       struct list_head *contexts)
109 {
110 	struct drm_i915_gem_object *obj;
111 	struct i915_gem_context *ctx;
112 	int pinned;
113 
114 	list_for_each_entry(obj, objects, st_link) {
115 		for (pinned = 0; pinned <= 1; pinned++) {
116 			list_for_each_entry(ctx, contexts, link) {
117 				struct i915_address_space *vm;
118 				struct i915_vma *vma;
119 				int err;
120 
121 				vm = i915_gem_context_get_vm_rcu(ctx);
122 				vma = checked_vma_instance(obj, vm, NULL);
123 				i915_vm_put(vm);
124 				if (IS_ERR(vma))
125 					return PTR_ERR(vma);
126 
127 				if (!assert_vma(vma, obj, ctx)) {
128 					pr_err("VMA lookup/create failed\n");
129 					return -EINVAL;
130 				}
131 
132 				if (!pinned) {
133 					err = i915_vma_pin(vma, 0, 0, PIN_USER);
134 					if (err) {
135 						pr_err("Failed to pin VMA\n");
136 						return err;
137 					}
138 				} else {
139 					i915_vma_unpin(vma);
140 				}
141 			}
142 		}
143 	}
144 
145 	return 0;
146 }
147 
148 static int igt_vma_create(void *arg)
149 {
150 	struct i915_ggtt *ggtt = arg;
151 	struct drm_i915_private *i915 = ggtt->vm.i915;
152 	struct drm_i915_gem_object *obj, *on;
153 	struct i915_gem_context *ctx, *cn;
154 	unsigned long num_obj, num_ctx;
155 	unsigned long no, nc;
156 	IGT_TIMEOUT(end_time);
157 	LIST_HEAD(contexts);
158 	LIST_HEAD(objects);
159 	int err = -ENOMEM;
160 
161 	/* Exercise creating many vma amonst many objections, checking the
162 	 * vma creation and lookup routines.
163 	 */
164 
165 	no = 0;
166 	for_each_prime_number(num_obj, ULONG_MAX - 1) {
167 		for (; no < num_obj; no++) {
168 			obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
169 			if (IS_ERR(obj))
170 				goto out;
171 
172 			list_add(&obj->st_link, &objects);
173 		}
174 
175 		nc = 0;
176 		for_each_prime_number(num_ctx, 2 * BITS_PER_LONG) {
177 			for (; nc < num_ctx; nc++) {
178 				ctx = mock_context(i915, "mock");
179 				if (!ctx)
180 					goto out;
181 
182 				list_move(&ctx->link, &contexts);
183 			}
184 
185 			err = create_vmas(i915, &objects, &contexts);
186 			if (err)
187 				goto out;
188 
189 			if (igt_timeout(end_time,
190 					"%s timed out: after %lu objects in %lu contexts\n",
191 					__func__, no, nc))
192 				goto end;
193 		}
194 
195 		list_for_each_entry_safe(ctx, cn, &contexts, link) {
196 			list_del_init(&ctx->link);
197 			mock_context_close(ctx);
198 		}
199 
200 		cond_resched();
201 	}
202 
203 end:
204 	/* Final pass to lookup all created contexts */
205 	err = create_vmas(i915, &objects, &contexts);
206 out:
207 	list_for_each_entry_safe(ctx, cn, &contexts, link) {
208 		list_del_init(&ctx->link);
209 		mock_context_close(ctx);
210 	}
211 
212 	list_for_each_entry_safe(obj, on, &objects, st_link)
213 		i915_gem_object_put(obj);
214 	return err;
215 }
216 
217 struct pin_mode {
218 	u64 size;
219 	u64 flags;
220 	bool (*assert)(const struct i915_vma *,
221 		       const struct pin_mode *mode,
222 		       int result);
223 	const char *string;
224 };
225 
226 static bool assert_pin_valid(const struct i915_vma *vma,
227 			     const struct pin_mode *mode,
228 			     int result)
229 {
230 	if (result)
231 		return false;
232 
233 	if (i915_vma_misplaced(vma, mode->size, 0, mode->flags))
234 		return false;
235 
236 	return true;
237 }
238 
239 __maybe_unused
240 static bool assert_pin_enospc(const struct i915_vma *vma,
241 			      const struct pin_mode *mode,
242 			      int result)
243 {
244 	return result == -ENOSPC;
245 }
246 
247 __maybe_unused
248 static bool assert_pin_einval(const struct i915_vma *vma,
249 			      const struct pin_mode *mode,
250 			      int result)
251 {
252 	return result == -EINVAL;
253 }
254 
255 static int igt_vma_pin1(void *arg)
256 {
257 	struct i915_ggtt *ggtt = arg;
258 	const struct pin_mode modes[] = {
259 #define VALID(sz, fl) { .size = (sz), .flags = (fl), .assert = assert_pin_valid, .string = #sz ", " #fl ", (valid) " }
260 #define __INVALID(sz, fl, check, eval) { .size = (sz), .flags = (fl), .assert = (check), .string = #sz ", " #fl ", (invalid " #eval ")" }
261 #define INVALID(sz, fl) __INVALID(sz, fl, assert_pin_einval, EINVAL)
262 #define NOSPACE(sz, fl) __INVALID(sz, fl, assert_pin_enospc, ENOSPC)
263 		VALID(0, PIN_GLOBAL),
264 		VALID(0, PIN_GLOBAL | PIN_MAPPABLE),
265 
266 		VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | 4096),
267 		VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | 8192),
268 		VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)),
269 		VALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)),
270 		VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->vm.total - 4096)),
271 
272 		VALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | (ggtt->mappable_end - 4096)),
273 		INVALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | ggtt->mappable_end),
274 		VALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | (ggtt->vm.total - 4096)),
275 		INVALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | ggtt->vm.total),
276 		INVALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | round_down(U64_MAX, PAGE_SIZE)),
277 
278 		VALID(4096, PIN_GLOBAL),
279 		VALID(8192, PIN_GLOBAL),
280 		VALID(ggtt->mappable_end - 4096, PIN_GLOBAL | PIN_MAPPABLE),
281 		VALID(ggtt->mappable_end, PIN_GLOBAL | PIN_MAPPABLE),
282 		NOSPACE(ggtt->mappable_end + 4096, PIN_GLOBAL | PIN_MAPPABLE),
283 		VALID(ggtt->vm.total - 4096, PIN_GLOBAL),
284 		VALID(ggtt->vm.total, PIN_GLOBAL),
285 		NOSPACE(ggtt->vm.total + 4096, PIN_GLOBAL),
286 		NOSPACE(round_down(U64_MAX, PAGE_SIZE), PIN_GLOBAL),
287 		INVALID(8192, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | (ggtt->mappable_end - 4096)),
288 		INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (ggtt->vm.total - 4096)),
289 		INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (round_down(U64_MAX, PAGE_SIZE) - 4096)),
290 
291 		VALID(8192, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)),
292 
293 #if !IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
294 		/* Misusing BIAS is a programming error (it is not controllable
295 		 * from userspace) so when debugging is enabled, it explodes.
296 		 * However, the tests are still quite interesting for checking
297 		 * variable start, end and size.
298 		 */
299 		NOSPACE(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | ggtt->mappable_end),
300 		NOSPACE(0, PIN_GLOBAL | PIN_OFFSET_BIAS | ggtt->vm.total),
301 		NOSPACE(8192, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)),
302 		NOSPACE(8192, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->vm.total - 4096)),
303 #endif
304 		{ },
305 #undef NOSPACE
306 #undef INVALID
307 #undef __INVALID
308 #undef VALID
309 	}, *m;
310 	struct drm_i915_gem_object *obj;
311 	struct i915_vma *vma;
312 	int err = -EINVAL;
313 
314 	/* Exercise all the weird and wonderful i915_vma_pin requests,
315 	 * focusing on error handling of boundary conditions.
316 	 */
317 
318 	GEM_BUG_ON(!drm_mm_clean(&ggtt->vm.mm));
319 
320 	obj = i915_gem_object_create_internal(ggtt->vm.i915, PAGE_SIZE);
321 	if (IS_ERR(obj))
322 		return PTR_ERR(obj);
323 
324 	vma = checked_vma_instance(obj, &ggtt->vm, NULL);
325 	if (IS_ERR(vma))
326 		goto out;
327 
328 	for (m = modes; m->assert; m++) {
329 		err = i915_vma_pin(vma, m->size, 0, m->flags);
330 		if (!m->assert(vma, m, err)) {
331 			pr_err("%s to pin single page into GGTT with mode[%d:%s]: size=%llx flags=%llx, err=%d\n",
332 			       m->assert == assert_pin_valid ? "Failed" : "Unexpectedly succeeded",
333 			       (int)(m - modes), m->string, m->size, m->flags,
334 			       err);
335 			if (!err)
336 				i915_vma_unpin(vma);
337 			err = -EINVAL;
338 			goto out;
339 		}
340 
341 		if (!err) {
342 			i915_vma_unpin(vma);
343 			err = i915_vma_unbind(vma);
344 			if (err) {
345 				pr_err("Failed to unbind single page from GGTT, err=%d\n", err);
346 				goto out;
347 			}
348 		}
349 
350 		cond_resched();
351 	}
352 
353 	err = 0;
354 out:
355 	i915_gem_object_put(obj);
356 	return err;
357 }
358 
359 static unsigned long rotated_index(const struct intel_rotation_info *r,
360 				   unsigned int n,
361 				   unsigned int x,
362 				   unsigned int y)
363 {
364 	return (r->plane[n].src_stride * (r->plane[n].height - y - 1) +
365 		r->plane[n].offset + x);
366 }
367 
368 static struct scatterlist *
369 assert_rotated(struct drm_i915_gem_object *obj,
370 	       const struct intel_rotation_info *r, unsigned int n,
371 	       struct scatterlist *sg)
372 {
373 	unsigned int x, y;
374 
375 	for (x = 0; x < r->plane[n].width; x++) {
376 		unsigned int left;
377 
378 		for (y = 0; y < r->plane[n].height; y++) {
379 			unsigned long src_idx;
380 			dma_addr_t src;
381 
382 			if (!sg) {
383 				pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n",
384 				       n, x, y);
385 				return ERR_PTR(-EINVAL);
386 			}
387 
388 			src_idx = rotated_index(r, n, x, y);
389 			src = i915_gem_object_get_dma_address(obj, src_idx);
390 
391 			if (sg_dma_len(sg) != PAGE_SIZE) {
392 				pr_err("Invalid sg.length, found %d, expected %lu for rotated page (%d, %d) [src index %lu]\n",
393 				       sg_dma_len(sg), PAGE_SIZE,
394 				       x, y, src_idx);
395 				return ERR_PTR(-EINVAL);
396 			}
397 
398 			if (sg_dma_address(sg) != src) {
399 				pr_err("Invalid address for rotated page (%d, %d) [src index %lu]\n",
400 				       x, y, src_idx);
401 				return ERR_PTR(-EINVAL);
402 			}
403 
404 			sg = sg_next(sg);
405 		}
406 
407 		left = (r->plane[n].dst_stride - y) * PAGE_SIZE;
408 
409 		if (!left)
410 			continue;
411 
412 		if (!sg) {
413 			pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n",
414 			       n, x, y);
415 			return ERR_PTR(-EINVAL);
416 		}
417 
418 		if (sg_dma_len(sg) != left) {
419 			pr_err("Invalid sg.length, found %d, expected %u for rotated page (%d, %d)\n",
420 			       sg_dma_len(sg), left, x, y);
421 			return ERR_PTR(-EINVAL);
422 		}
423 
424 		if (sg_dma_address(sg) != 0) {
425 			pr_err("Invalid address, found %pad, expected 0 for remapped page (%d, %d)\n",
426 			       &sg_dma_address(sg), x, y);
427 			return ERR_PTR(-EINVAL);
428 		}
429 
430 		sg = sg_next(sg);
431 	}
432 
433 	return sg;
434 }
435 
436 static unsigned long remapped_index(const struct intel_remapped_info *r,
437 				    unsigned int n,
438 				    unsigned int x,
439 				    unsigned int y)
440 {
441 	return (r->plane[n].src_stride * y +
442 		r->plane[n].offset + x);
443 }
444 
445 static struct scatterlist *
446 assert_remapped(struct drm_i915_gem_object *obj,
447 		const struct intel_remapped_info *r, unsigned int n,
448 		struct scatterlist *sg)
449 {
450 	unsigned int x, y;
451 	unsigned int left = 0;
452 	unsigned int offset;
453 
454 	for (y = 0; y < r->plane[n].height; y++) {
455 		for (x = 0; x < r->plane[n].width; x++) {
456 			unsigned long src_idx;
457 			dma_addr_t src;
458 
459 			if (!sg) {
460 				pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n",
461 				       n, x, y);
462 				return ERR_PTR(-EINVAL);
463 			}
464 			if (!left) {
465 				offset = 0;
466 				left = sg_dma_len(sg);
467 			}
468 
469 			src_idx = remapped_index(r, n, x, y);
470 			src = i915_gem_object_get_dma_address(obj, src_idx);
471 
472 			if (left < PAGE_SIZE || left & (PAGE_SIZE-1)) {
473 				pr_err("Invalid sg.length, found %d, expected %lu for remapped page (%d, %d) [src index %lu]\n",
474 				       sg_dma_len(sg), PAGE_SIZE,
475 				       x, y, src_idx);
476 				return ERR_PTR(-EINVAL);
477 			}
478 
479 			if (sg_dma_address(sg) + offset != src) {
480 				pr_err("Invalid address for remapped page (%d, %d) [src index %lu]\n",
481 				       x, y, src_idx);
482 				return ERR_PTR(-EINVAL);
483 			}
484 
485 			left -= PAGE_SIZE;
486 			offset += PAGE_SIZE;
487 
488 
489 			if (!left)
490 				sg = sg_next(sg);
491 		}
492 
493 		if (left) {
494 			pr_err("Unexpected sg tail with %d size for remapped page (%d, %d)\n",
495 			       left,
496 			       x, y);
497 			return ERR_PTR(-EINVAL);
498 		}
499 
500 		left = (r->plane[n].dst_stride - r->plane[n].width) * PAGE_SIZE;
501 
502 		if (!left)
503 			continue;
504 
505 		if (!sg) {
506 			pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n",
507 			       n, x, y);
508 			return ERR_PTR(-EINVAL);
509 		}
510 
511 		if (sg_dma_len(sg) != left) {
512 			pr_err("Invalid sg.length, found %u, expected %u for remapped page (%d, %d)\n",
513 			       sg_dma_len(sg), left,
514 			       x, y);
515 			return ERR_PTR(-EINVAL);
516 		}
517 
518 		if (sg_dma_address(sg) != 0) {
519 			pr_err("Invalid address, found %pad, expected 0 for remapped page (%d, %d)\n",
520 			       &sg_dma_address(sg),
521 			       x, y);
522 			return ERR_PTR(-EINVAL);
523 		}
524 
525 		sg = sg_next(sg);
526 		left = 0;
527 	}
528 
529 	return sg;
530 }
531 
532 static unsigned int remapped_size(enum i915_ggtt_view_type view_type,
533 				  const struct intel_remapped_plane_info *a,
534 				  const struct intel_remapped_plane_info *b)
535 {
536 
537 	if (view_type == I915_GGTT_VIEW_ROTATED)
538 		return a->dst_stride * a->width + b->dst_stride * b->width;
539 	else
540 		return a->dst_stride * a->height + b->dst_stride * b->height;
541 }
542 
543 static int igt_vma_rotate_remap(void *arg)
544 {
545 	struct i915_ggtt *ggtt = arg;
546 	struct i915_address_space *vm = &ggtt->vm;
547 	struct drm_i915_gem_object *obj;
548 	const struct intel_remapped_plane_info planes[] = {
549 		{ .width = 1, .height = 1, .src_stride = 1 },
550 		{ .width = 2, .height = 2, .src_stride = 2 },
551 		{ .width = 4, .height = 4, .src_stride = 4 },
552 		{ .width = 8, .height = 8, .src_stride = 8 },
553 
554 		{ .width = 3, .height = 5, .src_stride = 3 },
555 		{ .width = 3, .height = 5, .src_stride = 4 },
556 		{ .width = 3, .height = 5, .src_stride = 5 },
557 
558 		{ .width = 5, .height = 3, .src_stride = 5 },
559 		{ .width = 5, .height = 3, .src_stride = 7 },
560 		{ .width = 5, .height = 3, .src_stride = 9 },
561 
562 		{ .width = 4, .height = 6, .src_stride = 6 },
563 		{ .width = 6, .height = 4, .src_stride = 6 },
564 
565 		{ .width = 2, .height = 2, .src_stride = 2, .dst_stride = 2 },
566 		{ .width = 3, .height = 3, .src_stride = 3, .dst_stride = 4 },
567 		{ .width = 5, .height = 6, .src_stride = 7, .dst_stride = 8 },
568 
569 		{ }
570 	}, *a, *b;
571 	enum i915_ggtt_view_type types[] = {
572 		I915_GGTT_VIEW_ROTATED,
573 		I915_GGTT_VIEW_REMAPPED,
574 		0,
575 	}, *t;
576 	const unsigned int max_pages = 64;
577 	int err = -ENOMEM;
578 
579 	/* Create VMA for many different combinations of planes and check
580 	 * that the page layout within the rotated VMA match our expectations.
581 	 */
582 
583 	obj = i915_gem_object_create_internal(vm->i915, max_pages * PAGE_SIZE);
584 	if (IS_ERR(obj))
585 		goto out;
586 
587 	for (t = types; *t; t++) {
588 	for (a = planes; a->width; a++) {
589 		for (b = planes + ARRAY_SIZE(planes); b-- != planes; ) {
590 			struct i915_ggtt_view view = {
591 				.type = *t,
592 				.remapped.plane[0] = *a,
593 				.remapped.plane[1] = *b,
594 			};
595 			struct intel_remapped_plane_info *plane_info = view.remapped.plane;
596 			unsigned int n, max_offset;
597 
598 			max_offset = max(plane_info[0].src_stride * plane_info[0].height,
599 					 plane_info[1].src_stride * plane_info[1].height);
600 			GEM_BUG_ON(max_offset > max_pages);
601 			max_offset = max_pages - max_offset;
602 
603 			if (!plane_info[0].dst_stride)
604 				plane_info[0].dst_stride = view.type == I915_GGTT_VIEW_ROTATED ?
605 									plane_info[0].height :
606 									plane_info[0].width;
607 			if (!plane_info[1].dst_stride)
608 				plane_info[1].dst_stride = view.type == I915_GGTT_VIEW_ROTATED ?
609 									plane_info[1].height :
610 									plane_info[1].width;
611 
612 			for_each_prime_number_from(plane_info[0].offset, 0, max_offset) {
613 				for_each_prime_number_from(plane_info[1].offset, 0, max_offset) {
614 					struct scatterlist *sg;
615 					struct i915_vma *vma;
616 					unsigned int expected_pages;
617 
618 					vma = checked_vma_instance(obj, vm, &view);
619 					if (IS_ERR(vma)) {
620 						err = PTR_ERR(vma);
621 						goto out_object;
622 					}
623 
624 					err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
625 					if (err) {
626 						pr_err("Failed to pin VMA, err=%d\n", err);
627 						goto out_object;
628 					}
629 
630 					expected_pages = remapped_size(view.type, &plane_info[0], &plane_info[1]);
631 
632 					if (view.type == I915_GGTT_VIEW_ROTATED &&
633 					    vma->size != expected_pages * PAGE_SIZE) {
634 						pr_err("VMA is wrong size, expected %lu, found %llu\n",
635 						       PAGE_SIZE * expected_pages, vma->size);
636 						err = -EINVAL;
637 						goto out_object;
638 					}
639 
640 					if (view.type == I915_GGTT_VIEW_REMAPPED &&
641 					    vma->size > expected_pages * PAGE_SIZE) {
642 						pr_err("VMA is wrong size, expected %lu, found %llu\n",
643 						       PAGE_SIZE * expected_pages, vma->size);
644 						err = -EINVAL;
645 						goto out_object;
646 					}
647 
648 					if (vma->pages->nents > expected_pages) {
649 						pr_err("sg table is wrong sizeo, expected %u, found %u nents\n",
650 						       expected_pages, vma->pages->nents);
651 						err = -EINVAL;
652 						goto out_object;
653 					}
654 
655 					if (vma->node.size < vma->size) {
656 						pr_err("VMA binding too small, expected %llu, found %llu\n",
657 						       vma->size, vma->node.size);
658 						err = -EINVAL;
659 						goto out_object;
660 					}
661 
662 					if (vma->pages == obj->mm.pages) {
663 						pr_err("VMA using unrotated object pages!\n");
664 						err = -EINVAL;
665 						goto out_object;
666 					}
667 
668 					sg = vma->pages->sgl;
669 					for (n = 0; n < ARRAY_SIZE(view.rotated.plane); n++) {
670 						if (view.type == I915_GGTT_VIEW_ROTATED)
671 							sg = assert_rotated(obj, &view.rotated, n, sg);
672 						else
673 							sg = assert_remapped(obj, &view.remapped, n, sg);
674 						if (IS_ERR(sg)) {
675 							pr_err("Inconsistent %s VMA pages for plane %d: [(%d, %d, %d, %d, %d), (%d, %d, %d, %d, %d)]\n",
676 							       view.type == I915_GGTT_VIEW_ROTATED ?
677 							       "rotated" : "remapped", n,
678 							       plane_info[0].width,
679 							       plane_info[0].height,
680 							       plane_info[0].src_stride,
681 							       plane_info[0].dst_stride,
682 							       plane_info[0].offset,
683 							       plane_info[1].width,
684 							       plane_info[1].height,
685 							       plane_info[1].src_stride,
686 							       plane_info[1].dst_stride,
687 							       plane_info[1].offset);
688 							err = -EINVAL;
689 							goto out_object;
690 						}
691 					}
692 
693 					i915_vma_unpin(vma);
694 
695 					cond_resched();
696 				}
697 			}
698 		}
699 	}
700 	}
701 
702 out_object:
703 	i915_gem_object_put(obj);
704 out:
705 	return err;
706 }
707 
708 static bool assert_partial(struct drm_i915_gem_object *obj,
709 			   struct i915_vma *vma,
710 			   unsigned long offset,
711 			   unsigned long size)
712 {
713 	struct sgt_iter sgt;
714 	dma_addr_t dma;
715 
716 	for_each_sgt_daddr(dma, sgt, vma->pages) {
717 		dma_addr_t src;
718 
719 		if (!size) {
720 			pr_err("Partial scattergather list too long\n");
721 			return false;
722 		}
723 
724 		src = i915_gem_object_get_dma_address(obj, offset);
725 		if (src != dma) {
726 			pr_err("DMA mismatch for partial page offset %lu\n",
727 			       offset);
728 			return false;
729 		}
730 
731 		offset++;
732 		size--;
733 	}
734 
735 	return true;
736 }
737 
738 static bool assert_pin(struct i915_vma *vma,
739 		       struct i915_ggtt_view *view,
740 		       u64 size,
741 		       const char *name)
742 {
743 	bool ok = true;
744 
745 	if (vma->size != size) {
746 		pr_err("(%s) VMA is wrong size, expected %llu, found %llu\n",
747 		       name, size, vma->size);
748 		ok = false;
749 	}
750 
751 	if (vma->node.size < vma->size) {
752 		pr_err("(%s) VMA binding too small, expected %llu, found %llu\n",
753 		       name, vma->size, vma->node.size);
754 		ok = false;
755 	}
756 
757 	if (view && view->type != I915_GGTT_VIEW_NORMAL) {
758 		if (memcmp(&vma->ggtt_view, view, sizeof(*view))) {
759 			pr_err("(%s) VMA mismatch upon creation!\n",
760 			       name);
761 			ok = false;
762 		}
763 
764 		if (vma->pages == vma->obj->mm.pages) {
765 			pr_err("(%s) VMA using original object pages!\n",
766 			       name);
767 			ok = false;
768 		}
769 	} else {
770 		if (vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL) {
771 			pr_err("Not the normal ggtt view! Found %d\n",
772 			       vma->ggtt_view.type);
773 			ok = false;
774 		}
775 
776 		if (vma->pages != vma->obj->mm.pages) {
777 			pr_err("VMA not using object pages!\n");
778 			ok = false;
779 		}
780 	}
781 
782 	return ok;
783 }
784 
785 static int igt_vma_partial(void *arg)
786 {
787 	struct i915_ggtt *ggtt = arg;
788 	struct i915_address_space *vm = &ggtt->vm;
789 	const unsigned int npages = 1021; /* prime! */
790 	struct drm_i915_gem_object *obj;
791 	const struct phase {
792 		const char *name;
793 	} phases[] = {
794 		{ "create" },
795 		{ "lookup" },
796 		{ },
797 	}, *p;
798 	unsigned int sz, offset;
799 	struct i915_vma *vma;
800 	int err = -ENOMEM;
801 
802 	/* Create lots of different VMA for the object and check that
803 	 * we are returned the same VMA when we later request the same range.
804 	 */
805 
806 	obj = i915_gem_object_create_internal(vm->i915, npages * PAGE_SIZE);
807 	if (IS_ERR(obj))
808 		goto out;
809 
810 	for (p = phases; p->name; p++) { /* exercise both create/lookup */
811 		unsigned int count, nvma;
812 
813 		nvma = 0;
814 		for_each_prime_number_from(sz, 1, npages) {
815 			for_each_prime_number_from(offset, 0, npages - sz) {
816 				struct i915_ggtt_view view;
817 
818 				view.type = I915_GGTT_VIEW_PARTIAL;
819 				view.partial.offset = offset;
820 				view.partial.size = sz;
821 
822 				if (sz == npages)
823 					view.type = I915_GGTT_VIEW_NORMAL;
824 
825 				vma = checked_vma_instance(obj, vm, &view);
826 				if (IS_ERR(vma)) {
827 					err = PTR_ERR(vma);
828 					goto out_object;
829 				}
830 
831 				err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
832 				if (err)
833 					goto out_object;
834 
835 				if (!assert_pin(vma, &view, sz*PAGE_SIZE, p->name)) {
836 					pr_err("(%s) Inconsistent partial pinning for (offset=%d, size=%d)\n",
837 					       p->name, offset, sz);
838 					err = -EINVAL;
839 					goto out_object;
840 				}
841 
842 				if (!assert_partial(obj, vma, offset, sz)) {
843 					pr_err("(%s) Inconsistent partial pages for (offset=%d, size=%d)\n",
844 					       p->name, offset, sz);
845 					err = -EINVAL;
846 					goto out_object;
847 				}
848 
849 				i915_vma_unpin(vma);
850 				nvma++;
851 
852 				cond_resched();
853 			}
854 		}
855 
856 		count = 0;
857 		list_for_each_entry(vma, &obj->vma.list, obj_link)
858 			count++;
859 		if (count != nvma) {
860 			pr_err("(%s) All partial vma were not recorded on the obj->vma_list: found %u, expected %u\n",
861 			       p->name, count, nvma);
862 			err = -EINVAL;
863 			goto out_object;
864 		}
865 
866 		/* Check that we did create the whole object mapping */
867 		vma = checked_vma_instance(obj, vm, NULL);
868 		if (IS_ERR(vma)) {
869 			err = PTR_ERR(vma);
870 			goto out_object;
871 		}
872 
873 		err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
874 		if (err)
875 			goto out_object;
876 
877 		if (!assert_pin(vma, NULL, obj->base.size, p->name)) {
878 			pr_err("(%s) inconsistent full pin\n", p->name);
879 			err = -EINVAL;
880 			goto out_object;
881 		}
882 
883 		i915_vma_unpin(vma);
884 
885 		count = 0;
886 		list_for_each_entry(vma, &obj->vma.list, obj_link)
887 			count++;
888 		if (count != nvma) {
889 			pr_err("(%s) allocated an extra full vma!\n", p->name);
890 			err = -EINVAL;
891 			goto out_object;
892 		}
893 	}
894 
895 out_object:
896 	i915_gem_object_put(obj);
897 out:
898 	return err;
899 }
900 
901 int i915_vma_mock_selftests(void)
902 {
903 	static const struct i915_subtest tests[] = {
904 		SUBTEST(igt_vma_create),
905 		SUBTEST(igt_vma_pin1),
906 		SUBTEST(igt_vma_rotate_remap),
907 		SUBTEST(igt_vma_partial),
908 	};
909 	struct drm_i915_private *i915;
910 	struct i915_ggtt *ggtt;
911 	int err;
912 
913 	i915 = mock_gem_device();
914 	if (!i915)
915 		return -ENOMEM;
916 
917 	ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL);
918 	if (!ggtt) {
919 		err = -ENOMEM;
920 		goto out_put;
921 	}
922 	mock_init_ggtt(i915, ggtt);
923 
924 	err = i915_subtests(tests, ggtt);
925 
926 	mock_device_flush(i915);
927 	i915_gem_drain_freed_objects(i915);
928 	mock_fini_ggtt(ggtt);
929 	kfree(ggtt);
930 out_put:
931 	mock_destroy_device(i915);
932 	return err;
933 }
934 
935 static int igt_vma_remapped_gtt(void *arg)
936 {
937 	struct drm_i915_private *i915 = arg;
938 	const struct intel_remapped_plane_info planes[] = {
939 		{ .width = 1, .height = 1, .src_stride = 1 },
940 		{ .width = 2, .height = 2, .src_stride = 2 },
941 		{ .width = 4, .height = 4, .src_stride = 4 },
942 		{ .width = 8, .height = 8, .src_stride = 8 },
943 
944 		{ .width = 3, .height = 5, .src_stride = 3 },
945 		{ .width = 3, .height = 5, .src_stride = 4 },
946 		{ .width = 3, .height = 5, .src_stride = 5 },
947 
948 		{ .width = 5, .height = 3, .src_stride = 5 },
949 		{ .width = 5, .height = 3, .src_stride = 7 },
950 		{ .width = 5, .height = 3, .src_stride = 9 },
951 
952 		{ .width = 4, .height = 6, .src_stride = 6 },
953 		{ .width = 6, .height = 4, .src_stride = 6 },
954 
955 		{ .width = 2, .height = 2, .src_stride = 2, .dst_stride = 2 },
956 		{ .width = 3, .height = 3, .src_stride = 3, .dst_stride = 4 },
957 		{ .width = 5, .height = 6, .src_stride = 7, .dst_stride = 8 },
958 
959 		{ }
960 	}, *p;
961 	enum i915_ggtt_view_type types[] = {
962 		I915_GGTT_VIEW_ROTATED,
963 		I915_GGTT_VIEW_REMAPPED,
964 		0,
965 	}, *t;
966 	struct drm_i915_gem_object *obj;
967 	intel_wakeref_t wakeref;
968 	int err = 0;
969 
970 	obj = i915_gem_object_create_internal(i915, 10 * 10 * PAGE_SIZE);
971 	if (IS_ERR(obj))
972 		return PTR_ERR(obj);
973 
974 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
975 
976 	for (t = types; *t; t++) {
977 		for (p = planes; p->width; p++) {
978 			struct i915_ggtt_view view = {
979 				.type = *t,
980 				.rotated.plane[0] = *p,
981 			};
982 			struct intel_remapped_plane_info *plane_info = view.rotated.plane;
983 			struct i915_vma *vma;
984 			u32 __iomem *map;
985 			unsigned int x, y;
986 
987 			i915_gem_object_lock(obj, NULL);
988 			err = i915_gem_object_set_to_gtt_domain(obj, true);
989 			i915_gem_object_unlock(obj);
990 			if (err)
991 				goto out;
992 
993 			if (!plane_info[0].dst_stride)
994 				plane_info[0].dst_stride = *t == I915_GGTT_VIEW_ROTATED ?
995 								 p->height : p->width;
996 
997 			vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
998 			if (IS_ERR(vma)) {
999 				err = PTR_ERR(vma);
1000 				goto out;
1001 			}
1002 
1003 			GEM_BUG_ON(vma->ggtt_view.type != *t);
1004 
1005 			map = i915_vma_pin_iomap(vma);
1006 			i915_vma_unpin(vma);
1007 			if (IS_ERR(map)) {
1008 				err = PTR_ERR(map);
1009 				goto out;
1010 			}
1011 
1012 			for (y = 0 ; y < plane_info[0].height; y++) {
1013 				for (x = 0 ; x < plane_info[0].width; x++) {
1014 					unsigned int offset;
1015 					u32 val = y << 16 | x;
1016 
1017 					if (*t == I915_GGTT_VIEW_ROTATED)
1018 						offset = (x * plane_info[0].dst_stride + y) * PAGE_SIZE;
1019 					else
1020 						offset = (y * plane_info[0].dst_stride + x) * PAGE_SIZE;
1021 
1022 					iowrite32(val, &map[offset / sizeof(*map)]);
1023 				}
1024 			}
1025 
1026 			i915_vma_unpin_iomap(vma);
1027 
1028 			vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
1029 			if (IS_ERR(vma)) {
1030 				err = PTR_ERR(vma);
1031 				goto out;
1032 			}
1033 
1034 			GEM_BUG_ON(vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL);
1035 
1036 			map = i915_vma_pin_iomap(vma);
1037 			i915_vma_unpin(vma);
1038 			if (IS_ERR(map)) {
1039 				err = PTR_ERR(map);
1040 				goto out;
1041 			}
1042 
1043 			for (y = 0 ; y < plane_info[0].height; y++) {
1044 				for (x = 0 ; x < plane_info[0].width; x++) {
1045 					unsigned int offset, src_idx;
1046 					u32 exp = y << 16 | x;
1047 					u32 val;
1048 
1049 					if (*t == I915_GGTT_VIEW_ROTATED)
1050 						src_idx = rotated_index(&view.rotated, 0, x, y);
1051 					else
1052 						src_idx = remapped_index(&view.remapped, 0, x, y);
1053 					offset = src_idx * PAGE_SIZE;
1054 
1055 					val = ioread32(&map[offset / sizeof(*map)]);
1056 					if (val != exp) {
1057 						pr_err("%s VMA write test failed, expected 0x%x, found 0x%x\n",
1058 						       *t == I915_GGTT_VIEW_ROTATED ? "Rotated" : "Remapped",
1059 						       exp, val);
1060 						i915_vma_unpin_iomap(vma);
1061 						err = -EINVAL;
1062 						goto out;
1063 					}
1064 				}
1065 			}
1066 			i915_vma_unpin_iomap(vma);
1067 
1068 			cond_resched();
1069 		}
1070 	}
1071 
1072 out:
1073 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1074 	i915_gem_object_put(obj);
1075 
1076 	return err;
1077 }
1078 
1079 int i915_vma_live_selftests(struct drm_i915_private *i915)
1080 {
1081 	static const struct i915_subtest tests[] = {
1082 		SUBTEST(igt_vma_remapped_gtt),
1083 	};
1084 
1085 	return i915_subtests(tests, i915);
1086 }
1087