1 /*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include "gem/i915_gem_internal.h"
26 #include "gem/i915_gem_pm.h"
27 #include "gem/selftests/igt_gem_utils.h"
28 #include "gem/selftests/mock_context.h"
29 #include "gt/intel_gt.h"
30 #include "gt/intel_gt_print.h"
31
32 #include "i915_selftest.h"
33
34 #include "igt_flush_test.h"
35 #include "lib_sw_fence.h"
36 #include "mock_drm.h"
37 #include "mock_gem_device.h"
38
quirk_add(struct drm_i915_gem_object * obj,struct list_head * objects)39 static void quirk_add(struct drm_i915_gem_object *obj,
40 struct list_head *objects)
41 {
42 /* quirk is only for live tiled objects, use it to declare ownership */
43 GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
44 i915_gem_object_set_tiling_quirk(obj);
45 list_add(&obj->st_link, objects);
46 }
47
populate_ggtt(struct i915_ggtt * ggtt,struct list_head * objects)48 static int populate_ggtt(struct i915_ggtt *ggtt, struct list_head *objects)
49 {
50 struct drm_i915_gem_object *obj;
51 unsigned long count;
52
53 count = 0;
54 do {
55 struct i915_vma *vma;
56
57 obj = i915_gem_object_create_internal(ggtt->vm.i915,
58 I915_GTT_PAGE_SIZE);
59 if (IS_ERR(obj))
60 return PTR_ERR(obj);
61
62 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
63 if (IS_ERR(vma)) {
64 i915_gem_object_put(obj);
65 if (vma == ERR_PTR(-ENOSPC))
66 break;
67
68 return PTR_ERR(vma);
69 }
70
71 quirk_add(obj, objects);
72 count++;
73 } while (1);
74 pr_debug("Filled GGTT with %lu pages [%llu total]\n",
75 count, ggtt->vm.total / PAGE_SIZE);
76
77 if (list_empty(&ggtt->vm.bound_list)) {
78 pr_err("No objects on the GGTT inactive list!\n");
79 return -EINVAL;
80 }
81
82 return 0;
83 }
84
unpin_ggtt(struct i915_ggtt * ggtt)85 static void unpin_ggtt(struct i915_ggtt *ggtt)
86 {
87 struct i915_vma *vma;
88
89 list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
90 if (i915_gem_object_has_tiling_quirk(vma->obj))
91 i915_vma_unpin(vma);
92 }
93
cleanup_objects(struct i915_ggtt * ggtt,struct list_head * list)94 static void cleanup_objects(struct i915_ggtt *ggtt, struct list_head *list)
95 {
96 struct drm_i915_gem_object *obj, *on;
97
98 list_for_each_entry_safe(obj, on, list, st_link) {
99 GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj));
100 i915_gem_object_set_tiling_quirk(obj);
101 i915_gem_object_put(obj);
102 }
103
104 i915_gem_drain_freed_objects(ggtt->vm.i915);
105 }
106
igt_evict_something(void * arg)107 static int igt_evict_something(void *arg)
108 {
109 struct intel_gt *gt = arg;
110 struct i915_ggtt *ggtt = gt->ggtt;
111 LIST_HEAD(objects);
112 int err;
113
114 /* Fill the GGTT with pinned objects and try to evict one. */
115
116 err = populate_ggtt(ggtt, &objects);
117 if (err)
118 goto cleanup;
119
120 /* Everything is pinned, nothing should happen */
121 mutex_lock(&ggtt->vm.mutex);
122 err = i915_gem_evict_something(&ggtt->vm, NULL,
123 I915_GTT_PAGE_SIZE, 0, 0,
124 0, U64_MAX,
125 0);
126 mutex_unlock(&ggtt->vm.mutex);
127 if (err != -ENOSPC) {
128 pr_err("i915_gem_evict_something failed on a full GGTT with err=%d\n",
129 err);
130 goto cleanup;
131 }
132
133 unpin_ggtt(ggtt);
134
135 /* Everything is unpinned, we should be able to evict something */
136 mutex_lock(&ggtt->vm.mutex);
137 err = i915_gem_evict_something(&ggtt->vm, NULL,
138 I915_GTT_PAGE_SIZE, 0, 0,
139 0, U64_MAX,
140 0);
141 mutex_unlock(&ggtt->vm.mutex);
142 if (err) {
143 pr_err("i915_gem_evict_something failed on a full GGTT with err=%d\n",
144 err);
145 goto cleanup;
146 }
147
148 cleanup:
149 cleanup_objects(ggtt, &objects);
150 return err;
151 }
152
igt_overcommit(void * arg)153 static int igt_overcommit(void *arg)
154 {
155 struct intel_gt *gt = arg;
156 struct i915_ggtt *ggtt = gt->ggtt;
157 struct drm_i915_gem_object *obj;
158 struct i915_vma *vma;
159 LIST_HEAD(objects);
160 int err;
161
162 /* Fill the GGTT with pinned objects and then try to pin one more.
163 * We expect it to fail.
164 */
165
166 err = populate_ggtt(ggtt, &objects);
167 if (err)
168 goto cleanup;
169
170 obj = i915_gem_object_create_internal(gt->i915, I915_GTT_PAGE_SIZE);
171 if (IS_ERR(obj)) {
172 err = PTR_ERR(obj);
173 goto cleanup;
174 }
175
176 quirk_add(obj, &objects);
177
178 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 0);
179 if (vma != ERR_PTR(-ENOSPC)) {
180 pr_err("Failed to evict+insert, i915_gem_object_ggtt_pin returned err=%d\n", (int)PTR_ERR_OR_ZERO(vma));
181 err = -EINVAL;
182 goto cleanup;
183 }
184
185 cleanup:
186 cleanup_objects(ggtt, &objects);
187 return err;
188 }
189
igt_evict_for_vma(void * arg)190 static int igt_evict_for_vma(void *arg)
191 {
192 struct intel_gt *gt = arg;
193 struct i915_ggtt *ggtt = gt->ggtt;
194 struct drm_mm_node target = {
195 .start = 0,
196 .size = 4096,
197 };
198 LIST_HEAD(objects);
199 int err;
200
201 /* Fill the GGTT with pinned objects and try to evict a range. */
202
203 err = populate_ggtt(ggtt, &objects);
204 if (err)
205 goto cleanup;
206
207 /* Everything is pinned, nothing should happen */
208 mutex_lock(&ggtt->vm.mutex);
209 err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0);
210 mutex_unlock(&ggtt->vm.mutex);
211 if (err != -ENOSPC) {
212 pr_err("i915_gem_evict_for_node on a full GGTT returned err=%d\n",
213 err);
214 goto cleanup;
215 }
216
217 unpin_ggtt(ggtt);
218
219 /* Everything is unpinned, we should be able to evict the node */
220 mutex_lock(&ggtt->vm.mutex);
221 err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0);
222 mutex_unlock(&ggtt->vm.mutex);
223 if (err) {
224 pr_err("i915_gem_evict_for_node returned err=%d\n",
225 err);
226 goto cleanup;
227 }
228
229 cleanup:
230 cleanup_objects(ggtt, &objects);
231 return err;
232 }
233
mock_color_adjust(const struct drm_mm_node * node,unsigned long color,u64 * start,u64 * end)234 static void mock_color_adjust(const struct drm_mm_node *node,
235 unsigned long color,
236 u64 *start,
237 u64 *end)
238 {
239 }
240
igt_evict_for_cache_color(void * arg)241 static int igt_evict_for_cache_color(void *arg)
242 {
243 struct intel_gt *gt = arg;
244 struct i915_ggtt *ggtt = gt->ggtt;
245 const unsigned long flags = PIN_OFFSET_FIXED;
246 struct drm_mm_node target = {
247 .start = I915_GTT_PAGE_SIZE * 2,
248 .size = I915_GTT_PAGE_SIZE,
249 .color = i915_gem_get_pat_index(gt->i915, I915_CACHE_LLC),
250 };
251 struct drm_i915_gem_object *obj;
252 struct i915_vma *vma;
253 LIST_HEAD(objects);
254 int err;
255
256 /*
257 * Currently the use of color_adjust for the GGTT is limited to cache
258 * coloring and guard pages, and so the presence of mm.color_adjust for
259 * the GGTT is assumed to be i915_ggtt_color_adjust, hence using a mock
260 * color adjust will work just fine for our purposes.
261 */
262 ggtt->vm.mm.color_adjust = mock_color_adjust;
263 GEM_BUG_ON(!i915_vm_has_cache_coloring(&ggtt->vm));
264
265 obj = i915_gem_object_create_internal(gt->i915, I915_GTT_PAGE_SIZE);
266 if (IS_ERR(obj)) {
267 err = PTR_ERR(obj);
268 goto cleanup;
269 }
270 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
271 quirk_add(obj, &objects);
272
273 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
274 I915_GTT_PAGE_SIZE | flags);
275 if (IS_ERR(vma)) {
276 pr_err("[0]i915_gem_object_ggtt_pin failed\n");
277 err = PTR_ERR(vma);
278 goto cleanup;
279 }
280
281 obj = i915_gem_object_create_internal(gt->i915, I915_GTT_PAGE_SIZE);
282 if (IS_ERR(obj)) {
283 err = PTR_ERR(obj);
284 goto cleanup;
285 }
286 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
287 quirk_add(obj, &objects);
288
289 /* Neighbouring; same colour - should fit */
290 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
291 (I915_GTT_PAGE_SIZE * 2) | flags);
292 if (IS_ERR(vma)) {
293 pr_err("[1]i915_gem_object_ggtt_pin failed\n");
294 err = PTR_ERR(vma);
295 goto cleanup;
296 }
297
298 i915_vma_unpin(vma);
299
300 /* Remove just the second vma */
301 mutex_lock(&ggtt->vm.mutex);
302 err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0);
303 mutex_unlock(&ggtt->vm.mutex);
304 if (err) {
305 pr_err("[0]i915_gem_evict_for_node returned err=%d\n", err);
306 goto cleanup;
307 }
308
309 /* Attempt to remove the first *pinned* vma, by removing the (empty)
310 * neighbour -- this should fail.
311 */
312 target.color = i915_gem_get_pat_index(gt->i915, I915_CACHE_L3_LLC);
313
314 mutex_lock(&ggtt->vm.mutex);
315 err = i915_gem_evict_for_node(&ggtt->vm, NULL, &target, 0);
316 mutex_unlock(&ggtt->vm.mutex);
317 if (!err) {
318 pr_err("[1]i915_gem_evict_for_node returned err=%d\n", err);
319 err = -EINVAL;
320 goto cleanup;
321 }
322
323 err = 0;
324
325 cleanup:
326 unpin_ggtt(ggtt);
327 cleanup_objects(ggtt, &objects);
328 ggtt->vm.mm.color_adjust = NULL;
329 return err;
330 }
331
igt_evict_vm(void * arg)332 static int igt_evict_vm(void *arg)
333 {
334 struct intel_gt *gt = arg;
335 struct i915_ggtt *ggtt = gt->ggtt;
336 struct i915_gem_ww_ctx ww;
337 LIST_HEAD(objects);
338 int err;
339
340 /* Fill the GGTT with pinned objects and try to evict everything. */
341
342 err = populate_ggtt(ggtt, &objects);
343 if (err)
344 goto cleanup;
345
346 /* Everything is pinned, nothing should happen */
347 mutex_lock(&ggtt->vm.mutex);
348 err = i915_gem_evict_vm(&ggtt->vm, NULL, NULL);
349 mutex_unlock(&ggtt->vm.mutex);
350 if (err) {
351 pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n",
352 err);
353 goto cleanup;
354 }
355
356 unpin_ggtt(ggtt);
357
358 for_i915_gem_ww(&ww, err, false) {
359 mutex_lock(&ggtt->vm.mutex);
360 err = i915_gem_evict_vm(&ggtt->vm, &ww, NULL);
361 mutex_unlock(&ggtt->vm.mutex);
362 }
363
364 if (err) {
365 pr_err("i915_gem_evict_vm on a full GGTT returned err=%d]\n",
366 err);
367 goto cleanup;
368 }
369
370 cleanup:
371 cleanup_objects(ggtt, &objects);
372 return err;
373 }
374
igt_evict_contexts(void * arg)375 static int igt_evict_contexts(void *arg)
376 {
377 const u64 PRETEND_GGTT_SIZE = 16ull << 20;
378 struct intel_gt *gt = arg;
379 struct i915_ggtt *ggtt = gt->ggtt;
380 struct drm_i915_private *i915 = gt->i915;
381 struct intel_engine_cs *engine;
382 enum intel_engine_id id;
383 struct reserved {
384 struct drm_mm_node node;
385 struct reserved *next;
386 } *reserved = NULL;
387 intel_wakeref_t wakeref;
388 struct drm_mm_node hole;
389 unsigned long count;
390 int err;
391
392 /*
393 * The purpose of this test is to verify that we will trigger an
394 * eviction in the GGTT when constructing a request that requires
395 * additional space in the GGTT for pinning the context. This space
396 * is not directly tied to the request so reclaiming it requires
397 * extra work.
398 *
399 * As such this test is only meaningful for full-ppgtt environments
400 * where the GTT space of the request is separate from the GGTT
401 * allocation required to build the request.
402 */
403 if (!HAS_FULL_PPGTT(i915))
404 return 0;
405
406 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
407
408 /* Reserve a block so that we know we have enough to fit a few rq */
409 memset(&hole, 0, sizeof(hole));
410 mutex_lock(&ggtt->vm.mutex);
411 err = i915_gem_gtt_insert(&ggtt->vm, NULL, &hole,
412 PRETEND_GGTT_SIZE, 0, I915_COLOR_UNEVICTABLE,
413 0, ggtt->vm.total,
414 PIN_NOEVICT);
415 if (err)
416 goto out_locked;
417
418 /* Make the GGTT appear small by filling it with unevictable nodes */
419 count = 0;
420 do {
421 struct reserved *r;
422
423 mutex_unlock(&ggtt->vm.mutex);
424 r = kcalloc(1, sizeof(*r), GFP_KERNEL);
425 mutex_lock(&ggtt->vm.mutex);
426 if (!r) {
427 err = -ENOMEM;
428 goto out_locked;
429 }
430
431 if (i915_gem_gtt_insert(&ggtt->vm, NULL, &r->node,
432 1ul << 20, 0, I915_COLOR_UNEVICTABLE,
433 0, ggtt->vm.total,
434 PIN_NOEVICT)) {
435 kfree(r);
436 break;
437 }
438
439 r->next = reserved;
440 reserved = r;
441
442 count++;
443 } while (1);
444 drm_mm_remove_node(&hole);
445 mutex_unlock(&ggtt->vm.mutex);
446 pr_info("Filled GGTT with %lu 1MiB nodes\n", count);
447
448 /* Overfill the GGTT with context objects and so try to evict one. */
449 for_each_engine(engine, gt, id) {
450 struct i915_sw_fence fence;
451 struct i915_request *last = NULL;
452
453 count = 0;
454 onstack_fence_init(&fence);
455 do {
456 struct intel_context *ce;
457 struct i915_request *rq;
458
459 ce = intel_context_create(engine);
460 if (IS_ERR(ce))
461 break;
462
463 /* We will need some GGTT space for the rq's context */
464 igt_evict_ctl.fail_if_busy = true;
465 rq = intel_context_create_request(ce);
466 igt_evict_ctl.fail_if_busy = false;
467 intel_context_put(ce);
468
469 if (IS_ERR(rq)) {
470 /* When full, fail_if_busy will trigger EBUSY */
471 if (PTR_ERR(rq) != -EBUSY) {
472 pr_err("Unexpected error from request alloc (on %s): %d\n",
473 engine->name,
474 (int)PTR_ERR(rq));
475 err = PTR_ERR(rq);
476 }
477 break;
478 }
479
480 /* Keep every request/ctx pinned until we are full */
481 err = i915_sw_fence_await_sw_fence_gfp(&rq->submit,
482 &fence,
483 GFP_KERNEL);
484 if (err < 0)
485 break;
486
487 i915_request_add(rq);
488 count++;
489 if (last)
490 i915_request_put(last);
491 last = i915_request_get(rq);
492 err = 0;
493 } while(1);
494 onstack_fence_fini(&fence);
495 pr_info("Submitted %lu contexts/requests on %s\n",
496 count, engine->name);
497 if (err)
498 break;
499 if (last) {
500 if (i915_request_wait(last, 0, HZ) < 0) {
501 err = -EIO;
502 i915_request_put(last);
503 pr_err("Failed waiting for last request (on %s)",
504 engine->name);
505 break;
506 }
507 i915_request_put(last);
508 }
509 err = intel_gt_wait_for_idle(engine->gt, HZ * 3);
510 if (err) {
511 gt_err(engine->gt, "Failed to idle GT (on %s)",
512 engine->name);
513 break;
514 }
515 }
516
517 mutex_lock(&ggtt->vm.mutex);
518 out_locked:
519 if (igt_flush_test(i915))
520 err = -EIO;
521 while (reserved) {
522 struct reserved *next = reserved->next;
523
524 drm_mm_remove_node(&reserved->node);
525 kfree(reserved);
526
527 reserved = next;
528 }
529 if (drm_mm_node_allocated(&hole))
530 drm_mm_remove_node(&hole);
531 mutex_unlock(&ggtt->vm.mutex);
532 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
533
534 return err;
535 }
536
i915_gem_evict_mock_selftests(void)537 int i915_gem_evict_mock_selftests(void)
538 {
539 static const struct i915_subtest tests[] = {
540 SUBTEST(igt_evict_something),
541 SUBTEST(igt_evict_for_vma),
542 SUBTEST(igt_evict_for_cache_color),
543 SUBTEST(igt_evict_vm),
544 SUBTEST(igt_overcommit),
545 };
546 struct drm_i915_private *i915;
547 intel_wakeref_t wakeref;
548 int err = 0;
549
550 i915 = mock_gem_device();
551 if (!i915)
552 return -ENOMEM;
553
554 with_intel_runtime_pm(&i915->runtime_pm, wakeref)
555 err = i915_subtests(tests, to_gt(i915));
556
557 mock_destroy_device(i915);
558 return err;
559 }
560
i915_gem_evict_live_selftests(struct drm_i915_private * i915)561 int i915_gem_evict_live_selftests(struct drm_i915_private *i915)
562 {
563 static const struct i915_subtest tests[] = {
564 SUBTEST(igt_evict_contexts),
565 };
566
567 if (intel_gt_is_wedged(to_gt(i915)))
568 return 0;
569
570 return intel_gt_live_subtests(tests, to_gt(i915));
571 }
572