1 /* 2 * Copyright © 2008-2010 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * Chris Wilson <chris@chris-wilson.co.uuk> 26 * 27 */ 28 29 #include <drm/i915_drm.h> 30 31 #include "i915_drv.h" 32 #include "intel_drv.h" 33 #include "i915_trace.h" 34 35 I915_SELFTEST_DECLARE(static struct igt_evict_ctl { 36 bool fail_if_busy:1; 37 } igt_evict_ctl;) 38 39 static bool ggtt_is_idle(struct drm_i915_private *i915) 40 { 41 struct intel_engine_cs *engine; 42 enum intel_engine_id id; 43 44 if (i915->gt.active_requests) 45 return false; 46 47 for_each_engine(engine, i915, id) { 48 if (!intel_engine_has_kernel_context(engine)) 49 return false; 50 } 51 52 return true; 53 } 54 55 static int ggtt_flush(struct drm_i915_private *i915) 56 { 57 int err; 58 59 /* Not everything in the GGTT is tracked via vma (otherwise we 60 * could evict as required with minimal stalling) so we are forced 61 * to idle the GPU and explicitly retire outstanding requests in 62 * the hopes that we can then remove contexts and the like only 63 * bound by their active reference. 64 */ 65 err = i915_gem_switch_to_kernel_context(i915); 66 if (err) 67 return err; 68 69 err = i915_gem_wait_for_idle(i915, 70 I915_WAIT_INTERRUPTIBLE | 71 I915_WAIT_LOCKED, 72 MAX_SCHEDULE_TIMEOUT); 73 if (err) 74 return err; 75 76 GEM_BUG_ON(!ggtt_is_idle(i915)); 77 return 0; 78 } 79 80 static bool 81 mark_free(struct drm_mm_scan *scan, 82 struct i915_vma *vma, 83 unsigned int flags, 84 struct list_head *unwind) 85 { 86 if (i915_vma_is_pinned(vma)) 87 return false; 88 89 if (flags & PIN_NONFAULT && i915_vma_has_userfault(vma)) 90 return false; 91 92 list_add(&vma->evict_link, unwind); 93 return drm_mm_scan_add_block(scan, &vma->node); 94 } 95 96 /** 97 * i915_gem_evict_something - Evict vmas to make room for binding a new one 98 * @vm: address space to evict from 99 * @min_size: size of the desired free space 100 * @alignment: alignment constraint of the desired free space 101 * @cache_level: cache_level for the desired space 102 * @start: start (inclusive) of the range from which to evict objects 103 * @end: end (exclusive) of the range from which to evict objects 104 * @flags: additional flags to control the eviction algorithm 105 * 106 * This function will try to evict vmas until a free space satisfying the 107 * requirements is found. Callers must check first whether any such hole exists 108 * already before calling this function. 109 * 110 * This function is used by the object/vma binding code. 111 * 112 * Since this function is only used to free up virtual address space it only 113 * ignores pinned vmas, and not object where the backing storage itself is 114 * pinned. Hence obj->pages_pin_count does not protect against eviction. 115 * 116 * To clarify: This is for freeing up virtual address space, not for freeing 117 * memory in e.g. the shrinker. 118 */ 119 int 120 i915_gem_evict_something(struct i915_address_space *vm, 121 u64 min_size, u64 alignment, 122 unsigned cache_level, 123 u64 start, u64 end, 124 unsigned flags) 125 { 126 struct drm_i915_private *dev_priv = vm->i915; 127 struct drm_mm_scan scan; 128 struct list_head eviction_list; 129 struct i915_vma *vma, *next; 130 struct drm_mm_node *node; 131 enum drm_mm_insert_mode mode; 132 struct i915_vma *active; 133 int ret; 134 135 lockdep_assert_held(&vm->i915->drm.struct_mutex); 136 trace_i915_gem_evict(vm, min_size, alignment, flags); 137 138 /* 139 * The goal is to evict objects and amalgamate space in rough LRU order. 140 * Since both active and inactive objects reside on the same list, 141 * in a mix of creation and last scanned order, as we process the list 142 * we sort it into inactive/active, which keeps the active portion 143 * in a rough MRU order. 144 * 145 * The retirement sequence is thus: 146 * 1. Inactive objects (already retired, random order) 147 * 2. Active objects (will stall on unbinding, oldest scanned first) 148 */ 149 mode = DRM_MM_INSERT_BEST; 150 if (flags & PIN_HIGH) 151 mode = DRM_MM_INSERT_HIGH; 152 if (flags & PIN_MAPPABLE) 153 mode = DRM_MM_INSERT_LOW; 154 drm_mm_scan_init_with_range(&scan, &vm->mm, 155 min_size, alignment, cache_level, 156 start, end, mode); 157 158 /* 159 * Retire before we search the active list. Although we have 160 * reasonable accuracy in our retirement lists, we may have 161 * a stray pin (preventing eviction) that can only be resolved by 162 * retiring. 163 */ 164 if (!(flags & PIN_NONBLOCK)) 165 i915_retire_requests(dev_priv); 166 167 search_again: 168 active = NULL; 169 INIT_LIST_HEAD(&eviction_list); 170 list_for_each_entry_safe(vma, next, &vm->bound_list, vm_link) { 171 /* 172 * We keep this list in a rough least-recently scanned order 173 * of active elements (inactive elements are cheap to reap). 174 * New entries are added to the end, and we move anything we 175 * scan to the end. The assumption is that the working set 176 * of applications is either steady state (and thanks to the 177 * userspace bo cache it almost always is) or volatile and 178 * frequently replaced after a frame, which are self-evicting! 179 * Given that assumption, the MRU order of the scan list is 180 * fairly static, and keeping it in least-recently scan order 181 * is suitable. 182 * 183 * To notice when we complete one full cycle, we record the 184 * first active element seen, before moving it to the tail. 185 */ 186 if (i915_vma_is_active(vma)) { 187 if (vma == active) { 188 if (flags & PIN_NONBLOCK) 189 break; 190 191 active = ERR_PTR(-EAGAIN); 192 } 193 194 if (active != ERR_PTR(-EAGAIN)) { 195 if (!active) 196 active = vma; 197 198 list_move_tail(&vma->vm_link, &vm->bound_list); 199 continue; 200 } 201 } 202 203 if (mark_free(&scan, vma, flags, &eviction_list)) 204 goto found; 205 } 206 207 /* Nothing found, clean up and bail out! */ 208 list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { 209 ret = drm_mm_scan_remove_block(&scan, &vma->node); 210 BUG_ON(ret); 211 } 212 213 /* 214 * Can we unpin some objects such as idle hw contents, 215 * or pending flips? But since only the GGTT has global entries 216 * such as scanouts, rinbuffers and contexts, we can skip the 217 * purge when inspecting per-process local address spaces. 218 */ 219 if (!i915_is_ggtt(vm) || flags & PIN_NONBLOCK) 220 return -ENOSPC; 221 222 /* 223 * Not everything in the GGTT is tracked via VMA using 224 * i915_vma_move_to_active(), otherwise we could evict as required 225 * with minimal stalling. Instead we are forced to idle the GPU and 226 * explicitly retire outstanding requests which will then remove 227 * the pinning for active objects such as contexts and ring, 228 * enabling us to evict them on the next iteration. 229 * 230 * To ensure that all user contexts are evictable, we perform 231 * a switch to the perma-pinned kernel context. This all also gives 232 * us a termination condition, when the last retired context is 233 * the kernel's there is no more we can evict. 234 */ 235 if (!ggtt_is_idle(dev_priv)) { 236 if (I915_SELFTEST_ONLY(igt_evict_ctl.fail_if_busy)) 237 return -EBUSY; 238 239 ret = ggtt_flush(dev_priv); 240 if (ret) 241 return ret; 242 243 cond_resched(); 244 goto search_again; 245 } 246 247 /* 248 * If we still have pending pageflip completions, drop 249 * back to userspace to give our workqueues time to 250 * acquire our locks and unpin the old scanouts. 251 */ 252 return intel_has_pending_fb_unpin(dev_priv) ? -EAGAIN : -ENOSPC; 253 254 found: 255 /* drm_mm doesn't allow any other other operations while 256 * scanning, therefore store to-be-evicted objects on a 257 * temporary list and take a reference for all before 258 * calling unbind (which may remove the active reference 259 * of any of our objects, thus corrupting the list). 260 */ 261 list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { 262 if (drm_mm_scan_remove_block(&scan, &vma->node)) 263 __i915_vma_pin(vma); 264 else 265 list_del(&vma->evict_link); 266 } 267 268 /* Unbinding will emit any required flushes */ 269 ret = 0; 270 list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { 271 __i915_vma_unpin(vma); 272 if (ret == 0) 273 ret = i915_vma_unbind(vma); 274 } 275 276 while (ret == 0 && (node = drm_mm_scan_color_evict(&scan))) { 277 vma = container_of(node, struct i915_vma, node); 278 ret = i915_vma_unbind(vma); 279 } 280 281 return ret; 282 } 283 284 /** 285 * i915_gem_evict_for_vma - Evict vmas to make room for binding a new one 286 * @vm: address space to evict from 287 * @target: range (and color) to evict for 288 * @flags: additional flags to control the eviction algorithm 289 * 290 * This function will try to evict vmas that overlap the target node. 291 * 292 * To clarify: This is for freeing up virtual address space, not for freeing 293 * memory in e.g. the shrinker. 294 */ 295 int i915_gem_evict_for_node(struct i915_address_space *vm, 296 struct drm_mm_node *target, 297 unsigned int flags) 298 { 299 LIST_HEAD(eviction_list); 300 struct drm_mm_node *node; 301 u64 start = target->start; 302 u64 end = start + target->size; 303 struct i915_vma *vma, *next; 304 bool check_color; 305 int ret = 0; 306 307 lockdep_assert_held(&vm->i915->drm.struct_mutex); 308 GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE)); 309 GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE)); 310 311 trace_i915_gem_evict_node(vm, target, flags); 312 313 /* Retire before we search the active list. Although we have 314 * reasonable accuracy in our retirement lists, we may have 315 * a stray pin (preventing eviction) that can only be resolved by 316 * retiring. 317 */ 318 if (!(flags & PIN_NONBLOCK)) 319 i915_retire_requests(vm->i915); 320 321 check_color = vm->mm.color_adjust; 322 if (check_color) { 323 /* Expand search to cover neighbouring guard pages (or lack!) */ 324 if (start) 325 start -= I915_GTT_PAGE_SIZE; 326 327 /* Always look at the page afterwards to avoid the end-of-GTT */ 328 end += I915_GTT_PAGE_SIZE; 329 } 330 GEM_BUG_ON(start >= end); 331 332 drm_mm_for_each_node_in_range(node, &vm->mm, start, end) { 333 /* If we find any non-objects (!vma), we cannot evict them */ 334 if (node->color == I915_COLOR_UNEVICTABLE) { 335 ret = -ENOSPC; 336 break; 337 } 338 339 GEM_BUG_ON(!node->allocated); 340 vma = container_of(node, typeof(*vma), node); 341 342 /* If we are using coloring to insert guard pages between 343 * different cache domains within the address space, we have 344 * to check whether the objects on either side of our range 345 * abutt and conflict. If they are in conflict, then we evict 346 * those as well to make room for our guard pages. 347 */ 348 if (check_color) { 349 if (node->start + node->size == target->start) { 350 if (node->color == target->color) 351 continue; 352 } 353 if (node->start == target->start + target->size) { 354 if (node->color == target->color) 355 continue; 356 } 357 } 358 359 if (flags & PIN_NONBLOCK && 360 (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))) { 361 ret = -ENOSPC; 362 break; 363 } 364 365 if (flags & PIN_NONFAULT && i915_vma_has_userfault(vma)) { 366 ret = -ENOSPC; 367 break; 368 } 369 370 /* Overlap of objects in the same batch? */ 371 if (i915_vma_is_pinned(vma)) { 372 ret = -ENOSPC; 373 if (vma->exec_flags && 374 *vma->exec_flags & EXEC_OBJECT_PINNED) 375 ret = -EINVAL; 376 break; 377 } 378 379 /* Never show fear in the face of dragons! 380 * 381 * We cannot directly remove this node from within this 382 * iterator and as with i915_gem_evict_something() we employ 383 * the vma pin_count in order to prevent the action of 384 * unbinding one vma from freeing (by dropping its active 385 * reference) another in our eviction list. 386 */ 387 __i915_vma_pin(vma); 388 list_add(&vma->evict_link, &eviction_list); 389 } 390 391 list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { 392 __i915_vma_unpin(vma); 393 if (ret == 0) 394 ret = i915_vma_unbind(vma); 395 } 396 397 return ret; 398 } 399 400 /** 401 * i915_gem_evict_vm - Evict all idle vmas from a vm 402 * @vm: Address space to cleanse 403 * 404 * This function evicts all vmas from a vm. 405 * 406 * This is used by the execbuf code as a last-ditch effort to defragment the 407 * address space. 408 * 409 * To clarify: This is for freeing up virtual address space, not for freeing 410 * memory in e.g. the shrinker. 411 */ 412 int i915_gem_evict_vm(struct i915_address_space *vm) 413 { 414 struct list_head eviction_list; 415 struct i915_vma *vma, *next; 416 int ret; 417 418 lockdep_assert_held(&vm->i915->drm.struct_mutex); 419 trace_i915_gem_evict_vm(vm); 420 421 /* Switch back to the default context in order to unpin 422 * the existing context objects. However, such objects only 423 * pin themselves inside the global GTT and performing the 424 * switch otherwise is ineffective. 425 */ 426 if (i915_is_ggtt(vm)) { 427 ret = ggtt_flush(vm->i915); 428 if (ret) 429 return ret; 430 } 431 432 INIT_LIST_HEAD(&eviction_list); 433 mutex_lock(&vm->mutex); 434 list_for_each_entry(vma, &vm->bound_list, vm_link) { 435 if (i915_vma_is_pinned(vma)) 436 continue; 437 438 __i915_vma_pin(vma); 439 list_add(&vma->evict_link, &eviction_list); 440 } 441 mutex_unlock(&vm->mutex); 442 443 ret = 0; 444 list_for_each_entry_safe(vma, next, &eviction_list, evict_link) { 445 __i915_vma_unpin(vma); 446 if (ret == 0) 447 ret = i915_vma_unbind(vma); 448 } 449 return ret; 450 } 451 452 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 453 #include "selftests/i915_gem_evict.c" 454 #endif 455