1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2008,2010 Intel Corporation 5 */ 6 7 #include <linux/intel-iommu.h> 8 #include <linux/dma-resv.h> 9 #include <linux/sync_file.h> 10 #include <linux/uaccess.h> 11 12 #include <drm/drm_syncobj.h> 13 14 #include "display/intel_frontbuffer.h" 15 16 #include "gem/i915_gem_ioctls.h" 17 #include "gt/intel_context.h" 18 #include "gt/intel_gt.h" 19 #include "gt/intel_gt_buffer_pool.h" 20 #include "gt/intel_gt_pm.h" 21 #include "gt/intel_ring.h" 22 23 #include "i915_drv.h" 24 #include "i915_gem_clflush.h" 25 #include "i915_gem_context.h" 26 #include "i915_gem_ioctls.h" 27 #include "i915_sw_fence_work.h" 28 #include "i915_trace.h" 29 30 struct eb_vma { 31 struct i915_vma *vma; 32 unsigned int flags; 33 34 /** This vma's place in the execbuf reservation list */ 35 struct drm_i915_gem_exec_object2 *exec; 36 struct list_head bind_link; 37 struct list_head reloc_link; 38 39 struct hlist_node node; 40 u32 handle; 41 }; 42 43 struct eb_vma_array { 44 struct kref kref; 45 struct eb_vma vma[]; 46 }; 47 48 enum { 49 FORCE_CPU_RELOC = 1, 50 FORCE_GTT_RELOC, 51 FORCE_GPU_RELOC, 52 #define DBG_FORCE_RELOC 0 /* choose one of the above! */ 53 }; 54 55 #define __EXEC_OBJECT_HAS_PIN BIT(31) 56 #define __EXEC_OBJECT_HAS_FENCE BIT(30) 57 #define __EXEC_OBJECT_NEEDS_MAP BIT(29) 58 #define __EXEC_OBJECT_NEEDS_BIAS BIT(28) 59 #define __EXEC_OBJECT_INTERNAL_FLAGS (~0u << 28) /* all of the above */ 60 61 #define __EXEC_HAS_RELOC BIT(31) 62 #define __EXEC_INTERNAL_FLAGS (~0u << 31) 63 #define UPDATE PIN_OFFSET_FIXED 64 65 #define BATCH_OFFSET_BIAS (256*1024) 66 67 #define __I915_EXEC_ILLEGAL_FLAGS \ 68 (__I915_EXEC_UNKNOWN_FLAGS | \ 69 I915_EXEC_CONSTANTS_MASK | \ 70 I915_EXEC_RESOURCE_STREAMER) 71 72 /* Catch emission of unexpected errors for CI! */ 73 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) 74 #undef EINVAL 75 #define EINVAL ({ \ 76 DRM_DEBUG_DRIVER("EINVAL at %s:%d\n", __func__, __LINE__); \ 77 22; \ 78 }) 79 #endif 80 81 /** 82 * DOC: User command execution 83 * 84 * Userspace submits commands to be executed on the GPU as an instruction 85 * stream within a GEM object we call a batchbuffer. This instructions may 86 * refer to other GEM objects containing auxiliary state such as kernels, 87 * samplers, render targets and even secondary batchbuffers. Userspace does 88 * not know where in the GPU memory these objects reside and so before the 89 * batchbuffer is passed to the GPU for execution, those addresses in the 90 * batchbuffer and auxiliary objects are updated. This is known as relocation, 91 * or patching. To try and avoid having to relocate each object on the next 92 * execution, userspace is told the location of those objects in this pass, 93 * but this remains just a hint as the kernel may choose a new location for 94 * any object in the future. 95 * 96 * At the level of talking to the hardware, submitting a batchbuffer for the 97 * GPU to execute is to add content to a buffer from which the HW 98 * command streamer is reading. 99 * 100 * 1. Add a command to load the HW context. For Logical Ring Contexts, i.e. 101 * Execlists, this command is not placed on the same buffer as the 102 * remaining items. 103 * 104 * 2. Add a command to invalidate caches to the buffer. 105 * 106 * 3. Add a batchbuffer start command to the buffer; the start command is 107 * essentially a token together with the GPU address of the batchbuffer 108 * to be executed. 109 * 110 * 4. Add a pipeline flush to the buffer. 111 * 112 * 5. Add a memory write command to the buffer to record when the GPU 113 * is done executing the batchbuffer. The memory write writes the 114 * global sequence number of the request, ``i915_request::global_seqno``; 115 * the i915 driver uses the current value in the register to determine 116 * if the GPU has completed the batchbuffer. 117 * 118 * 6. Add a user interrupt command to the buffer. This command instructs 119 * the GPU to issue an interrupt when the command, pipeline flush and 120 * memory write are completed. 121 * 122 * 7. Inform the hardware of the additional commands added to the buffer 123 * (by updating the tail pointer). 124 * 125 * Processing an execbuf ioctl is conceptually split up into a few phases. 126 * 127 * 1. Validation - Ensure all the pointers, handles and flags are valid. 128 * 2. Reservation - Assign GPU address space for every object 129 * 3. Relocation - Update any addresses to point to the final locations 130 * 4. Serialisation - Order the request with respect to its dependencies 131 * 5. Construction - Construct a request to execute the batchbuffer 132 * 6. Submission (at some point in the future execution) 133 * 134 * Reserving resources for the execbuf is the most complicated phase. We 135 * neither want to have to migrate the object in the address space, nor do 136 * we want to have to update any relocations pointing to this object. Ideally, 137 * we want to leave the object where it is and for all the existing relocations 138 * to match. If the object is given a new address, or if userspace thinks the 139 * object is elsewhere, we have to parse all the relocation entries and update 140 * the addresses. Userspace can set the I915_EXEC_NORELOC flag to hint that 141 * all the target addresses in all of its objects match the value in the 142 * relocation entries and that they all match the presumed offsets given by the 143 * list of execbuffer objects. Using this knowledge, we know that if we haven't 144 * moved any buffers, all the relocation entries are valid and we can skip 145 * the update. (If userspace is wrong, the likely outcome is an impromptu GPU 146 * hang.) The requirement for using I915_EXEC_NO_RELOC are: 147 * 148 * The addresses written in the objects must match the corresponding 149 * reloc.presumed_offset which in turn must match the corresponding 150 * execobject.offset. 151 * 152 * Any render targets written to in the batch must be flagged with 153 * EXEC_OBJECT_WRITE. 154 * 155 * To avoid stalling, execobject.offset should match the current 156 * address of that object within the active context. 157 * 158 * The reservation is done is multiple phases. First we try and keep any 159 * object already bound in its current location - so as long as meets the 160 * constraints imposed by the new execbuffer. Any object left unbound after the 161 * first pass is then fitted into any available idle space. If an object does 162 * not fit, all objects are removed from the reservation and the process rerun 163 * after sorting the objects into a priority order (more difficult to fit 164 * objects are tried first). Failing that, the entire VM is cleared and we try 165 * to fit the execbuf once last time before concluding that it simply will not 166 * fit. 167 * 168 * A small complication to all of this is that we allow userspace not only to 169 * specify an alignment and a size for the object in the address space, but 170 * we also allow userspace to specify the exact offset. This objects are 171 * simpler to place (the location is known a priori) all we have to do is make 172 * sure the space is available. 173 * 174 * Once all the objects are in place, patching up the buried pointers to point 175 * to the final locations is a fairly simple job of walking over the relocation 176 * entry arrays, looking up the right address and rewriting the value into 177 * the object. Simple! ... The relocation entries are stored in user memory 178 * and so to access them we have to copy them into a local buffer. That copy 179 * has to avoid taking any pagefaults as they may lead back to a GEM object 180 * requiring the struct_mutex (i.e. recursive deadlock). So once again we split 181 * the relocation into multiple passes. First we try to do everything within an 182 * atomic context (avoid the pagefaults) which requires that we never wait. If 183 * we detect that we may wait, or if we need to fault, then we have to fallback 184 * to a slower path. The slowpath has to drop the mutex. (Can you hear alarm 185 * bells yet?) Dropping the mutex means that we lose all the state we have 186 * built up so far for the execbuf and we must reset any global data. However, 187 * we do leave the objects pinned in their final locations - which is a 188 * potential issue for concurrent execbufs. Once we have left the mutex, we can 189 * allocate and copy all the relocation entries into a large array at our 190 * leisure, reacquire the mutex, reclaim all the objects and other state and 191 * then proceed to update any incorrect addresses with the objects. 192 * 193 * As we process the relocation entries, we maintain a record of whether the 194 * object is being written to. Using NORELOC, we expect userspace to provide 195 * this information instead. We also check whether we can skip the relocation 196 * by comparing the expected value inside the relocation entry with the target's 197 * final address. If they differ, we have to map the current object and rewrite 198 * the 4 or 8 byte pointer within. 199 * 200 * Serialising an execbuf is quite simple according to the rules of the GEM 201 * ABI. Execution within each context is ordered by the order of submission. 202 * Writes to any GEM object are in order of submission and are exclusive. Reads 203 * from a GEM object are unordered with respect to other reads, but ordered by 204 * writes. A write submitted after a read cannot occur before the read, and 205 * similarly any read submitted after a write cannot occur before the write. 206 * Writes are ordered between engines such that only one write occurs at any 207 * time (completing any reads beforehand) - using semaphores where available 208 * and CPU serialisation otherwise. Other GEM access obey the same rules, any 209 * write (either via mmaps using set-domain, or via pwrite) must flush all GPU 210 * reads before starting, and any read (either using set-domain or pread) must 211 * flush all GPU writes before starting. (Note we only employ a barrier before, 212 * we currently rely on userspace not concurrently starting a new execution 213 * whilst reading or writing to an object. This may be an advantage or not 214 * depending on how much you trust userspace not to shoot themselves in the 215 * foot.) Serialisation may just result in the request being inserted into 216 * a DAG awaiting its turn, but most simple is to wait on the CPU until 217 * all dependencies are resolved. 218 * 219 * After all of that, is just a matter of closing the request and handing it to 220 * the hardware (well, leaving it in a queue to be executed). However, we also 221 * offer the ability for batchbuffers to be run with elevated privileges so 222 * that they access otherwise hidden registers. (Used to adjust L3 cache etc.) 223 * Before any batch is given extra privileges we first must check that it 224 * contains no nefarious instructions, we check that each instruction is from 225 * our whitelist and all registers are also from an allowed list. We first 226 * copy the user's batchbuffer to a shadow (so that the user doesn't have 227 * access to it, either by the CPU or GPU as we scan it) and then parse each 228 * instruction. If everything is ok, we set a flag telling the hardware to run 229 * the batchbuffer in trusted mode, otherwise the ioctl is rejected. 230 */ 231 232 struct i915_execbuffer { 233 struct drm_i915_private *i915; /** i915 backpointer */ 234 struct drm_file *file; /** per-file lookup tables and limits */ 235 struct drm_i915_gem_execbuffer2 *args; /** ioctl parameters */ 236 struct drm_i915_gem_exec_object2 *exec; /** ioctl execobj[] */ 237 struct eb_vma *vma; 238 239 struct intel_engine_cs *engine; /** engine to queue the request to */ 240 struct intel_context *context; /* logical state for the request */ 241 struct i915_gem_context *gem_context; /** caller's context */ 242 243 struct i915_request *request; /** our request to build */ 244 struct eb_vma *batch; /** identity of the batch obj/vma */ 245 struct i915_vma *trampoline; /** trampoline used for chaining */ 246 247 /** actual size of execobj[] as we may extend it for the cmdparser */ 248 unsigned int buffer_count; 249 250 /** list of vma not yet bound during reservation phase */ 251 struct list_head unbound; 252 253 /** list of vma that have execobj.relocation_count */ 254 struct list_head relocs; 255 256 /** 257 * Track the most recently used object for relocations, as we 258 * frequently have to perform multiple relocations within the same 259 * obj/page 260 */ 261 struct reloc_cache { 262 struct drm_mm_node node; /** temporary GTT binding */ 263 unsigned long vaddr; /** Current kmap address */ 264 unsigned long page; /** Currently mapped page index */ 265 unsigned int gen; /** Cached value of INTEL_GEN */ 266 bool use_64bit_reloc : 1; 267 bool has_llc : 1; 268 bool has_fence : 1; 269 bool needs_unfenced : 1; 270 271 struct i915_vma *target; 272 struct i915_request *rq; 273 struct i915_vma *rq_vma; 274 u32 *rq_cmd; 275 unsigned int rq_size; 276 } reloc_cache; 277 278 u64 invalid_flags; /** Set of execobj.flags that are invalid */ 279 u32 context_flags; /** Set of execobj.flags to insert from the ctx */ 280 281 u32 batch_start_offset; /** Location within object of batch */ 282 u32 batch_len; /** Length of batch within object */ 283 u32 batch_flags; /** Flags composed for emit_bb_start() */ 284 285 /** 286 * Indicate either the size of the hastable used to resolve 287 * relocation handles, or if negative that we are using a direct 288 * index into the execobj[]. 289 */ 290 int lut_size; 291 struct hlist_head *buckets; /** ht for relocation handles */ 292 struct eb_vma_array *array; 293 }; 294 295 static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb) 296 { 297 return intel_engine_requires_cmd_parser(eb->engine) || 298 (intel_engine_using_cmd_parser(eb->engine) && 299 eb->args->batch_len); 300 } 301 302 static struct eb_vma_array *eb_vma_array_create(unsigned int count) 303 { 304 struct eb_vma_array *arr; 305 306 arr = kvmalloc(struct_size(arr, vma, count), GFP_KERNEL | __GFP_NOWARN); 307 if (!arr) 308 return NULL; 309 310 kref_init(&arr->kref); 311 arr->vma[0].vma = NULL; 312 313 return arr; 314 } 315 316 static inline void eb_unreserve_vma(struct eb_vma *ev) 317 { 318 struct i915_vma *vma = ev->vma; 319 320 if (unlikely(ev->flags & __EXEC_OBJECT_HAS_FENCE)) 321 __i915_vma_unpin_fence(vma); 322 323 if (ev->flags & __EXEC_OBJECT_HAS_PIN) 324 __i915_vma_unpin(vma); 325 326 ev->flags &= ~(__EXEC_OBJECT_HAS_PIN | 327 __EXEC_OBJECT_HAS_FENCE); 328 } 329 330 static void eb_vma_array_destroy(struct kref *kref) 331 { 332 struct eb_vma_array *arr = container_of(kref, typeof(*arr), kref); 333 struct eb_vma *ev = arr->vma; 334 335 while (ev->vma) { 336 eb_unreserve_vma(ev); 337 i915_vma_put(ev->vma); 338 ev++; 339 } 340 341 kvfree(arr); 342 } 343 344 static void eb_vma_array_put(struct eb_vma_array *arr) 345 { 346 kref_put(&arr->kref, eb_vma_array_destroy); 347 } 348 349 static int eb_create(struct i915_execbuffer *eb) 350 { 351 /* Allocate an extra slot for use by the command parser + sentinel */ 352 eb->array = eb_vma_array_create(eb->buffer_count + 2); 353 if (!eb->array) 354 return -ENOMEM; 355 356 eb->vma = eb->array->vma; 357 358 if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) { 359 unsigned int size = 1 + ilog2(eb->buffer_count); 360 361 /* 362 * Without a 1:1 association between relocation handles and 363 * the execobject[] index, we instead create a hashtable. 364 * We size it dynamically based on available memory, starting 365 * first with 1:1 assocative hash and scaling back until 366 * the allocation succeeds. 367 * 368 * Later on we use a positive lut_size to indicate we are 369 * using this hashtable, and a negative value to indicate a 370 * direct lookup. 371 */ 372 do { 373 gfp_t flags; 374 375 /* While we can still reduce the allocation size, don't 376 * raise a warning and allow the allocation to fail. 377 * On the last pass though, we want to try as hard 378 * as possible to perform the allocation and warn 379 * if it fails. 380 */ 381 flags = GFP_KERNEL; 382 if (size > 1) 383 flags |= __GFP_NORETRY | __GFP_NOWARN; 384 385 eb->buckets = kzalloc(sizeof(struct hlist_head) << size, 386 flags); 387 if (eb->buckets) 388 break; 389 } while (--size); 390 391 if (unlikely(!size)) { 392 eb_vma_array_put(eb->array); 393 return -ENOMEM; 394 } 395 396 eb->lut_size = size; 397 } else { 398 eb->lut_size = -eb->buffer_count; 399 } 400 401 return 0; 402 } 403 404 static bool 405 eb_vma_misplaced(const struct drm_i915_gem_exec_object2 *entry, 406 const struct i915_vma *vma, 407 unsigned int flags) 408 { 409 if (vma->node.size < entry->pad_to_size) 410 return true; 411 412 if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment)) 413 return true; 414 415 if (flags & EXEC_OBJECT_PINNED && 416 vma->node.start != entry->offset) 417 return true; 418 419 if (flags & __EXEC_OBJECT_NEEDS_BIAS && 420 vma->node.start < BATCH_OFFSET_BIAS) 421 return true; 422 423 if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) && 424 (vma->node.start + vma->node.size - 1) >> 32) 425 return true; 426 427 if (flags & __EXEC_OBJECT_NEEDS_MAP && 428 !i915_vma_is_map_and_fenceable(vma)) 429 return true; 430 431 return false; 432 } 433 434 static u64 eb_pin_flags(const struct drm_i915_gem_exec_object2 *entry, 435 unsigned int exec_flags) 436 { 437 u64 pin_flags = 0; 438 439 if (exec_flags & EXEC_OBJECT_NEEDS_GTT) 440 pin_flags |= PIN_GLOBAL; 441 442 /* 443 * Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset, 444 * limit address to the first 4GBs for unflagged objects. 445 */ 446 if (!(exec_flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS)) 447 pin_flags |= PIN_ZONE_4G; 448 449 if (exec_flags & __EXEC_OBJECT_NEEDS_MAP) 450 pin_flags |= PIN_MAPPABLE; 451 452 if (exec_flags & EXEC_OBJECT_PINNED) 453 pin_flags |= entry->offset | PIN_OFFSET_FIXED; 454 else if (exec_flags & __EXEC_OBJECT_NEEDS_BIAS) 455 pin_flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS; 456 457 return pin_flags; 458 } 459 460 static inline bool 461 eb_pin_vma(struct i915_execbuffer *eb, 462 const struct drm_i915_gem_exec_object2 *entry, 463 struct eb_vma *ev) 464 { 465 struct i915_vma *vma = ev->vma; 466 u64 pin_flags; 467 468 if (vma->node.size) 469 pin_flags = vma->node.start; 470 else 471 pin_flags = entry->offset & PIN_OFFSET_MASK; 472 473 pin_flags |= PIN_USER | PIN_NOEVICT | PIN_OFFSET_FIXED; 474 if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_GTT)) 475 pin_flags |= PIN_GLOBAL; 476 477 /* Attempt to reuse the current location if available */ 478 if (unlikely(i915_vma_pin(vma, 0, 0, pin_flags))) { 479 if (entry->flags & EXEC_OBJECT_PINNED) 480 return false; 481 482 /* Failing that pick any _free_ space if suitable */ 483 if (unlikely(i915_vma_pin(vma, 484 entry->pad_to_size, 485 entry->alignment, 486 eb_pin_flags(entry, ev->flags) | 487 PIN_USER | PIN_NOEVICT))) 488 return false; 489 } 490 491 if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) { 492 if (unlikely(i915_vma_pin_fence(vma))) { 493 i915_vma_unpin(vma); 494 return false; 495 } 496 497 if (vma->fence) 498 ev->flags |= __EXEC_OBJECT_HAS_FENCE; 499 } 500 501 ev->flags |= __EXEC_OBJECT_HAS_PIN; 502 return !eb_vma_misplaced(entry, vma, ev->flags); 503 } 504 505 static int 506 eb_validate_vma(struct i915_execbuffer *eb, 507 struct drm_i915_gem_exec_object2 *entry, 508 struct i915_vma *vma) 509 { 510 if (unlikely(entry->flags & eb->invalid_flags)) 511 return -EINVAL; 512 513 if (unlikely(entry->alignment && 514 !is_power_of_2_u64(entry->alignment))) 515 return -EINVAL; 516 517 /* 518 * Offset can be used as input (EXEC_OBJECT_PINNED), reject 519 * any non-page-aligned or non-canonical addresses. 520 */ 521 if (unlikely(entry->flags & EXEC_OBJECT_PINNED && 522 entry->offset != gen8_canonical_addr(entry->offset & I915_GTT_PAGE_MASK))) 523 return -EINVAL; 524 525 /* pad_to_size was once a reserved field, so sanitize it */ 526 if (entry->flags & EXEC_OBJECT_PAD_TO_SIZE) { 527 if (unlikely(offset_in_page(entry->pad_to_size))) 528 return -EINVAL; 529 } else { 530 entry->pad_to_size = 0; 531 } 532 /* 533 * From drm_mm perspective address space is continuous, 534 * so from this point we're always using non-canonical 535 * form internally. 536 */ 537 entry->offset = gen8_noncanonical_addr(entry->offset); 538 539 if (!eb->reloc_cache.has_fence) { 540 entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE; 541 } else { 542 if ((entry->flags & EXEC_OBJECT_NEEDS_FENCE || 543 eb->reloc_cache.needs_unfenced) && 544 i915_gem_object_is_tiled(vma->obj)) 545 entry->flags |= EXEC_OBJECT_NEEDS_GTT | __EXEC_OBJECT_NEEDS_MAP; 546 } 547 548 if (!(entry->flags & EXEC_OBJECT_PINNED)) 549 entry->flags |= eb->context_flags; 550 551 return 0; 552 } 553 554 static void 555 eb_add_vma(struct i915_execbuffer *eb, 556 unsigned int i, unsigned batch_idx, 557 struct i915_vma *vma) 558 { 559 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; 560 struct eb_vma *ev = &eb->vma[i]; 561 562 GEM_BUG_ON(i915_vma_is_closed(vma)); 563 564 ev->vma = vma; 565 ev->exec = entry; 566 ev->flags = entry->flags; 567 568 if (eb->lut_size > 0) { 569 ev->handle = entry->handle; 570 hlist_add_head(&ev->node, 571 &eb->buckets[hash_32(entry->handle, 572 eb->lut_size)]); 573 } 574 575 if (entry->relocation_count) 576 list_add_tail(&ev->reloc_link, &eb->relocs); 577 578 /* 579 * SNA is doing fancy tricks with compressing batch buffers, which leads 580 * to negative relocation deltas. Usually that works out ok since the 581 * relocate address is still positive, except when the batch is placed 582 * very low in the GTT. Ensure this doesn't happen. 583 * 584 * Note that actual hangs have only been observed on gen7, but for 585 * paranoia do it everywhere. 586 */ 587 if (i == batch_idx) { 588 if (entry->relocation_count && 589 !(ev->flags & EXEC_OBJECT_PINNED)) 590 ev->flags |= __EXEC_OBJECT_NEEDS_BIAS; 591 if (eb->reloc_cache.has_fence) 592 ev->flags |= EXEC_OBJECT_NEEDS_FENCE; 593 594 eb->batch = ev; 595 } 596 597 if (eb_pin_vma(eb, entry, ev)) { 598 if (entry->offset != vma->node.start) { 599 entry->offset = vma->node.start | UPDATE; 600 eb->args->flags |= __EXEC_HAS_RELOC; 601 } 602 } else { 603 eb_unreserve_vma(ev); 604 list_add_tail(&ev->bind_link, &eb->unbound); 605 } 606 } 607 608 static inline int use_cpu_reloc(const struct reloc_cache *cache, 609 const struct drm_i915_gem_object *obj) 610 { 611 if (!i915_gem_object_has_struct_page(obj)) 612 return false; 613 614 if (DBG_FORCE_RELOC == FORCE_CPU_RELOC) 615 return true; 616 617 if (DBG_FORCE_RELOC == FORCE_GTT_RELOC) 618 return false; 619 620 return (cache->has_llc || 621 obj->cache_dirty || 622 obj->cache_level != I915_CACHE_NONE); 623 } 624 625 static int eb_reserve_vma(const struct i915_execbuffer *eb, 626 struct eb_vma *ev, 627 u64 pin_flags) 628 { 629 struct drm_i915_gem_exec_object2 *entry = ev->exec; 630 struct i915_vma *vma = ev->vma; 631 int err; 632 633 if (drm_mm_node_allocated(&vma->node) && 634 eb_vma_misplaced(entry, vma, ev->flags)) { 635 err = i915_vma_unbind(vma); 636 if (err) 637 return err; 638 } 639 640 err = i915_vma_pin(vma, 641 entry->pad_to_size, entry->alignment, 642 eb_pin_flags(entry, ev->flags) | pin_flags); 643 if (err) 644 return err; 645 646 if (entry->offset != vma->node.start) { 647 entry->offset = vma->node.start | UPDATE; 648 eb->args->flags |= __EXEC_HAS_RELOC; 649 } 650 651 if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) { 652 err = i915_vma_pin_fence(vma); 653 if (unlikely(err)) { 654 i915_vma_unpin(vma); 655 return err; 656 } 657 658 if (vma->fence) 659 ev->flags |= __EXEC_OBJECT_HAS_FENCE; 660 } 661 662 ev->flags |= __EXEC_OBJECT_HAS_PIN; 663 GEM_BUG_ON(eb_vma_misplaced(entry, vma, ev->flags)); 664 665 return 0; 666 } 667 668 static int eb_reserve(struct i915_execbuffer *eb) 669 { 670 const unsigned int count = eb->buffer_count; 671 unsigned int pin_flags = PIN_USER | PIN_NONBLOCK; 672 struct list_head last; 673 struct eb_vma *ev; 674 unsigned int i, pass; 675 int err = 0; 676 677 /* 678 * Attempt to pin all of the buffers into the GTT. 679 * This is done in 3 phases: 680 * 681 * 1a. Unbind all objects that do not match the GTT constraints for 682 * the execbuffer (fenceable, mappable, alignment etc). 683 * 1b. Increment pin count for already bound objects. 684 * 2. Bind new objects. 685 * 3. Decrement pin count. 686 * 687 * This avoid unnecessary unbinding of later objects in order to make 688 * room for the earlier objects *unless* we need to defragment. 689 */ 690 691 if (mutex_lock_interruptible(&eb->i915->drm.struct_mutex)) 692 return -EINTR; 693 694 pass = 0; 695 do { 696 list_for_each_entry(ev, &eb->unbound, bind_link) { 697 err = eb_reserve_vma(eb, ev, pin_flags); 698 if (err) 699 break; 700 } 701 if (!(err == -ENOSPC || err == -EAGAIN)) 702 break; 703 704 /* Resort *all* the objects into priority order */ 705 INIT_LIST_HEAD(&eb->unbound); 706 INIT_LIST_HEAD(&last); 707 for (i = 0; i < count; i++) { 708 unsigned int flags; 709 710 ev = &eb->vma[i]; 711 flags = ev->flags; 712 if (flags & EXEC_OBJECT_PINNED && 713 flags & __EXEC_OBJECT_HAS_PIN) 714 continue; 715 716 eb_unreserve_vma(ev); 717 718 if (flags & EXEC_OBJECT_PINNED) 719 /* Pinned must have their slot */ 720 list_add(&ev->bind_link, &eb->unbound); 721 else if (flags & __EXEC_OBJECT_NEEDS_MAP) 722 /* Map require the lowest 256MiB (aperture) */ 723 list_add_tail(&ev->bind_link, &eb->unbound); 724 else if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS)) 725 /* Prioritise 4GiB region for restricted bo */ 726 list_add(&ev->bind_link, &last); 727 else 728 list_add_tail(&ev->bind_link, &last); 729 } 730 list_splice_tail(&last, &eb->unbound); 731 732 if (err == -EAGAIN) { 733 mutex_unlock(&eb->i915->drm.struct_mutex); 734 flush_workqueue(eb->i915->mm.userptr_wq); 735 mutex_lock(&eb->i915->drm.struct_mutex); 736 continue; 737 } 738 739 switch (pass++) { 740 case 0: 741 break; 742 743 case 1: 744 /* Too fragmented, unbind everything and retry */ 745 mutex_lock(&eb->context->vm->mutex); 746 err = i915_gem_evict_vm(eb->context->vm); 747 mutex_unlock(&eb->context->vm->mutex); 748 if (err) 749 goto unlock; 750 break; 751 752 default: 753 err = -ENOSPC; 754 goto unlock; 755 } 756 757 pin_flags = PIN_USER; 758 } while (1); 759 760 unlock: 761 mutex_unlock(&eb->i915->drm.struct_mutex); 762 return err; 763 } 764 765 static unsigned int eb_batch_index(const struct i915_execbuffer *eb) 766 { 767 if (eb->args->flags & I915_EXEC_BATCH_FIRST) 768 return 0; 769 else 770 return eb->buffer_count - 1; 771 } 772 773 static int eb_select_context(struct i915_execbuffer *eb) 774 { 775 struct i915_gem_context *ctx; 776 777 ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1); 778 if (unlikely(!ctx)) 779 return -ENOENT; 780 781 eb->gem_context = ctx; 782 if (rcu_access_pointer(ctx->vm)) 783 eb->invalid_flags |= EXEC_OBJECT_NEEDS_GTT; 784 785 eb->context_flags = 0; 786 if (test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags)) 787 eb->context_flags |= __EXEC_OBJECT_NEEDS_BIAS; 788 789 return 0; 790 } 791 792 static int __eb_add_lut(struct i915_execbuffer *eb, 793 u32 handle, struct i915_vma *vma) 794 { 795 struct i915_gem_context *ctx = eb->gem_context; 796 struct i915_lut_handle *lut; 797 int err; 798 799 lut = i915_lut_handle_alloc(); 800 if (unlikely(!lut)) 801 return -ENOMEM; 802 803 i915_vma_get(vma); 804 if (!atomic_fetch_inc(&vma->open_count)) 805 i915_vma_reopen(vma); 806 lut->handle = handle; 807 lut->ctx = ctx; 808 809 /* Check that the context hasn't been closed in the meantime */ 810 err = -EINTR; 811 if (!mutex_lock_interruptible(&ctx->mutex)) { 812 err = -ENOENT; 813 if (likely(!i915_gem_context_is_closed(ctx))) 814 err = radix_tree_insert(&ctx->handles_vma, handle, vma); 815 if (err == 0) { /* And nor has this handle */ 816 struct drm_i915_gem_object *obj = vma->obj; 817 818 i915_gem_object_lock(obj); 819 if (idr_find(&eb->file->object_idr, handle) == obj) { 820 list_add(&lut->obj_link, &obj->lut_list); 821 } else { 822 radix_tree_delete(&ctx->handles_vma, handle); 823 err = -ENOENT; 824 } 825 i915_gem_object_unlock(obj); 826 } 827 mutex_unlock(&ctx->mutex); 828 } 829 if (unlikely(err)) 830 goto err; 831 832 return 0; 833 834 err: 835 i915_vma_close(vma); 836 i915_vma_put(vma); 837 i915_lut_handle_free(lut); 838 return err; 839 } 840 841 static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle) 842 { 843 do { 844 struct drm_i915_gem_object *obj; 845 struct i915_vma *vma; 846 int err; 847 848 rcu_read_lock(); 849 vma = radix_tree_lookup(&eb->gem_context->handles_vma, handle); 850 if (likely(vma)) 851 vma = i915_vma_tryget(vma); 852 rcu_read_unlock(); 853 if (likely(vma)) 854 return vma; 855 856 obj = i915_gem_object_lookup(eb->file, handle); 857 if (unlikely(!obj)) 858 return ERR_PTR(-ENOENT); 859 860 vma = i915_vma_instance(obj, eb->context->vm, NULL); 861 if (IS_ERR(vma)) { 862 i915_gem_object_put(obj); 863 return vma; 864 } 865 866 err = __eb_add_lut(eb, handle, vma); 867 if (likely(!err)) 868 return vma; 869 870 i915_gem_object_put(obj); 871 if (err != -EEXIST) 872 return ERR_PTR(err); 873 } while (1); 874 } 875 876 static int eb_lookup_vmas(struct i915_execbuffer *eb) 877 { 878 unsigned int batch = eb_batch_index(eb); 879 unsigned int i; 880 int err = 0; 881 882 INIT_LIST_HEAD(&eb->relocs); 883 INIT_LIST_HEAD(&eb->unbound); 884 885 for (i = 0; i < eb->buffer_count; i++) { 886 struct i915_vma *vma; 887 888 vma = eb_lookup_vma(eb, eb->exec[i].handle); 889 if (IS_ERR(vma)) { 890 err = PTR_ERR(vma); 891 break; 892 } 893 894 err = eb_validate_vma(eb, &eb->exec[i], vma); 895 if (unlikely(err)) { 896 i915_vma_put(vma); 897 break; 898 } 899 900 eb_add_vma(eb, i, batch, vma); 901 } 902 903 eb->vma[i].vma = NULL; 904 return err; 905 } 906 907 static struct eb_vma * 908 eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle) 909 { 910 if (eb->lut_size < 0) { 911 if (handle >= -eb->lut_size) 912 return NULL; 913 return &eb->vma[handle]; 914 } else { 915 struct hlist_head *head; 916 struct eb_vma *ev; 917 918 head = &eb->buckets[hash_32(handle, eb->lut_size)]; 919 hlist_for_each_entry(ev, head, node) { 920 if (ev->handle == handle) 921 return ev; 922 } 923 return NULL; 924 } 925 } 926 927 static void eb_destroy(const struct i915_execbuffer *eb) 928 { 929 GEM_BUG_ON(eb->reloc_cache.rq); 930 931 if (eb->array) 932 eb_vma_array_put(eb->array); 933 934 if (eb->lut_size > 0) 935 kfree(eb->buckets); 936 } 937 938 static inline u64 939 relocation_target(const struct drm_i915_gem_relocation_entry *reloc, 940 const struct i915_vma *target) 941 { 942 return gen8_canonical_addr((int)reloc->delta + target->node.start); 943 } 944 945 static void reloc_cache_init(struct reloc_cache *cache, 946 struct drm_i915_private *i915) 947 { 948 cache->page = -1; 949 cache->vaddr = 0; 950 /* Must be a variable in the struct to allow GCC to unroll. */ 951 cache->gen = INTEL_GEN(i915); 952 cache->has_llc = HAS_LLC(i915); 953 cache->use_64bit_reloc = HAS_64BIT_RELOC(i915); 954 cache->has_fence = cache->gen < 4; 955 cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment; 956 cache->node.flags = 0; 957 cache->rq = NULL; 958 cache->target = NULL; 959 } 960 961 static inline void *unmask_page(unsigned long p) 962 { 963 return (void *)(uintptr_t)(p & PAGE_MASK); 964 } 965 966 static inline unsigned int unmask_flags(unsigned long p) 967 { 968 return p & ~PAGE_MASK; 969 } 970 971 #define KMAP 0x4 /* after CLFLUSH_FLAGS */ 972 973 static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache) 974 { 975 struct drm_i915_private *i915 = 976 container_of(cache, struct i915_execbuffer, reloc_cache)->i915; 977 return &i915->ggtt; 978 } 979 980 #define RELOC_TAIL 4 981 982 static int reloc_gpu_chain(struct reloc_cache *cache) 983 { 984 struct intel_gt_buffer_pool_node *pool; 985 struct i915_request *rq = cache->rq; 986 struct i915_vma *batch; 987 u32 *cmd; 988 int err; 989 990 pool = intel_gt_get_buffer_pool(rq->engine->gt, PAGE_SIZE); 991 if (IS_ERR(pool)) 992 return PTR_ERR(pool); 993 994 batch = i915_vma_instance(pool->obj, rq->context->vm, NULL); 995 if (IS_ERR(batch)) { 996 err = PTR_ERR(batch); 997 goto out_pool; 998 } 999 1000 err = i915_vma_pin(batch, 0, 0, PIN_USER | PIN_NONBLOCK); 1001 if (err) 1002 goto out_pool; 1003 1004 GEM_BUG_ON(cache->rq_size + RELOC_TAIL > PAGE_SIZE / sizeof(u32)); 1005 cmd = cache->rq_cmd + cache->rq_size; 1006 *cmd++ = MI_ARB_CHECK; 1007 if (cache->gen >= 8) 1008 *cmd++ = MI_BATCH_BUFFER_START_GEN8; 1009 else if (cache->gen >= 6) 1010 *cmd++ = MI_BATCH_BUFFER_START; 1011 else 1012 *cmd++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT; 1013 *cmd++ = lower_32_bits(batch->node.start); 1014 *cmd++ = upper_32_bits(batch->node.start); /* Always 0 for gen<8 */ 1015 i915_gem_object_flush_map(cache->rq_vma->obj); 1016 i915_gem_object_unpin_map(cache->rq_vma->obj); 1017 cache->rq_vma = NULL; 1018 1019 err = intel_gt_buffer_pool_mark_active(pool, rq); 1020 if (err == 0) { 1021 i915_vma_lock(batch); 1022 err = i915_request_await_object(rq, batch->obj, false); 1023 if (err == 0) 1024 err = i915_vma_move_to_active(batch, rq, 0); 1025 i915_vma_unlock(batch); 1026 } 1027 i915_vma_unpin(batch); 1028 if (err) 1029 goto out_pool; 1030 1031 cmd = i915_gem_object_pin_map(batch->obj, 1032 cache->has_llc ? 1033 I915_MAP_FORCE_WB : 1034 I915_MAP_FORCE_WC); 1035 if (IS_ERR(cmd)) { 1036 err = PTR_ERR(cmd); 1037 goto out_pool; 1038 } 1039 1040 /* Return with batch mapping (cmd) still pinned */ 1041 cache->rq_cmd = cmd; 1042 cache->rq_size = 0; 1043 cache->rq_vma = batch; 1044 1045 out_pool: 1046 intel_gt_buffer_pool_put(pool); 1047 return err; 1048 } 1049 1050 static unsigned int reloc_bb_flags(const struct reloc_cache *cache) 1051 { 1052 return cache->gen > 5 ? 0 : I915_DISPATCH_SECURE; 1053 } 1054 1055 static int reloc_gpu_flush(struct reloc_cache *cache) 1056 { 1057 struct i915_request *rq; 1058 int err; 1059 1060 rq = fetch_and_zero(&cache->rq); 1061 if (!rq) 1062 return 0; 1063 1064 if (cache->rq_vma) { 1065 struct drm_i915_gem_object *obj = cache->rq_vma->obj; 1066 1067 GEM_BUG_ON(cache->rq_size >= obj->base.size / sizeof(u32)); 1068 cache->rq_cmd[cache->rq_size++] = MI_BATCH_BUFFER_END; 1069 1070 __i915_gem_object_flush_map(obj, 1071 0, sizeof(u32) * cache->rq_size); 1072 i915_gem_object_unpin_map(obj); 1073 } 1074 1075 err = 0; 1076 if (rq->engine->emit_init_breadcrumb) 1077 err = rq->engine->emit_init_breadcrumb(rq); 1078 if (!err) 1079 err = rq->engine->emit_bb_start(rq, 1080 rq->batch->node.start, 1081 PAGE_SIZE, 1082 reloc_bb_flags(cache)); 1083 if (err) 1084 i915_request_set_error_once(rq, err); 1085 1086 intel_gt_chipset_flush(rq->engine->gt); 1087 i915_request_add(rq); 1088 1089 return err; 1090 } 1091 1092 static void reloc_cache_reset(struct reloc_cache *cache) 1093 { 1094 void *vaddr; 1095 1096 if (!cache->vaddr) 1097 return; 1098 1099 vaddr = unmask_page(cache->vaddr); 1100 if (cache->vaddr & KMAP) { 1101 if (cache->vaddr & CLFLUSH_AFTER) 1102 mb(); 1103 1104 kunmap_atomic(vaddr); 1105 i915_gem_object_finish_access((struct drm_i915_gem_object *)cache->node.mm); 1106 } else { 1107 struct i915_ggtt *ggtt = cache_to_ggtt(cache); 1108 1109 intel_gt_flush_ggtt_writes(ggtt->vm.gt); 1110 io_mapping_unmap_atomic((void __iomem *)vaddr); 1111 1112 if (drm_mm_node_allocated(&cache->node)) { 1113 ggtt->vm.clear_range(&ggtt->vm, 1114 cache->node.start, 1115 cache->node.size); 1116 mutex_lock(&ggtt->vm.mutex); 1117 drm_mm_remove_node(&cache->node); 1118 mutex_unlock(&ggtt->vm.mutex); 1119 } else { 1120 i915_vma_unpin((struct i915_vma *)cache->node.mm); 1121 } 1122 } 1123 1124 cache->vaddr = 0; 1125 cache->page = -1; 1126 } 1127 1128 static void *reloc_kmap(struct drm_i915_gem_object *obj, 1129 struct reloc_cache *cache, 1130 unsigned long page) 1131 { 1132 void *vaddr; 1133 1134 if (cache->vaddr) { 1135 kunmap_atomic(unmask_page(cache->vaddr)); 1136 } else { 1137 unsigned int flushes; 1138 int err; 1139 1140 err = i915_gem_object_prepare_write(obj, &flushes); 1141 if (err) 1142 return ERR_PTR(err); 1143 1144 BUILD_BUG_ON(KMAP & CLFLUSH_FLAGS); 1145 BUILD_BUG_ON((KMAP | CLFLUSH_FLAGS) & PAGE_MASK); 1146 1147 cache->vaddr = flushes | KMAP; 1148 cache->node.mm = (void *)obj; 1149 if (flushes) 1150 mb(); 1151 } 1152 1153 vaddr = kmap_atomic(i915_gem_object_get_dirty_page(obj, page)); 1154 cache->vaddr = unmask_flags(cache->vaddr) | (unsigned long)vaddr; 1155 cache->page = page; 1156 1157 return vaddr; 1158 } 1159 1160 static void *reloc_iomap(struct drm_i915_gem_object *obj, 1161 struct reloc_cache *cache, 1162 unsigned long page) 1163 { 1164 struct i915_ggtt *ggtt = cache_to_ggtt(cache); 1165 unsigned long offset; 1166 void *vaddr; 1167 1168 if (cache->vaddr) { 1169 intel_gt_flush_ggtt_writes(ggtt->vm.gt); 1170 io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr)); 1171 } else { 1172 struct i915_vma *vma; 1173 int err; 1174 1175 if (i915_gem_object_is_tiled(obj)) 1176 return ERR_PTR(-EINVAL); 1177 1178 if (use_cpu_reloc(cache, obj)) 1179 return NULL; 1180 1181 i915_gem_object_lock(obj); 1182 err = i915_gem_object_set_to_gtt_domain(obj, true); 1183 i915_gem_object_unlock(obj); 1184 if (err) 1185 return ERR_PTR(err); 1186 1187 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, 1188 PIN_MAPPABLE | 1189 PIN_NONBLOCK /* NOWARN */ | 1190 PIN_NOEVICT); 1191 if (IS_ERR(vma)) { 1192 memset(&cache->node, 0, sizeof(cache->node)); 1193 mutex_lock(&ggtt->vm.mutex); 1194 err = drm_mm_insert_node_in_range 1195 (&ggtt->vm.mm, &cache->node, 1196 PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE, 1197 0, ggtt->mappable_end, 1198 DRM_MM_INSERT_LOW); 1199 mutex_unlock(&ggtt->vm.mutex); 1200 if (err) /* no inactive aperture space, use cpu reloc */ 1201 return NULL; 1202 } else { 1203 cache->node.start = vma->node.start; 1204 cache->node.mm = (void *)vma; 1205 } 1206 } 1207 1208 offset = cache->node.start; 1209 if (drm_mm_node_allocated(&cache->node)) { 1210 ggtt->vm.insert_page(&ggtt->vm, 1211 i915_gem_object_get_dma_address(obj, page), 1212 offset, I915_CACHE_NONE, 0); 1213 } else { 1214 offset += page << PAGE_SHIFT; 1215 } 1216 1217 vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->iomap, 1218 offset); 1219 cache->page = page; 1220 cache->vaddr = (unsigned long)vaddr; 1221 1222 return vaddr; 1223 } 1224 1225 static void *reloc_vaddr(struct drm_i915_gem_object *obj, 1226 struct reloc_cache *cache, 1227 unsigned long page) 1228 { 1229 void *vaddr; 1230 1231 if (cache->page == page) { 1232 vaddr = unmask_page(cache->vaddr); 1233 } else { 1234 vaddr = NULL; 1235 if ((cache->vaddr & KMAP) == 0) 1236 vaddr = reloc_iomap(obj, cache, page); 1237 if (!vaddr) 1238 vaddr = reloc_kmap(obj, cache, page); 1239 } 1240 1241 return vaddr; 1242 } 1243 1244 static void clflush_write32(u32 *addr, u32 value, unsigned int flushes) 1245 { 1246 if (unlikely(flushes & (CLFLUSH_BEFORE | CLFLUSH_AFTER))) { 1247 if (flushes & CLFLUSH_BEFORE) { 1248 clflushopt(addr); 1249 mb(); 1250 } 1251 1252 *addr = value; 1253 1254 /* 1255 * Writes to the same cacheline are serialised by the CPU 1256 * (including clflush). On the write path, we only require 1257 * that it hits memory in an orderly fashion and place 1258 * mb barriers at the start and end of the relocation phase 1259 * to ensure ordering of clflush wrt to the system. 1260 */ 1261 if (flushes & CLFLUSH_AFTER) 1262 clflushopt(addr); 1263 } else 1264 *addr = value; 1265 } 1266 1267 static int reloc_move_to_gpu(struct i915_request *rq, struct i915_vma *vma) 1268 { 1269 struct drm_i915_gem_object *obj = vma->obj; 1270 int err; 1271 1272 i915_vma_lock(vma); 1273 1274 if (obj->cache_dirty & ~obj->cache_coherent) 1275 i915_gem_clflush_object(obj, 0); 1276 obj->write_domain = 0; 1277 1278 err = i915_request_await_object(rq, vma->obj, true); 1279 if (err == 0) 1280 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE); 1281 1282 i915_vma_unlock(vma); 1283 1284 return err; 1285 } 1286 1287 static int __reloc_gpu_alloc(struct i915_execbuffer *eb, 1288 struct intel_engine_cs *engine, 1289 unsigned int len) 1290 { 1291 struct reloc_cache *cache = &eb->reloc_cache; 1292 struct intel_gt_buffer_pool_node *pool; 1293 struct i915_request *rq; 1294 struct i915_vma *batch; 1295 u32 *cmd; 1296 int err; 1297 1298 pool = intel_gt_get_buffer_pool(engine->gt, PAGE_SIZE); 1299 if (IS_ERR(pool)) 1300 return PTR_ERR(pool); 1301 1302 cmd = i915_gem_object_pin_map(pool->obj, 1303 cache->has_llc ? 1304 I915_MAP_FORCE_WB : 1305 I915_MAP_FORCE_WC); 1306 if (IS_ERR(cmd)) { 1307 err = PTR_ERR(cmd); 1308 goto out_pool; 1309 } 1310 1311 batch = i915_vma_instance(pool->obj, eb->context->vm, NULL); 1312 if (IS_ERR(batch)) { 1313 err = PTR_ERR(batch); 1314 goto err_unmap; 1315 } 1316 1317 err = i915_vma_pin(batch, 0, 0, PIN_USER | PIN_NONBLOCK); 1318 if (err) 1319 goto err_unmap; 1320 1321 if (engine == eb->context->engine) { 1322 rq = i915_request_create(eb->context); 1323 } else { 1324 struct intel_context *ce; 1325 1326 ce = intel_context_create(engine); 1327 if (IS_ERR(ce)) { 1328 err = PTR_ERR(ce); 1329 goto err_unpin; 1330 } 1331 1332 i915_vm_put(ce->vm); 1333 ce->vm = i915_vm_get(eb->context->vm); 1334 1335 rq = intel_context_create_request(ce); 1336 intel_context_put(ce); 1337 } 1338 if (IS_ERR(rq)) { 1339 err = PTR_ERR(rq); 1340 goto err_unpin; 1341 } 1342 1343 err = intel_gt_buffer_pool_mark_active(pool, rq); 1344 if (err) 1345 goto err_request; 1346 1347 i915_vma_lock(batch); 1348 err = i915_request_await_object(rq, batch->obj, false); 1349 if (err == 0) 1350 err = i915_vma_move_to_active(batch, rq, 0); 1351 i915_vma_unlock(batch); 1352 if (err) 1353 goto skip_request; 1354 1355 rq->batch = batch; 1356 i915_vma_unpin(batch); 1357 1358 cache->rq = rq; 1359 cache->rq_cmd = cmd; 1360 cache->rq_size = 0; 1361 cache->rq_vma = batch; 1362 1363 /* Return with batch mapping (cmd) still pinned */ 1364 goto out_pool; 1365 1366 skip_request: 1367 i915_request_set_error_once(rq, err); 1368 err_request: 1369 i915_request_add(rq); 1370 err_unpin: 1371 i915_vma_unpin(batch); 1372 err_unmap: 1373 i915_gem_object_unpin_map(pool->obj); 1374 out_pool: 1375 intel_gt_buffer_pool_put(pool); 1376 return err; 1377 } 1378 1379 static bool reloc_can_use_engine(const struct intel_engine_cs *engine) 1380 { 1381 return engine->class != VIDEO_DECODE_CLASS || !IS_GEN(engine->i915, 6); 1382 } 1383 1384 static u32 *reloc_gpu(struct i915_execbuffer *eb, 1385 struct i915_vma *vma, 1386 unsigned int len) 1387 { 1388 struct reloc_cache *cache = &eb->reloc_cache; 1389 u32 *cmd; 1390 int err; 1391 1392 if (unlikely(!cache->rq)) { 1393 struct intel_engine_cs *engine = eb->engine; 1394 1395 if (!reloc_can_use_engine(engine)) { 1396 engine = engine->gt->engine_class[COPY_ENGINE_CLASS][0]; 1397 if (!engine) 1398 return ERR_PTR(-ENODEV); 1399 } 1400 1401 err = __reloc_gpu_alloc(eb, engine, len); 1402 if (unlikely(err)) 1403 return ERR_PTR(err); 1404 } 1405 1406 if (vma != cache->target) { 1407 err = reloc_move_to_gpu(cache->rq, vma); 1408 if (unlikely(err)) { 1409 i915_request_set_error_once(cache->rq, err); 1410 return ERR_PTR(err); 1411 } 1412 1413 cache->target = vma; 1414 } 1415 1416 if (unlikely(cache->rq_size + len > 1417 PAGE_SIZE / sizeof(u32) - RELOC_TAIL)) { 1418 err = reloc_gpu_chain(cache); 1419 if (unlikely(err)) { 1420 i915_request_set_error_once(cache->rq, err); 1421 return ERR_PTR(err); 1422 } 1423 } 1424 1425 GEM_BUG_ON(cache->rq_size + len >= PAGE_SIZE / sizeof(u32)); 1426 cmd = cache->rq_cmd + cache->rq_size; 1427 cache->rq_size += len; 1428 1429 return cmd; 1430 } 1431 1432 static inline bool use_reloc_gpu(struct i915_vma *vma) 1433 { 1434 if (DBG_FORCE_RELOC == FORCE_GPU_RELOC) 1435 return true; 1436 1437 if (DBG_FORCE_RELOC) 1438 return false; 1439 1440 return !dma_resv_test_signaled_rcu(vma->resv, true); 1441 } 1442 1443 static unsigned long vma_phys_addr(struct i915_vma *vma, u32 offset) 1444 { 1445 struct page *page; 1446 unsigned long addr; 1447 1448 GEM_BUG_ON(vma->pages != vma->obj->mm.pages); 1449 1450 page = i915_gem_object_get_page(vma->obj, offset >> PAGE_SHIFT); 1451 addr = PFN_PHYS(page_to_pfn(page)); 1452 GEM_BUG_ON(overflows_type(addr, u32)); /* expected dma32 */ 1453 1454 return addr + offset_in_page(offset); 1455 } 1456 1457 static bool __reloc_entry_gpu(struct i915_execbuffer *eb, 1458 struct i915_vma *vma, 1459 u64 offset, 1460 u64 target_addr) 1461 { 1462 const unsigned int gen = eb->reloc_cache.gen; 1463 unsigned int len; 1464 u32 *batch; 1465 u64 addr; 1466 1467 if (gen >= 8) 1468 len = offset & 7 ? 8 : 5; 1469 else if (gen >= 4) 1470 len = 4; 1471 else 1472 len = 3; 1473 1474 batch = reloc_gpu(eb, vma, len); 1475 if (IS_ERR(batch)) 1476 return false; 1477 1478 addr = gen8_canonical_addr(vma->node.start + offset); 1479 if (gen >= 8) { 1480 if (offset & 7) { 1481 *batch++ = MI_STORE_DWORD_IMM_GEN4; 1482 *batch++ = lower_32_bits(addr); 1483 *batch++ = upper_32_bits(addr); 1484 *batch++ = lower_32_bits(target_addr); 1485 1486 addr = gen8_canonical_addr(addr + 4); 1487 1488 *batch++ = MI_STORE_DWORD_IMM_GEN4; 1489 *batch++ = lower_32_bits(addr); 1490 *batch++ = upper_32_bits(addr); 1491 *batch++ = upper_32_bits(target_addr); 1492 } else { 1493 *batch++ = (MI_STORE_DWORD_IMM_GEN4 | (1 << 21)) + 1; 1494 *batch++ = lower_32_bits(addr); 1495 *batch++ = upper_32_bits(addr); 1496 *batch++ = lower_32_bits(target_addr); 1497 *batch++ = upper_32_bits(target_addr); 1498 } 1499 } else if (gen >= 6) { 1500 *batch++ = MI_STORE_DWORD_IMM_GEN4; 1501 *batch++ = 0; 1502 *batch++ = addr; 1503 *batch++ = target_addr; 1504 } else if (IS_I965G(eb->i915)) { 1505 *batch++ = MI_STORE_DWORD_IMM_GEN4; 1506 *batch++ = 0; 1507 *batch++ = vma_phys_addr(vma, offset); 1508 *batch++ = target_addr; 1509 } else if (gen >= 4) { 1510 *batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 1511 *batch++ = 0; 1512 *batch++ = addr; 1513 *batch++ = target_addr; 1514 } else if (gen >= 3 && 1515 !(IS_I915G(eb->i915) || IS_I915GM(eb->i915))) { 1516 *batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL; 1517 *batch++ = addr; 1518 *batch++ = target_addr; 1519 } else { 1520 *batch++ = MI_STORE_DWORD_IMM; 1521 *batch++ = vma_phys_addr(vma, offset); 1522 *batch++ = target_addr; 1523 } 1524 1525 return true; 1526 } 1527 1528 static bool reloc_entry_gpu(struct i915_execbuffer *eb, 1529 struct i915_vma *vma, 1530 u64 offset, 1531 u64 target_addr) 1532 { 1533 if (eb->reloc_cache.vaddr) 1534 return false; 1535 1536 if (!use_reloc_gpu(vma)) 1537 return false; 1538 1539 return __reloc_entry_gpu(eb, vma, offset, target_addr); 1540 } 1541 1542 static u64 1543 relocate_entry(struct i915_vma *vma, 1544 const struct drm_i915_gem_relocation_entry *reloc, 1545 struct i915_execbuffer *eb, 1546 const struct i915_vma *target) 1547 { 1548 u64 target_addr = relocation_target(reloc, target); 1549 u64 offset = reloc->offset; 1550 1551 if (!reloc_entry_gpu(eb, vma, offset, target_addr)) { 1552 bool wide = eb->reloc_cache.use_64bit_reloc; 1553 void *vaddr; 1554 1555 repeat: 1556 vaddr = reloc_vaddr(vma->obj, 1557 &eb->reloc_cache, 1558 offset >> PAGE_SHIFT); 1559 if (IS_ERR(vaddr)) 1560 return PTR_ERR(vaddr); 1561 1562 GEM_BUG_ON(!IS_ALIGNED(offset, sizeof(u32))); 1563 clflush_write32(vaddr + offset_in_page(offset), 1564 lower_32_bits(target_addr), 1565 eb->reloc_cache.vaddr); 1566 1567 if (wide) { 1568 offset += sizeof(u32); 1569 target_addr >>= 32; 1570 wide = false; 1571 goto repeat; 1572 } 1573 } 1574 1575 return target->node.start | UPDATE; 1576 } 1577 1578 static u64 1579 eb_relocate_entry(struct i915_execbuffer *eb, 1580 struct eb_vma *ev, 1581 const struct drm_i915_gem_relocation_entry *reloc) 1582 { 1583 struct drm_i915_private *i915 = eb->i915; 1584 struct eb_vma *target; 1585 int err; 1586 1587 /* we've already hold a reference to all valid objects */ 1588 target = eb_get_vma(eb, reloc->target_handle); 1589 if (unlikely(!target)) 1590 return -ENOENT; 1591 1592 /* Validate that the target is in a valid r/w GPU domain */ 1593 if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) { 1594 drm_dbg(&i915->drm, "reloc with multiple write domains: " 1595 "target %d offset %d " 1596 "read %08x write %08x", 1597 reloc->target_handle, 1598 (int) reloc->offset, 1599 reloc->read_domains, 1600 reloc->write_domain); 1601 return -EINVAL; 1602 } 1603 if (unlikely((reloc->write_domain | reloc->read_domains) 1604 & ~I915_GEM_GPU_DOMAINS)) { 1605 drm_dbg(&i915->drm, "reloc with read/write non-GPU domains: " 1606 "target %d offset %d " 1607 "read %08x write %08x", 1608 reloc->target_handle, 1609 (int) reloc->offset, 1610 reloc->read_domains, 1611 reloc->write_domain); 1612 return -EINVAL; 1613 } 1614 1615 if (reloc->write_domain) { 1616 target->flags |= EXEC_OBJECT_WRITE; 1617 1618 /* 1619 * Sandybridge PPGTT errata: We need a global gtt mapping 1620 * for MI and pipe_control writes because the gpu doesn't 1621 * properly redirect them through the ppgtt for non_secure 1622 * batchbuffers. 1623 */ 1624 if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION && 1625 IS_GEN(eb->i915, 6)) { 1626 err = i915_vma_bind(target->vma, 1627 target->vma->obj->cache_level, 1628 PIN_GLOBAL, NULL); 1629 if (err) 1630 return err; 1631 } 1632 } 1633 1634 /* 1635 * If the relocation already has the right value in it, no 1636 * more work needs to be done. 1637 */ 1638 if (!DBG_FORCE_RELOC && 1639 gen8_canonical_addr(target->vma->node.start) == reloc->presumed_offset) 1640 return 0; 1641 1642 /* Check that the relocation address is valid... */ 1643 if (unlikely(reloc->offset > 1644 ev->vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) { 1645 drm_dbg(&i915->drm, "Relocation beyond object bounds: " 1646 "target %d offset %d size %d.\n", 1647 reloc->target_handle, 1648 (int)reloc->offset, 1649 (int)ev->vma->size); 1650 return -EINVAL; 1651 } 1652 if (unlikely(reloc->offset & 3)) { 1653 drm_dbg(&i915->drm, "Relocation not 4-byte aligned: " 1654 "target %d offset %d.\n", 1655 reloc->target_handle, 1656 (int)reloc->offset); 1657 return -EINVAL; 1658 } 1659 1660 /* 1661 * If we write into the object, we need to force the synchronisation 1662 * barrier, either with an asynchronous clflush or if we executed the 1663 * patching using the GPU (though that should be serialised by the 1664 * timeline). To be completely sure, and since we are required to 1665 * do relocations we are already stalling, disable the user's opt 1666 * out of our synchronisation. 1667 */ 1668 ev->flags &= ~EXEC_OBJECT_ASYNC; 1669 1670 /* and update the user's relocation entry */ 1671 return relocate_entry(ev->vma, reloc, eb, target->vma); 1672 } 1673 1674 static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev) 1675 { 1676 #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry)) 1677 struct drm_i915_gem_relocation_entry stack[N_RELOC(512)]; 1678 const struct drm_i915_gem_exec_object2 *entry = ev->exec; 1679 struct drm_i915_gem_relocation_entry __user *urelocs = 1680 u64_to_user_ptr(entry->relocs_ptr); 1681 unsigned long remain = entry->relocation_count; 1682 1683 if (unlikely(remain > N_RELOC(ULONG_MAX))) 1684 return -EINVAL; 1685 1686 /* 1687 * We must check that the entire relocation array is safe 1688 * to read. However, if the array is not writable the user loses 1689 * the updated relocation values. 1690 */ 1691 if (unlikely(!access_ok(urelocs, remain * sizeof(*urelocs)))) 1692 return -EFAULT; 1693 1694 do { 1695 struct drm_i915_gem_relocation_entry *r = stack; 1696 unsigned int count = 1697 min_t(unsigned long, remain, ARRAY_SIZE(stack)); 1698 unsigned int copied; 1699 1700 /* 1701 * This is the fast path and we cannot handle a pagefault 1702 * whilst holding the struct mutex lest the user pass in the 1703 * relocations contained within a mmaped bo. For in such a case 1704 * we, the page fault handler would call i915_gem_fault() and 1705 * we would try to acquire the struct mutex again. Obviously 1706 * this is bad and so lockdep complains vehemently. 1707 */ 1708 copied = __copy_from_user(r, urelocs, count * sizeof(r[0])); 1709 if (unlikely(copied)) { 1710 remain = -EFAULT; 1711 goto out; 1712 } 1713 1714 remain -= count; 1715 do { 1716 u64 offset = eb_relocate_entry(eb, ev, r); 1717 1718 if (likely(offset == 0)) { 1719 } else if ((s64)offset < 0) { 1720 remain = (int)offset; 1721 goto out; 1722 } else { 1723 /* 1724 * Note that reporting an error now 1725 * leaves everything in an inconsistent 1726 * state as we have *already* changed 1727 * the relocation value inside the 1728 * object. As we have not changed the 1729 * reloc.presumed_offset or will not 1730 * change the execobject.offset, on the 1731 * call we may not rewrite the value 1732 * inside the object, leaving it 1733 * dangling and causing a GPU hang. Unless 1734 * userspace dynamically rebuilds the 1735 * relocations on each execbuf rather than 1736 * presume a static tree. 1737 * 1738 * We did previously check if the relocations 1739 * were writable (access_ok), an error now 1740 * would be a strange race with mprotect, 1741 * having already demonstrated that we 1742 * can read from this userspace address. 1743 */ 1744 offset = gen8_canonical_addr(offset & ~UPDATE); 1745 __put_user(offset, 1746 &urelocs[r - stack].presumed_offset); 1747 } 1748 } while (r++, --count); 1749 urelocs += ARRAY_SIZE(stack); 1750 } while (remain); 1751 out: 1752 reloc_cache_reset(&eb->reloc_cache); 1753 return remain; 1754 } 1755 1756 static int eb_relocate(struct i915_execbuffer *eb) 1757 { 1758 int err; 1759 1760 err = eb_lookup_vmas(eb); 1761 if (err) 1762 return err; 1763 1764 if (!list_empty(&eb->unbound)) { 1765 err = eb_reserve(eb); 1766 if (err) 1767 return err; 1768 } 1769 1770 /* The objects are in their final locations, apply the relocations. */ 1771 if (eb->args->flags & __EXEC_HAS_RELOC) { 1772 struct eb_vma *ev; 1773 int flush; 1774 1775 list_for_each_entry(ev, &eb->relocs, reloc_link) { 1776 err = eb_relocate_vma(eb, ev); 1777 if (err) 1778 break; 1779 } 1780 1781 flush = reloc_gpu_flush(&eb->reloc_cache); 1782 if (!err) 1783 err = flush; 1784 } 1785 1786 return err; 1787 } 1788 1789 static int eb_move_to_gpu(struct i915_execbuffer *eb) 1790 { 1791 const unsigned int count = eb->buffer_count; 1792 struct ww_acquire_ctx acquire; 1793 unsigned int i; 1794 int err = 0; 1795 1796 ww_acquire_init(&acquire, &reservation_ww_class); 1797 1798 for (i = 0; i < count; i++) { 1799 struct eb_vma *ev = &eb->vma[i]; 1800 struct i915_vma *vma = ev->vma; 1801 1802 err = ww_mutex_lock_interruptible(&vma->resv->lock, &acquire); 1803 if (err == -EDEADLK) { 1804 GEM_BUG_ON(i == 0); 1805 do { 1806 int j = i - 1; 1807 1808 ww_mutex_unlock(&eb->vma[j].vma->resv->lock); 1809 1810 swap(eb->vma[i], eb->vma[j]); 1811 } while (--i); 1812 1813 err = ww_mutex_lock_slow_interruptible(&vma->resv->lock, 1814 &acquire); 1815 } 1816 if (err) 1817 break; 1818 } 1819 ww_acquire_done(&acquire); 1820 1821 while (i--) { 1822 struct eb_vma *ev = &eb->vma[i]; 1823 struct i915_vma *vma = ev->vma; 1824 unsigned int flags = ev->flags; 1825 struct drm_i915_gem_object *obj = vma->obj; 1826 1827 assert_vma_held(vma); 1828 1829 if (flags & EXEC_OBJECT_CAPTURE) { 1830 struct i915_capture_list *capture; 1831 1832 capture = kmalloc(sizeof(*capture), GFP_KERNEL); 1833 if (capture) { 1834 capture->next = eb->request->capture_list; 1835 capture->vma = vma; 1836 eb->request->capture_list = capture; 1837 } 1838 } 1839 1840 /* 1841 * If the GPU is not _reading_ through the CPU cache, we need 1842 * to make sure that any writes (both previous GPU writes from 1843 * before a change in snooping levels and normal CPU writes) 1844 * caught in that cache are flushed to main memory. 1845 * 1846 * We want to say 1847 * obj->cache_dirty && 1848 * !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ) 1849 * but gcc's optimiser doesn't handle that as well and emits 1850 * two jumps instead of one. Maybe one day... 1851 */ 1852 if (unlikely(obj->cache_dirty & ~obj->cache_coherent)) { 1853 if (i915_gem_clflush_object(obj, 0)) 1854 flags &= ~EXEC_OBJECT_ASYNC; 1855 } 1856 1857 if (err == 0 && !(flags & EXEC_OBJECT_ASYNC)) { 1858 err = i915_request_await_object 1859 (eb->request, obj, flags & EXEC_OBJECT_WRITE); 1860 } 1861 1862 if (err == 0) 1863 err = i915_vma_move_to_active(vma, eb->request, flags); 1864 1865 i915_vma_unlock(vma); 1866 eb_unreserve_vma(ev); 1867 } 1868 ww_acquire_fini(&acquire); 1869 1870 eb_vma_array_put(fetch_and_zero(&eb->array)); 1871 1872 if (unlikely(err)) 1873 goto err_skip; 1874 1875 /* Unconditionally flush any chipset caches (for streaming writes). */ 1876 intel_gt_chipset_flush(eb->engine->gt); 1877 return 0; 1878 1879 err_skip: 1880 i915_request_set_error_once(eb->request, err); 1881 return err; 1882 } 1883 1884 static int i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec) 1885 { 1886 if (exec->flags & __I915_EXEC_ILLEGAL_FLAGS) 1887 return -EINVAL; 1888 1889 /* Kernel clipping was a DRI1 misfeature */ 1890 if (!(exec->flags & I915_EXEC_FENCE_ARRAY)) { 1891 if (exec->num_cliprects || exec->cliprects_ptr) 1892 return -EINVAL; 1893 } 1894 1895 if (exec->DR4 == 0xffffffff) { 1896 DRM_DEBUG("UXA submitting garbage DR4, fixing up\n"); 1897 exec->DR4 = 0; 1898 } 1899 if (exec->DR1 || exec->DR4) 1900 return -EINVAL; 1901 1902 if ((exec->batch_start_offset | exec->batch_len) & 0x7) 1903 return -EINVAL; 1904 1905 return 0; 1906 } 1907 1908 static int i915_reset_gen7_sol_offsets(struct i915_request *rq) 1909 { 1910 u32 *cs; 1911 int i; 1912 1913 if (!IS_GEN(rq->engine->i915, 7) || rq->engine->id != RCS0) { 1914 drm_dbg(&rq->engine->i915->drm, "sol reset is gen7/rcs only\n"); 1915 return -EINVAL; 1916 } 1917 1918 cs = intel_ring_begin(rq, 4 * 2 + 2); 1919 if (IS_ERR(cs)) 1920 return PTR_ERR(cs); 1921 1922 *cs++ = MI_LOAD_REGISTER_IMM(4); 1923 for (i = 0; i < 4; i++) { 1924 *cs++ = i915_mmio_reg_offset(GEN7_SO_WRITE_OFFSET(i)); 1925 *cs++ = 0; 1926 } 1927 *cs++ = MI_NOOP; 1928 intel_ring_advance(rq, cs); 1929 1930 return 0; 1931 } 1932 1933 static struct i915_vma * 1934 shadow_batch_pin(struct drm_i915_gem_object *obj, 1935 struct i915_address_space *vm, 1936 unsigned int flags) 1937 { 1938 struct i915_vma *vma; 1939 int err; 1940 1941 vma = i915_vma_instance(obj, vm, NULL); 1942 if (IS_ERR(vma)) 1943 return vma; 1944 1945 err = i915_vma_pin(vma, 0, 0, flags); 1946 if (err) 1947 return ERR_PTR(err); 1948 1949 return vma; 1950 } 1951 1952 struct eb_parse_work { 1953 struct dma_fence_work base; 1954 struct intel_engine_cs *engine; 1955 struct i915_vma *batch; 1956 struct i915_vma *shadow; 1957 struct i915_vma *trampoline; 1958 unsigned int batch_offset; 1959 unsigned int batch_length; 1960 }; 1961 1962 static int __eb_parse(struct dma_fence_work *work) 1963 { 1964 struct eb_parse_work *pw = container_of(work, typeof(*pw), base); 1965 1966 return intel_engine_cmd_parser(pw->engine, 1967 pw->batch, 1968 pw->batch_offset, 1969 pw->batch_length, 1970 pw->shadow, 1971 pw->trampoline); 1972 } 1973 1974 static void __eb_parse_release(struct dma_fence_work *work) 1975 { 1976 struct eb_parse_work *pw = container_of(work, typeof(*pw), base); 1977 1978 if (pw->trampoline) 1979 i915_active_release(&pw->trampoline->active); 1980 i915_active_release(&pw->shadow->active); 1981 i915_active_release(&pw->batch->active); 1982 } 1983 1984 static const struct dma_fence_work_ops eb_parse_ops = { 1985 .name = "eb_parse", 1986 .work = __eb_parse, 1987 .release = __eb_parse_release, 1988 }; 1989 1990 static int eb_parse_pipeline(struct i915_execbuffer *eb, 1991 struct i915_vma *shadow, 1992 struct i915_vma *trampoline) 1993 { 1994 struct eb_parse_work *pw; 1995 int err; 1996 1997 pw = kzalloc(sizeof(*pw), GFP_KERNEL); 1998 if (!pw) 1999 return -ENOMEM; 2000 2001 err = i915_active_acquire(&eb->batch->vma->active); 2002 if (err) 2003 goto err_free; 2004 2005 err = i915_active_acquire(&shadow->active); 2006 if (err) 2007 goto err_batch; 2008 2009 if (trampoline) { 2010 err = i915_active_acquire(&trampoline->active); 2011 if (err) 2012 goto err_shadow; 2013 } 2014 2015 dma_fence_work_init(&pw->base, &eb_parse_ops); 2016 2017 pw->engine = eb->engine; 2018 pw->batch = eb->batch->vma; 2019 pw->batch_offset = eb->batch_start_offset; 2020 pw->batch_length = eb->batch_len; 2021 pw->shadow = shadow; 2022 pw->trampoline = trampoline; 2023 2024 err = dma_resv_lock_interruptible(pw->batch->resv, NULL); 2025 if (err) 2026 goto err_trampoline; 2027 2028 err = dma_resv_reserve_shared(pw->batch->resv, 1); 2029 if (err) 2030 goto err_batch_unlock; 2031 2032 /* Wait for all writes (and relocs) into the batch to complete */ 2033 err = i915_sw_fence_await_reservation(&pw->base.chain, 2034 pw->batch->resv, NULL, false, 2035 0, I915_FENCE_GFP); 2036 if (err < 0) 2037 goto err_batch_unlock; 2038 2039 /* Keep the batch alive and unwritten as we parse */ 2040 dma_resv_add_shared_fence(pw->batch->resv, &pw->base.dma); 2041 2042 dma_resv_unlock(pw->batch->resv); 2043 2044 /* Force execution to wait for completion of the parser */ 2045 dma_resv_lock(shadow->resv, NULL); 2046 dma_resv_add_excl_fence(shadow->resv, &pw->base.dma); 2047 dma_resv_unlock(shadow->resv); 2048 2049 dma_fence_work_commit_imm(&pw->base); 2050 return 0; 2051 2052 err_batch_unlock: 2053 dma_resv_unlock(pw->batch->resv); 2054 err_trampoline: 2055 if (trampoline) 2056 i915_active_release(&trampoline->active); 2057 err_shadow: 2058 i915_active_release(&shadow->active); 2059 err_batch: 2060 i915_active_release(&eb->batch->vma->active); 2061 err_free: 2062 kfree(pw); 2063 return err; 2064 } 2065 2066 static int eb_parse(struct i915_execbuffer *eb) 2067 { 2068 struct drm_i915_private *i915 = eb->i915; 2069 struct intel_gt_buffer_pool_node *pool; 2070 struct i915_vma *shadow, *trampoline; 2071 unsigned int len; 2072 int err; 2073 2074 if (!eb_use_cmdparser(eb)) 2075 return 0; 2076 2077 len = eb->batch_len; 2078 if (!CMDPARSER_USES_GGTT(eb->i915)) { 2079 /* 2080 * ppGTT backed shadow buffers must be mapped RO, to prevent 2081 * post-scan tampering 2082 */ 2083 if (!eb->context->vm->has_read_only) { 2084 drm_dbg(&i915->drm, 2085 "Cannot prevent post-scan tampering without RO capable vm\n"); 2086 return -EINVAL; 2087 } 2088 } else { 2089 len += I915_CMD_PARSER_TRAMPOLINE_SIZE; 2090 } 2091 2092 pool = intel_gt_get_buffer_pool(eb->engine->gt, len); 2093 if (IS_ERR(pool)) 2094 return PTR_ERR(pool); 2095 2096 shadow = shadow_batch_pin(pool->obj, eb->context->vm, PIN_USER); 2097 if (IS_ERR(shadow)) { 2098 err = PTR_ERR(shadow); 2099 goto err; 2100 } 2101 i915_gem_object_set_readonly(shadow->obj); 2102 2103 trampoline = NULL; 2104 if (CMDPARSER_USES_GGTT(eb->i915)) { 2105 trampoline = shadow; 2106 2107 shadow = shadow_batch_pin(pool->obj, 2108 &eb->engine->gt->ggtt->vm, 2109 PIN_GLOBAL); 2110 if (IS_ERR(shadow)) { 2111 err = PTR_ERR(shadow); 2112 shadow = trampoline; 2113 goto err_shadow; 2114 } 2115 2116 eb->batch_flags |= I915_DISPATCH_SECURE; 2117 } 2118 2119 err = eb_parse_pipeline(eb, shadow, trampoline); 2120 if (err) 2121 goto err_trampoline; 2122 2123 eb->vma[eb->buffer_count].vma = i915_vma_get(shadow); 2124 eb->vma[eb->buffer_count].flags = __EXEC_OBJECT_HAS_PIN; 2125 eb->batch = &eb->vma[eb->buffer_count++]; 2126 eb->vma[eb->buffer_count].vma = NULL; 2127 2128 eb->trampoline = trampoline; 2129 eb->batch_start_offset = 0; 2130 2131 shadow->private = pool; 2132 return 0; 2133 2134 err_trampoline: 2135 if (trampoline) 2136 i915_vma_unpin(trampoline); 2137 err_shadow: 2138 i915_vma_unpin(shadow); 2139 err: 2140 intel_gt_buffer_pool_put(pool); 2141 return err; 2142 } 2143 2144 static void 2145 add_to_client(struct i915_request *rq, struct drm_file *file) 2146 { 2147 struct drm_i915_file_private *file_priv = file->driver_priv; 2148 2149 rq->file_priv = file_priv; 2150 2151 spin_lock(&file_priv->mm.lock); 2152 list_add_tail(&rq->client_link, &file_priv->mm.request_list); 2153 spin_unlock(&file_priv->mm.lock); 2154 } 2155 2156 static int eb_submit(struct i915_execbuffer *eb, struct i915_vma *batch) 2157 { 2158 int err; 2159 2160 err = eb_move_to_gpu(eb); 2161 if (err) 2162 return err; 2163 2164 if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) { 2165 err = i915_reset_gen7_sol_offsets(eb->request); 2166 if (err) 2167 return err; 2168 } 2169 2170 /* 2171 * After we completed waiting for other engines (using HW semaphores) 2172 * then we can signal that this request/batch is ready to run. This 2173 * allows us to determine if the batch is still waiting on the GPU 2174 * or actually running by checking the breadcrumb. 2175 */ 2176 if (eb->engine->emit_init_breadcrumb) { 2177 err = eb->engine->emit_init_breadcrumb(eb->request); 2178 if (err) 2179 return err; 2180 } 2181 2182 err = eb->engine->emit_bb_start(eb->request, 2183 batch->node.start + 2184 eb->batch_start_offset, 2185 eb->batch_len, 2186 eb->batch_flags); 2187 if (err) 2188 return err; 2189 2190 if (eb->trampoline) { 2191 GEM_BUG_ON(eb->batch_start_offset); 2192 err = eb->engine->emit_bb_start(eb->request, 2193 eb->trampoline->node.start + 2194 eb->batch_len, 2195 0, 0); 2196 if (err) 2197 return err; 2198 } 2199 2200 if (intel_context_nopreempt(eb->context)) 2201 __set_bit(I915_FENCE_FLAG_NOPREEMPT, &eb->request->fence.flags); 2202 2203 return 0; 2204 } 2205 2206 static int num_vcs_engines(const struct drm_i915_private *i915) 2207 { 2208 return hweight64(INTEL_INFO(i915)->engine_mask & 2209 GENMASK_ULL(VCS0 + I915_MAX_VCS - 1, VCS0)); 2210 } 2211 2212 /* 2213 * Find one BSD ring to dispatch the corresponding BSD command. 2214 * The engine index is returned. 2215 */ 2216 static unsigned int 2217 gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv, 2218 struct drm_file *file) 2219 { 2220 struct drm_i915_file_private *file_priv = file->driver_priv; 2221 2222 /* Check whether the file_priv has already selected one ring. */ 2223 if ((int)file_priv->bsd_engine < 0) 2224 file_priv->bsd_engine = 2225 get_random_int() % num_vcs_engines(dev_priv); 2226 2227 return file_priv->bsd_engine; 2228 } 2229 2230 static const enum intel_engine_id user_ring_map[] = { 2231 [I915_EXEC_DEFAULT] = RCS0, 2232 [I915_EXEC_RENDER] = RCS0, 2233 [I915_EXEC_BLT] = BCS0, 2234 [I915_EXEC_BSD] = VCS0, 2235 [I915_EXEC_VEBOX] = VECS0 2236 }; 2237 2238 static struct i915_request *eb_throttle(struct intel_context *ce) 2239 { 2240 struct intel_ring *ring = ce->ring; 2241 struct intel_timeline *tl = ce->timeline; 2242 struct i915_request *rq; 2243 2244 /* 2245 * Completely unscientific finger-in-the-air estimates for suitable 2246 * maximum user request size (to avoid blocking) and then backoff. 2247 */ 2248 if (intel_ring_update_space(ring) >= PAGE_SIZE) 2249 return NULL; 2250 2251 /* 2252 * Find a request that after waiting upon, there will be at least half 2253 * the ring available. The hysteresis allows us to compete for the 2254 * shared ring and should mean that we sleep less often prior to 2255 * claiming our resources, but not so long that the ring completely 2256 * drains before we can submit our next request. 2257 */ 2258 list_for_each_entry(rq, &tl->requests, link) { 2259 if (rq->ring != ring) 2260 continue; 2261 2262 if (__intel_ring_space(rq->postfix, 2263 ring->emit, ring->size) > ring->size / 2) 2264 break; 2265 } 2266 if (&rq->link == &tl->requests) 2267 return NULL; /* weird, we will check again later for real */ 2268 2269 return i915_request_get(rq); 2270 } 2271 2272 static int __eb_pin_engine(struct i915_execbuffer *eb, struct intel_context *ce) 2273 { 2274 struct intel_timeline *tl; 2275 struct i915_request *rq; 2276 int err; 2277 2278 /* 2279 * ABI: Before userspace accesses the GPU (e.g. execbuffer), report 2280 * EIO if the GPU is already wedged. 2281 */ 2282 err = intel_gt_terminally_wedged(ce->engine->gt); 2283 if (err) 2284 return err; 2285 2286 if (unlikely(intel_context_is_banned(ce))) 2287 return -EIO; 2288 2289 /* 2290 * Pinning the contexts may generate requests in order to acquire 2291 * GGTT space, so do this first before we reserve a seqno for 2292 * ourselves. 2293 */ 2294 err = intel_context_pin(ce); 2295 if (err) 2296 return err; 2297 2298 /* 2299 * Take a local wakeref for preparing to dispatch the execbuf as 2300 * we expect to access the hardware fairly frequently in the 2301 * process, and require the engine to be kept awake between accesses. 2302 * Upon dispatch, we acquire another prolonged wakeref that we hold 2303 * until the timeline is idle, which in turn releases the wakeref 2304 * taken on the engine, and the parent device. 2305 */ 2306 tl = intel_context_timeline_lock(ce); 2307 if (IS_ERR(tl)) { 2308 err = PTR_ERR(tl); 2309 goto err_unpin; 2310 } 2311 2312 intel_context_enter(ce); 2313 rq = eb_throttle(ce); 2314 2315 intel_context_timeline_unlock(tl); 2316 2317 if (rq) { 2318 bool nonblock = eb->file->filp->f_flags & O_NONBLOCK; 2319 long timeout; 2320 2321 timeout = MAX_SCHEDULE_TIMEOUT; 2322 if (nonblock) 2323 timeout = 0; 2324 2325 timeout = i915_request_wait(rq, 2326 I915_WAIT_INTERRUPTIBLE, 2327 timeout); 2328 i915_request_put(rq); 2329 2330 if (timeout < 0) { 2331 err = nonblock ? -EWOULDBLOCK : timeout; 2332 goto err_exit; 2333 } 2334 } 2335 2336 eb->engine = ce->engine; 2337 eb->context = ce; 2338 return 0; 2339 2340 err_exit: 2341 mutex_lock(&tl->mutex); 2342 intel_context_exit(ce); 2343 intel_context_timeline_unlock(tl); 2344 err_unpin: 2345 intel_context_unpin(ce); 2346 return err; 2347 } 2348 2349 static void eb_unpin_engine(struct i915_execbuffer *eb) 2350 { 2351 struct intel_context *ce = eb->context; 2352 struct intel_timeline *tl = ce->timeline; 2353 2354 mutex_lock(&tl->mutex); 2355 intel_context_exit(ce); 2356 mutex_unlock(&tl->mutex); 2357 2358 intel_context_unpin(ce); 2359 } 2360 2361 static unsigned int 2362 eb_select_legacy_ring(struct i915_execbuffer *eb, 2363 struct drm_file *file, 2364 struct drm_i915_gem_execbuffer2 *args) 2365 { 2366 struct drm_i915_private *i915 = eb->i915; 2367 unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK; 2368 2369 if (user_ring_id != I915_EXEC_BSD && 2370 (args->flags & I915_EXEC_BSD_MASK)) { 2371 drm_dbg(&i915->drm, 2372 "execbuf with non bsd ring but with invalid " 2373 "bsd dispatch flags: %d\n", (int)(args->flags)); 2374 return -1; 2375 } 2376 2377 if (user_ring_id == I915_EXEC_BSD && num_vcs_engines(i915) > 1) { 2378 unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK; 2379 2380 if (bsd_idx == I915_EXEC_BSD_DEFAULT) { 2381 bsd_idx = gen8_dispatch_bsd_engine(i915, file); 2382 } else if (bsd_idx >= I915_EXEC_BSD_RING1 && 2383 bsd_idx <= I915_EXEC_BSD_RING2) { 2384 bsd_idx >>= I915_EXEC_BSD_SHIFT; 2385 bsd_idx--; 2386 } else { 2387 drm_dbg(&i915->drm, 2388 "execbuf with unknown bsd ring: %u\n", 2389 bsd_idx); 2390 return -1; 2391 } 2392 2393 return _VCS(bsd_idx); 2394 } 2395 2396 if (user_ring_id >= ARRAY_SIZE(user_ring_map)) { 2397 drm_dbg(&i915->drm, "execbuf with unknown ring: %u\n", 2398 user_ring_id); 2399 return -1; 2400 } 2401 2402 return user_ring_map[user_ring_id]; 2403 } 2404 2405 static int 2406 eb_pin_engine(struct i915_execbuffer *eb, 2407 struct drm_file *file, 2408 struct drm_i915_gem_execbuffer2 *args) 2409 { 2410 struct intel_context *ce; 2411 unsigned int idx; 2412 int err; 2413 2414 if (i915_gem_context_user_engines(eb->gem_context)) 2415 idx = args->flags & I915_EXEC_RING_MASK; 2416 else 2417 idx = eb_select_legacy_ring(eb, file, args); 2418 2419 ce = i915_gem_context_get_engine(eb->gem_context, idx); 2420 if (IS_ERR(ce)) 2421 return PTR_ERR(ce); 2422 2423 err = __eb_pin_engine(eb, ce); 2424 intel_context_put(ce); 2425 2426 return err; 2427 } 2428 2429 static void 2430 __free_fence_array(struct drm_syncobj **fences, unsigned int n) 2431 { 2432 while (n--) 2433 drm_syncobj_put(ptr_mask_bits(fences[n], 2)); 2434 kvfree(fences); 2435 } 2436 2437 static struct drm_syncobj ** 2438 get_fence_array(struct drm_i915_gem_execbuffer2 *args, 2439 struct drm_file *file) 2440 { 2441 const unsigned long nfences = args->num_cliprects; 2442 struct drm_i915_gem_exec_fence __user *user; 2443 struct drm_syncobj **fences; 2444 unsigned long n; 2445 int err; 2446 2447 if (!(args->flags & I915_EXEC_FENCE_ARRAY)) 2448 return NULL; 2449 2450 /* Check multiplication overflow for access_ok() and kvmalloc_array() */ 2451 BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long)); 2452 if (nfences > min_t(unsigned long, 2453 ULONG_MAX / sizeof(*user), 2454 SIZE_MAX / sizeof(*fences))) 2455 return ERR_PTR(-EINVAL); 2456 2457 user = u64_to_user_ptr(args->cliprects_ptr); 2458 if (!access_ok(user, nfences * sizeof(*user))) 2459 return ERR_PTR(-EFAULT); 2460 2461 fences = kvmalloc_array(nfences, sizeof(*fences), 2462 __GFP_NOWARN | GFP_KERNEL); 2463 if (!fences) 2464 return ERR_PTR(-ENOMEM); 2465 2466 for (n = 0; n < nfences; n++) { 2467 struct drm_i915_gem_exec_fence fence; 2468 struct drm_syncobj *syncobj; 2469 2470 if (__copy_from_user(&fence, user++, sizeof(fence))) { 2471 err = -EFAULT; 2472 goto err; 2473 } 2474 2475 if (fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS) { 2476 err = -EINVAL; 2477 goto err; 2478 } 2479 2480 syncobj = drm_syncobj_find(file, fence.handle); 2481 if (!syncobj) { 2482 DRM_DEBUG("Invalid syncobj handle provided\n"); 2483 err = -ENOENT; 2484 goto err; 2485 } 2486 2487 BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) & 2488 ~__I915_EXEC_FENCE_UNKNOWN_FLAGS); 2489 2490 fences[n] = ptr_pack_bits(syncobj, fence.flags, 2); 2491 } 2492 2493 return fences; 2494 2495 err: 2496 __free_fence_array(fences, n); 2497 return ERR_PTR(err); 2498 } 2499 2500 static void 2501 put_fence_array(struct drm_i915_gem_execbuffer2 *args, 2502 struct drm_syncobj **fences) 2503 { 2504 if (fences) 2505 __free_fence_array(fences, args->num_cliprects); 2506 } 2507 2508 static int 2509 await_fence_array(struct i915_execbuffer *eb, 2510 struct drm_syncobj **fences) 2511 { 2512 const unsigned int nfences = eb->args->num_cliprects; 2513 unsigned int n; 2514 int err; 2515 2516 for (n = 0; n < nfences; n++) { 2517 struct drm_syncobj *syncobj; 2518 struct dma_fence *fence; 2519 unsigned int flags; 2520 2521 syncobj = ptr_unpack_bits(fences[n], &flags, 2); 2522 if (!(flags & I915_EXEC_FENCE_WAIT)) 2523 continue; 2524 2525 fence = drm_syncobj_fence_get(syncobj); 2526 if (!fence) 2527 return -EINVAL; 2528 2529 err = i915_request_await_dma_fence(eb->request, fence); 2530 dma_fence_put(fence); 2531 if (err < 0) 2532 return err; 2533 } 2534 2535 return 0; 2536 } 2537 2538 static void 2539 signal_fence_array(struct i915_execbuffer *eb, 2540 struct drm_syncobj **fences) 2541 { 2542 const unsigned int nfences = eb->args->num_cliprects; 2543 struct dma_fence * const fence = &eb->request->fence; 2544 unsigned int n; 2545 2546 for (n = 0; n < nfences; n++) { 2547 struct drm_syncobj *syncobj; 2548 unsigned int flags; 2549 2550 syncobj = ptr_unpack_bits(fences[n], &flags, 2); 2551 if (!(flags & I915_EXEC_FENCE_SIGNAL)) 2552 continue; 2553 2554 drm_syncobj_replace_fence(syncobj, fence); 2555 } 2556 } 2557 2558 static void retire_requests(struct intel_timeline *tl, struct i915_request *end) 2559 { 2560 struct i915_request *rq, *rn; 2561 2562 list_for_each_entry_safe(rq, rn, &tl->requests, link) 2563 if (rq == end || !i915_request_retire(rq)) 2564 break; 2565 } 2566 2567 static void eb_request_add(struct i915_execbuffer *eb) 2568 { 2569 struct i915_request *rq = eb->request; 2570 struct intel_timeline * const tl = i915_request_timeline(rq); 2571 struct i915_sched_attr attr = {}; 2572 struct i915_request *prev; 2573 2574 lockdep_assert_held(&tl->mutex); 2575 lockdep_unpin_lock(&tl->mutex, rq->cookie); 2576 2577 trace_i915_request_add(rq); 2578 2579 prev = __i915_request_commit(rq); 2580 2581 /* Check that the context wasn't destroyed before submission */ 2582 if (likely(!intel_context_is_closed(eb->context))) { 2583 attr = eb->gem_context->sched; 2584 } else { 2585 /* Serialise with context_close via the add_to_timeline */ 2586 i915_request_set_error_once(rq, -ENOENT); 2587 __i915_request_skip(rq); 2588 } 2589 2590 __i915_request_queue(rq, &attr); 2591 2592 /* Try to clean up the client's timeline after submitting the request */ 2593 if (prev) 2594 retire_requests(tl, prev); 2595 2596 mutex_unlock(&tl->mutex); 2597 } 2598 2599 static int 2600 i915_gem_do_execbuffer(struct drm_device *dev, 2601 struct drm_file *file, 2602 struct drm_i915_gem_execbuffer2 *args, 2603 struct drm_i915_gem_exec_object2 *exec, 2604 struct drm_syncobj **fences) 2605 { 2606 struct drm_i915_private *i915 = to_i915(dev); 2607 struct i915_execbuffer eb; 2608 struct dma_fence *in_fence = NULL; 2609 struct sync_file *out_fence = NULL; 2610 struct i915_vma *batch; 2611 int out_fence_fd = -1; 2612 int err; 2613 2614 BUILD_BUG_ON(__EXEC_INTERNAL_FLAGS & ~__I915_EXEC_ILLEGAL_FLAGS); 2615 BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS & 2616 ~__EXEC_OBJECT_UNKNOWN_FLAGS); 2617 2618 eb.i915 = i915; 2619 eb.file = file; 2620 eb.args = args; 2621 if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC)) 2622 args->flags |= __EXEC_HAS_RELOC; 2623 2624 eb.exec = exec; 2625 2626 eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS; 2627 reloc_cache_init(&eb.reloc_cache, eb.i915); 2628 2629 eb.buffer_count = args->buffer_count; 2630 eb.batch_start_offset = args->batch_start_offset; 2631 eb.batch_len = args->batch_len; 2632 eb.trampoline = NULL; 2633 2634 eb.batch_flags = 0; 2635 if (args->flags & I915_EXEC_SECURE) { 2636 if (INTEL_GEN(i915) >= 11) 2637 return -ENODEV; 2638 2639 /* Return -EPERM to trigger fallback code on old binaries. */ 2640 if (!HAS_SECURE_BATCHES(i915)) 2641 return -EPERM; 2642 2643 if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN)) 2644 return -EPERM; 2645 2646 eb.batch_flags |= I915_DISPATCH_SECURE; 2647 } 2648 if (args->flags & I915_EXEC_IS_PINNED) 2649 eb.batch_flags |= I915_DISPATCH_PINNED; 2650 2651 #define IN_FENCES (I915_EXEC_FENCE_IN | I915_EXEC_FENCE_SUBMIT) 2652 if (args->flags & IN_FENCES) { 2653 if ((args->flags & IN_FENCES) == IN_FENCES) 2654 return -EINVAL; 2655 2656 in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2)); 2657 if (!in_fence) 2658 return -EINVAL; 2659 } 2660 #undef IN_FENCES 2661 2662 if (args->flags & I915_EXEC_FENCE_OUT) { 2663 out_fence_fd = get_unused_fd_flags(O_CLOEXEC); 2664 if (out_fence_fd < 0) { 2665 err = out_fence_fd; 2666 goto err_in_fence; 2667 } 2668 } 2669 2670 err = eb_create(&eb); 2671 if (err) 2672 goto err_out_fence; 2673 2674 GEM_BUG_ON(!eb.lut_size); 2675 2676 err = eb_select_context(&eb); 2677 if (unlikely(err)) 2678 goto err_destroy; 2679 2680 err = eb_pin_engine(&eb, file, args); 2681 if (unlikely(err)) 2682 goto err_context; 2683 2684 err = eb_relocate(&eb); 2685 if (err) { 2686 /* 2687 * If the user expects the execobject.offset and 2688 * reloc.presumed_offset to be an exact match, 2689 * as for using NO_RELOC, then we cannot update 2690 * the execobject.offset until we have completed 2691 * relocation. 2692 */ 2693 args->flags &= ~__EXEC_HAS_RELOC; 2694 goto err_vma; 2695 } 2696 2697 if (unlikely(eb.batch->flags & EXEC_OBJECT_WRITE)) { 2698 drm_dbg(&i915->drm, 2699 "Attempting to use self-modifying batch buffer\n"); 2700 err = -EINVAL; 2701 goto err_vma; 2702 } 2703 2704 if (range_overflows_t(u64, 2705 eb.batch_start_offset, eb.batch_len, 2706 eb.batch->vma->size)) { 2707 drm_dbg(&i915->drm, "Attempting to use out-of-bounds batch\n"); 2708 err = -EINVAL; 2709 goto err_vma; 2710 } 2711 2712 if (eb.batch_len == 0) 2713 eb.batch_len = eb.batch->vma->size - eb.batch_start_offset; 2714 2715 err = eb_parse(&eb); 2716 if (err) 2717 goto err_vma; 2718 2719 /* 2720 * snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure 2721 * batch" bit. Hence we need to pin secure batches into the global gtt. 2722 * hsw should have this fixed, but bdw mucks it up again. */ 2723 batch = eb.batch->vma; 2724 if (eb.batch_flags & I915_DISPATCH_SECURE) { 2725 struct i915_vma *vma; 2726 2727 /* 2728 * So on first glance it looks freaky that we pin the batch here 2729 * outside of the reservation loop. But: 2730 * - The batch is already pinned into the relevant ppgtt, so we 2731 * already have the backing storage fully allocated. 2732 * - No other BO uses the global gtt (well contexts, but meh), 2733 * so we don't really have issues with multiple objects not 2734 * fitting due to fragmentation. 2735 * So this is actually safe. 2736 */ 2737 vma = i915_gem_object_ggtt_pin(batch->obj, NULL, 0, 0, 0); 2738 if (IS_ERR(vma)) { 2739 err = PTR_ERR(vma); 2740 goto err_parse; 2741 } 2742 2743 batch = vma; 2744 } 2745 2746 /* All GPU relocation batches must be submitted prior to the user rq */ 2747 GEM_BUG_ON(eb.reloc_cache.rq); 2748 2749 /* Allocate a request for this batch buffer nice and early. */ 2750 eb.request = i915_request_create(eb.context); 2751 if (IS_ERR(eb.request)) { 2752 err = PTR_ERR(eb.request); 2753 goto err_batch_unpin; 2754 } 2755 2756 if (in_fence) { 2757 if (args->flags & I915_EXEC_FENCE_SUBMIT) 2758 err = i915_request_await_execution(eb.request, 2759 in_fence, 2760 eb.engine->bond_execute); 2761 else 2762 err = i915_request_await_dma_fence(eb.request, 2763 in_fence); 2764 if (err < 0) 2765 goto err_request; 2766 } 2767 2768 if (fences) { 2769 err = await_fence_array(&eb, fences); 2770 if (err) 2771 goto err_request; 2772 } 2773 2774 if (out_fence_fd != -1) { 2775 out_fence = sync_file_create(&eb.request->fence); 2776 if (!out_fence) { 2777 err = -ENOMEM; 2778 goto err_request; 2779 } 2780 } 2781 2782 /* 2783 * Whilst this request exists, batch_obj will be on the 2784 * active_list, and so will hold the active reference. Only when this 2785 * request is retired will the the batch_obj be moved onto the 2786 * inactive_list and lose its active reference. Hence we do not need 2787 * to explicitly hold another reference here. 2788 */ 2789 eb.request->batch = batch; 2790 if (batch->private) 2791 intel_gt_buffer_pool_mark_active(batch->private, eb.request); 2792 2793 trace_i915_request_queue(eb.request, eb.batch_flags); 2794 err = eb_submit(&eb, batch); 2795 err_request: 2796 add_to_client(eb.request, file); 2797 i915_request_get(eb.request); 2798 eb_request_add(&eb); 2799 2800 if (fences) 2801 signal_fence_array(&eb, fences); 2802 2803 if (out_fence) { 2804 if (err == 0) { 2805 fd_install(out_fence_fd, out_fence->file); 2806 args->rsvd2 &= GENMASK_ULL(31, 0); /* keep in-fence */ 2807 args->rsvd2 |= (u64)out_fence_fd << 32; 2808 out_fence_fd = -1; 2809 } else { 2810 fput(out_fence->file); 2811 } 2812 } 2813 i915_request_put(eb.request); 2814 2815 err_batch_unpin: 2816 if (eb.batch_flags & I915_DISPATCH_SECURE) 2817 i915_vma_unpin(batch); 2818 err_parse: 2819 if (batch->private) 2820 intel_gt_buffer_pool_put(batch->private); 2821 err_vma: 2822 if (eb.trampoline) 2823 i915_vma_unpin(eb.trampoline); 2824 eb_unpin_engine(&eb); 2825 err_context: 2826 i915_gem_context_put(eb.gem_context); 2827 err_destroy: 2828 eb_destroy(&eb); 2829 err_out_fence: 2830 if (out_fence_fd != -1) 2831 put_unused_fd(out_fence_fd); 2832 err_in_fence: 2833 dma_fence_put(in_fence); 2834 return err; 2835 } 2836 2837 static size_t eb_element_size(void) 2838 { 2839 return sizeof(struct drm_i915_gem_exec_object2); 2840 } 2841 2842 static bool check_buffer_count(size_t count) 2843 { 2844 const size_t sz = eb_element_size(); 2845 2846 /* 2847 * When using LUT_HANDLE, we impose a limit of INT_MAX for the lookup 2848 * array size (see eb_create()). Otherwise, we can accept an array as 2849 * large as can be addressed (though use large arrays at your peril)! 2850 */ 2851 2852 return !(count < 1 || count > INT_MAX || count > SIZE_MAX / sz - 1); 2853 } 2854 2855 /* 2856 * Legacy execbuffer just creates an exec2 list from the original exec object 2857 * list array and passes it to the real function. 2858 */ 2859 int 2860 i915_gem_execbuffer_ioctl(struct drm_device *dev, void *data, 2861 struct drm_file *file) 2862 { 2863 struct drm_i915_private *i915 = to_i915(dev); 2864 struct drm_i915_gem_execbuffer *args = data; 2865 struct drm_i915_gem_execbuffer2 exec2; 2866 struct drm_i915_gem_exec_object *exec_list = NULL; 2867 struct drm_i915_gem_exec_object2 *exec2_list = NULL; 2868 const size_t count = args->buffer_count; 2869 unsigned int i; 2870 int err; 2871 2872 if (!check_buffer_count(count)) { 2873 drm_dbg(&i915->drm, "execbuf2 with %zd buffers\n", count); 2874 return -EINVAL; 2875 } 2876 2877 exec2.buffers_ptr = args->buffers_ptr; 2878 exec2.buffer_count = args->buffer_count; 2879 exec2.batch_start_offset = args->batch_start_offset; 2880 exec2.batch_len = args->batch_len; 2881 exec2.DR1 = args->DR1; 2882 exec2.DR4 = args->DR4; 2883 exec2.num_cliprects = args->num_cliprects; 2884 exec2.cliprects_ptr = args->cliprects_ptr; 2885 exec2.flags = I915_EXEC_RENDER; 2886 i915_execbuffer2_set_context_id(exec2, 0); 2887 2888 err = i915_gem_check_execbuffer(&exec2); 2889 if (err) 2890 return err; 2891 2892 /* Copy in the exec list from userland */ 2893 exec_list = kvmalloc_array(count, sizeof(*exec_list), 2894 __GFP_NOWARN | GFP_KERNEL); 2895 exec2_list = kvmalloc_array(count, eb_element_size(), 2896 __GFP_NOWARN | GFP_KERNEL); 2897 if (exec_list == NULL || exec2_list == NULL) { 2898 drm_dbg(&i915->drm, 2899 "Failed to allocate exec list for %d buffers\n", 2900 args->buffer_count); 2901 kvfree(exec_list); 2902 kvfree(exec2_list); 2903 return -ENOMEM; 2904 } 2905 err = copy_from_user(exec_list, 2906 u64_to_user_ptr(args->buffers_ptr), 2907 sizeof(*exec_list) * count); 2908 if (err) { 2909 drm_dbg(&i915->drm, "copy %d exec entries failed %d\n", 2910 args->buffer_count, err); 2911 kvfree(exec_list); 2912 kvfree(exec2_list); 2913 return -EFAULT; 2914 } 2915 2916 for (i = 0; i < args->buffer_count; i++) { 2917 exec2_list[i].handle = exec_list[i].handle; 2918 exec2_list[i].relocation_count = exec_list[i].relocation_count; 2919 exec2_list[i].relocs_ptr = exec_list[i].relocs_ptr; 2920 exec2_list[i].alignment = exec_list[i].alignment; 2921 exec2_list[i].offset = exec_list[i].offset; 2922 if (INTEL_GEN(to_i915(dev)) < 4) 2923 exec2_list[i].flags = EXEC_OBJECT_NEEDS_FENCE; 2924 else 2925 exec2_list[i].flags = 0; 2926 } 2927 2928 err = i915_gem_do_execbuffer(dev, file, &exec2, exec2_list, NULL); 2929 if (exec2.flags & __EXEC_HAS_RELOC) { 2930 struct drm_i915_gem_exec_object __user *user_exec_list = 2931 u64_to_user_ptr(args->buffers_ptr); 2932 2933 /* Copy the new buffer offsets back to the user's exec list. */ 2934 for (i = 0; i < args->buffer_count; i++) { 2935 if (!(exec2_list[i].offset & UPDATE)) 2936 continue; 2937 2938 exec2_list[i].offset = 2939 gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK); 2940 exec2_list[i].offset &= PIN_OFFSET_MASK; 2941 if (__copy_to_user(&user_exec_list[i].offset, 2942 &exec2_list[i].offset, 2943 sizeof(user_exec_list[i].offset))) 2944 break; 2945 } 2946 } 2947 2948 kvfree(exec_list); 2949 kvfree(exec2_list); 2950 return err; 2951 } 2952 2953 int 2954 i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data, 2955 struct drm_file *file) 2956 { 2957 struct drm_i915_private *i915 = to_i915(dev); 2958 struct drm_i915_gem_execbuffer2 *args = data; 2959 struct drm_i915_gem_exec_object2 *exec2_list; 2960 struct drm_syncobj **fences = NULL; 2961 const size_t count = args->buffer_count; 2962 int err; 2963 2964 if (!check_buffer_count(count)) { 2965 drm_dbg(&i915->drm, "execbuf2 with %zd buffers\n", count); 2966 return -EINVAL; 2967 } 2968 2969 err = i915_gem_check_execbuffer(args); 2970 if (err) 2971 return err; 2972 2973 exec2_list = kvmalloc_array(count, eb_element_size(), 2974 __GFP_NOWARN | GFP_KERNEL); 2975 if (exec2_list == NULL) { 2976 drm_dbg(&i915->drm, "Failed to allocate exec list for %zd buffers\n", 2977 count); 2978 return -ENOMEM; 2979 } 2980 if (copy_from_user(exec2_list, 2981 u64_to_user_ptr(args->buffers_ptr), 2982 sizeof(*exec2_list) * count)) { 2983 drm_dbg(&i915->drm, "copy %zd exec entries failed\n", count); 2984 kvfree(exec2_list); 2985 return -EFAULT; 2986 } 2987 2988 if (args->flags & I915_EXEC_FENCE_ARRAY) { 2989 fences = get_fence_array(args, file); 2990 if (IS_ERR(fences)) { 2991 kvfree(exec2_list); 2992 return PTR_ERR(fences); 2993 } 2994 } 2995 2996 err = i915_gem_do_execbuffer(dev, file, args, exec2_list, fences); 2997 2998 /* 2999 * Now that we have begun execution of the batchbuffer, we ignore 3000 * any new error after this point. Also given that we have already 3001 * updated the associated relocations, we try to write out the current 3002 * object locations irrespective of any error. 3003 */ 3004 if (args->flags & __EXEC_HAS_RELOC) { 3005 struct drm_i915_gem_exec_object2 __user *user_exec_list = 3006 u64_to_user_ptr(args->buffers_ptr); 3007 unsigned int i; 3008 3009 /* Copy the new buffer offsets back to the user's exec list. */ 3010 /* 3011 * Note: count * sizeof(*user_exec_list) does not overflow, 3012 * because we checked 'count' in check_buffer_count(). 3013 * 3014 * And this range already got effectively checked earlier 3015 * when we did the "copy_from_user()" above. 3016 */ 3017 if (!user_access_begin(user_exec_list, count * sizeof(*user_exec_list))) 3018 goto end; 3019 3020 for (i = 0; i < args->buffer_count; i++) { 3021 if (!(exec2_list[i].offset & UPDATE)) 3022 continue; 3023 3024 exec2_list[i].offset = 3025 gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK); 3026 unsafe_put_user(exec2_list[i].offset, 3027 &user_exec_list[i].offset, 3028 end_user); 3029 } 3030 end_user: 3031 user_access_end(); 3032 end:; 3033 } 3034 3035 args->flags &= ~__I915_EXEC_UNKNOWN_FLAGS; 3036 put_fence_array(args, fences); 3037 kvfree(exec2_list); 3038 return err; 3039 } 3040 3041 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 3042 #include "selftests/i915_gem_execbuffer.c" 3043 #endif 3044