1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2008,2010 Intel Corporation 5 */ 6 7 #include <linux/intel-iommu.h> 8 #include <linux/dma-resv.h> 9 #include <linux/sync_file.h> 10 #include <linux/uaccess.h> 11 12 #include <drm/drm_syncobj.h> 13 14 #include "display/intel_frontbuffer.h" 15 16 #include "gem/i915_gem_ioctls.h" 17 #include "gt/intel_context.h" 18 #include "gt/intel_gpu_commands.h" 19 #include "gt/intel_gt.h" 20 #include "gt/intel_gt_buffer_pool.h" 21 #include "gt/intel_gt_pm.h" 22 #include "gt/intel_ring.h" 23 24 #include "pxp/intel_pxp.h" 25 26 #include "i915_drv.h" 27 #include "i915_gem_clflush.h" 28 #include "i915_gem_context.h" 29 #include "i915_gem_ioctls.h" 30 #include "i915_trace.h" 31 #include "i915_user_extensions.h" 32 33 struct eb_vma { 34 struct i915_vma *vma; 35 unsigned int flags; 36 37 /** This vma's place in the execbuf reservation list */ 38 struct drm_i915_gem_exec_object2 *exec; 39 struct list_head bind_link; 40 struct list_head reloc_link; 41 42 struct hlist_node node; 43 u32 handle; 44 }; 45 46 enum { 47 FORCE_CPU_RELOC = 1, 48 FORCE_GTT_RELOC, 49 FORCE_GPU_RELOC, 50 #define DBG_FORCE_RELOC 0 /* choose one of the above! */ 51 }; 52 53 /* __EXEC_OBJECT_NO_RESERVE is BIT(31), defined in i915_vma.h */ 54 #define __EXEC_OBJECT_HAS_PIN BIT(30) 55 #define __EXEC_OBJECT_HAS_FENCE BIT(29) 56 #define __EXEC_OBJECT_USERPTR_INIT BIT(28) 57 #define __EXEC_OBJECT_NEEDS_MAP BIT(27) 58 #define __EXEC_OBJECT_NEEDS_BIAS BIT(26) 59 #define __EXEC_OBJECT_INTERNAL_FLAGS (~0u << 26) /* all of the above + */ 60 #define __EXEC_OBJECT_RESERVED (__EXEC_OBJECT_HAS_PIN | __EXEC_OBJECT_HAS_FENCE) 61 62 #define __EXEC_HAS_RELOC BIT(31) 63 #define __EXEC_ENGINE_PINNED BIT(30) 64 #define __EXEC_USERPTR_USED BIT(29) 65 #define __EXEC_INTERNAL_FLAGS (~0u << 29) 66 #define UPDATE PIN_OFFSET_FIXED 67 68 #define BATCH_OFFSET_BIAS (256*1024) 69 70 #define __I915_EXEC_ILLEGAL_FLAGS \ 71 (__I915_EXEC_UNKNOWN_FLAGS | \ 72 I915_EXEC_CONSTANTS_MASK | \ 73 I915_EXEC_RESOURCE_STREAMER) 74 75 /* Catch emission of unexpected errors for CI! */ 76 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) 77 #undef EINVAL 78 #define EINVAL ({ \ 79 DRM_DEBUG_DRIVER("EINVAL at %s:%d\n", __func__, __LINE__); \ 80 22; \ 81 }) 82 #endif 83 84 /** 85 * DOC: User command execution 86 * 87 * Userspace submits commands to be executed on the GPU as an instruction 88 * stream within a GEM object we call a batchbuffer. This instructions may 89 * refer to other GEM objects containing auxiliary state such as kernels, 90 * samplers, render targets and even secondary batchbuffers. Userspace does 91 * not know where in the GPU memory these objects reside and so before the 92 * batchbuffer is passed to the GPU for execution, those addresses in the 93 * batchbuffer and auxiliary objects are updated. This is known as relocation, 94 * or patching. To try and avoid having to relocate each object on the next 95 * execution, userspace is told the location of those objects in this pass, 96 * but this remains just a hint as the kernel may choose a new location for 97 * any object in the future. 98 * 99 * At the level of talking to the hardware, submitting a batchbuffer for the 100 * GPU to execute is to add content to a buffer from which the HW 101 * command streamer is reading. 102 * 103 * 1. Add a command to load the HW context. For Logical Ring Contexts, i.e. 104 * Execlists, this command is not placed on the same buffer as the 105 * remaining items. 106 * 107 * 2. Add a command to invalidate caches to the buffer. 108 * 109 * 3. Add a batchbuffer start command to the buffer; the start command is 110 * essentially a token together with the GPU address of the batchbuffer 111 * to be executed. 112 * 113 * 4. Add a pipeline flush to the buffer. 114 * 115 * 5. Add a memory write command to the buffer to record when the GPU 116 * is done executing the batchbuffer. The memory write writes the 117 * global sequence number of the request, ``i915_request::global_seqno``; 118 * the i915 driver uses the current value in the register to determine 119 * if the GPU has completed the batchbuffer. 120 * 121 * 6. Add a user interrupt command to the buffer. This command instructs 122 * the GPU to issue an interrupt when the command, pipeline flush and 123 * memory write are completed. 124 * 125 * 7. Inform the hardware of the additional commands added to the buffer 126 * (by updating the tail pointer). 127 * 128 * Processing an execbuf ioctl is conceptually split up into a few phases. 129 * 130 * 1. Validation - Ensure all the pointers, handles and flags are valid. 131 * 2. Reservation - Assign GPU address space for every object 132 * 3. Relocation - Update any addresses to point to the final locations 133 * 4. Serialisation - Order the request with respect to its dependencies 134 * 5. Construction - Construct a request to execute the batchbuffer 135 * 6. Submission (at some point in the future execution) 136 * 137 * Reserving resources for the execbuf is the most complicated phase. We 138 * neither want to have to migrate the object in the address space, nor do 139 * we want to have to update any relocations pointing to this object. Ideally, 140 * we want to leave the object where it is and for all the existing relocations 141 * to match. If the object is given a new address, or if userspace thinks the 142 * object is elsewhere, we have to parse all the relocation entries and update 143 * the addresses. Userspace can set the I915_EXEC_NORELOC flag to hint that 144 * all the target addresses in all of its objects match the value in the 145 * relocation entries and that they all match the presumed offsets given by the 146 * list of execbuffer objects. Using this knowledge, we know that if we haven't 147 * moved any buffers, all the relocation entries are valid and we can skip 148 * the update. (If userspace is wrong, the likely outcome is an impromptu GPU 149 * hang.) The requirement for using I915_EXEC_NO_RELOC are: 150 * 151 * The addresses written in the objects must match the corresponding 152 * reloc.presumed_offset which in turn must match the corresponding 153 * execobject.offset. 154 * 155 * Any render targets written to in the batch must be flagged with 156 * EXEC_OBJECT_WRITE. 157 * 158 * To avoid stalling, execobject.offset should match the current 159 * address of that object within the active context. 160 * 161 * The reservation is done is multiple phases. First we try and keep any 162 * object already bound in its current location - so as long as meets the 163 * constraints imposed by the new execbuffer. Any object left unbound after the 164 * first pass is then fitted into any available idle space. If an object does 165 * not fit, all objects are removed from the reservation and the process rerun 166 * after sorting the objects into a priority order (more difficult to fit 167 * objects are tried first). Failing that, the entire VM is cleared and we try 168 * to fit the execbuf once last time before concluding that it simply will not 169 * fit. 170 * 171 * A small complication to all of this is that we allow userspace not only to 172 * specify an alignment and a size for the object in the address space, but 173 * we also allow userspace to specify the exact offset. This objects are 174 * simpler to place (the location is known a priori) all we have to do is make 175 * sure the space is available. 176 * 177 * Once all the objects are in place, patching up the buried pointers to point 178 * to the final locations is a fairly simple job of walking over the relocation 179 * entry arrays, looking up the right address and rewriting the value into 180 * the object. Simple! ... The relocation entries are stored in user memory 181 * and so to access them we have to copy them into a local buffer. That copy 182 * has to avoid taking any pagefaults as they may lead back to a GEM object 183 * requiring the struct_mutex (i.e. recursive deadlock). So once again we split 184 * the relocation into multiple passes. First we try to do everything within an 185 * atomic context (avoid the pagefaults) which requires that we never wait. If 186 * we detect that we may wait, or if we need to fault, then we have to fallback 187 * to a slower path. The slowpath has to drop the mutex. (Can you hear alarm 188 * bells yet?) Dropping the mutex means that we lose all the state we have 189 * built up so far for the execbuf and we must reset any global data. However, 190 * we do leave the objects pinned in their final locations - which is a 191 * potential issue for concurrent execbufs. Once we have left the mutex, we can 192 * allocate and copy all the relocation entries into a large array at our 193 * leisure, reacquire the mutex, reclaim all the objects and other state and 194 * then proceed to update any incorrect addresses with the objects. 195 * 196 * As we process the relocation entries, we maintain a record of whether the 197 * object is being written to. Using NORELOC, we expect userspace to provide 198 * this information instead. We also check whether we can skip the relocation 199 * by comparing the expected value inside the relocation entry with the target's 200 * final address. If they differ, we have to map the current object and rewrite 201 * the 4 or 8 byte pointer within. 202 * 203 * Serialising an execbuf is quite simple according to the rules of the GEM 204 * ABI. Execution within each context is ordered by the order of submission. 205 * Writes to any GEM object are in order of submission and are exclusive. Reads 206 * from a GEM object are unordered with respect to other reads, but ordered by 207 * writes. A write submitted after a read cannot occur before the read, and 208 * similarly any read submitted after a write cannot occur before the write. 209 * Writes are ordered between engines such that only one write occurs at any 210 * time (completing any reads beforehand) - using semaphores where available 211 * and CPU serialisation otherwise. Other GEM access obey the same rules, any 212 * write (either via mmaps using set-domain, or via pwrite) must flush all GPU 213 * reads before starting, and any read (either using set-domain or pread) must 214 * flush all GPU writes before starting. (Note we only employ a barrier before, 215 * we currently rely on userspace not concurrently starting a new execution 216 * whilst reading or writing to an object. This may be an advantage or not 217 * depending on how much you trust userspace not to shoot themselves in the 218 * foot.) Serialisation may just result in the request being inserted into 219 * a DAG awaiting its turn, but most simple is to wait on the CPU until 220 * all dependencies are resolved. 221 * 222 * After all of that, is just a matter of closing the request and handing it to 223 * the hardware (well, leaving it in a queue to be executed). However, we also 224 * offer the ability for batchbuffers to be run with elevated privileges so 225 * that they access otherwise hidden registers. (Used to adjust L3 cache etc.) 226 * Before any batch is given extra privileges we first must check that it 227 * contains no nefarious instructions, we check that each instruction is from 228 * our whitelist and all registers are also from an allowed list. We first 229 * copy the user's batchbuffer to a shadow (so that the user doesn't have 230 * access to it, either by the CPU or GPU as we scan it) and then parse each 231 * instruction. If everything is ok, we set a flag telling the hardware to run 232 * the batchbuffer in trusted mode, otherwise the ioctl is rejected. 233 */ 234 235 struct eb_fence { 236 struct drm_syncobj *syncobj; /* Use with ptr_mask_bits() */ 237 struct dma_fence *dma_fence; 238 u64 value; 239 struct dma_fence_chain *chain_fence; 240 }; 241 242 struct i915_execbuffer { 243 struct drm_i915_private *i915; /** i915 backpointer */ 244 struct drm_file *file; /** per-file lookup tables and limits */ 245 struct drm_i915_gem_execbuffer2 *args; /** ioctl parameters */ 246 struct drm_i915_gem_exec_object2 *exec; /** ioctl execobj[] */ 247 struct eb_vma *vma; 248 249 struct intel_gt *gt; /* gt for the execbuf */ 250 struct intel_context *context; /* logical state for the request */ 251 struct i915_gem_context *gem_context; /** caller's context */ 252 253 /** our requests to build */ 254 struct i915_request *requests[MAX_ENGINE_INSTANCE + 1]; 255 /** identity of the batch obj/vma */ 256 struct eb_vma *batches[MAX_ENGINE_INSTANCE + 1]; 257 struct i915_vma *trampoline; /** trampoline used for chaining */ 258 259 /** used for excl fence in dma_resv objects when > 1 BB submitted */ 260 struct dma_fence *composite_fence; 261 262 /** actual size of execobj[] as we may extend it for the cmdparser */ 263 unsigned int buffer_count; 264 265 /* number of batches in execbuf IOCTL */ 266 unsigned int num_batches; 267 268 /** list of vma not yet bound during reservation phase */ 269 struct list_head unbound; 270 271 /** list of vma that have execobj.relocation_count */ 272 struct list_head relocs; 273 274 struct i915_gem_ww_ctx ww; 275 276 /** 277 * Track the most recently used object for relocations, as we 278 * frequently have to perform multiple relocations within the same 279 * obj/page 280 */ 281 struct reloc_cache { 282 struct drm_mm_node node; /** temporary GTT binding */ 283 unsigned long vaddr; /** Current kmap address */ 284 unsigned long page; /** Currently mapped page index */ 285 unsigned int graphics_ver; /** Cached value of GRAPHICS_VER */ 286 bool use_64bit_reloc : 1; 287 bool has_llc : 1; 288 bool has_fence : 1; 289 bool needs_unfenced : 1; 290 } reloc_cache; 291 292 u64 invalid_flags; /** Set of execobj.flags that are invalid */ 293 294 /** Length of batch within object */ 295 u64 batch_len[MAX_ENGINE_INSTANCE + 1]; 296 u32 batch_start_offset; /** Location within object of batch */ 297 u32 batch_flags; /** Flags composed for emit_bb_start() */ 298 struct intel_gt_buffer_pool_node *batch_pool; /** pool node for batch buffer */ 299 300 /** 301 * Indicate either the size of the hastable used to resolve 302 * relocation handles, or if negative that we are using a direct 303 * index into the execobj[]. 304 */ 305 int lut_size; 306 struct hlist_head *buckets; /** ht for relocation handles */ 307 308 struct eb_fence *fences; 309 unsigned long num_fences; 310 }; 311 312 static int eb_parse(struct i915_execbuffer *eb); 313 static int eb_pin_engine(struct i915_execbuffer *eb, bool throttle); 314 static void eb_unpin_engine(struct i915_execbuffer *eb); 315 316 static inline bool eb_use_cmdparser(const struct i915_execbuffer *eb) 317 { 318 return intel_engine_requires_cmd_parser(eb->context->engine) || 319 (intel_engine_using_cmd_parser(eb->context->engine) && 320 eb->args->batch_len); 321 } 322 323 static int eb_create(struct i915_execbuffer *eb) 324 { 325 if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) { 326 unsigned int size = 1 + ilog2(eb->buffer_count); 327 328 /* 329 * Without a 1:1 association between relocation handles and 330 * the execobject[] index, we instead create a hashtable. 331 * We size it dynamically based on available memory, starting 332 * first with 1:1 assocative hash and scaling back until 333 * the allocation succeeds. 334 * 335 * Later on we use a positive lut_size to indicate we are 336 * using this hashtable, and a negative value to indicate a 337 * direct lookup. 338 */ 339 do { 340 gfp_t flags; 341 342 /* While we can still reduce the allocation size, don't 343 * raise a warning and allow the allocation to fail. 344 * On the last pass though, we want to try as hard 345 * as possible to perform the allocation and warn 346 * if it fails. 347 */ 348 flags = GFP_KERNEL; 349 if (size > 1) 350 flags |= __GFP_NORETRY | __GFP_NOWARN; 351 352 eb->buckets = kzalloc(sizeof(struct hlist_head) << size, 353 flags); 354 if (eb->buckets) 355 break; 356 } while (--size); 357 358 if (unlikely(!size)) 359 return -ENOMEM; 360 361 eb->lut_size = size; 362 } else { 363 eb->lut_size = -eb->buffer_count; 364 } 365 366 return 0; 367 } 368 369 static bool 370 eb_vma_misplaced(const struct drm_i915_gem_exec_object2 *entry, 371 const struct i915_vma *vma, 372 unsigned int flags) 373 { 374 if (vma->node.size < entry->pad_to_size) 375 return true; 376 377 if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment)) 378 return true; 379 380 if (flags & EXEC_OBJECT_PINNED && 381 vma->node.start != entry->offset) 382 return true; 383 384 if (flags & __EXEC_OBJECT_NEEDS_BIAS && 385 vma->node.start < BATCH_OFFSET_BIAS) 386 return true; 387 388 if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS) && 389 (vma->node.start + vma->node.size + 4095) >> 32) 390 return true; 391 392 if (flags & __EXEC_OBJECT_NEEDS_MAP && 393 !i915_vma_is_map_and_fenceable(vma)) 394 return true; 395 396 return false; 397 } 398 399 static u64 eb_pin_flags(const struct drm_i915_gem_exec_object2 *entry, 400 unsigned int exec_flags) 401 { 402 u64 pin_flags = 0; 403 404 if (exec_flags & EXEC_OBJECT_NEEDS_GTT) 405 pin_flags |= PIN_GLOBAL; 406 407 /* 408 * Wa32bitGeneralStateOffset & Wa32bitInstructionBaseOffset, 409 * limit address to the first 4GBs for unflagged objects. 410 */ 411 if (!(exec_flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS)) 412 pin_flags |= PIN_ZONE_4G; 413 414 if (exec_flags & __EXEC_OBJECT_NEEDS_MAP) 415 pin_flags |= PIN_MAPPABLE; 416 417 if (exec_flags & EXEC_OBJECT_PINNED) 418 pin_flags |= entry->offset | PIN_OFFSET_FIXED; 419 else if (exec_flags & __EXEC_OBJECT_NEEDS_BIAS) 420 pin_flags |= BATCH_OFFSET_BIAS | PIN_OFFSET_BIAS; 421 422 return pin_flags; 423 } 424 425 static inline int 426 eb_pin_vma(struct i915_execbuffer *eb, 427 const struct drm_i915_gem_exec_object2 *entry, 428 struct eb_vma *ev) 429 { 430 struct i915_vma *vma = ev->vma; 431 u64 pin_flags; 432 int err; 433 434 if (vma->node.size) 435 pin_flags = vma->node.start; 436 else 437 pin_flags = entry->offset & PIN_OFFSET_MASK; 438 439 pin_flags |= PIN_USER | PIN_NOEVICT | PIN_OFFSET_FIXED; 440 if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_GTT)) 441 pin_flags |= PIN_GLOBAL; 442 443 /* Attempt to reuse the current location if available */ 444 err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, pin_flags); 445 if (err == -EDEADLK) 446 return err; 447 448 if (unlikely(err)) { 449 if (entry->flags & EXEC_OBJECT_PINNED) 450 return err; 451 452 /* Failing that pick any _free_ space if suitable */ 453 err = i915_vma_pin_ww(vma, &eb->ww, 454 entry->pad_to_size, 455 entry->alignment, 456 eb_pin_flags(entry, ev->flags) | 457 PIN_USER | PIN_NOEVICT); 458 if (unlikely(err)) 459 return err; 460 } 461 462 if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) { 463 err = i915_vma_pin_fence(vma); 464 if (unlikely(err)) { 465 i915_vma_unpin(vma); 466 return err; 467 } 468 469 if (vma->fence) 470 ev->flags |= __EXEC_OBJECT_HAS_FENCE; 471 } 472 473 ev->flags |= __EXEC_OBJECT_HAS_PIN; 474 if (eb_vma_misplaced(entry, vma, ev->flags)) 475 return -EBADSLT; 476 477 return 0; 478 } 479 480 static inline void 481 eb_unreserve_vma(struct eb_vma *ev) 482 { 483 if (!(ev->flags & __EXEC_OBJECT_HAS_PIN)) 484 return; 485 486 if (unlikely(ev->flags & __EXEC_OBJECT_HAS_FENCE)) 487 __i915_vma_unpin_fence(ev->vma); 488 489 __i915_vma_unpin(ev->vma); 490 ev->flags &= ~__EXEC_OBJECT_RESERVED; 491 } 492 493 static int 494 eb_validate_vma(struct i915_execbuffer *eb, 495 struct drm_i915_gem_exec_object2 *entry, 496 struct i915_vma *vma) 497 { 498 /* Relocations are disallowed for all platforms after TGL-LP. This 499 * also covers all platforms with local memory. 500 */ 501 if (entry->relocation_count && 502 GRAPHICS_VER(eb->i915) >= 12 && !IS_TIGERLAKE(eb->i915)) 503 return -EINVAL; 504 505 if (unlikely(entry->flags & eb->invalid_flags)) 506 return -EINVAL; 507 508 if (unlikely(entry->alignment && 509 !is_power_of_2_u64(entry->alignment))) 510 return -EINVAL; 511 512 /* 513 * Offset can be used as input (EXEC_OBJECT_PINNED), reject 514 * any non-page-aligned or non-canonical addresses. 515 */ 516 if (unlikely(entry->flags & EXEC_OBJECT_PINNED && 517 entry->offset != gen8_canonical_addr(entry->offset & I915_GTT_PAGE_MASK))) 518 return -EINVAL; 519 520 /* pad_to_size was once a reserved field, so sanitize it */ 521 if (entry->flags & EXEC_OBJECT_PAD_TO_SIZE) { 522 if (unlikely(offset_in_page(entry->pad_to_size))) 523 return -EINVAL; 524 } else { 525 entry->pad_to_size = 0; 526 } 527 /* 528 * From drm_mm perspective address space is continuous, 529 * so from this point we're always using non-canonical 530 * form internally. 531 */ 532 entry->offset = gen8_noncanonical_addr(entry->offset); 533 534 if (!eb->reloc_cache.has_fence) { 535 entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE; 536 } else { 537 if ((entry->flags & EXEC_OBJECT_NEEDS_FENCE || 538 eb->reloc_cache.needs_unfenced) && 539 i915_gem_object_is_tiled(vma->obj)) 540 entry->flags |= EXEC_OBJECT_NEEDS_GTT | __EXEC_OBJECT_NEEDS_MAP; 541 } 542 543 return 0; 544 } 545 546 static inline bool 547 is_batch_buffer(struct i915_execbuffer *eb, unsigned int buffer_idx) 548 { 549 return eb->args->flags & I915_EXEC_BATCH_FIRST ? 550 buffer_idx < eb->num_batches : 551 buffer_idx >= eb->args->buffer_count - eb->num_batches; 552 } 553 554 static int 555 eb_add_vma(struct i915_execbuffer *eb, 556 unsigned int *current_batch, 557 unsigned int i, 558 struct i915_vma *vma) 559 { 560 struct drm_i915_private *i915 = eb->i915; 561 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; 562 struct eb_vma *ev = &eb->vma[i]; 563 564 ev->vma = vma; 565 ev->exec = entry; 566 ev->flags = entry->flags; 567 568 if (eb->lut_size > 0) { 569 ev->handle = entry->handle; 570 hlist_add_head(&ev->node, 571 &eb->buckets[hash_32(entry->handle, 572 eb->lut_size)]); 573 } 574 575 if (entry->relocation_count) 576 list_add_tail(&ev->reloc_link, &eb->relocs); 577 578 /* 579 * SNA is doing fancy tricks with compressing batch buffers, which leads 580 * to negative relocation deltas. Usually that works out ok since the 581 * relocate address is still positive, except when the batch is placed 582 * very low in the GTT. Ensure this doesn't happen. 583 * 584 * Note that actual hangs have only been observed on gen7, but for 585 * paranoia do it everywhere. 586 */ 587 if (is_batch_buffer(eb, i)) { 588 if (entry->relocation_count && 589 !(ev->flags & EXEC_OBJECT_PINNED)) 590 ev->flags |= __EXEC_OBJECT_NEEDS_BIAS; 591 if (eb->reloc_cache.has_fence) 592 ev->flags |= EXEC_OBJECT_NEEDS_FENCE; 593 594 eb->batches[*current_batch] = ev; 595 596 if (unlikely(ev->flags & EXEC_OBJECT_WRITE)) { 597 drm_dbg(&i915->drm, 598 "Attempting to use self-modifying batch buffer\n"); 599 return -EINVAL; 600 } 601 602 if (range_overflows_t(u64, 603 eb->batch_start_offset, 604 eb->args->batch_len, 605 ev->vma->size)) { 606 drm_dbg(&i915->drm, "Attempting to use out-of-bounds batch\n"); 607 return -EINVAL; 608 } 609 610 if (eb->args->batch_len == 0) 611 eb->batch_len[*current_batch] = ev->vma->size - 612 eb->batch_start_offset; 613 else 614 eb->batch_len[*current_batch] = eb->args->batch_len; 615 if (unlikely(eb->batch_len[*current_batch] == 0)) { /* impossible! */ 616 drm_dbg(&i915->drm, "Invalid batch length\n"); 617 return -EINVAL; 618 } 619 620 ++*current_batch; 621 } 622 623 return 0; 624 } 625 626 static inline int use_cpu_reloc(const struct reloc_cache *cache, 627 const struct drm_i915_gem_object *obj) 628 { 629 if (!i915_gem_object_has_struct_page(obj)) 630 return false; 631 632 if (DBG_FORCE_RELOC == FORCE_CPU_RELOC) 633 return true; 634 635 if (DBG_FORCE_RELOC == FORCE_GTT_RELOC) 636 return false; 637 638 return (cache->has_llc || 639 obj->cache_dirty || 640 obj->cache_level != I915_CACHE_NONE); 641 } 642 643 static int eb_reserve_vma(struct i915_execbuffer *eb, 644 struct eb_vma *ev, 645 u64 pin_flags) 646 { 647 struct drm_i915_gem_exec_object2 *entry = ev->exec; 648 struct i915_vma *vma = ev->vma; 649 int err; 650 651 if (drm_mm_node_allocated(&vma->node) && 652 eb_vma_misplaced(entry, vma, ev->flags)) { 653 err = i915_vma_unbind(vma); 654 if (err) 655 return err; 656 } 657 658 err = i915_vma_pin_ww(vma, &eb->ww, 659 entry->pad_to_size, entry->alignment, 660 eb_pin_flags(entry, ev->flags) | pin_flags); 661 if (err) 662 return err; 663 664 if (entry->offset != vma->node.start) { 665 entry->offset = vma->node.start | UPDATE; 666 eb->args->flags |= __EXEC_HAS_RELOC; 667 } 668 669 if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) { 670 err = i915_vma_pin_fence(vma); 671 if (unlikely(err)) { 672 i915_vma_unpin(vma); 673 return err; 674 } 675 676 if (vma->fence) 677 ev->flags |= __EXEC_OBJECT_HAS_FENCE; 678 } 679 680 ev->flags |= __EXEC_OBJECT_HAS_PIN; 681 GEM_BUG_ON(eb_vma_misplaced(entry, vma, ev->flags)); 682 683 return 0; 684 } 685 686 static int eb_reserve(struct i915_execbuffer *eb) 687 { 688 const unsigned int count = eb->buffer_count; 689 unsigned int pin_flags = PIN_USER | PIN_NONBLOCK; 690 struct list_head last; 691 struct eb_vma *ev; 692 unsigned int i, pass; 693 int err = 0; 694 695 /* 696 * Attempt to pin all of the buffers into the GTT. 697 * This is done in 3 phases: 698 * 699 * 1a. Unbind all objects that do not match the GTT constraints for 700 * the execbuffer (fenceable, mappable, alignment etc). 701 * 1b. Increment pin count for already bound objects. 702 * 2. Bind new objects. 703 * 3. Decrement pin count. 704 * 705 * This avoid unnecessary unbinding of later objects in order to make 706 * room for the earlier objects *unless* we need to defragment. 707 */ 708 pass = 0; 709 do { 710 list_for_each_entry(ev, &eb->unbound, bind_link) { 711 err = eb_reserve_vma(eb, ev, pin_flags); 712 if (err) 713 break; 714 } 715 if (err != -ENOSPC) 716 return err; 717 718 /* Resort *all* the objects into priority order */ 719 INIT_LIST_HEAD(&eb->unbound); 720 INIT_LIST_HEAD(&last); 721 for (i = 0; i < count; i++) { 722 unsigned int flags; 723 724 ev = &eb->vma[i]; 725 flags = ev->flags; 726 if (flags & EXEC_OBJECT_PINNED && 727 flags & __EXEC_OBJECT_HAS_PIN) 728 continue; 729 730 eb_unreserve_vma(ev); 731 732 if (flags & EXEC_OBJECT_PINNED) 733 /* Pinned must have their slot */ 734 list_add(&ev->bind_link, &eb->unbound); 735 else if (flags & __EXEC_OBJECT_NEEDS_MAP) 736 /* Map require the lowest 256MiB (aperture) */ 737 list_add_tail(&ev->bind_link, &eb->unbound); 738 else if (!(flags & EXEC_OBJECT_SUPPORTS_48B_ADDRESS)) 739 /* Prioritise 4GiB region for restricted bo */ 740 list_add(&ev->bind_link, &last); 741 else 742 list_add_tail(&ev->bind_link, &last); 743 } 744 list_splice_tail(&last, &eb->unbound); 745 746 switch (pass++) { 747 case 0: 748 break; 749 750 case 1: 751 /* Too fragmented, unbind everything and retry */ 752 mutex_lock(&eb->context->vm->mutex); 753 err = i915_gem_evict_vm(eb->context->vm); 754 mutex_unlock(&eb->context->vm->mutex); 755 if (err) 756 return err; 757 break; 758 759 default: 760 return -ENOSPC; 761 } 762 763 pin_flags = PIN_USER; 764 } while (1); 765 } 766 767 static int eb_select_context(struct i915_execbuffer *eb) 768 { 769 struct i915_gem_context *ctx; 770 771 ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1); 772 if (unlikely(IS_ERR(ctx))) 773 return PTR_ERR(ctx); 774 775 eb->gem_context = ctx; 776 if (i915_gem_context_has_full_ppgtt(ctx)) 777 eb->invalid_flags |= EXEC_OBJECT_NEEDS_GTT; 778 779 return 0; 780 } 781 782 static int __eb_add_lut(struct i915_execbuffer *eb, 783 u32 handle, struct i915_vma *vma) 784 { 785 struct i915_gem_context *ctx = eb->gem_context; 786 struct i915_lut_handle *lut; 787 int err; 788 789 lut = i915_lut_handle_alloc(); 790 if (unlikely(!lut)) 791 return -ENOMEM; 792 793 i915_vma_get(vma); 794 if (!atomic_fetch_inc(&vma->open_count)) 795 i915_vma_reopen(vma); 796 lut->handle = handle; 797 lut->ctx = ctx; 798 799 /* Check that the context hasn't been closed in the meantime */ 800 err = -EINTR; 801 if (!mutex_lock_interruptible(&ctx->lut_mutex)) { 802 if (likely(!i915_gem_context_is_closed(ctx))) 803 err = radix_tree_insert(&ctx->handles_vma, handle, vma); 804 else 805 err = -ENOENT; 806 if (err == 0) { /* And nor has this handle */ 807 struct drm_i915_gem_object *obj = vma->obj; 808 809 spin_lock(&obj->lut_lock); 810 if (idr_find(&eb->file->object_idr, handle) == obj) { 811 list_add(&lut->obj_link, &obj->lut_list); 812 } else { 813 radix_tree_delete(&ctx->handles_vma, handle); 814 err = -ENOENT; 815 } 816 spin_unlock(&obj->lut_lock); 817 } 818 mutex_unlock(&ctx->lut_mutex); 819 } 820 if (unlikely(err)) 821 goto err; 822 823 return 0; 824 825 err: 826 i915_vma_close(vma); 827 i915_vma_put(vma); 828 i915_lut_handle_free(lut); 829 return err; 830 } 831 832 static struct i915_vma *eb_lookup_vma(struct i915_execbuffer *eb, u32 handle) 833 { 834 struct i915_address_space *vm = eb->context->vm; 835 836 do { 837 struct drm_i915_gem_object *obj; 838 struct i915_vma *vma; 839 int err; 840 841 rcu_read_lock(); 842 vma = radix_tree_lookup(&eb->gem_context->handles_vma, handle); 843 if (likely(vma && vma->vm == vm)) 844 vma = i915_vma_tryget(vma); 845 rcu_read_unlock(); 846 if (likely(vma)) 847 return vma; 848 849 obj = i915_gem_object_lookup(eb->file, handle); 850 if (unlikely(!obj)) 851 return ERR_PTR(-ENOENT); 852 853 /* 854 * If the user has opted-in for protected-object tracking, make 855 * sure the object encryption can be used. 856 * We only need to do this when the object is first used with 857 * this context, because the context itself will be banned when 858 * the protected objects become invalid. 859 */ 860 if (i915_gem_context_uses_protected_content(eb->gem_context) && 861 i915_gem_object_is_protected(obj)) { 862 err = intel_pxp_key_check(&vm->gt->pxp, obj, true); 863 if (err) { 864 i915_gem_object_put(obj); 865 return ERR_PTR(err); 866 } 867 } 868 869 vma = i915_vma_instance(obj, vm, NULL); 870 if (IS_ERR(vma)) { 871 i915_gem_object_put(obj); 872 return vma; 873 } 874 875 err = __eb_add_lut(eb, handle, vma); 876 if (likely(!err)) 877 return vma; 878 879 i915_gem_object_put(obj); 880 if (err != -EEXIST) 881 return ERR_PTR(err); 882 } while (1); 883 } 884 885 static int eb_lookup_vmas(struct i915_execbuffer *eb) 886 { 887 unsigned int i, current_batch = 0; 888 int err = 0; 889 890 INIT_LIST_HEAD(&eb->relocs); 891 892 for (i = 0; i < eb->buffer_count; i++) { 893 struct i915_vma *vma; 894 895 vma = eb_lookup_vma(eb, eb->exec[i].handle); 896 if (IS_ERR(vma)) { 897 err = PTR_ERR(vma); 898 goto err; 899 } 900 901 err = eb_validate_vma(eb, &eb->exec[i], vma); 902 if (unlikely(err)) { 903 i915_vma_put(vma); 904 goto err; 905 } 906 907 err = eb_add_vma(eb, ¤t_batch, i, vma); 908 if (err) 909 return err; 910 911 if (i915_gem_object_is_userptr(vma->obj)) { 912 err = i915_gem_object_userptr_submit_init(vma->obj); 913 if (err) { 914 if (i + 1 < eb->buffer_count) { 915 /* 916 * Execbuffer code expects last vma entry to be NULL, 917 * since we already initialized this entry, 918 * set the next value to NULL or we mess up 919 * cleanup handling. 920 */ 921 eb->vma[i + 1].vma = NULL; 922 } 923 924 return err; 925 } 926 927 eb->vma[i].flags |= __EXEC_OBJECT_USERPTR_INIT; 928 eb->args->flags |= __EXEC_USERPTR_USED; 929 } 930 } 931 932 return 0; 933 934 err: 935 eb->vma[i].vma = NULL; 936 return err; 937 } 938 939 static int eb_lock_vmas(struct i915_execbuffer *eb) 940 { 941 unsigned int i; 942 int err; 943 944 for (i = 0; i < eb->buffer_count; i++) { 945 struct eb_vma *ev = &eb->vma[i]; 946 struct i915_vma *vma = ev->vma; 947 948 err = i915_gem_object_lock(vma->obj, &eb->ww); 949 if (err) 950 return err; 951 } 952 953 return 0; 954 } 955 956 static int eb_validate_vmas(struct i915_execbuffer *eb) 957 { 958 unsigned int i; 959 int err; 960 961 INIT_LIST_HEAD(&eb->unbound); 962 963 err = eb_lock_vmas(eb); 964 if (err) 965 return err; 966 967 for (i = 0; i < eb->buffer_count; i++) { 968 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; 969 struct eb_vma *ev = &eb->vma[i]; 970 struct i915_vma *vma = ev->vma; 971 972 err = eb_pin_vma(eb, entry, ev); 973 if (err == -EDEADLK) 974 return err; 975 976 if (!err) { 977 if (entry->offset != vma->node.start) { 978 entry->offset = vma->node.start | UPDATE; 979 eb->args->flags |= __EXEC_HAS_RELOC; 980 } 981 } else { 982 eb_unreserve_vma(ev); 983 984 list_add_tail(&ev->bind_link, &eb->unbound); 985 if (drm_mm_node_allocated(&vma->node)) { 986 err = i915_vma_unbind(vma); 987 if (err) 988 return err; 989 } 990 } 991 992 if (!(ev->flags & EXEC_OBJECT_WRITE)) { 993 err = dma_resv_reserve_shared(vma->resv, 1); 994 if (err) 995 return err; 996 } 997 998 GEM_BUG_ON(drm_mm_node_allocated(&vma->node) && 999 eb_vma_misplaced(&eb->exec[i], vma, ev->flags)); 1000 } 1001 1002 if (!list_empty(&eb->unbound)) 1003 return eb_reserve(eb); 1004 1005 return 0; 1006 } 1007 1008 static struct eb_vma * 1009 eb_get_vma(const struct i915_execbuffer *eb, unsigned long handle) 1010 { 1011 if (eb->lut_size < 0) { 1012 if (handle >= -eb->lut_size) 1013 return NULL; 1014 return &eb->vma[handle]; 1015 } else { 1016 struct hlist_head *head; 1017 struct eb_vma *ev; 1018 1019 head = &eb->buckets[hash_32(handle, eb->lut_size)]; 1020 hlist_for_each_entry(ev, head, node) { 1021 if (ev->handle == handle) 1022 return ev; 1023 } 1024 return NULL; 1025 } 1026 } 1027 1028 static void eb_release_vmas(struct i915_execbuffer *eb, bool final) 1029 { 1030 const unsigned int count = eb->buffer_count; 1031 unsigned int i; 1032 1033 for (i = 0; i < count; i++) { 1034 struct eb_vma *ev = &eb->vma[i]; 1035 struct i915_vma *vma = ev->vma; 1036 1037 if (!vma) 1038 break; 1039 1040 eb_unreserve_vma(ev); 1041 1042 if (final) 1043 i915_vma_put(vma); 1044 } 1045 1046 eb_unpin_engine(eb); 1047 } 1048 1049 static void eb_destroy(const struct i915_execbuffer *eb) 1050 { 1051 if (eb->lut_size > 0) 1052 kfree(eb->buckets); 1053 } 1054 1055 static inline u64 1056 relocation_target(const struct drm_i915_gem_relocation_entry *reloc, 1057 const struct i915_vma *target) 1058 { 1059 return gen8_canonical_addr((int)reloc->delta + target->node.start); 1060 } 1061 1062 static void reloc_cache_init(struct reloc_cache *cache, 1063 struct drm_i915_private *i915) 1064 { 1065 cache->page = -1; 1066 cache->vaddr = 0; 1067 /* Must be a variable in the struct to allow GCC to unroll. */ 1068 cache->graphics_ver = GRAPHICS_VER(i915); 1069 cache->has_llc = HAS_LLC(i915); 1070 cache->use_64bit_reloc = HAS_64BIT_RELOC(i915); 1071 cache->has_fence = cache->graphics_ver < 4; 1072 cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment; 1073 cache->node.flags = 0; 1074 } 1075 1076 static inline void *unmask_page(unsigned long p) 1077 { 1078 return (void *)(uintptr_t)(p & PAGE_MASK); 1079 } 1080 1081 static inline unsigned int unmask_flags(unsigned long p) 1082 { 1083 return p & ~PAGE_MASK; 1084 } 1085 1086 #define KMAP 0x4 /* after CLFLUSH_FLAGS */ 1087 1088 static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache) 1089 { 1090 struct drm_i915_private *i915 = 1091 container_of(cache, struct i915_execbuffer, reloc_cache)->i915; 1092 return &i915->ggtt; 1093 } 1094 1095 static void reloc_cache_reset(struct reloc_cache *cache, struct i915_execbuffer *eb) 1096 { 1097 void *vaddr; 1098 1099 if (!cache->vaddr) 1100 return; 1101 1102 vaddr = unmask_page(cache->vaddr); 1103 if (cache->vaddr & KMAP) { 1104 struct drm_i915_gem_object *obj = 1105 (struct drm_i915_gem_object *)cache->node.mm; 1106 if (cache->vaddr & CLFLUSH_AFTER) 1107 mb(); 1108 1109 kunmap_atomic(vaddr); 1110 i915_gem_object_finish_access(obj); 1111 } else { 1112 struct i915_ggtt *ggtt = cache_to_ggtt(cache); 1113 1114 intel_gt_flush_ggtt_writes(ggtt->vm.gt); 1115 io_mapping_unmap_atomic((void __iomem *)vaddr); 1116 1117 if (drm_mm_node_allocated(&cache->node)) { 1118 ggtt->vm.clear_range(&ggtt->vm, 1119 cache->node.start, 1120 cache->node.size); 1121 mutex_lock(&ggtt->vm.mutex); 1122 drm_mm_remove_node(&cache->node); 1123 mutex_unlock(&ggtt->vm.mutex); 1124 } else { 1125 i915_vma_unpin((struct i915_vma *)cache->node.mm); 1126 } 1127 } 1128 1129 cache->vaddr = 0; 1130 cache->page = -1; 1131 } 1132 1133 static void *reloc_kmap(struct drm_i915_gem_object *obj, 1134 struct reloc_cache *cache, 1135 unsigned long pageno) 1136 { 1137 void *vaddr; 1138 struct page *page; 1139 1140 if (cache->vaddr) { 1141 kunmap_atomic(unmask_page(cache->vaddr)); 1142 } else { 1143 unsigned int flushes; 1144 int err; 1145 1146 err = i915_gem_object_prepare_write(obj, &flushes); 1147 if (err) 1148 return ERR_PTR(err); 1149 1150 BUILD_BUG_ON(KMAP & CLFLUSH_FLAGS); 1151 BUILD_BUG_ON((KMAP | CLFLUSH_FLAGS) & PAGE_MASK); 1152 1153 cache->vaddr = flushes | KMAP; 1154 cache->node.mm = (void *)obj; 1155 if (flushes) 1156 mb(); 1157 } 1158 1159 page = i915_gem_object_get_page(obj, pageno); 1160 if (!obj->mm.dirty) 1161 set_page_dirty(page); 1162 1163 vaddr = kmap_atomic(page); 1164 cache->vaddr = unmask_flags(cache->vaddr) | (unsigned long)vaddr; 1165 cache->page = pageno; 1166 1167 return vaddr; 1168 } 1169 1170 static void *reloc_iomap(struct drm_i915_gem_object *obj, 1171 struct i915_execbuffer *eb, 1172 unsigned long page) 1173 { 1174 struct reloc_cache *cache = &eb->reloc_cache; 1175 struct i915_ggtt *ggtt = cache_to_ggtt(cache); 1176 unsigned long offset; 1177 void *vaddr; 1178 1179 if (cache->vaddr) { 1180 intel_gt_flush_ggtt_writes(ggtt->vm.gt); 1181 io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr)); 1182 } else { 1183 struct i915_vma *vma; 1184 int err; 1185 1186 if (i915_gem_object_is_tiled(obj)) 1187 return ERR_PTR(-EINVAL); 1188 1189 if (use_cpu_reloc(cache, obj)) 1190 return NULL; 1191 1192 err = i915_gem_object_set_to_gtt_domain(obj, true); 1193 if (err) 1194 return ERR_PTR(err); 1195 1196 vma = i915_gem_object_ggtt_pin_ww(obj, &eb->ww, NULL, 0, 0, 1197 PIN_MAPPABLE | 1198 PIN_NONBLOCK /* NOWARN */ | 1199 PIN_NOEVICT); 1200 if (vma == ERR_PTR(-EDEADLK)) 1201 return vma; 1202 1203 if (IS_ERR(vma)) { 1204 memset(&cache->node, 0, sizeof(cache->node)); 1205 mutex_lock(&ggtt->vm.mutex); 1206 err = drm_mm_insert_node_in_range 1207 (&ggtt->vm.mm, &cache->node, 1208 PAGE_SIZE, 0, I915_COLOR_UNEVICTABLE, 1209 0, ggtt->mappable_end, 1210 DRM_MM_INSERT_LOW); 1211 mutex_unlock(&ggtt->vm.mutex); 1212 if (err) /* no inactive aperture space, use cpu reloc */ 1213 return NULL; 1214 } else { 1215 cache->node.start = vma->node.start; 1216 cache->node.mm = (void *)vma; 1217 } 1218 } 1219 1220 offset = cache->node.start; 1221 if (drm_mm_node_allocated(&cache->node)) { 1222 ggtt->vm.insert_page(&ggtt->vm, 1223 i915_gem_object_get_dma_address(obj, page), 1224 offset, I915_CACHE_NONE, 0); 1225 } else { 1226 offset += page << PAGE_SHIFT; 1227 } 1228 1229 vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->iomap, 1230 offset); 1231 cache->page = page; 1232 cache->vaddr = (unsigned long)vaddr; 1233 1234 return vaddr; 1235 } 1236 1237 static void *reloc_vaddr(struct drm_i915_gem_object *obj, 1238 struct i915_execbuffer *eb, 1239 unsigned long page) 1240 { 1241 struct reloc_cache *cache = &eb->reloc_cache; 1242 void *vaddr; 1243 1244 if (cache->page == page) { 1245 vaddr = unmask_page(cache->vaddr); 1246 } else { 1247 vaddr = NULL; 1248 if ((cache->vaddr & KMAP) == 0) 1249 vaddr = reloc_iomap(obj, eb, page); 1250 if (!vaddr) 1251 vaddr = reloc_kmap(obj, cache, page); 1252 } 1253 1254 return vaddr; 1255 } 1256 1257 static void clflush_write32(u32 *addr, u32 value, unsigned int flushes) 1258 { 1259 if (unlikely(flushes & (CLFLUSH_BEFORE | CLFLUSH_AFTER))) { 1260 if (flushes & CLFLUSH_BEFORE) { 1261 clflushopt(addr); 1262 mb(); 1263 } 1264 1265 *addr = value; 1266 1267 /* 1268 * Writes to the same cacheline are serialised by the CPU 1269 * (including clflush). On the write path, we only require 1270 * that it hits memory in an orderly fashion and place 1271 * mb barriers at the start and end of the relocation phase 1272 * to ensure ordering of clflush wrt to the system. 1273 */ 1274 if (flushes & CLFLUSH_AFTER) 1275 clflushopt(addr); 1276 } else 1277 *addr = value; 1278 } 1279 1280 static u64 1281 relocate_entry(struct i915_vma *vma, 1282 const struct drm_i915_gem_relocation_entry *reloc, 1283 struct i915_execbuffer *eb, 1284 const struct i915_vma *target) 1285 { 1286 u64 target_addr = relocation_target(reloc, target); 1287 u64 offset = reloc->offset; 1288 bool wide = eb->reloc_cache.use_64bit_reloc; 1289 void *vaddr; 1290 1291 repeat: 1292 vaddr = reloc_vaddr(vma->obj, eb, 1293 offset >> PAGE_SHIFT); 1294 if (IS_ERR(vaddr)) 1295 return PTR_ERR(vaddr); 1296 1297 GEM_BUG_ON(!IS_ALIGNED(offset, sizeof(u32))); 1298 clflush_write32(vaddr + offset_in_page(offset), 1299 lower_32_bits(target_addr), 1300 eb->reloc_cache.vaddr); 1301 1302 if (wide) { 1303 offset += sizeof(u32); 1304 target_addr >>= 32; 1305 wide = false; 1306 goto repeat; 1307 } 1308 1309 return target->node.start | UPDATE; 1310 } 1311 1312 static u64 1313 eb_relocate_entry(struct i915_execbuffer *eb, 1314 struct eb_vma *ev, 1315 const struct drm_i915_gem_relocation_entry *reloc) 1316 { 1317 struct drm_i915_private *i915 = eb->i915; 1318 struct eb_vma *target; 1319 int err; 1320 1321 /* we've already hold a reference to all valid objects */ 1322 target = eb_get_vma(eb, reloc->target_handle); 1323 if (unlikely(!target)) 1324 return -ENOENT; 1325 1326 /* Validate that the target is in a valid r/w GPU domain */ 1327 if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) { 1328 drm_dbg(&i915->drm, "reloc with multiple write domains: " 1329 "target %d offset %d " 1330 "read %08x write %08x", 1331 reloc->target_handle, 1332 (int) reloc->offset, 1333 reloc->read_domains, 1334 reloc->write_domain); 1335 return -EINVAL; 1336 } 1337 if (unlikely((reloc->write_domain | reloc->read_domains) 1338 & ~I915_GEM_GPU_DOMAINS)) { 1339 drm_dbg(&i915->drm, "reloc with read/write non-GPU domains: " 1340 "target %d offset %d " 1341 "read %08x write %08x", 1342 reloc->target_handle, 1343 (int) reloc->offset, 1344 reloc->read_domains, 1345 reloc->write_domain); 1346 return -EINVAL; 1347 } 1348 1349 if (reloc->write_domain) { 1350 target->flags |= EXEC_OBJECT_WRITE; 1351 1352 /* 1353 * Sandybridge PPGTT errata: We need a global gtt mapping 1354 * for MI and pipe_control writes because the gpu doesn't 1355 * properly redirect them through the ppgtt for non_secure 1356 * batchbuffers. 1357 */ 1358 if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION && 1359 GRAPHICS_VER(eb->i915) == 6) { 1360 err = i915_vma_bind(target->vma, 1361 target->vma->obj->cache_level, 1362 PIN_GLOBAL, NULL); 1363 if (err) 1364 return err; 1365 } 1366 } 1367 1368 /* 1369 * If the relocation already has the right value in it, no 1370 * more work needs to be done. 1371 */ 1372 if (!DBG_FORCE_RELOC && 1373 gen8_canonical_addr(target->vma->node.start) == reloc->presumed_offset) 1374 return 0; 1375 1376 /* Check that the relocation address is valid... */ 1377 if (unlikely(reloc->offset > 1378 ev->vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) { 1379 drm_dbg(&i915->drm, "Relocation beyond object bounds: " 1380 "target %d offset %d size %d.\n", 1381 reloc->target_handle, 1382 (int)reloc->offset, 1383 (int)ev->vma->size); 1384 return -EINVAL; 1385 } 1386 if (unlikely(reloc->offset & 3)) { 1387 drm_dbg(&i915->drm, "Relocation not 4-byte aligned: " 1388 "target %d offset %d.\n", 1389 reloc->target_handle, 1390 (int)reloc->offset); 1391 return -EINVAL; 1392 } 1393 1394 /* 1395 * If we write into the object, we need to force the synchronisation 1396 * barrier, either with an asynchronous clflush or if we executed the 1397 * patching using the GPU (though that should be serialised by the 1398 * timeline). To be completely sure, and since we are required to 1399 * do relocations we are already stalling, disable the user's opt 1400 * out of our synchronisation. 1401 */ 1402 ev->flags &= ~EXEC_OBJECT_ASYNC; 1403 1404 /* and update the user's relocation entry */ 1405 return relocate_entry(ev->vma, reloc, eb, target->vma); 1406 } 1407 1408 static int eb_relocate_vma(struct i915_execbuffer *eb, struct eb_vma *ev) 1409 { 1410 #define N_RELOC(x) ((x) / sizeof(struct drm_i915_gem_relocation_entry)) 1411 struct drm_i915_gem_relocation_entry stack[N_RELOC(512)]; 1412 const struct drm_i915_gem_exec_object2 *entry = ev->exec; 1413 struct drm_i915_gem_relocation_entry __user *urelocs = 1414 u64_to_user_ptr(entry->relocs_ptr); 1415 unsigned long remain = entry->relocation_count; 1416 1417 if (unlikely(remain > N_RELOC(ULONG_MAX))) 1418 return -EINVAL; 1419 1420 /* 1421 * We must check that the entire relocation array is safe 1422 * to read. However, if the array is not writable the user loses 1423 * the updated relocation values. 1424 */ 1425 if (unlikely(!access_ok(urelocs, remain * sizeof(*urelocs)))) 1426 return -EFAULT; 1427 1428 do { 1429 struct drm_i915_gem_relocation_entry *r = stack; 1430 unsigned int count = 1431 min_t(unsigned long, remain, ARRAY_SIZE(stack)); 1432 unsigned int copied; 1433 1434 /* 1435 * This is the fast path and we cannot handle a pagefault 1436 * whilst holding the struct mutex lest the user pass in the 1437 * relocations contained within a mmaped bo. For in such a case 1438 * we, the page fault handler would call i915_gem_fault() and 1439 * we would try to acquire the struct mutex again. Obviously 1440 * this is bad and so lockdep complains vehemently. 1441 */ 1442 pagefault_disable(); 1443 copied = __copy_from_user_inatomic(r, urelocs, count * sizeof(r[0])); 1444 pagefault_enable(); 1445 if (unlikely(copied)) { 1446 remain = -EFAULT; 1447 goto out; 1448 } 1449 1450 remain -= count; 1451 do { 1452 u64 offset = eb_relocate_entry(eb, ev, r); 1453 1454 if (likely(offset == 0)) { 1455 } else if ((s64)offset < 0) { 1456 remain = (int)offset; 1457 goto out; 1458 } else { 1459 /* 1460 * Note that reporting an error now 1461 * leaves everything in an inconsistent 1462 * state as we have *already* changed 1463 * the relocation value inside the 1464 * object. As we have not changed the 1465 * reloc.presumed_offset or will not 1466 * change the execobject.offset, on the 1467 * call we may not rewrite the value 1468 * inside the object, leaving it 1469 * dangling and causing a GPU hang. Unless 1470 * userspace dynamically rebuilds the 1471 * relocations on each execbuf rather than 1472 * presume a static tree. 1473 * 1474 * We did previously check if the relocations 1475 * were writable (access_ok), an error now 1476 * would be a strange race with mprotect, 1477 * having already demonstrated that we 1478 * can read from this userspace address. 1479 */ 1480 offset = gen8_canonical_addr(offset & ~UPDATE); 1481 __put_user(offset, 1482 &urelocs[r - stack].presumed_offset); 1483 } 1484 } while (r++, --count); 1485 urelocs += ARRAY_SIZE(stack); 1486 } while (remain); 1487 out: 1488 reloc_cache_reset(&eb->reloc_cache, eb); 1489 return remain; 1490 } 1491 1492 static int 1493 eb_relocate_vma_slow(struct i915_execbuffer *eb, struct eb_vma *ev) 1494 { 1495 const struct drm_i915_gem_exec_object2 *entry = ev->exec; 1496 struct drm_i915_gem_relocation_entry *relocs = 1497 u64_to_ptr(typeof(*relocs), entry->relocs_ptr); 1498 unsigned int i; 1499 int err; 1500 1501 for (i = 0; i < entry->relocation_count; i++) { 1502 u64 offset = eb_relocate_entry(eb, ev, &relocs[i]); 1503 1504 if ((s64)offset < 0) { 1505 err = (int)offset; 1506 goto err; 1507 } 1508 } 1509 err = 0; 1510 err: 1511 reloc_cache_reset(&eb->reloc_cache, eb); 1512 return err; 1513 } 1514 1515 static int check_relocations(const struct drm_i915_gem_exec_object2 *entry) 1516 { 1517 const char __user *addr, *end; 1518 unsigned long size; 1519 char __maybe_unused c; 1520 1521 size = entry->relocation_count; 1522 if (size == 0) 1523 return 0; 1524 1525 if (size > N_RELOC(ULONG_MAX)) 1526 return -EINVAL; 1527 1528 addr = u64_to_user_ptr(entry->relocs_ptr); 1529 size *= sizeof(struct drm_i915_gem_relocation_entry); 1530 if (!access_ok(addr, size)) 1531 return -EFAULT; 1532 1533 end = addr + size; 1534 for (; addr < end; addr += PAGE_SIZE) { 1535 int err = __get_user(c, addr); 1536 if (err) 1537 return err; 1538 } 1539 return __get_user(c, end - 1); 1540 } 1541 1542 static int eb_copy_relocations(const struct i915_execbuffer *eb) 1543 { 1544 struct drm_i915_gem_relocation_entry *relocs; 1545 const unsigned int count = eb->buffer_count; 1546 unsigned int i; 1547 int err; 1548 1549 for (i = 0; i < count; i++) { 1550 const unsigned int nreloc = eb->exec[i].relocation_count; 1551 struct drm_i915_gem_relocation_entry __user *urelocs; 1552 unsigned long size; 1553 unsigned long copied; 1554 1555 if (nreloc == 0) 1556 continue; 1557 1558 err = check_relocations(&eb->exec[i]); 1559 if (err) 1560 goto err; 1561 1562 urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr); 1563 size = nreloc * sizeof(*relocs); 1564 1565 relocs = kvmalloc_array(size, 1, GFP_KERNEL); 1566 if (!relocs) { 1567 err = -ENOMEM; 1568 goto err; 1569 } 1570 1571 /* copy_from_user is limited to < 4GiB */ 1572 copied = 0; 1573 do { 1574 unsigned int len = 1575 min_t(u64, BIT_ULL(31), size - copied); 1576 1577 if (__copy_from_user((char *)relocs + copied, 1578 (char __user *)urelocs + copied, 1579 len)) 1580 goto end; 1581 1582 copied += len; 1583 } while (copied < size); 1584 1585 /* 1586 * As we do not update the known relocation offsets after 1587 * relocating (due to the complexities in lock handling), 1588 * we need to mark them as invalid now so that we force the 1589 * relocation processing next time. Just in case the target 1590 * object is evicted and then rebound into its old 1591 * presumed_offset before the next execbuffer - if that 1592 * happened we would make the mistake of assuming that the 1593 * relocations were valid. 1594 */ 1595 if (!user_access_begin(urelocs, size)) 1596 goto end; 1597 1598 for (copied = 0; copied < nreloc; copied++) 1599 unsafe_put_user(-1, 1600 &urelocs[copied].presumed_offset, 1601 end_user); 1602 user_access_end(); 1603 1604 eb->exec[i].relocs_ptr = (uintptr_t)relocs; 1605 } 1606 1607 return 0; 1608 1609 end_user: 1610 user_access_end(); 1611 end: 1612 kvfree(relocs); 1613 err = -EFAULT; 1614 err: 1615 while (i--) { 1616 relocs = u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr); 1617 if (eb->exec[i].relocation_count) 1618 kvfree(relocs); 1619 } 1620 return err; 1621 } 1622 1623 static int eb_prefault_relocations(const struct i915_execbuffer *eb) 1624 { 1625 const unsigned int count = eb->buffer_count; 1626 unsigned int i; 1627 1628 for (i = 0; i < count; i++) { 1629 int err; 1630 1631 err = check_relocations(&eb->exec[i]); 1632 if (err) 1633 return err; 1634 } 1635 1636 return 0; 1637 } 1638 1639 static int eb_reinit_userptr(struct i915_execbuffer *eb) 1640 { 1641 const unsigned int count = eb->buffer_count; 1642 unsigned int i; 1643 int ret; 1644 1645 if (likely(!(eb->args->flags & __EXEC_USERPTR_USED))) 1646 return 0; 1647 1648 for (i = 0; i < count; i++) { 1649 struct eb_vma *ev = &eb->vma[i]; 1650 1651 if (!i915_gem_object_is_userptr(ev->vma->obj)) 1652 continue; 1653 1654 ret = i915_gem_object_userptr_submit_init(ev->vma->obj); 1655 if (ret) 1656 return ret; 1657 1658 ev->flags |= __EXEC_OBJECT_USERPTR_INIT; 1659 } 1660 1661 return 0; 1662 } 1663 1664 static noinline int eb_relocate_parse_slow(struct i915_execbuffer *eb) 1665 { 1666 bool have_copy = false; 1667 struct eb_vma *ev; 1668 int err = 0; 1669 1670 repeat: 1671 if (signal_pending(current)) { 1672 err = -ERESTARTSYS; 1673 goto out; 1674 } 1675 1676 /* We may process another execbuffer during the unlock... */ 1677 eb_release_vmas(eb, false); 1678 i915_gem_ww_ctx_fini(&eb->ww); 1679 1680 /* 1681 * We take 3 passes through the slowpatch. 1682 * 1683 * 1 - we try to just prefault all the user relocation entries and 1684 * then attempt to reuse the atomic pagefault disabled fast path again. 1685 * 1686 * 2 - we copy the user entries to a local buffer here outside of the 1687 * local and allow ourselves to wait upon any rendering before 1688 * relocations 1689 * 1690 * 3 - we already have a local copy of the relocation entries, but 1691 * were interrupted (EAGAIN) whilst waiting for the objects, try again. 1692 */ 1693 if (!err) { 1694 err = eb_prefault_relocations(eb); 1695 } else if (!have_copy) { 1696 err = eb_copy_relocations(eb); 1697 have_copy = err == 0; 1698 } else { 1699 cond_resched(); 1700 err = 0; 1701 } 1702 1703 if (!err) 1704 err = eb_reinit_userptr(eb); 1705 1706 i915_gem_ww_ctx_init(&eb->ww, true); 1707 if (err) 1708 goto out; 1709 1710 /* reacquire the objects */ 1711 repeat_validate: 1712 err = eb_pin_engine(eb, false); 1713 if (err) 1714 goto err; 1715 1716 err = eb_validate_vmas(eb); 1717 if (err) 1718 goto err; 1719 1720 GEM_BUG_ON(!eb->batches[0]); 1721 1722 list_for_each_entry(ev, &eb->relocs, reloc_link) { 1723 if (!have_copy) { 1724 err = eb_relocate_vma(eb, ev); 1725 if (err) 1726 break; 1727 } else { 1728 err = eb_relocate_vma_slow(eb, ev); 1729 if (err) 1730 break; 1731 } 1732 } 1733 1734 if (err == -EDEADLK) 1735 goto err; 1736 1737 if (err && !have_copy) 1738 goto repeat; 1739 1740 if (err) 1741 goto err; 1742 1743 /* as last step, parse the command buffer */ 1744 err = eb_parse(eb); 1745 if (err) 1746 goto err; 1747 1748 /* 1749 * Leave the user relocations as are, this is the painfully slow path, 1750 * and we want to avoid the complication of dropping the lock whilst 1751 * having buffers reserved in the aperture and so causing spurious 1752 * ENOSPC for random operations. 1753 */ 1754 1755 err: 1756 if (err == -EDEADLK) { 1757 eb_release_vmas(eb, false); 1758 err = i915_gem_ww_ctx_backoff(&eb->ww); 1759 if (!err) 1760 goto repeat_validate; 1761 } 1762 1763 if (err == -EAGAIN) 1764 goto repeat; 1765 1766 out: 1767 if (have_copy) { 1768 const unsigned int count = eb->buffer_count; 1769 unsigned int i; 1770 1771 for (i = 0; i < count; i++) { 1772 const struct drm_i915_gem_exec_object2 *entry = 1773 &eb->exec[i]; 1774 struct drm_i915_gem_relocation_entry *relocs; 1775 1776 if (!entry->relocation_count) 1777 continue; 1778 1779 relocs = u64_to_ptr(typeof(*relocs), entry->relocs_ptr); 1780 kvfree(relocs); 1781 } 1782 } 1783 1784 return err; 1785 } 1786 1787 static int eb_relocate_parse(struct i915_execbuffer *eb) 1788 { 1789 int err; 1790 bool throttle = true; 1791 1792 retry: 1793 err = eb_pin_engine(eb, throttle); 1794 if (err) { 1795 if (err != -EDEADLK) 1796 return err; 1797 1798 goto err; 1799 } 1800 1801 /* only throttle once, even if we didn't need to throttle */ 1802 throttle = false; 1803 1804 err = eb_validate_vmas(eb); 1805 if (err == -EAGAIN) 1806 goto slow; 1807 else if (err) 1808 goto err; 1809 1810 /* The objects are in their final locations, apply the relocations. */ 1811 if (eb->args->flags & __EXEC_HAS_RELOC) { 1812 struct eb_vma *ev; 1813 1814 list_for_each_entry(ev, &eb->relocs, reloc_link) { 1815 err = eb_relocate_vma(eb, ev); 1816 if (err) 1817 break; 1818 } 1819 1820 if (err == -EDEADLK) 1821 goto err; 1822 else if (err) 1823 goto slow; 1824 } 1825 1826 if (!err) 1827 err = eb_parse(eb); 1828 1829 err: 1830 if (err == -EDEADLK) { 1831 eb_release_vmas(eb, false); 1832 err = i915_gem_ww_ctx_backoff(&eb->ww); 1833 if (!err) 1834 goto retry; 1835 } 1836 1837 return err; 1838 1839 slow: 1840 err = eb_relocate_parse_slow(eb); 1841 if (err) 1842 /* 1843 * If the user expects the execobject.offset and 1844 * reloc.presumed_offset to be an exact match, 1845 * as for using NO_RELOC, then we cannot update 1846 * the execobject.offset until we have completed 1847 * relocation. 1848 */ 1849 eb->args->flags &= ~__EXEC_HAS_RELOC; 1850 1851 return err; 1852 } 1853 1854 /* 1855 * Using two helper loops for the order of which requests / batches are created 1856 * and added the to backend. Requests are created in order from the parent to 1857 * the last child. Requests are added in the reverse order, from the last child 1858 * to parent. This is done for locking reasons as the timeline lock is acquired 1859 * during request creation and released when the request is added to the 1860 * backend. To make lockdep happy (see intel_context_timeline_lock) this must be 1861 * the ordering. 1862 */ 1863 #define for_each_batch_create_order(_eb, _i) \ 1864 for ((_i) = 0; (_i) < (_eb)->num_batches; ++(_i)) 1865 #define for_each_batch_add_order(_eb, _i) \ 1866 BUILD_BUG_ON(!typecheck(int, _i)); \ 1867 for ((_i) = (_eb)->num_batches - 1; (_i) >= 0; --(_i)) 1868 1869 static struct i915_request * 1870 eb_find_first_request_added(struct i915_execbuffer *eb) 1871 { 1872 int i; 1873 1874 for_each_batch_add_order(eb, i) 1875 if (eb->requests[i]) 1876 return eb->requests[i]; 1877 1878 GEM_BUG_ON("Request not found"); 1879 1880 return NULL; 1881 } 1882 1883 static int eb_move_to_gpu(struct i915_execbuffer *eb) 1884 { 1885 const unsigned int count = eb->buffer_count; 1886 unsigned int i = count; 1887 int err = 0, j; 1888 1889 while (i--) { 1890 struct eb_vma *ev = &eb->vma[i]; 1891 struct i915_vma *vma = ev->vma; 1892 unsigned int flags = ev->flags; 1893 struct drm_i915_gem_object *obj = vma->obj; 1894 1895 assert_vma_held(vma); 1896 1897 if (flags & EXEC_OBJECT_CAPTURE) { 1898 struct i915_capture_list *capture; 1899 1900 for_each_batch_create_order(eb, j) { 1901 if (!eb->requests[j]) 1902 break; 1903 1904 capture = kmalloc(sizeof(*capture), GFP_KERNEL); 1905 if (capture) { 1906 capture->next = 1907 eb->requests[j]->capture_list; 1908 capture->vma = vma; 1909 eb->requests[j]->capture_list = capture; 1910 } 1911 } 1912 } 1913 1914 /* 1915 * If the GPU is not _reading_ through the CPU cache, we need 1916 * to make sure that any writes (both previous GPU writes from 1917 * before a change in snooping levels and normal CPU writes) 1918 * caught in that cache are flushed to main memory. 1919 * 1920 * We want to say 1921 * obj->cache_dirty && 1922 * !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ) 1923 * but gcc's optimiser doesn't handle that as well and emits 1924 * two jumps instead of one. Maybe one day... 1925 */ 1926 if (unlikely(obj->cache_dirty & ~obj->cache_coherent)) { 1927 if (i915_gem_clflush_object(obj, 0)) 1928 flags &= ~EXEC_OBJECT_ASYNC; 1929 } 1930 1931 /* We only need to await on the first request */ 1932 if (err == 0 && !(flags & EXEC_OBJECT_ASYNC)) { 1933 err = i915_request_await_object 1934 (eb_find_first_request_added(eb), obj, 1935 flags & EXEC_OBJECT_WRITE); 1936 } 1937 1938 for_each_batch_add_order(eb, j) { 1939 if (err) 1940 break; 1941 if (!eb->requests[j]) 1942 continue; 1943 1944 err = _i915_vma_move_to_active(vma, eb->requests[j], 1945 j ? NULL : 1946 eb->composite_fence ? 1947 eb->composite_fence : 1948 &eb->requests[j]->fence, 1949 flags | __EXEC_OBJECT_NO_RESERVE); 1950 } 1951 } 1952 1953 #ifdef CONFIG_MMU_NOTIFIER 1954 if (!err && (eb->args->flags & __EXEC_USERPTR_USED)) { 1955 read_lock(&eb->i915->mm.notifier_lock); 1956 1957 /* 1958 * count is always at least 1, otherwise __EXEC_USERPTR_USED 1959 * could not have been set 1960 */ 1961 for (i = 0; i < count; i++) { 1962 struct eb_vma *ev = &eb->vma[i]; 1963 struct drm_i915_gem_object *obj = ev->vma->obj; 1964 1965 if (!i915_gem_object_is_userptr(obj)) 1966 continue; 1967 1968 err = i915_gem_object_userptr_submit_done(obj); 1969 if (err) 1970 break; 1971 } 1972 1973 read_unlock(&eb->i915->mm.notifier_lock); 1974 } 1975 #endif 1976 1977 if (unlikely(err)) 1978 goto err_skip; 1979 1980 /* Unconditionally flush any chipset caches (for streaming writes). */ 1981 intel_gt_chipset_flush(eb->gt); 1982 return 0; 1983 1984 err_skip: 1985 for_each_batch_create_order(eb, j) { 1986 if (!eb->requests[j]) 1987 break; 1988 1989 i915_request_set_error_once(eb->requests[j], err); 1990 } 1991 return err; 1992 } 1993 1994 static int i915_gem_check_execbuffer(struct drm_i915_gem_execbuffer2 *exec) 1995 { 1996 if (exec->flags & __I915_EXEC_ILLEGAL_FLAGS) 1997 return -EINVAL; 1998 1999 /* Kernel clipping was a DRI1 misfeature */ 2000 if (!(exec->flags & (I915_EXEC_FENCE_ARRAY | 2001 I915_EXEC_USE_EXTENSIONS))) { 2002 if (exec->num_cliprects || exec->cliprects_ptr) 2003 return -EINVAL; 2004 } 2005 2006 if (exec->DR4 == 0xffffffff) { 2007 DRM_DEBUG("UXA submitting garbage DR4, fixing up\n"); 2008 exec->DR4 = 0; 2009 } 2010 if (exec->DR1 || exec->DR4) 2011 return -EINVAL; 2012 2013 if ((exec->batch_start_offset | exec->batch_len) & 0x7) 2014 return -EINVAL; 2015 2016 return 0; 2017 } 2018 2019 static int i915_reset_gen7_sol_offsets(struct i915_request *rq) 2020 { 2021 u32 *cs; 2022 int i; 2023 2024 if (GRAPHICS_VER(rq->engine->i915) != 7 || rq->engine->id != RCS0) { 2025 drm_dbg(&rq->engine->i915->drm, "sol reset is gen7/rcs only\n"); 2026 return -EINVAL; 2027 } 2028 2029 cs = intel_ring_begin(rq, 4 * 2 + 2); 2030 if (IS_ERR(cs)) 2031 return PTR_ERR(cs); 2032 2033 *cs++ = MI_LOAD_REGISTER_IMM(4); 2034 for (i = 0; i < 4; i++) { 2035 *cs++ = i915_mmio_reg_offset(GEN7_SO_WRITE_OFFSET(i)); 2036 *cs++ = 0; 2037 } 2038 *cs++ = MI_NOOP; 2039 intel_ring_advance(rq, cs); 2040 2041 return 0; 2042 } 2043 2044 static struct i915_vma * 2045 shadow_batch_pin(struct i915_execbuffer *eb, 2046 struct drm_i915_gem_object *obj, 2047 struct i915_address_space *vm, 2048 unsigned int flags) 2049 { 2050 struct i915_vma *vma; 2051 int err; 2052 2053 vma = i915_vma_instance(obj, vm, NULL); 2054 if (IS_ERR(vma)) 2055 return vma; 2056 2057 err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, flags); 2058 if (err) 2059 return ERR_PTR(err); 2060 2061 return vma; 2062 } 2063 2064 static struct i915_vma *eb_dispatch_secure(struct i915_execbuffer *eb, struct i915_vma *vma) 2065 { 2066 /* 2067 * snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure 2068 * batch" bit. Hence we need to pin secure batches into the global gtt. 2069 * hsw should have this fixed, but bdw mucks it up again. */ 2070 if (eb->batch_flags & I915_DISPATCH_SECURE) 2071 return i915_gem_object_ggtt_pin_ww(vma->obj, &eb->ww, NULL, 0, 0, 0); 2072 2073 return NULL; 2074 } 2075 2076 static int eb_parse(struct i915_execbuffer *eb) 2077 { 2078 struct drm_i915_private *i915 = eb->i915; 2079 struct intel_gt_buffer_pool_node *pool = eb->batch_pool; 2080 struct i915_vma *shadow, *trampoline, *batch; 2081 unsigned long len; 2082 int err; 2083 2084 if (!eb_use_cmdparser(eb)) { 2085 batch = eb_dispatch_secure(eb, eb->batches[0]->vma); 2086 if (IS_ERR(batch)) 2087 return PTR_ERR(batch); 2088 2089 goto secure_batch; 2090 } 2091 2092 if (intel_context_is_parallel(eb->context)) 2093 return -EINVAL; 2094 2095 len = eb->batch_len[0]; 2096 if (!CMDPARSER_USES_GGTT(eb->i915)) { 2097 /* 2098 * ppGTT backed shadow buffers must be mapped RO, to prevent 2099 * post-scan tampering 2100 */ 2101 if (!eb->context->vm->has_read_only) { 2102 drm_dbg(&i915->drm, 2103 "Cannot prevent post-scan tampering without RO capable vm\n"); 2104 return -EINVAL; 2105 } 2106 } else { 2107 len += I915_CMD_PARSER_TRAMPOLINE_SIZE; 2108 } 2109 if (unlikely(len < eb->batch_len[0])) /* last paranoid check of overflow */ 2110 return -EINVAL; 2111 2112 if (!pool) { 2113 pool = intel_gt_get_buffer_pool(eb->gt, len, 2114 I915_MAP_WB); 2115 if (IS_ERR(pool)) 2116 return PTR_ERR(pool); 2117 eb->batch_pool = pool; 2118 } 2119 2120 err = i915_gem_object_lock(pool->obj, &eb->ww); 2121 if (err) 2122 goto err; 2123 2124 shadow = shadow_batch_pin(eb, pool->obj, eb->context->vm, PIN_USER); 2125 if (IS_ERR(shadow)) { 2126 err = PTR_ERR(shadow); 2127 goto err; 2128 } 2129 intel_gt_buffer_pool_mark_used(pool); 2130 i915_gem_object_set_readonly(shadow->obj); 2131 shadow->private = pool; 2132 2133 trampoline = NULL; 2134 if (CMDPARSER_USES_GGTT(eb->i915)) { 2135 trampoline = shadow; 2136 2137 shadow = shadow_batch_pin(eb, pool->obj, 2138 &eb->gt->ggtt->vm, 2139 PIN_GLOBAL); 2140 if (IS_ERR(shadow)) { 2141 err = PTR_ERR(shadow); 2142 shadow = trampoline; 2143 goto err_shadow; 2144 } 2145 shadow->private = pool; 2146 2147 eb->batch_flags |= I915_DISPATCH_SECURE; 2148 } 2149 2150 batch = eb_dispatch_secure(eb, shadow); 2151 if (IS_ERR(batch)) { 2152 err = PTR_ERR(batch); 2153 goto err_trampoline; 2154 } 2155 2156 err = dma_resv_reserve_shared(shadow->resv, 1); 2157 if (err) 2158 goto err_trampoline; 2159 2160 err = intel_engine_cmd_parser(eb->context->engine, 2161 eb->batches[0]->vma, 2162 eb->batch_start_offset, 2163 eb->batch_len[0], 2164 shadow, trampoline); 2165 if (err) 2166 goto err_unpin_batch; 2167 2168 eb->batches[0] = &eb->vma[eb->buffer_count++]; 2169 eb->batches[0]->vma = i915_vma_get(shadow); 2170 eb->batches[0]->flags = __EXEC_OBJECT_HAS_PIN; 2171 2172 eb->trampoline = trampoline; 2173 eb->batch_start_offset = 0; 2174 2175 secure_batch: 2176 if (batch) { 2177 if (intel_context_is_parallel(eb->context)) 2178 return -EINVAL; 2179 2180 eb->batches[0] = &eb->vma[eb->buffer_count++]; 2181 eb->batches[0]->flags = __EXEC_OBJECT_HAS_PIN; 2182 eb->batches[0]->vma = i915_vma_get(batch); 2183 } 2184 return 0; 2185 2186 err_unpin_batch: 2187 if (batch) 2188 i915_vma_unpin(batch); 2189 err_trampoline: 2190 if (trampoline) 2191 i915_vma_unpin(trampoline); 2192 err_shadow: 2193 i915_vma_unpin(shadow); 2194 err: 2195 return err; 2196 } 2197 2198 static int eb_request_submit(struct i915_execbuffer *eb, 2199 struct i915_request *rq, 2200 struct i915_vma *batch, 2201 u64 batch_len) 2202 { 2203 int err; 2204 2205 if (intel_context_nopreempt(rq->context)) 2206 __set_bit(I915_FENCE_FLAG_NOPREEMPT, &rq->fence.flags); 2207 2208 if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) { 2209 err = i915_reset_gen7_sol_offsets(rq); 2210 if (err) 2211 return err; 2212 } 2213 2214 /* 2215 * After we completed waiting for other engines (using HW semaphores) 2216 * then we can signal that this request/batch is ready to run. This 2217 * allows us to determine if the batch is still waiting on the GPU 2218 * or actually running by checking the breadcrumb. 2219 */ 2220 if (rq->context->engine->emit_init_breadcrumb) { 2221 err = rq->context->engine->emit_init_breadcrumb(rq); 2222 if (err) 2223 return err; 2224 } 2225 2226 err = rq->context->engine->emit_bb_start(rq, 2227 batch->node.start + 2228 eb->batch_start_offset, 2229 batch_len, 2230 eb->batch_flags); 2231 if (err) 2232 return err; 2233 2234 if (eb->trampoline) { 2235 GEM_BUG_ON(intel_context_is_parallel(rq->context)); 2236 GEM_BUG_ON(eb->batch_start_offset); 2237 err = rq->context->engine->emit_bb_start(rq, 2238 eb->trampoline->node.start + 2239 batch_len, 0, 0); 2240 if (err) 2241 return err; 2242 } 2243 2244 return 0; 2245 } 2246 2247 static int eb_submit(struct i915_execbuffer *eb) 2248 { 2249 unsigned int i; 2250 int err; 2251 2252 err = eb_move_to_gpu(eb); 2253 2254 for_each_batch_create_order(eb, i) { 2255 if (!eb->requests[i]) 2256 break; 2257 2258 trace_i915_request_queue(eb->requests[i], eb->batch_flags); 2259 if (!err) 2260 err = eb_request_submit(eb, eb->requests[i], 2261 eb->batches[i]->vma, 2262 eb->batch_len[i]); 2263 } 2264 2265 return err; 2266 } 2267 2268 static int num_vcs_engines(const struct drm_i915_private *i915) 2269 { 2270 return hweight_long(VDBOX_MASK(&i915->gt)); 2271 } 2272 2273 /* 2274 * Find one BSD ring to dispatch the corresponding BSD command. 2275 * The engine index is returned. 2276 */ 2277 static unsigned int 2278 gen8_dispatch_bsd_engine(struct drm_i915_private *dev_priv, 2279 struct drm_file *file) 2280 { 2281 struct drm_i915_file_private *file_priv = file->driver_priv; 2282 2283 /* Check whether the file_priv has already selected one ring. */ 2284 if ((int)file_priv->bsd_engine < 0) 2285 file_priv->bsd_engine = 2286 get_random_int() % num_vcs_engines(dev_priv); 2287 2288 return file_priv->bsd_engine; 2289 } 2290 2291 static const enum intel_engine_id user_ring_map[] = { 2292 [I915_EXEC_DEFAULT] = RCS0, 2293 [I915_EXEC_RENDER] = RCS0, 2294 [I915_EXEC_BLT] = BCS0, 2295 [I915_EXEC_BSD] = VCS0, 2296 [I915_EXEC_VEBOX] = VECS0 2297 }; 2298 2299 static struct i915_request *eb_throttle(struct i915_execbuffer *eb, struct intel_context *ce) 2300 { 2301 struct intel_ring *ring = ce->ring; 2302 struct intel_timeline *tl = ce->timeline; 2303 struct i915_request *rq; 2304 2305 /* 2306 * Completely unscientific finger-in-the-air estimates for suitable 2307 * maximum user request size (to avoid blocking) and then backoff. 2308 */ 2309 if (intel_ring_update_space(ring) >= PAGE_SIZE) 2310 return NULL; 2311 2312 /* 2313 * Find a request that after waiting upon, there will be at least half 2314 * the ring available. The hysteresis allows us to compete for the 2315 * shared ring and should mean that we sleep less often prior to 2316 * claiming our resources, but not so long that the ring completely 2317 * drains before we can submit our next request. 2318 */ 2319 list_for_each_entry(rq, &tl->requests, link) { 2320 if (rq->ring != ring) 2321 continue; 2322 2323 if (__intel_ring_space(rq->postfix, 2324 ring->emit, ring->size) > ring->size / 2) 2325 break; 2326 } 2327 if (&rq->link == &tl->requests) 2328 return NULL; /* weird, we will check again later for real */ 2329 2330 return i915_request_get(rq); 2331 } 2332 2333 static int eb_pin_timeline(struct i915_execbuffer *eb, struct intel_context *ce, 2334 bool throttle) 2335 { 2336 struct intel_timeline *tl; 2337 struct i915_request *rq = NULL; 2338 2339 /* 2340 * Take a local wakeref for preparing to dispatch the execbuf as 2341 * we expect to access the hardware fairly frequently in the 2342 * process, and require the engine to be kept awake between accesses. 2343 * Upon dispatch, we acquire another prolonged wakeref that we hold 2344 * until the timeline is idle, which in turn releases the wakeref 2345 * taken on the engine, and the parent device. 2346 */ 2347 tl = intel_context_timeline_lock(ce); 2348 if (IS_ERR(tl)) 2349 return PTR_ERR(tl); 2350 2351 intel_context_enter(ce); 2352 if (throttle) 2353 rq = eb_throttle(eb, ce); 2354 intel_context_timeline_unlock(tl); 2355 2356 if (rq) { 2357 bool nonblock = eb->file->filp->f_flags & O_NONBLOCK; 2358 long timeout = nonblock ? 0 : MAX_SCHEDULE_TIMEOUT; 2359 2360 if (i915_request_wait(rq, I915_WAIT_INTERRUPTIBLE, 2361 timeout) < 0) { 2362 i915_request_put(rq); 2363 2364 tl = intel_context_timeline_lock(ce); 2365 intel_context_exit(ce); 2366 intel_context_timeline_unlock(tl); 2367 2368 if (nonblock) 2369 return -EWOULDBLOCK; 2370 else 2371 return -EINTR; 2372 } 2373 i915_request_put(rq); 2374 } 2375 2376 return 0; 2377 } 2378 2379 static int eb_pin_engine(struct i915_execbuffer *eb, bool throttle) 2380 { 2381 struct intel_context *ce = eb->context, *child; 2382 int err; 2383 int i = 0, j = 0; 2384 2385 GEM_BUG_ON(eb->args->flags & __EXEC_ENGINE_PINNED); 2386 2387 if (unlikely(intel_context_is_banned(ce))) 2388 return -EIO; 2389 2390 /* 2391 * Pinning the contexts may generate requests in order to acquire 2392 * GGTT space, so do this first before we reserve a seqno for 2393 * ourselves. 2394 */ 2395 err = intel_context_pin_ww(ce, &eb->ww); 2396 if (err) 2397 return err; 2398 for_each_child(ce, child) { 2399 err = intel_context_pin_ww(child, &eb->ww); 2400 GEM_BUG_ON(err); /* perma-pinned should incr a counter */ 2401 } 2402 2403 for_each_child(ce, child) { 2404 err = eb_pin_timeline(eb, child, throttle); 2405 if (err) 2406 goto unwind; 2407 ++i; 2408 } 2409 err = eb_pin_timeline(eb, ce, throttle); 2410 if (err) 2411 goto unwind; 2412 2413 eb->args->flags |= __EXEC_ENGINE_PINNED; 2414 return 0; 2415 2416 unwind: 2417 for_each_child(ce, child) { 2418 if (j++ < i) { 2419 mutex_lock(&child->timeline->mutex); 2420 intel_context_exit(child); 2421 mutex_unlock(&child->timeline->mutex); 2422 } 2423 } 2424 for_each_child(ce, child) 2425 intel_context_unpin(child); 2426 intel_context_unpin(ce); 2427 return err; 2428 } 2429 2430 static void eb_unpin_engine(struct i915_execbuffer *eb) 2431 { 2432 struct intel_context *ce = eb->context, *child; 2433 2434 if (!(eb->args->flags & __EXEC_ENGINE_PINNED)) 2435 return; 2436 2437 eb->args->flags &= ~__EXEC_ENGINE_PINNED; 2438 2439 for_each_child(ce, child) { 2440 mutex_lock(&child->timeline->mutex); 2441 intel_context_exit(child); 2442 mutex_unlock(&child->timeline->mutex); 2443 2444 intel_context_unpin(child); 2445 } 2446 2447 mutex_lock(&ce->timeline->mutex); 2448 intel_context_exit(ce); 2449 mutex_unlock(&ce->timeline->mutex); 2450 2451 intel_context_unpin(ce); 2452 } 2453 2454 static unsigned int 2455 eb_select_legacy_ring(struct i915_execbuffer *eb) 2456 { 2457 struct drm_i915_private *i915 = eb->i915; 2458 struct drm_i915_gem_execbuffer2 *args = eb->args; 2459 unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK; 2460 2461 if (user_ring_id != I915_EXEC_BSD && 2462 (args->flags & I915_EXEC_BSD_MASK)) { 2463 drm_dbg(&i915->drm, 2464 "execbuf with non bsd ring but with invalid " 2465 "bsd dispatch flags: %d\n", (int)(args->flags)); 2466 return -1; 2467 } 2468 2469 if (user_ring_id == I915_EXEC_BSD && num_vcs_engines(i915) > 1) { 2470 unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK; 2471 2472 if (bsd_idx == I915_EXEC_BSD_DEFAULT) { 2473 bsd_idx = gen8_dispatch_bsd_engine(i915, eb->file); 2474 } else if (bsd_idx >= I915_EXEC_BSD_RING1 && 2475 bsd_idx <= I915_EXEC_BSD_RING2) { 2476 bsd_idx >>= I915_EXEC_BSD_SHIFT; 2477 bsd_idx--; 2478 } else { 2479 drm_dbg(&i915->drm, 2480 "execbuf with unknown bsd ring: %u\n", 2481 bsd_idx); 2482 return -1; 2483 } 2484 2485 return _VCS(bsd_idx); 2486 } 2487 2488 if (user_ring_id >= ARRAY_SIZE(user_ring_map)) { 2489 drm_dbg(&i915->drm, "execbuf with unknown ring: %u\n", 2490 user_ring_id); 2491 return -1; 2492 } 2493 2494 return user_ring_map[user_ring_id]; 2495 } 2496 2497 static int 2498 eb_select_engine(struct i915_execbuffer *eb) 2499 { 2500 struct intel_context *ce, *child; 2501 unsigned int idx; 2502 int err; 2503 2504 if (i915_gem_context_user_engines(eb->gem_context)) 2505 idx = eb->args->flags & I915_EXEC_RING_MASK; 2506 else 2507 idx = eb_select_legacy_ring(eb); 2508 2509 ce = i915_gem_context_get_engine(eb->gem_context, idx); 2510 if (IS_ERR(ce)) 2511 return PTR_ERR(ce); 2512 2513 if (intel_context_is_parallel(ce)) { 2514 if (eb->buffer_count < ce->parallel.number_children + 1) { 2515 intel_context_put(ce); 2516 return -EINVAL; 2517 } 2518 if (eb->batch_start_offset || eb->args->batch_len) { 2519 intel_context_put(ce); 2520 return -EINVAL; 2521 } 2522 } 2523 eb->num_batches = ce->parallel.number_children + 1; 2524 2525 for_each_child(ce, child) 2526 intel_context_get(child); 2527 intel_gt_pm_get(ce->engine->gt); 2528 2529 if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) { 2530 err = intel_context_alloc_state(ce); 2531 if (err) 2532 goto err; 2533 } 2534 for_each_child(ce, child) { 2535 if (!test_bit(CONTEXT_ALLOC_BIT, &child->flags)) { 2536 err = intel_context_alloc_state(child); 2537 if (err) 2538 goto err; 2539 } 2540 } 2541 2542 /* 2543 * ABI: Before userspace accesses the GPU (e.g. execbuffer), report 2544 * EIO if the GPU is already wedged. 2545 */ 2546 err = intel_gt_terminally_wedged(ce->engine->gt); 2547 if (err) 2548 goto err; 2549 2550 eb->context = ce; 2551 eb->gt = ce->engine->gt; 2552 2553 /* 2554 * Make sure engine pool stays alive even if we call intel_context_put 2555 * during ww handling. The pool is destroyed when last pm reference 2556 * is dropped, which breaks our -EDEADLK handling. 2557 */ 2558 return err; 2559 2560 err: 2561 intel_gt_pm_put(ce->engine->gt); 2562 for_each_child(ce, child) 2563 intel_context_put(child); 2564 intel_context_put(ce); 2565 return err; 2566 } 2567 2568 static void 2569 eb_put_engine(struct i915_execbuffer *eb) 2570 { 2571 struct intel_context *child; 2572 2573 intel_gt_pm_put(eb->gt); 2574 for_each_child(eb->context, child) 2575 intel_context_put(child); 2576 intel_context_put(eb->context); 2577 } 2578 2579 static void 2580 __free_fence_array(struct eb_fence *fences, unsigned int n) 2581 { 2582 while (n--) { 2583 drm_syncobj_put(ptr_mask_bits(fences[n].syncobj, 2)); 2584 dma_fence_put(fences[n].dma_fence); 2585 dma_fence_chain_free(fences[n].chain_fence); 2586 } 2587 kvfree(fences); 2588 } 2589 2590 static int 2591 add_timeline_fence_array(struct i915_execbuffer *eb, 2592 const struct drm_i915_gem_execbuffer_ext_timeline_fences *timeline_fences) 2593 { 2594 struct drm_i915_gem_exec_fence __user *user_fences; 2595 u64 __user *user_values; 2596 struct eb_fence *f; 2597 u64 nfences; 2598 int err = 0; 2599 2600 nfences = timeline_fences->fence_count; 2601 if (!nfences) 2602 return 0; 2603 2604 /* Check multiplication overflow for access_ok() and kvmalloc_array() */ 2605 BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long)); 2606 if (nfences > min_t(unsigned long, 2607 ULONG_MAX / sizeof(*user_fences), 2608 SIZE_MAX / sizeof(*f)) - eb->num_fences) 2609 return -EINVAL; 2610 2611 user_fences = u64_to_user_ptr(timeline_fences->handles_ptr); 2612 if (!access_ok(user_fences, nfences * sizeof(*user_fences))) 2613 return -EFAULT; 2614 2615 user_values = u64_to_user_ptr(timeline_fences->values_ptr); 2616 if (!access_ok(user_values, nfences * sizeof(*user_values))) 2617 return -EFAULT; 2618 2619 f = krealloc(eb->fences, 2620 (eb->num_fences + nfences) * sizeof(*f), 2621 __GFP_NOWARN | GFP_KERNEL); 2622 if (!f) 2623 return -ENOMEM; 2624 2625 eb->fences = f; 2626 f += eb->num_fences; 2627 2628 BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) & 2629 ~__I915_EXEC_FENCE_UNKNOWN_FLAGS); 2630 2631 while (nfences--) { 2632 struct drm_i915_gem_exec_fence user_fence; 2633 struct drm_syncobj *syncobj; 2634 struct dma_fence *fence = NULL; 2635 u64 point; 2636 2637 if (__copy_from_user(&user_fence, 2638 user_fences++, 2639 sizeof(user_fence))) 2640 return -EFAULT; 2641 2642 if (user_fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS) 2643 return -EINVAL; 2644 2645 if (__get_user(point, user_values++)) 2646 return -EFAULT; 2647 2648 syncobj = drm_syncobj_find(eb->file, user_fence.handle); 2649 if (!syncobj) { 2650 DRM_DEBUG("Invalid syncobj handle provided\n"); 2651 return -ENOENT; 2652 } 2653 2654 fence = drm_syncobj_fence_get(syncobj); 2655 2656 if (!fence && user_fence.flags && 2657 !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) { 2658 DRM_DEBUG("Syncobj handle has no fence\n"); 2659 drm_syncobj_put(syncobj); 2660 return -EINVAL; 2661 } 2662 2663 if (fence) 2664 err = dma_fence_chain_find_seqno(&fence, point); 2665 2666 if (err && !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) { 2667 DRM_DEBUG("Syncobj handle missing requested point %llu\n", point); 2668 dma_fence_put(fence); 2669 drm_syncobj_put(syncobj); 2670 return err; 2671 } 2672 2673 /* 2674 * A point might have been signaled already and 2675 * garbage collected from the timeline. In this case 2676 * just ignore the point and carry on. 2677 */ 2678 if (!fence && !(user_fence.flags & I915_EXEC_FENCE_SIGNAL)) { 2679 drm_syncobj_put(syncobj); 2680 continue; 2681 } 2682 2683 /* 2684 * For timeline syncobjs we need to preallocate chains for 2685 * later signaling. 2686 */ 2687 if (point != 0 && user_fence.flags & I915_EXEC_FENCE_SIGNAL) { 2688 /* 2689 * Waiting and signaling the same point (when point != 2690 * 0) would break the timeline. 2691 */ 2692 if (user_fence.flags & I915_EXEC_FENCE_WAIT) { 2693 DRM_DEBUG("Trying to wait & signal the same timeline point.\n"); 2694 dma_fence_put(fence); 2695 drm_syncobj_put(syncobj); 2696 return -EINVAL; 2697 } 2698 2699 f->chain_fence = dma_fence_chain_alloc(); 2700 if (!f->chain_fence) { 2701 drm_syncobj_put(syncobj); 2702 dma_fence_put(fence); 2703 return -ENOMEM; 2704 } 2705 } else { 2706 f->chain_fence = NULL; 2707 } 2708 2709 f->syncobj = ptr_pack_bits(syncobj, user_fence.flags, 2); 2710 f->dma_fence = fence; 2711 f->value = point; 2712 f++; 2713 eb->num_fences++; 2714 } 2715 2716 return 0; 2717 } 2718 2719 static int add_fence_array(struct i915_execbuffer *eb) 2720 { 2721 struct drm_i915_gem_execbuffer2 *args = eb->args; 2722 struct drm_i915_gem_exec_fence __user *user; 2723 unsigned long num_fences = args->num_cliprects; 2724 struct eb_fence *f; 2725 2726 if (!(args->flags & I915_EXEC_FENCE_ARRAY)) 2727 return 0; 2728 2729 if (!num_fences) 2730 return 0; 2731 2732 /* Check multiplication overflow for access_ok() and kvmalloc_array() */ 2733 BUILD_BUG_ON(sizeof(size_t) > sizeof(unsigned long)); 2734 if (num_fences > min_t(unsigned long, 2735 ULONG_MAX / sizeof(*user), 2736 SIZE_MAX / sizeof(*f) - eb->num_fences)) 2737 return -EINVAL; 2738 2739 user = u64_to_user_ptr(args->cliprects_ptr); 2740 if (!access_ok(user, num_fences * sizeof(*user))) 2741 return -EFAULT; 2742 2743 f = krealloc(eb->fences, 2744 (eb->num_fences + num_fences) * sizeof(*f), 2745 __GFP_NOWARN | GFP_KERNEL); 2746 if (!f) 2747 return -ENOMEM; 2748 2749 eb->fences = f; 2750 f += eb->num_fences; 2751 while (num_fences--) { 2752 struct drm_i915_gem_exec_fence user_fence; 2753 struct drm_syncobj *syncobj; 2754 struct dma_fence *fence = NULL; 2755 2756 if (__copy_from_user(&user_fence, user++, sizeof(user_fence))) 2757 return -EFAULT; 2758 2759 if (user_fence.flags & __I915_EXEC_FENCE_UNKNOWN_FLAGS) 2760 return -EINVAL; 2761 2762 syncobj = drm_syncobj_find(eb->file, user_fence.handle); 2763 if (!syncobj) { 2764 DRM_DEBUG("Invalid syncobj handle provided\n"); 2765 return -ENOENT; 2766 } 2767 2768 if (user_fence.flags & I915_EXEC_FENCE_WAIT) { 2769 fence = drm_syncobj_fence_get(syncobj); 2770 if (!fence) { 2771 DRM_DEBUG("Syncobj handle has no fence\n"); 2772 drm_syncobj_put(syncobj); 2773 return -EINVAL; 2774 } 2775 } 2776 2777 BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) & 2778 ~__I915_EXEC_FENCE_UNKNOWN_FLAGS); 2779 2780 f->syncobj = ptr_pack_bits(syncobj, user_fence.flags, 2); 2781 f->dma_fence = fence; 2782 f->value = 0; 2783 f->chain_fence = NULL; 2784 f++; 2785 eb->num_fences++; 2786 } 2787 2788 return 0; 2789 } 2790 2791 static void put_fence_array(struct eb_fence *fences, int num_fences) 2792 { 2793 if (fences) 2794 __free_fence_array(fences, num_fences); 2795 } 2796 2797 static int 2798 await_fence_array(struct i915_execbuffer *eb, 2799 struct i915_request *rq) 2800 { 2801 unsigned int n; 2802 int err; 2803 2804 for (n = 0; n < eb->num_fences; n++) { 2805 struct drm_syncobj *syncobj; 2806 unsigned int flags; 2807 2808 syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2); 2809 2810 if (!eb->fences[n].dma_fence) 2811 continue; 2812 2813 err = i915_request_await_dma_fence(rq, eb->fences[n].dma_fence); 2814 if (err < 0) 2815 return err; 2816 } 2817 2818 return 0; 2819 } 2820 2821 static void signal_fence_array(const struct i915_execbuffer *eb, 2822 struct dma_fence * const fence) 2823 { 2824 unsigned int n; 2825 2826 for (n = 0; n < eb->num_fences; n++) { 2827 struct drm_syncobj *syncobj; 2828 unsigned int flags; 2829 2830 syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2); 2831 if (!(flags & I915_EXEC_FENCE_SIGNAL)) 2832 continue; 2833 2834 if (eb->fences[n].chain_fence) { 2835 drm_syncobj_add_point(syncobj, 2836 eb->fences[n].chain_fence, 2837 fence, 2838 eb->fences[n].value); 2839 /* 2840 * The chain's ownership is transferred to the 2841 * timeline. 2842 */ 2843 eb->fences[n].chain_fence = NULL; 2844 } else { 2845 drm_syncobj_replace_fence(syncobj, fence); 2846 } 2847 } 2848 } 2849 2850 static int 2851 parse_timeline_fences(struct i915_user_extension __user *ext, void *data) 2852 { 2853 struct i915_execbuffer *eb = data; 2854 struct drm_i915_gem_execbuffer_ext_timeline_fences timeline_fences; 2855 2856 if (copy_from_user(&timeline_fences, ext, sizeof(timeline_fences))) 2857 return -EFAULT; 2858 2859 return add_timeline_fence_array(eb, &timeline_fences); 2860 } 2861 2862 static void retire_requests(struct intel_timeline *tl, struct i915_request *end) 2863 { 2864 struct i915_request *rq, *rn; 2865 2866 list_for_each_entry_safe(rq, rn, &tl->requests, link) 2867 if (rq == end || !i915_request_retire(rq)) 2868 break; 2869 } 2870 2871 static int eb_request_add(struct i915_execbuffer *eb, struct i915_request *rq, 2872 int err, bool last_parallel) 2873 { 2874 struct intel_timeline * const tl = i915_request_timeline(rq); 2875 struct i915_sched_attr attr = {}; 2876 struct i915_request *prev; 2877 2878 lockdep_assert_held(&tl->mutex); 2879 lockdep_unpin_lock(&tl->mutex, rq->cookie); 2880 2881 trace_i915_request_add(rq); 2882 2883 prev = __i915_request_commit(rq); 2884 2885 /* Check that the context wasn't destroyed before submission */ 2886 if (likely(!intel_context_is_closed(eb->context))) { 2887 attr = eb->gem_context->sched; 2888 } else { 2889 /* Serialise with context_close via the add_to_timeline */ 2890 i915_request_set_error_once(rq, -ENOENT); 2891 __i915_request_skip(rq); 2892 err = -ENOENT; /* override any transient errors */ 2893 } 2894 2895 if (intel_context_is_parallel(eb->context)) { 2896 if (err) { 2897 __i915_request_skip(rq); 2898 set_bit(I915_FENCE_FLAG_SKIP_PARALLEL, 2899 &rq->fence.flags); 2900 } 2901 if (last_parallel) 2902 set_bit(I915_FENCE_FLAG_SUBMIT_PARALLEL, 2903 &rq->fence.flags); 2904 } 2905 2906 __i915_request_queue(rq, &attr); 2907 2908 /* Try to clean up the client's timeline after submitting the request */ 2909 if (prev) 2910 retire_requests(tl, prev); 2911 2912 mutex_unlock(&tl->mutex); 2913 2914 return err; 2915 } 2916 2917 static int eb_requests_add(struct i915_execbuffer *eb, int err) 2918 { 2919 int i; 2920 2921 /* 2922 * We iterate in reverse order of creation to release timeline mutexes in 2923 * same order. 2924 */ 2925 for_each_batch_add_order(eb, i) { 2926 struct i915_request *rq = eb->requests[i]; 2927 2928 if (!rq) 2929 continue; 2930 err |= eb_request_add(eb, rq, err, i == 0); 2931 } 2932 2933 return err; 2934 } 2935 2936 static const i915_user_extension_fn execbuf_extensions[] = { 2937 [DRM_I915_GEM_EXECBUFFER_EXT_TIMELINE_FENCES] = parse_timeline_fences, 2938 }; 2939 2940 static int 2941 parse_execbuf2_extensions(struct drm_i915_gem_execbuffer2 *args, 2942 struct i915_execbuffer *eb) 2943 { 2944 if (!(args->flags & I915_EXEC_USE_EXTENSIONS)) 2945 return 0; 2946 2947 /* The execbuf2 extension mechanism reuses cliprects_ptr. So we cannot 2948 * have another flag also using it at the same time. 2949 */ 2950 if (eb->args->flags & I915_EXEC_FENCE_ARRAY) 2951 return -EINVAL; 2952 2953 if (args->num_cliprects != 0) 2954 return -EINVAL; 2955 2956 return i915_user_extensions(u64_to_user_ptr(args->cliprects_ptr), 2957 execbuf_extensions, 2958 ARRAY_SIZE(execbuf_extensions), 2959 eb); 2960 } 2961 2962 static void eb_requests_get(struct i915_execbuffer *eb) 2963 { 2964 unsigned int i; 2965 2966 for_each_batch_create_order(eb, i) { 2967 if (!eb->requests[i]) 2968 break; 2969 2970 i915_request_get(eb->requests[i]); 2971 } 2972 } 2973 2974 static void eb_requests_put(struct i915_execbuffer *eb) 2975 { 2976 unsigned int i; 2977 2978 for_each_batch_create_order(eb, i) { 2979 if (!eb->requests[i]) 2980 break; 2981 2982 i915_request_put(eb->requests[i]); 2983 } 2984 } 2985 2986 static struct sync_file * 2987 eb_composite_fence_create(struct i915_execbuffer *eb, int out_fence_fd) 2988 { 2989 struct sync_file *out_fence = NULL; 2990 struct dma_fence_array *fence_array; 2991 struct dma_fence **fences; 2992 unsigned int i; 2993 2994 GEM_BUG_ON(!intel_context_is_parent(eb->context)); 2995 2996 fences = kmalloc_array(eb->num_batches, sizeof(*fences), GFP_KERNEL); 2997 if (!fences) 2998 return ERR_PTR(-ENOMEM); 2999 3000 for_each_batch_create_order(eb, i) { 3001 fences[i] = &eb->requests[i]->fence; 3002 __set_bit(I915_FENCE_FLAG_COMPOSITE, 3003 &eb->requests[i]->fence.flags); 3004 } 3005 3006 fence_array = dma_fence_array_create(eb->num_batches, 3007 fences, 3008 eb->context->parallel.fence_context, 3009 eb->context->parallel.seqno, 3010 false); 3011 if (!fence_array) { 3012 kfree(fences); 3013 return ERR_PTR(-ENOMEM); 3014 } 3015 3016 /* Move ownership to the dma_fence_array created above */ 3017 for_each_batch_create_order(eb, i) 3018 dma_fence_get(fences[i]); 3019 3020 if (out_fence_fd != -1) { 3021 out_fence = sync_file_create(&fence_array->base); 3022 /* sync_file now owns fence_arry, drop creation ref */ 3023 dma_fence_put(&fence_array->base); 3024 if (!out_fence) 3025 return ERR_PTR(-ENOMEM); 3026 } 3027 3028 eb->composite_fence = &fence_array->base; 3029 3030 return out_fence; 3031 } 3032 3033 static struct sync_file * 3034 eb_fences_add(struct i915_execbuffer *eb, struct i915_request *rq, 3035 struct dma_fence *in_fence, int out_fence_fd) 3036 { 3037 struct sync_file *out_fence = NULL; 3038 int err; 3039 3040 if (unlikely(eb->gem_context->syncobj)) { 3041 struct dma_fence *fence; 3042 3043 fence = drm_syncobj_fence_get(eb->gem_context->syncobj); 3044 err = i915_request_await_dma_fence(rq, fence); 3045 dma_fence_put(fence); 3046 if (err) 3047 return ERR_PTR(err); 3048 } 3049 3050 if (in_fence) { 3051 if (eb->args->flags & I915_EXEC_FENCE_SUBMIT) 3052 err = i915_request_await_execution(rq, in_fence); 3053 else 3054 err = i915_request_await_dma_fence(rq, in_fence); 3055 if (err < 0) 3056 return ERR_PTR(err); 3057 } 3058 3059 if (eb->fences) { 3060 err = await_fence_array(eb, rq); 3061 if (err) 3062 return ERR_PTR(err); 3063 } 3064 3065 if (intel_context_is_parallel(eb->context)) { 3066 out_fence = eb_composite_fence_create(eb, out_fence_fd); 3067 if (IS_ERR(out_fence)) 3068 return ERR_PTR(-ENOMEM); 3069 } else if (out_fence_fd != -1) { 3070 out_fence = sync_file_create(&rq->fence); 3071 if (!out_fence) 3072 return ERR_PTR(-ENOMEM); 3073 } 3074 3075 return out_fence; 3076 } 3077 3078 static struct intel_context * 3079 eb_find_context(struct i915_execbuffer *eb, unsigned int context_number) 3080 { 3081 struct intel_context *child; 3082 3083 if (likely(context_number == 0)) 3084 return eb->context; 3085 3086 for_each_child(eb->context, child) 3087 if (!--context_number) 3088 return child; 3089 3090 GEM_BUG_ON("Context not found"); 3091 3092 return NULL; 3093 } 3094 3095 static struct sync_file * 3096 eb_requests_create(struct i915_execbuffer *eb, struct dma_fence *in_fence, 3097 int out_fence_fd) 3098 { 3099 struct sync_file *out_fence = NULL; 3100 unsigned int i; 3101 3102 for_each_batch_create_order(eb, i) { 3103 /* Allocate a request for this batch buffer nice and early. */ 3104 eb->requests[i] = i915_request_create(eb_find_context(eb, i)); 3105 if (IS_ERR(eb->requests[i])) { 3106 out_fence = ERR_PTR(PTR_ERR(eb->requests[i])); 3107 eb->requests[i] = NULL; 3108 return out_fence; 3109 } 3110 3111 /* 3112 * Only the first request added (committed to backend) has to 3113 * take the in fences into account as all subsequent requests 3114 * will have fences inserted inbetween them. 3115 */ 3116 if (i + 1 == eb->num_batches) { 3117 out_fence = eb_fences_add(eb, eb->requests[i], 3118 in_fence, out_fence_fd); 3119 if (IS_ERR(out_fence)) 3120 return out_fence; 3121 } 3122 3123 /* 3124 * Whilst this request exists, batch_obj will be on the 3125 * active_list, and so will hold the active reference. Only when 3126 * this request is retired will the batch_obj be moved onto 3127 * the inactive_list and lose its active reference. Hence we do 3128 * not need to explicitly hold another reference here. 3129 */ 3130 eb->requests[i]->batch = eb->batches[i]->vma; 3131 if (eb->batch_pool) { 3132 GEM_BUG_ON(intel_context_is_parallel(eb->context)); 3133 intel_gt_buffer_pool_mark_active(eb->batch_pool, 3134 eb->requests[i]); 3135 } 3136 } 3137 3138 return out_fence; 3139 } 3140 3141 static int 3142 i915_gem_do_execbuffer(struct drm_device *dev, 3143 struct drm_file *file, 3144 struct drm_i915_gem_execbuffer2 *args, 3145 struct drm_i915_gem_exec_object2 *exec) 3146 { 3147 struct drm_i915_private *i915 = to_i915(dev); 3148 struct i915_execbuffer eb; 3149 struct dma_fence *in_fence = NULL; 3150 struct sync_file *out_fence = NULL; 3151 int out_fence_fd = -1; 3152 int err; 3153 3154 BUILD_BUG_ON(__EXEC_INTERNAL_FLAGS & ~__I915_EXEC_ILLEGAL_FLAGS); 3155 BUILD_BUG_ON(__EXEC_OBJECT_INTERNAL_FLAGS & 3156 ~__EXEC_OBJECT_UNKNOWN_FLAGS); 3157 3158 eb.i915 = i915; 3159 eb.file = file; 3160 eb.args = args; 3161 if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC)) 3162 args->flags |= __EXEC_HAS_RELOC; 3163 3164 eb.exec = exec; 3165 eb.vma = (struct eb_vma *)(exec + args->buffer_count + 1); 3166 eb.vma[0].vma = NULL; 3167 eb.batch_pool = NULL; 3168 3169 eb.invalid_flags = __EXEC_OBJECT_UNKNOWN_FLAGS; 3170 reloc_cache_init(&eb.reloc_cache, eb.i915); 3171 3172 eb.buffer_count = args->buffer_count; 3173 eb.batch_start_offset = args->batch_start_offset; 3174 eb.trampoline = NULL; 3175 3176 eb.fences = NULL; 3177 eb.num_fences = 0; 3178 3179 memset(eb.requests, 0, sizeof(struct i915_request *) * 3180 ARRAY_SIZE(eb.requests)); 3181 eb.composite_fence = NULL; 3182 3183 eb.batch_flags = 0; 3184 if (args->flags & I915_EXEC_SECURE) { 3185 if (GRAPHICS_VER(i915) >= 11) 3186 return -ENODEV; 3187 3188 /* Return -EPERM to trigger fallback code on old binaries. */ 3189 if (!HAS_SECURE_BATCHES(i915)) 3190 return -EPERM; 3191 3192 if (!drm_is_current_master(file) || !capable(CAP_SYS_ADMIN)) 3193 return -EPERM; 3194 3195 eb.batch_flags |= I915_DISPATCH_SECURE; 3196 } 3197 if (args->flags & I915_EXEC_IS_PINNED) 3198 eb.batch_flags |= I915_DISPATCH_PINNED; 3199 3200 err = parse_execbuf2_extensions(args, &eb); 3201 if (err) 3202 goto err_ext; 3203 3204 err = add_fence_array(&eb); 3205 if (err) 3206 goto err_ext; 3207 3208 #define IN_FENCES (I915_EXEC_FENCE_IN | I915_EXEC_FENCE_SUBMIT) 3209 if (args->flags & IN_FENCES) { 3210 if ((args->flags & IN_FENCES) == IN_FENCES) 3211 return -EINVAL; 3212 3213 in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2)); 3214 if (!in_fence) { 3215 err = -EINVAL; 3216 goto err_ext; 3217 } 3218 } 3219 #undef IN_FENCES 3220 3221 if (args->flags & I915_EXEC_FENCE_OUT) { 3222 out_fence_fd = get_unused_fd_flags(O_CLOEXEC); 3223 if (out_fence_fd < 0) { 3224 err = out_fence_fd; 3225 goto err_in_fence; 3226 } 3227 } 3228 3229 err = eb_create(&eb); 3230 if (err) 3231 goto err_out_fence; 3232 3233 GEM_BUG_ON(!eb.lut_size); 3234 3235 err = eb_select_context(&eb); 3236 if (unlikely(err)) 3237 goto err_destroy; 3238 3239 err = eb_select_engine(&eb); 3240 if (unlikely(err)) 3241 goto err_context; 3242 3243 err = eb_lookup_vmas(&eb); 3244 if (err) { 3245 eb_release_vmas(&eb, true); 3246 goto err_engine; 3247 } 3248 3249 i915_gem_ww_ctx_init(&eb.ww, true); 3250 3251 err = eb_relocate_parse(&eb); 3252 if (err) { 3253 /* 3254 * If the user expects the execobject.offset and 3255 * reloc.presumed_offset to be an exact match, 3256 * as for using NO_RELOC, then we cannot update 3257 * the execobject.offset until we have completed 3258 * relocation. 3259 */ 3260 args->flags &= ~__EXEC_HAS_RELOC; 3261 goto err_vma; 3262 } 3263 3264 ww_acquire_done(&eb.ww.ctx); 3265 3266 out_fence = eb_requests_create(&eb, in_fence, out_fence_fd); 3267 if (IS_ERR(out_fence)) { 3268 err = PTR_ERR(out_fence); 3269 if (eb.requests[0]) 3270 goto err_request; 3271 else 3272 goto err_vma; 3273 } 3274 3275 err = eb_submit(&eb); 3276 3277 err_request: 3278 eb_requests_get(&eb); 3279 err = eb_requests_add(&eb, err); 3280 3281 if (eb.fences) 3282 signal_fence_array(&eb, eb.composite_fence ? 3283 eb.composite_fence : 3284 &eb.requests[0]->fence); 3285 3286 if (out_fence) { 3287 if (err == 0) { 3288 fd_install(out_fence_fd, out_fence->file); 3289 args->rsvd2 &= GENMASK_ULL(31, 0); /* keep in-fence */ 3290 args->rsvd2 |= (u64)out_fence_fd << 32; 3291 out_fence_fd = -1; 3292 } else { 3293 fput(out_fence->file); 3294 } 3295 } 3296 3297 if (unlikely(eb.gem_context->syncobj)) { 3298 drm_syncobj_replace_fence(eb.gem_context->syncobj, 3299 eb.composite_fence ? 3300 eb.composite_fence : 3301 &eb.requests[0]->fence); 3302 } 3303 3304 if (!out_fence && eb.composite_fence) 3305 dma_fence_put(eb.composite_fence); 3306 3307 eb_requests_put(&eb); 3308 3309 err_vma: 3310 eb_release_vmas(&eb, true); 3311 if (eb.trampoline) 3312 i915_vma_unpin(eb.trampoline); 3313 WARN_ON(err == -EDEADLK); 3314 i915_gem_ww_ctx_fini(&eb.ww); 3315 3316 if (eb.batch_pool) 3317 intel_gt_buffer_pool_put(eb.batch_pool); 3318 err_engine: 3319 eb_put_engine(&eb); 3320 err_context: 3321 i915_gem_context_put(eb.gem_context); 3322 err_destroy: 3323 eb_destroy(&eb); 3324 err_out_fence: 3325 if (out_fence_fd != -1) 3326 put_unused_fd(out_fence_fd); 3327 err_in_fence: 3328 dma_fence_put(in_fence); 3329 err_ext: 3330 put_fence_array(eb.fences, eb.num_fences); 3331 return err; 3332 } 3333 3334 static size_t eb_element_size(void) 3335 { 3336 return sizeof(struct drm_i915_gem_exec_object2) + sizeof(struct eb_vma); 3337 } 3338 3339 static bool check_buffer_count(size_t count) 3340 { 3341 const size_t sz = eb_element_size(); 3342 3343 /* 3344 * When using LUT_HANDLE, we impose a limit of INT_MAX for the lookup 3345 * array size (see eb_create()). Otherwise, we can accept an array as 3346 * large as can be addressed (though use large arrays at your peril)! 3347 */ 3348 3349 return !(count < 1 || count > INT_MAX || count > SIZE_MAX / sz - 1); 3350 } 3351 3352 int 3353 i915_gem_execbuffer2_ioctl(struct drm_device *dev, void *data, 3354 struct drm_file *file) 3355 { 3356 struct drm_i915_private *i915 = to_i915(dev); 3357 struct drm_i915_gem_execbuffer2 *args = data; 3358 struct drm_i915_gem_exec_object2 *exec2_list; 3359 const size_t count = args->buffer_count; 3360 int err; 3361 3362 if (!check_buffer_count(count)) { 3363 drm_dbg(&i915->drm, "execbuf2 with %zd buffers\n", count); 3364 return -EINVAL; 3365 } 3366 3367 err = i915_gem_check_execbuffer(args); 3368 if (err) 3369 return err; 3370 3371 /* Allocate extra slots for use by the command parser */ 3372 exec2_list = kvmalloc_array(count + 2, eb_element_size(), 3373 __GFP_NOWARN | GFP_KERNEL); 3374 if (exec2_list == NULL) { 3375 drm_dbg(&i915->drm, "Failed to allocate exec list for %zd buffers\n", 3376 count); 3377 return -ENOMEM; 3378 } 3379 if (copy_from_user(exec2_list, 3380 u64_to_user_ptr(args->buffers_ptr), 3381 sizeof(*exec2_list) * count)) { 3382 drm_dbg(&i915->drm, "copy %zd exec entries failed\n", count); 3383 kvfree(exec2_list); 3384 return -EFAULT; 3385 } 3386 3387 err = i915_gem_do_execbuffer(dev, file, args, exec2_list); 3388 3389 /* 3390 * Now that we have begun execution of the batchbuffer, we ignore 3391 * any new error after this point. Also given that we have already 3392 * updated the associated relocations, we try to write out the current 3393 * object locations irrespective of any error. 3394 */ 3395 if (args->flags & __EXEC_HAS_RELOC) { 3396 struct drm_i915_gem_exec_object2 __user *user_exec_list = 3397 u64_to_user_ptr(args->buffers_ptr); 3398 unsigned int i; 3399 3400 /* Copy the new buffer offsets back to the user's exec list. */ 3401 /* 3402 * Note: count * sizeof(*user_exec_list) does not overflow, 3403 * because we checked 'count' in check_buffer_count(). 3404 * 3405 * And this range already got effectively checked earlier 3406 * when we did the "copy_from_user()" above. 3407 */ 3408 if (!user_write_access_begin(user_exec_list, 3409 count * sizeof(*user_exec_list))) 3410 goto end; 3411 3412 for (i = 0; i < args->buffer_count; i++) { 3413 if (!(exec2_list[i].offset & UPDATE)) 3414 continue; 3415 3416 exec2_list[i].offset = 3417 gen8_canonical_addr(exec2_list[i].offset & PIN_OFFSET_MASK); 3418 unsafe_put_user(exec2_list[i].offset, 3419 &user_exec_list[i].offset, 3420 end_user); 3421 } 3422 end_user: 3423 user_write_access_end(); 3424 end:; 3425 } 3426 3427 args->flags &= ~__I915_EXEC_UNKNOWN_FLAGS; 3428 kvfree(exec2_list); 3429 return err; 3430 } 3431