1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2011-2012 Intel Corporation 5 */ 6 7 /* 8 * This file implements HW context support. On gen5+ a HW context consists of an 9 * opaque GPU object which is referenced at times of context saves and restores. 10 * With RC6 enabled, the context is also referenced as the GPU enters and exists 11 * from RC6 (GPU has it's own internal power context, except on gen5). Though 12 * something like a context does exist for the media ring, the code only 13 * supports contexts for the render ring. 14 * 15 * In software, there is a distinction between contexts created by the user, 16 * and the default HW context. The default HW context is used by GPU clients 17 * that do not request setup of their own hardware context. The default 18 * context's state is never restored to help prevent programming errors. This 19 * would happen if a client ran and piggy-backed off another clients GPU state. 20 * The default context only exists to give the GPU some offset to load as the 21 * current to invoke a save of the context we actually care about. In fact, the 22 * code could likely be constructed, albeit in a more complicated fashion, to 23 * never use the default context, though that limits the driver's ability to 24 * swap out, and/or destroy other contexts. 25 * 26 * All other contexts are created as a request by the GPU client. These contexts 27 * store GPU state, and thus allow GPU clients to not re-emit state (and 28 * potentially query certain state) at any time. The kernel driver makes 29 * certain that the appropriate commands are inserted. 30 * 31 * The context life cycle is semi-complicated in that context BOs may live 32 * longer than the context itself because of the way the hardware, and object 33 * tracking works. Below is a very crude representation of the state machine 34 * describing the context life. 35 * refcount pincount active 36 * S0: initial state 0 0 0 37 * S1: context created 1 0 0 38 * S2: context is currently running 2 1 X 39 * S3: GPU referenced, but not current 2 0 1 40 * S4: context is current, but destroyed 1 1 0 41 * S5: like S3, but destroyed 1 0 1 42 * 43 * The most common (but not all) transitions: 44 * S0->S1: client creates a context 45 * S1->S2: client submits execbuf with context 46 * S2->S3: other clients submits execbuf with context 47 * S3->S1: context object was retired 48 * S3->S2: clients submits another execbuf 49 * S2->S4: context destroy called with current context 50 * S3->S5->S0: destroy path 51 * S4->S5->S0: destroy path on current context 52 * 53 * There are two confusing terms used above: 54 * The "current context" means the context which is currently running on the 55 * GPU. The GPU has loaded its state already and has stored away the gtt 56 * offset of the BO. The GPU is not actively referencing the data at this 57 * offset, but it will on the next context switch. The only way to avoid this 58 * is to do a GPU reset. 59 * 60 * An "active context' is one which was previously the "current context" and is 61 * on the active list waiting for the next context switch to occur. Until this 62 * happens, the object must remain at the same gtt offset. It is therefore 63 * possible to destroy a context, but it is still active. 64 * 65 */ 66 67 #include <linux/log2.h> 68 #include <linux/nospec.h> 69 70 #include "gt/gen6_ppgtt.h" 71 #include "gt/intel_context.h" 72 #include "gt/intel_context_param.h" 73 #include "gt/intel_engine_heartbeat.h" 74 #include "gt/intel_engine_user.h" 75 #include "gt/intel_execlists_submission.h" /* virtual_engine */ 76 #include "gt/intel_gpu_commands.h" 77 #include "gt/intel_ring.h" 78 79 #include "i915_gem_context.h" 80 #include "i915_globals.h" 81 #include "i915_trace.h" 82 #include "i915_user_extensions.h" 83 84 #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1 85 86 static struct i915_global_gem_context { 87 struct i915_global base; 88 struct kmem_cache *slab_luts; 89 } global; 90 91 struct i915_lut_handle *i915_lut_handle_alloc(void) 92 { 93 return kmem_cache_alloc(global.slab_luts, GFP_KERNEL); 94 } 95 96 void i915_lut_handle_free(struct i915_lut_handle *lut) 97 { 98 return kmem_cache_free(global.slab_luts, lut); 99 } 100 101 static void lut_close(struct i915_gem_context *ctx) 102 { 103 struct radix_tree_iter iter; 104 void __rcu **slot; 105 106 mutex_lock(&ctx->lut_mutex); 107 rcu_read_lock(); 108 radix_tree_for_each_slot(slot, &ctx->handles_vma, &iter, 0) { 109 struct i915_vma *vma = rcu_dereference_raw(*slot); 110 struct drm_i915_gem_object *obj = vma->obj; 111 struct i915_lut_handle *lut; 112 113 if (!kref_get_unless_zero(&obj->base.refcount)) 114 continue; 115 116 spin_lock(&obj->lut_lock); 117 list_for_each_entry(lut, &obj->lut_list, obj_link) { 118 if (lut->ctx != ctx) 119 continue; 120 121 if (lut->handle != iter.index) 122 continue; 123 124 list_del(&lut->obj_link); 125 break; 126 } 127 spin_unlock(&obj->lut_lock); 128 129 if (&lut->obj_link != &obj->lut_list) { 130 i915_lut_handle_free(lut); 131 radix_tree_iter_delete(&ctx->handles_vma, &iter, slot); 132 i915_vma_close(vma); 133 i915_gem_object_put(obj); 134 } 135 136 i915_gem_object_put(obj); 137 } 138 rcu_read_unlock(); 139 mutex_unlock(&ctx->lut_mutex); 140 } 141 142 static struct intel_context * 143 lookup_user_engine(struct i915_gem_context *ctx, 144 unsigned long flags, 145 const struct i915_engine_class_instance *ci) 146 #define LOOKUP_USER_INDEX BIT(0) 147 { 148 int idx; 149 150 if (!!(flags & LOOKUP_USER_INDEX) != i915_gem_context_user_engines(ctx)) 151 return ERR_PTR(-EINVAL); 152 153 if (!i915_gem_context_user_engines(ctx)) { 154 struct intel_engine_cs *engine; 155 156 engine = intel_engine_lookup_user(ctx->i915, 157 ci->engine_class, 158 ci->engine_instance); 159 if (!engine) 160 return ERR_PTR(-EINVAL); 161 162 idx = engine->legacy_idx; 163 } else { 164 idx = ci->engine_instance; 165 } 166 167 return i915_gem_context_get_engine(ctx, idx); 168 } 169 170 static struct i915_address_space * 171 context_get_vm_rcu(struct i915_gem_context *ctx) 172 { 173 GEM_BUG_ON(!rcu_access_pointer(ctx->vm)); 174 175 do { 176 struct i915_address_space *vm; 177 178 /* 179 * We do not allow downgrading from full-ppgtt [to a shared 180 * global gtt], so ctx->vm cannot become NULL. 181 */ 182 vm = rcu_dereference(ctx->vm); 183 if (!kref_get_unless_zero(&vm->ref)) 184 continue; 185 186 /* 187 * This ppgtt may have be reallocated between 188 * the read and the kref, and reassigned to a third 189 * context. In order to avoid inadvertent sharing 190 * of this ppgtt with that third context (and not 191 * src), we have to confirm that we have the same 192 * ppgtt after passing through the strong memory 193 * barrier implied by a successful 194 * kref_get_unless_zero(). 195 * 196 * Once we have acquired the current ppgtt of ctx, 197 * we no longer care if it is released from ctx, as 198 * it cannot be reallocated elsewhere. 199 */ 200 201 if (vm == rcu_access_pointer(ctx->vm)) 202 return rcu_pointer_handoff(vm); 203 204 i915_vm_put(vm); 205 } while (1); 206 } 207 208 static void intel_context_set_gem(struct intel_context *ce, 209 struct i915_gem_context *ctx) 210 { 211 GEM_BUG_ON(rcu_access_pointer(ce->gem_context)); 212 RCU_INIT_POINTER(ce->gem_context, ctx); 213 214 if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) 215 ce->ring = __intel_context_ring_size(SZ_16K); 216 217 if (rcu_access_pointer(ctx->vm)) { 218 struct i915_address_space *vm; 219 220 rcu_read_lock(); 221 vm = context_get_vm_rcu(ctx); /* hmm */ 222 rcu_read_unlock(); 223 224 i915_vm_put(ce->vm); 225 ce->vm = vm; 226 } 227 228 GEM_BUG_ON(ce->timeline); 229 if (ctx->timeline) 230 ce->timeline = intel_timeline_get(ctx->timeline); 231 232 if (ctx->sched.priority >= I915_PRIORITY_NORMAL && 233 intel_engine_has_timeslices(ce->engine)) 234 __set_bit(CONTEXT_USE_SEMAPHORES, &ce->flags); 235 236 intel_context_set_watchdog_us(ce, ctx->watchdog.timeout_us); 237 } 238 239 static void __free_engines(struct i915_gem_engines *e, unsigned int count) 240 { 241 while (count--) { 242 if (!e->engines[count]) 243 continue; 244 245 intel_context_put(e->engines[count]); 246 } 247 kfree(e); 248 } 249 250 static void free_engines(struct i915_gem_engines *e) 251 { 252 __free_engines(e, e->num_engines); 253 } 254 255 static void free_engines_rcu(struct rcu_head *rcu) 256 { 257 struct i915_gem_engines *engines = 258 container_of(rcu, struct i915_gem_engines, rcu); 259 260 i915_sw_fence_fini(&engines->fence); 261 free_engines(engines); 262 } 263 264 static int __i915_sw_fence_call 265 engines_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) 266 { 267 struct i915_gem_engines *engines = 268 container_of(fence, typeof(*engines), fence); 269 270 switch (state) { 271 case FENCE_COMPLETE: 272 if (!list_empty(&engines->link)) { 273 struct i915_gem_context *ctx = engines->ctx; 274 unsigned long flags; 275 276 spin_lock_irqsave(&ctx->stale.lock, flags); 277 list_del(&engines->link); 278 spin_unlock_irqrestore(&ctx->stale.lock, flags); 279 } 280 i915_gem_context_put(engines->ctx); 281 break; 282 283 case FENCE_FREE: 284 init_rcu_head(&engines->rcu); 285 call_rcu(&engines->rcu, free_engines_rcu); 286 break; 287 } 288 289 return NOTIFY_DONE; 290 } 291 292 static struct i915_gem_engines *alloc_engines(unsigned int count) 293 { 294 struct i915_gem_engines *e; 295 296 e = kzalloc(struct_size(e, engines, count), GFP_KERNEL); 297 if (!e) 298 return NULL; 299 300 i915_sw_fence_init(&e->fence, engines_notify); 301 return e; 302 } 303 304 static struct i915_gem_engines *default_engines(struct i915_gem_context *ctx) 305 { 306 const struct intel_gt *gt = &ctx->i915->gt; 307 struct intel_engine_cs *engine; 308 struct i915_gem_engines *e; 309 enum intel_engine_id id; 310 311 e = alloc_engines(I915_NUM_ENGINES); 312 if (!e) 313 return ERR_PTR(-ENOMEM); 314 315 for_each_engine(engine, gt, id) { 316 struct intel_context *ce; 317 318 if (engine->legacy_idx == INVALID_ENGINE) 319 continue; 320 321 GEM_BUG_ON(engine->legacy_idx >= I915_NUM_ENGINES); 322 GEM_BUG_ON(e->engines[engine->legacy_idx]); 323 324 ce = intel_context_create(engine); 325 if (IS_ERR(ce)) { 326 __free_engines(e, e->num_engines + 1); 327 return ERR_CAST(ce); 328 } 329 330 intel_context_set_gem(ce, ctx); 331 332 e->engines[engine->legacy_idx] = ce; 333 e->num_engines = max(e->num_engines, engine->legacy_idx); 334 } 335 e->num_engines++; 336 337 return e; 338 } 339 340 void i915_gem_context_release(struct kref *ref) 341 { 342 struct i915_gem_context *ctx = container_of(ref, typeof(*ctx), ref); 343 344 trace_i915_context_free(ctx); 345 GEM_BUG_ON(!i915_gem_context_is_closed(ctx)); 346 347 mutex_destroy(&ctx->engines_mutex); 348 mutex_destroy(&ctx->lut_mutex); 349 350 if (ctx->timeline) 351 intel_timeline_put(ctx->timeline); 352 353 put_pid(ctx->pid); 354 mutex_destroy(&ctx->mutex); 355 356 kfree_rcu(ctx, rcu); 357 } 358 359 static inline struct i915_gem_engines * 360 __context_engines_static(const struct i915_gem_context *ctx) 361 { 362 return rcu_dereference_protected(ctx->engines, true); 363 } 364 365 static void __reset_context(struct i915_gem_context *ctx, 366 struct intel_engine_cs *engine) 367 { 368 intel_gt_handle_error(engine->gt, engine->mask, 0, 369 "context closure in %s", ctx->name); 370 } 371 372 static bool __cancel_engine(struct intel_engine_cs *engine) 373 { 374 /* 375 * Send a "high priority pulse" down the engine to cause the 376 * current request to be momentarily preempted. (If it fails to 377 * be preempted, it will be reset). As we have marked our context 378 * as banned, any incomplete request, including any running, will 379 * be skipped following the preemption. 380 * 381 * If there is no hangchecking (one of the reasons why we try to 382 * cancel the context) and no forced preemption, there may be no 383 * means by which we reset the GPU and evict the persistent hog. 384 * Ergo if we are unable to inject a preemptive pulse that can 385 * kill the banned context, we fallback to doing a local reset 386 * instead. 387 */ 388 return intel_engine_pulse(engine) == 0; 389 } 390 391 static struct intel_engine_cs *active_engine(struct intel_context *ce) 392 { 393 struct intel_engine_cs *engine = NULL; 394 struct i915_request *rq; 395 396 if (intel_context_has_inflight(ce)) 397 return intel_context_inflight(ce); 398 399 if (!ce->timeline) 400 return NULL; 401 402 /* 403 * rq->link is only SLAB_TYPESAFE_BY_RCU, we need to hold a reference 404 * to the request to prevent it being transferred to a new timeline 405 * (and onto a new timeline->requests list). 406 */ 407 rcu_read_lock(); 408 list_for_each_entry_reverse(rq, &ce->timeline->requests, link) { 409 bool found; 410 411 /* timeline is already completed upto this point? */ 412 if (!i915_request_get_rcu(rq)) 413 break; 414 415 /* Check with the backend if the request is inflight */ 416 found = true; 417 if (likely(rcu_access_pointer(rq->timeline) == ce->timeline)) 418 found = i915_request_active_engine(rq, &engine); 419 420 i915_request_put(rq); 421 if (found) 422 break; 423 } 424 rcu_read_unlock(); 425 426 return engine; 427 } 428 429 static void kill_engines(struct i915_gem_engines *engines, bool ban) 430 { 431 struct i915_gem_engines_iter it; 432 struct intel_context *ce; 433 434 /* 435 * Map the user's engine back to the actual engines; one virtual 436 * engine will be mapped to multiple engines, and using ctx->engine[] 437 * the same engine may be have multiple instances in the user's map. 438 * However, we only care about pending requests, so only include 439 * engines on which there are incomplete requests. 440 */ 441 for_each_gem_engine(ce, engines, it) { 442 struct intel_engine_cs *engine; 443 444 if (ban && intel_context_set_banned(ce)) 445 continue; 446 447 /* 448 * Check the current active state of this context; if we 449 * are currently executing on the GPU we need to evict 450 * ourselves. On the other hand, if we haven't yet been 451 * submitted to the GPU or if everything is complete, 452 * we have nothing to do. 453 */ 454 engine = active_engine(ce); 455 456 /* First attempt to gracefully cancel the context */ 457 if (engine && !__cancel_engine(engine) && ban) 458 /* 459 * If we are unable to send a preemptive pulse to bump 460 * the context from the GPU, we have to resort to a full 461 * reset. We hope the collateral damage is worth it. 462 */ 463 __reset_context(engines->ctx, engine); 464 } 465 } 466 467 static void kill_context(struct i915_gem_context *ctx) 468 { 469 bool ban = (!i915_gem_context_is_persistent(ctx) || 470 !ctx->i915->params.enable_hangcheck); 471 struct i915_gem_engines *pos, *next; 472 473 spin_lock_irq(&ctx->stale.lock); 474 GEM_BUG_ON(!i915_gem_context_is_closed(ctx)); 475 list_for_each_entry_safe(pos, next, &ctx->stale.engines, link) { 476 if (!i915_sw_fence_await(&pos->fence)) { 477 list_del_init(&pos->link); 478 continue; 479 } 480 481 spin_unlock_irq(&ctx->stale.lock); 482 483 kill_engines(pos, ban); 484 485 spin_lock_irq(&ctx->stale.lock); 486 GEM_BUG_ON(i915_sw_fence_signaled(&pos->fence)); 487 list_safe_reset_next(pos, next, link); 488 list_del_init(&pos->link); /* decouple from FENCE_COMPLETE */ 489 490 i915_sw_fence_complete(&pos->fence); 491 } 492 spin_unlock_irq(&ctx->stale.lock); 493 } 494 495 static void engines_idle_release(struct i915_gem_context *ctx, 496 struct i915_gem_engines *engines) 497 { 498 struct i915_gem_engines_iter it; 499 struct intel_context *ce; 500 501 INIT_LIST_HEAD(&engines->link); 502 503 engines->ctx = i915_gem_context_get(ctx); 504 505 for_each_gem_engine(ce, engines, it) { 506 int err; 507 508 /* serialises with execbuf */ 509 set_bit(CONTEXT_CLOSED_BIT, &ce->flags); 510 if (!intel_context_pin_if_active(ce)) 511 continue; 512 513 /* Wait until context is finally scheduled out and retired */ 514 err = i915_sw_fence_await_active(&engines->fence, 515 &ce->active, 516 I915_ACTIVE_AWAIT_BARRIER); 517 intel_context_unpin(ce); 518 if (err) 519 goto kill; 520 } 521 522 spin_lock_irq(&ctx->stale.lock); 523 if (!i915_gem_context_is_closed(ctx)) 524 list_add_tail(&engines->link, &ctx->stale.engines); 525 spin_unlock_irq(&ctx->stale.lock); 526 527 kill: 528 if (list_empty(&engines->link)) /* raced, already closed */ 529 kill_engines(engines, true); 530 531 i915_sw_fence_commit(&engines->fence); 532 } 533 534 static void set_closed_name(struct i915_gem_context *ctx) 535 { 536 char *s; 537 538 /* Replace '[]' with '<>' to indicate closed in debug prints */ 539 540 s = strrchr(ctx->name, '['); 541 if (!s) 542 return; 543 544 *s = '<'; 545 546 s = strchr(s + 1, ']'); 547 if (s) 548 *s = '>'; 549 } 550 551 static void context_close(struct i915_gem_context *ctx) 552 { 553 struct i915_address_space *vm; 554 555 /* Flush any concurrent set_engines() */ 556 mutex_lock(&ctx->engines_mutex); 557 engines_idle_release(ctx, rcu_replace_pointer(ctx->engines, NULL, 1)); 558 i915_gem_context_set_closed(ctx); 559 mutex_unlock(&ctx->engines_mutex); 560 561 mutex_lock(&ctx->mutex); 562 563 set_closed_name(ctx); 564 565 vm = i915_gem_context_vm(ctx); 566 if (vm) 567 i915_vm_close(vm); 568 569 ctx->file_priv = ERR_PTR(-EBADF); 570 571 /* 572 * The LUT uses the VMA as a backpointer to unref the object, 573 * so we need to clear the LUT before we close all the VMA (inside 574 * the ppgtt). 575 */ 576 lut_close(ctx); 577 578 spin_lock(&ctx->i915->gem.contexts.lock); 579 list_del(&ctx->link); 580 spin_unlock(&ctx->i915->gem.contexts.lock); 581 582 mutex_unlock(&ctx->mutex); 583 584 /* 585 * If the user has disabled hangchecking, we can not be sure that 586 * the batches will ever complete after the context is closed, 587 * keeping the context and all resources pinned forever. So in this 588 * case we opt to forcibly kill off all remaining requests on 589 * context close. 590 */ 591 kill_context(ctx); 592 593 i915_gem_context_put(ctx); 594 } 595 596 static int __context_set_persistence(struct i915_gem_context *ctx, bool state) 597 { 598 if (i915_gem_context_is_persistent(ctx) == state) 599 return 0; 600 601 if (state) { 602 /* 603 * Only contexts that are short-lived [that will expire or be 604 * reset] are allowed to survive past termination. We require 605 * hangcheck to ensure that the persistent requests are healthy. 606 */ 607 if (!ctx->i915->params.enable_hangcheck) 608 return -EINVAL; 609 610 i915_gem_context_set_persistence(ctx); 611 } else { 612 /* To cancel a context we use "preempt-to-idle" */ 613 if (!(ctx->i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION)) 614 return -ENODEV; 615 616 /* 617 * If the cancel fails, we then need to reset, cleanly! 618 * 619 * If the per-engine reset fails, all hope is lost! We resort 620 * to a full GPU reset in that unlikely case, but realistically 621 * if the engine could not reset, the full reset does not fare 622 * much better. The damage has been done. 623 * 624 * However, if we cannot reset an engine by itself, we cannot 625 * cleanup a hanging persistent context without causing 626 * colateral damage, and we should not pretend we can by 627 * exposing the interface. 628 */ 629 if (!intel_has_reset_engine(&ctx->i915->gt)) 630 return -ENODEV; 631 632 i915_gem_context_clear_persistence(ctx); 633 } 634 635 return 0; 636 } 637 638 static struct i915_gem_context * 639 __create_context(struct drm_i915_private *i915) 640 { 641 struct i915_gem_context *ctx; 642 struct i915_gem_engines *e; 643 int err; 644 int i; 645 646 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 647 if (!ctx) 648 return ERR_PTR(-ENOMEM); 649 650 kref_init(&ctx->ref); 651 ctx->i915 = i915; 652 ctx->sched.priority = I915_PRIORITY_NORMAL; 653 mutex_init(&ctx->mutex); 654 INIT_LIST_HEAD(&ctx->link); 655 656 spin_lock_init(&ctx->stale.lock); 657 INIT_LIST_HEAD(&ctx->stale.engines); 658 659 mutex_init(&ctx->engines_mutex); 660 e = default_engines(ctx); 661 if (IS_ERR(e)) { 662 err = PTR_ERR(e); 663 goto err_free; 664 } 665 RCU_INIT_POINTER(ctx->engines, e); 666 667 INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL); 668 mutex_init(&ctx->lut_mutex); 669 670 /* NB: Mark all slices as needing a remap so that when the context first 671 * loads it will restore whatever remap state already exists. If there 672 * is no remap info, it will be a NOP. */ 673 ctx->remap_slice = ALL_L3_SLICES(i915); 674 675 i915_gem_context_set_bannable(ctx); 676 i915_gem_context_set_recoverable(ctx); 677 __context_set_persistence(ctx, true /* cgroup hook? */); 678 679 for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++) 680 ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES; 681 682 return ctx; 683 684 err_free: 685 kfree(ctx); 686 return ERR_PTR(err); 687 } 688 689 static inline struct i915_gem_engines * 690 __context_engines_await(const struct i915_gem_context *ctx, 691 bool *user_engines) 692 { 693 struct i915_gem_engines *engines; 694 695 rcu_read_lock(); 696 do { 697 engines = rcu_dereference(ctx->engines); 698 GEM_BUG_ON(!engines); 699 700 if (user_engines) 701 *user_engines = i915_gem_context_user_engines(ctx); 702 703 /* successful await => strong mb */ 704 if (unlikely(!i915_sw_fence_await(&engines->fence))) 705 continue; 706 707 if (likely(engines == rcu_access_pointer(ctx->engines))) 708 break; 709 710 i915_sw_fence_complete(&engines->fence); 711 } while (1); 712 rcu_read_unlock(); 713 714 return engines; 715 } 716 717 static int 718 context_apply_all(struct i915_gem_context *ctx, 719 int (*fn)(struct intel_context *ce, void *data), 720 void *data) 721 { 722 struct i915_gem_engines_iter it; 723 struct i915_gem_engines *e; 724 struct intel_context *ce; 725 int err = 0; 726 727 e = __context_engines_await(ctx, NULL); 728 for_each_gem_engine(ce, e, it) { 729 err = fn(ce, data); 730 if (err) 731 break; 732 } 733 i915_sw_fence_complete(&e->fence); 734 735 return err; 736 } 737 738 static int __apply_ppgtt(struct intel_context *ce, void *vm) 739 { 740 i915_vm_put(ce->vm); 741 ce->vm = i915_vm_get(vm); 742 return 0; 743 } 744 745 static struct i915_address_space * 746 __set_ppgtt(struct i915_gem_context *ctx, struct i915_address_space *vm) 747 { 748 struct i915_address_space *old; 749 750 old = rcu_replace_pointer(ctx->vm, 751 i915_vm_open(vm), 752 lockdep_is_held(&ctx->mutex)); 753 GEM_BUG_ON(old && i915_vm_is_4lvl(vm) != i915_vm_is_4lvl(old)); 754 755 context_apply_all(ctx, __apply_ppgtt, vm); 756 757 return old; 758 } 759 760 static void __assign_ppgtt(struct i915_gem_context *ctx, 761 struct i915_address_space *vm) 762 { 763 if (vm == rcu_access_pointer(ctx->vm)) 764 return; 765 766 vm = __set_ppgtt(ctx, vm); 767 if (vm) 768 i915_vm_close(vm); 769 } 770 771 static void __set_timeline(struct intel_timeline **dst, 772 struct intel_timeline *src) 773 { 774 struct intel_timeline *old = *dst; 775 776 *dst = src ? intel_timeline_get(src) : NULL; 777 778 if (old) 779 intel_timeline_put(old); 780 } 781 782 static int __apply_timeline(struct intel_context *ce, void *timeline) 783 { 784 __set_timeline(&ce->timeline, timeline); 785 return 0; 786 } 787 788 static void __assign_timeline(struct i915_gem_context *ctx, 789 struct intel_timeline *timeline) 790 { 791 __set_timeline(&ctx->timeline, timeline); 792 context_apply_all(ctx, __apply_timeline, timeline); 793 } 794 795 static int __apply_watchdog(struct intel_context *ce, void *timeout_us) 796 { 797 return intel_context_set_watchdog_us(ce, (uintptr_t)timeout_us); 798 } 799 800 static int 801 __set_watchdog(struct i915_gem_context *ctx, unsigned long timeout_us) 802 { 803 int ret; 804 805 ret = context_apply_all(ctx, __apply_watchdog, 806 (void *)(uintptr_t)timeout_us); 807 if (!ret) 808 ctx->watchdog.timeout_us = timeout_us; 809 810 return ret; 811 } 812 813 static void __set_default_fence_expiry(struct i915_gem_context *ctx) 814 { 815 struct drm_i915_private *i915 = ctx->i915; 816 int ret; 817 818 if (!IS_ACTIVE(CONFIG_DRM_I915_REQUEST_TIMEOUT) || 819 !i915->params.request_timeout_ms) 820 return; 821 822 /* Default expiry for user fences. */ 823 ret = __set_watchdog(ctx, i915->params.request_timeout_ms * 1000); 824 if (ret) 825 drm_notice(&i915->drm, 826 "Failed to configure default fence expiry! (%d)", 827 ret); 828 } 829 830 static struct i915_gem_context * 831 i915_gem_create_context(struct drm_i915_private *i915, unsigned int flags) 832 { 833 struct i915_gem_context *ctx; 834 835 if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE && 836 !HAS_EXECLISTS(i915)) 837 return ERR_PTR(-EINVAL); 838 839 ctx = __create_context(i915); 840 if (IS_ERR(ctx)) 841 return ctx; 842 843 if (HAS_FULL_PPGTT(i915)) { 844 struct i915_ppgtt *ppgtt; 845 846 ppgtt = i915_ppgtt_create(&i915->gt); 847 if (IS_ERR(ppgtt)) { 848 drm_dbg(&i915->drm, "PPGTT setup failed (%ld)\n", 849 PTR_ERR(ppgtt)); 850 context_close(ctx); 851 return ERR_CAST(ppgtt); 852 } 853 854 mutex_lock(&ctx->mutex); 855 __assign_ppgtt(ctx, &ppgtt->vm); 856 mutex_unlock(&ctx->mutex); 857 858 i915_vm_put(&ppgtt->vm); 859 } 860 861 if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE) { 862 struct intel_timeline *timeline; 863 864 timeline = intel_timeline_create(&i915->gt); 865 if (IS_ERR(timeline)) { 866 context_close(ctx); 867 return ERR_CAST(timeline); 868 } 869 870 __assign_timeline(ctx, timeline); 871 intel_timeline_put(timeline); 872 } 873 874 __set_default_fence_expiry(ctx); 875 876 trace_i915_context_create(ctx); 877 878 return ctx; 879 } 880 881 static void init_contexts(struct i915_gem_contexts *gc) 882 { 883 spin_lock_init(&gc->lock); 884 INIT_LIST_HEAD(&gc->list); 885 } 886 887 void i915_gem_init__contexts(struct drm_i915_private *i915) 888 { 889 init_contexts(&i915->gem.contexts); 890 } 891 892 static int gem_context_register(struct i915_gem_context *ctx, 893 struct drm_i915_file_private *fpriv, 894 u32 *id) 895 { 896 struct drm_i915_private *i915 = ctx->i915; 897 struct i915_address_space *vm; 898 int ret; 899 900 ctx->file_priv = fpriv; 901 902 mutex_lock(&ctx->mutex); 903 vm = i915_gem_context_vm(ctx); 904 if (vm) 905 WRITE_ONCE(vm->file, fpriv); /* XXX */ 906 mutex_unlock(&ctx->mutex); 907 908 ctx->pid = get_task_pid(current, PIDTYPE_PID); 909 snprintf(ctx->name, sizeof(ctx->name), "%s[%d]", 910 current->comm, pid_nr(ctx->pid)); 911 912 /* And finally expose ourselves to userspace via the idr */ 913 ret = xa_alloc(&fpriv->context_xa, id, ctx, xa_limit_32b, GFP_KERNEL); 914 if (ret) 915 goto err_pid; 916 917 spin_lock(&i915->gem.contexts.lock); 918 list_add_tail(&ctx->link, &i915->gem.contexts.list); 919 spin_unlock(&i915->gem.contexts.lock); 920 921 return 0; 922 923 err_pid: 924 put_pid(fetch_and_zero(&ctx->pid)); 925 return ret; 926 } 927 928 int i915_gem_context_open(struct drm_i915_private *i915, 929 struct drm_file *file) 930 { 931 struct drm_i915_file_private *file_priv = file->driver_priv; 932 struct i915_gem_context *ctx; 933 int err; 934 u32 id; 935 936 xa_init_flags(&file_priv->context_xa, XA_FLAGS_ALLOC); 937 938 /* 0 reserved for invalid/unassigned ppgtt */ 939 xa_init_flags(&file_priv->vm_xa, XA_FLAGS_ALLOC1); 940 941 ctx = i915_gem_create_context(i915, 0); 942 if (IS_ERR(ctx)) { 943 err = PTR_ERR(ctx); 944 goto err; 945 } 946 947 err = gem_context_register(ctx, file_priv, &id); 948 if (err < 0) 949 goto err_ctx; 950 951 GEM_BUG_ON(id); 952 return 0; 953 954 err_ctx: 955 context_close(ctx); 956 err: 957 xa_destroy(&file_priv->vm_xa); 958 xa_destroy(&file_priv->context_xa); 959 return err; 960 } 961 962 void i915_gem_context_close(struct drm_file *file) 963 { 964 struct drm_i915_file_private *file_priv = file->driver_priv; 965 struct i915_address_space *vm; 966 struct i915_gem_context *ctx; 967 unsigned long idx; 968 969 xa_for_each(&file_priv->context_xa, idx, ctx) 970 context_close(ctx); 971 xa_destroy(&file_priv->context_xa); 972 973 xa_for_each(&file_priv->vm_xa, idx, vm) 974 i915_vm_put(vm); 975 xa_destroy(&file_priv->vm_xa); 976 } 977 978 int i915_gem_vm_create_ioctl(struct drm_device *dev, void *data, 979 struct drm_file *file) 980 { 981 struct drm_i915_private *i915 = to_i915(dev); 982 struct drm_i915_gem_vm_control *args = data; 983 struct drm_i915_file_private *file_priv = file->driver_priv; 984 struct i915_ppgtt *ppgtt; 985 u32 id; 986 int err; 987 988 if (!HAS_FULL_PPGTT(i915)) 989 return -ENODEV; 990 991 if (args->flags) 992 return -EINVAL; 993 994 ppgtt = i915_ppgtt_create(&i915->gt); 995 if (IS_ERR(ppgtt)) 996 return PTR_ERR(ppgtt); 997 998 ppgtt->vm.file = file_priv; 999 1000 if (args->extensions) { 1001 err = i915_user_extensions(u64_to_user_ptr(args->extensions), 1002 NULL, 0, 1003 ppgtt); 1004 if (err) 1005 goto err_put; 1006 } 1007 1008 err = xa_alloc(&file_priv->vm_xa, &id, &ppgtt->vm, 1009 xa_limit_32b, GFP_KERNEL); 1010 if (err) 1011 goto err_put; 1012 1013 GEM_BUG_ON(id == 0); /* reserved for invalid/unassigned ppgtt */ 1014 args->vm_id = id; 1015 return 0; 1016 1017 err_put: 1018 i915_vm_put(&ppgtt->vm); 1019 return err; 1020 } 1021 1022 int i915_gem_vm_destroy_ioctl(struct drm_device *dev, void *data, 1023 struct drm_file *file) 1024 { 1025 struct drm_i915_file_private *file_priv = file->driver_priv; 1026 struct drm_i915_gem_vm_control *args = data; 1027 struct i915_address_space *vm; 1028 1029 if (args->flags) 1030 return -EINVAL; 1031 1032 if (args->extensions) 1033 return -EINVAL; 1034 1035 vm = xa_erase(&file_priv->vm_xa, args->vm_id); 1036 if (!vm) 1037 return -ENOENT; 1038 1039 i915_vm_put(vm); 1040 return 0; 1041 } 1042 1043 struct context_barrier_task { 1044 struct i915_active base; 1045 void (*task)(void *data); 1046 void *data; 1047 }; 1048 1049 static void cb_retire(struct i915_active *base) 1050 { 1051 struct context_barrier_task *cb = container_of(base, typeof(*cb), base); 1052 1053 if (cb->task) 1054 cb->task(cb->data); 1055 1056 i915_active_fini(&cb->base); 1057 kfree(cb); 1058 } 1059 1060 I915_SELFTEST_DECLARE(static intel_engine_mask_t context_barrier_inject_fault); 1061 static int context_barrier_task(struct i915_gem_context *ctx, 1062 intel_engine_mask_t engines, 1063 bool (*skip)(struct intel_context *ce, void *data), 1064 int (*pin)(struct intel_context *ce, struct i915_gem_ww_ctx *ww, void *data), 1065 int (*emit)(struct i915_request *rq, void *data), 1066 void (*task)(void *data), 1067 void *data) 1068 { 1069 struct context_barrier_task *cb; 1070 struct i915_gem_engines_iter it; 1071 struct i915_gem_engines *e; 1072 struct i915_gem_ww_ctx ww; 1073 struct intel_context *ce; 1074 int err = 0; 1075 1076 GEM_BUG_ON(!task); 1077 1078 cb = kmalloc(sizeof(*cb), GFP_KERNEL); 1079 if (!cb) 1080 return -ENOMEM; 1081 1082 i915_active_init(&cb->base, NULL, cb_retire, 0); 1083 err = i915_active_acquire(&cb->base); 1084 if (err) { 1085 kfree(cb); 1086 return err; 1087 } 1088 1089 e = __context_engines_await(ctx, NULL); 1090 if (!e) { 1091 i915_active_release(&cb->base); 1092 return -ENOENT; 1093 } 1094 1095 for_each_gem_engine(ce, e, it) { 1096 struct i915_request *rq; 1097 1098 if (I915_SELFTEST_ONLY(context_barrier_inject_fault & 1099 ce->engine->mask)) { 1100 err = -ENXIO; 1101 break; 1102 } 1103 1104 if (!(ce->engine->mask & engines)) 1105 continue; 1106 1107 if (skip && skip(ce, data)) 1108 continue; 1109 1110 i915_gem_ww_ctx_init(&ww, true); 1111 retry: 1112 err = intel_context_pin_ww(ce, &ww); 1113 if (err) 1114 goto err; 1115 1116 if (pin) 1117 err = pin(ce, &ww, data); 1118 if (err) 1119 goto err_unpin; 1120 1121 rq = i915_request_create(ce); 1122 if (IS_ERR(rq)) { 1123 err = PTR_ERR(rq); 1124 goto err_unpin; 1125 } 1126 1127 err = 0; 1128 if (emit) 1129 err = emit(rq, data); 1130 if (err == 0) 1131 err = i915_active_add_request(&cb->base, rq); 1132 1133 i915_request_add(rq); 1134 err_unpin: 1135 intel_context_unpin(ce); 1136 err: 1137 if (err == -EDEADLK) { 1138 err = i915_gem_ww_ctx_backoff(&ww); 1139 if (!err) 1140 goto retry; 1141 } 1142 i915_gem_ww_ctx_fini(&ww); 1143 1144 if (err) 1145 break; 1146 } 1147 i915_sw_fence_complete(&e->fence); 1148 1149 cb->task = err ? NULL : task; /* caller needs to unwind instead */ 1150 cb->data = data; 1151 1152 i915_active_release(&cb->base); 1153 1154 return err; 1155 } 1156 1157 static int get_ppgtt(struct drm_i915_file_private *file_priv, 1158 struct i915_gem_context *ctx, 1159 struct drm_i915_gem_context_param *args) 1160 { 1161 struct i915_address_space *vm; 1162 int err; 1163 u32 id; 1164 1165 if (!rcu_access_pointer(ctx->vm)) 1166 return -ENODEV; 1167 1168 rcu_read_lock(); 1169 vm = context_get_vm_rcu(ctx); 1170 rcu_read_unlock(); 1171 if (!vm) 1172 return -ENODEV; 1173 1174 err = xa_alloc(&file_priv->vm_xa, &id, vm, xa_limit_32b, GFP_KERNEL); 1175 if (err) 1176 goto err_put; 1177 1178 i915_vm_open(vm); 1179 1180 GEM_BUG_ON(id == 0); /* reserved for invalid/unassigned ppgtt */ 1181 args->value = id; 1182 args->size = 0; 1183 1184 err_put: 1185 i915_vm_put(vm); 1186 return err; 1187 } 1188 1189 static void set_ppgtt_barrier(void *data) 1190 { 1191 struct i915_address_space *old = data; 1192 1193 if (GRAPHICS_VER(old->i915) < 8) 1194 gen6_ppgtt_unpin_all(i915_vm_to_ppgtt(old)); 1195 1196 i915_vm_close(old); 1197 } 1198 1199 static int pin_ppgtt_update(struct intel_context *ce, struct i915_gem_ww_ctx *ww, void *data) 1200 { 1201 struct i915_address_space *vm = ce->vm; 1202 1203 if (!HAS_LOGICAL_RING_CONTEXTS(vm->i915)) 1204 /* ppGTT is not part of the legacy context image */ 1205 return gen6_ppgtt_pin(i915_vm_to_ppgtt(vm), ww); 1206 1207 return 0; 1208 } 1209 1210 static int emit_ppgtt_update(struct i915_request *rq, void *data) 1211 { 1212 struct i915_address_space *vm = rq->context->vm; 1213 struct intel_engine_cs *engine = rq->engine; 1214 u32 base = engine->mmio_base; 1215 u32 *cs; 1216 int i; 1217 1218 if (i915_vm_is_4lvl(vm)) { 1219 struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1220 const dma_addr_t pd_daddr = px_dma(ppgtt->pd); 1221 1222 cs = intel_ring_begin(rq, 6); 1223 if (IS_ERR(cs)) 1224 return PTR_ERR(cs); 1225 1226 *cs++ = MI_LOAD_REGISTER_IMM(2); 1227 1228 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 0)); 1229 *cs++ = upper_32_bits(pd_daddr); 1230 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 0)); 1231 *cs++ = lower_32_bits(pd_daddr); 1232 1233 *cs++ = MI_NOOP; 1234 intel_ring_advance(rq, cs); 1235 } else if (HAS_LOGICAL_RING_CONTEXTS(engine->i915)) { 1236 struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1237 int err; 1238 1239 /* Magic required to prevent forcewake errors! */ 1240 err = engine->emit_flush(rq, EMIT_INVALIDATE); 1241 if (err) 1242 return err; 1243 1244 cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2); 1245 if (IS_ERR(cs)) 1246 return PTR_ERR(cs); 1247 1248 *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED; 1249 for (i = GEN8_3LVL_PDPES; i--; ) { 1250 const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); 1251 1252 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i)); 1253 *cs++ = upper_32_bits(pd_daddr); 1254 *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i)); 1255 *cs++ = lower_32_bits(pd_daddr); 1256 } 1257 *cs++ = MI_NOOP; 1258 intel_ring_advance(rq, cs); 1259 } 1260 1261 return 0; 1262 } 1263 1264 static bool skip_ppgtt_update(struct intel_context *ce, void *data) 1265 { 1266 if (HAS_LOGICAL_RING_CONTEXTS(ce->engine->i915)) 1267 return !ce->state; 1268 else 1269 return !atomic_read(&ce->pin_count); 1270 } 1271 1272 static int set_ppgtt(struct drm_i915_file_private *file_priv, 1273 struct i915_gem_context *ctx, 1274 struct drm_i915_gem_context_param *args) 1275 { 1276 struct i915_address_space *vm, *old; 1277 int err; 1278 1279 if (args->size) 1280 return -EINVAL; 1281 1282 if (!rcu_access_pointer(ctx->vm)) 1283 return -ENODEV; 1284 1285 if (upper_32_bits(args->value)) 1286 return -ENOENT; 1287 1288 rcu_read_lock(); 1289 vm = xa_load(&file_priv->vm_xa, args->value); 1290 if (vm && !kref_get_unless_zero(&vm->ref)) 1291 vm = NULL; 1292 rcu_read_unlock(); 1293 if (!vm) 1294 return -ENOENT; 1295 1296 err = mutex_lock_interruptible(&ctx->mutex); 1297 if (err) 1298 goto out; 1299 1300 if (i915_gem_context_is_closed(ctx)) { 1301 err = -ENOENT; 1302 goto unlock; 1303 } 1304 1305 if (vm == rcu_access_pointer(ctx->vm)) 1306 goto unlock; 1307 1308 old = __set_ppgtt(ctx, vm); 1309 1310 /* Teardown the existing obj:vma cache, it will have to be rebuilt. */ 1311 lut_close(ctx); 1312 1313 /* 1314 * We need to flush any requests using the current ppgtt before 1315 * we release it as the requests do not hold a reference themselves, 1316 * only indirectly through the context. 1317 */ 1318 err = context_barrier_task(ctx, ALL_ENGINES, 1319 skip_ppgtt_update, 1320 pin_ppgtt_update, 1321 emit_ppgtt_update, 1322 set_ppgtt_barrier, 1323 old); 1324 if (err) { 1325 i915_vm_close(__set_ppgtt(ctx, old)); 1326 i915_vm_close(old); 1327 lut_close(ctx); /* force a rebuild of the old obj:vma cache */ 1328 } 1329 1330 unlock: 1331 mutex_unlock(&ctx->mutex); 1332 out: 1333 i915_vm_put(vm); 1334 return err; 1335 } 1336 1337 int 1338 i915_gem_user_to_context_sseu(struct intel_gt *gt, 1339 const struct drm_i915_gem_context_param_sseu *user, 1340 struct intel_sseu *context) 1341 { 1342 const struct sseu_dev_info *device = >->info.sseu; 1343 struct drm_i915_private *i915 = gt->i915; 1344 1345 /* No zeros in any field. */ 1346 if (!user->slice_mask || !user->subslice_mask || 1347 !user->min_eus_per_subslice || !user->max_eus_per_subslice) 1348 return -EINVAL; 1349 1350 /* Max > min. */ 1351 if (user->max_eus_per_subslice < user->min_eus_per_subslice) 1352 return -EINVAL; 1353 1354 /* 1355 * Some future proofing on the types since the uAPI is wider than the 1356 * current internal implementation. 1357 */ 1358 if (overflows_type(user->slice_mask, context->slice_mask) || 1359 overflows_type(user->subslice_mask, context->subslice_mask) || 1360 overflows_type(user->min_eus_per_subslice, 1361 context->min_eus_per_subslice) || 1362 overflows_type(user->max_eus_per_subslice, 1363 context->max_eus_per_subslice)) 1364 return -EINVAL; 1365 1366 /* Check validity against hardware. */ 1367 if (user->slice_mask & ~device->slice_mask) 1368 return -EINVAL; 1369 1370 if (user->subslice_mask & ~device->subslice_mask[0]) 1371 return -EINVAL; 1372 1373 if (user->max_eus_per_subslice > device->max_eus_per_subslice) 1374 return -EINVAL; 1375 1376 context->slice_mask = user->slice_mask; 1377 context->subslice_mask = user->subslice_mask; 1378 context->min_eus_per_subslice = user->min_eus_per_subslice; 1379 context->max_eus_per_subslice = user->max_eus_per_subslice; 1380 1381 /* Part specific restrictions. */ 1382 if (GRAPHICS_VER(i915) == 11) { 1383 unsigned int hw_s = hweight8(device->slice_mask); 1384 unsigned int hw_ss_per_s = hweight8(device->subslice_mask[0]); 1385 unsigned int req_s = hweight8(context->slice_mask); 1386 unsigned int req_ss = hweight8(context->subslice_mask); 1387 1388 /* 1389 * Only full subslice enablement is possible if more than one 1390 * slice is turned on. 1391 */ 1392 if (req_s > 1 && req_ss != hw_ss_per_s) 1393 return -EINVAL; 1394 1395 /* 1396 * If more than four (SScount bitfield limit) subslices are 1397 * requested then the number has to be even. 1398 */ 1399 if (req_ss > 4 && (req_ss & 1)) 1400 return -EINVAL; 1401 1402 /* 1403 * If only one slice is enabled and subslice count is below the 1404 * device full enablement, it must be at most half of the all 1405 * available subslices. 1406 */ 1407 if (req_s == 1 && req_ss < hw_ss_per_s && 1408 req_ss > (hw_ss_per_s / 2)) 1409 return -EINVAL; 1410 1411 /* ABI restriction - VME use case only. */ 1412 1413 /* All slices or one slice only. */ 1414 if (req_s != 1 && req_s != hw_s) 1415 return -EINVAL; 1416 1417 /* 1418 * Half subslices or full enablement only when one slice is 1419 * enabled. 1420 */ 1421 if (req_s == 1 && 1422 (req_ss != hw_ss_per_s && req_ss != (hw_ss_per_s / 2))) 1423 return -EINVAL; 1424 1425 /* No EU configuration changes. */ 1426 if ((user->min_eus_per_subslice != 1427 device->max_eus_per_subslice) || 1428 (user->max_eus_per_subslice != 1429 device->max_eus_per_subslice)) 1430 return -EINVAL; 1431 } 1432 1433 return 0; 1434 } 1435 1436 static int set_sseu(struct i915_gem_context *ctx, 1437 struct drm_i915_gem_context_param *args) 1438 { 1439 struct drm_i915_private *i915 = ctx->i915; 1440 struct drm_i915_gem_context_param_sseu user_sseu; 1441 struct intel_context *ce; 1442 struct intel_sseu sseu; 1443 unsigned long lookup; 1444 int ret; 1445 1446 if (args->size < sizeof(user_sseu)) 1447 return -EINVAL; 1448 1449 if (GRAPHICS_VER(i915) != 11) 1450 return -ENODEV; 1451 1452 if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), 1453 sizeof(user_sseu))) 1454 return -EFAULT; 1455 1456 if (user_sseu.rsvd) 1457 return -EINVAL; 1458 1459 if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) 1460 return -EINVAL; 1461 1462 lookup = 0; 1463 if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) 1464 lookup |= LOOKUP_USER_INDEX; 1465 1466 ce = lookup_user_engine(ctx, lookup, &user_sseu.engine); 1467 if (IS_ERR(ce)) 1468 return PTR_ERR(ce); 1469 1470 /* Only render engine supports RPCS configuration. */ 1471 if (ce->engine->class != RENDER_CLASS) { 1472 ret = -ENODEV; 1473 goto out_ce; 1474 } 1475 1476 ret = i915_gem_user_to_context_sseu(ce->engine->gt, &user_sseu, &sseu); 1477 if (ret) 1478 goto out_ce; 1479 1480 ret = intel_context_reconfigure_sseu(ce, sseu); 1481 if (ret) 1482 goto out_ce; 1483 1484 args->size = sizeof(user_sseu); 1485 1486 out_ce: 1487 intel_context_put(ce); 1488 return ret; 1489 } 1490 1491 struct set_engines { 1492 struct i915_gem_context *ctx; 1493 struct i915_gem_engines *engines; 1494 }; 1495 1496 static int 1497 set_engines__load_balance(struct i915_user_extension __user *base, void *data) 1498 { 1499 struct i915_context_engines_load_balance __user *ext = 1500 container_of_user(base, typeof(*ext), base); 1501 const struct set_engines *set = data; 1502 struct drm_i915_private *i915 = set->ctx->i915; 1503 struct intel_engine_cs *stack[16]; 1504 struct intel_engine_cs **siblings; 1505 struct intel_context *ce; 1506 u16 num_siblings, idx; 1507 unsigned int n; 1508 int err; 1509 1510 if (!HAS_EXECLISTS(i915)) 1511 return -ENODEV; 1512 1513 if (intel_uc_uses_guc_submission(&i915->gt.uc)) 1514 return -ENODEV; /* not implement yet */ 1515 1516 if (get_user(idx, &ext->engine_index)) 1517 return -EFAULT; 1518 1519 if (idx >= set->engines->num_engines) { 1520 drm_dbg(&i915->drm, "Invalid placement value, %d >= %d\n", 1521 idx, set->engines->num_engines); 1522 return -EINVAL; 1523 } 1524 1525 idx = array_index_nospec(idx, set->engines->num_engines); 1526 if (set->engines->engines[idx]) { 1527 drm_dbg(&i915->drm, 1528 "Invalid placement[%d], already occupied\n", idx); 1529 return -EEXIST; 1530 } 1531 1532 if (get_user(num_siblings, &ext->num_siblings)) 1533 return -EFAULT; 1534 1535 err = check_user_mbz(&ext->flags); 1536 if (err) 1537 return err; 1538 1539 err = check_user_mbz(&ext->mbz64); 1540 if (err) 1541 return err; 1542 1543 siblings = stack; 1544 if (num_siblings > ARRAY_SIZE(stack)) { 1545 siblings = kmalloc_array(num_siblings, 1546 sizeof(*siblings), 1547 GFP_KERNEL); 1548 if (!siblings) 1549 return -ENOMEM; 1550 } 1551 1552 for (n = 0; n < num_siblings; n++) { 1553 struct i915_engine_class_instance ci; 1554 1555 if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) { 1556 err = -EFAULT; 1557 goto out_siblings; 1558 } 1559 1560 siblings[n] = intel_engine_lookup_user(i915, 1561 ci.engine_class, 1562 ci.engine_instance); 1563 if (!siblings[n]) { 1564 drm_dbg(&i915->drm, 1565 "Invalid sibling[%d]: { class:%d, inst:%d }\n", 1566 n, ci.engine_class, ci.engine_instance); 1567 err = -EINVAL; 1568 goto out_siblings; 1569 } 1570 } 1571 1572 ce = intel_execlists_create_virtual(siblings, n); 1573 if (IS_ERR(ce)) { 1574 err = PTR_ERR(ce); 1575 goto out_siblings; 1576 } 1577 1578 intel_context_set_gem(ce, set->ctx); 1579 1580 if (cmpxchg(&set->engines->engines[idx], NULL, ce)) { 1581 intel_context_put(ce); 1582 err = -EEXIST; 1583 goto out_siblings; 1584 } 1585 1586 out_siblings: 1587 if (siblings != stack) 1588 kfree(siblings); 1589 1590 return err; 1591 } 1592 1593 static int 1594 set_engines__bond(struct i915_user_extension __user *base, void *data) 1595 { 1596 struct i915_context_engines_bond __user *ext = 1597 container_of_user(base, typeof(*ext), base); 1598 const struct set_engines *set = data; 1599 struct drm_i915_private *i915 = set->ctx->i915; 1600 struct i915_engine_class_instance ci; 1601 struct intel_engine_cs *virtual; 1602 struct intel_engine_cs *master; 1603 u16 idx, num_bonds; 1604 int err, n; 1605 1606 if (get_user(idx, &ext->virtual_index)) 1607 return -EFAULT; 1608 1609 if (idx >= set->engines->num_engines) { 1610 drm_dbg(&i915->drm, 1611 "Invalid index for virtual engine: %d >= %d\n", 1612 idx, set->engines->num_engines); 1613 return -EINVAL; 1614 } 1615 1616 idx = array_index_nospec(idx, set->engines->num_engines); 1617 if (!set->engines->engines[idx]) { 1618 drm_dbg(&i915->drm, "Invalid engine at %d\n", idx); 1619 return -EINVAL; 1620 } 1621 virtual = set->engines->engines[idx]->engine; 1622 1623 err = check_user_mbz(&ext->flags); 1624 if (err) 1625 return err; 1626 1627 for (n = 0; n < ARRAY_SIZE(ext->mbz64); n++) { 1628 err = check_user_mbz(&ext->mbz64[n]); 1629 if (err) 1630 return err; 1631 } 1632 1633 if (copy_from_user(&ci, &ext->master, sizeof(ci))) 1634 return -EFAULT; 1635 1636 master = intel_engine_lookup_user(i915, 1637 ci.engine_class, ci.engine_instance); 1638 if (!master) { 1639 drm_dbg(&i915->drm, 1640 "Unrecognised master engine: { class:%u, instance:%u }\n", 1641 ci.engine_class, ci.engine_instance); 1642 return -EINVAL; 1643 } 1644 1645 if (get_user(num_bonds, &ext->num_bonds)) 1646 return -EFAULT; 1647 1648 for (n = 0; n < num_bonds; n++) { 1649 struct intel_engine_cs *bond; 1650 1651 if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) 1652 return -EFAULT; 1653 1654 bond = intel_engine_lookup_user(i915, 1655 ci.engine_class, 1656 ci.engine_instance); 1657 if (!bond) { 1658 drm_dbg(&i915->drm, 1659 "Unrecognised engine[%d] for bonding: { class:%d, instance: %d }\n", 1660 n, ci.engine_class, ci.engine_instance); 1661 return -EINVAL; 1662 } 1663 1664 /* 1665 * A non-virtual engine has no siblings to choose between; and 1666 * a submit fence will always be directed to the one engine. 1667 */ 1668 if (intel_engine_is_virtual(virtual)) { 1669 err = intel_virtual_engine_attach_bond(virtual, 1670 master, 1671 bond); 1672 if (err) 1673 return err; 1674 } 1675 } 1676 1677 return 0; 1678 } 1679 1680 static const i915_user_extension_fn set_engines__extensions[] = { 1681 [I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE] = set_engines__load_balance, 1682 [I915_CONTEXT_ENGINES_EXT_BOND] = set_engines__bond, 1683 }; 1684 1685 static int 1686 set_engines(struct i915_gem_context *ctx, 1687 const struct drm_i915_gem_context_param *args) 1688 { 1689 struct drm_i915_private *i915 = ctx->i915; 1690 struct i915_context_param_engines __user *user = 1691 u64_to_user_ptr(args->value); 1692 struct set_engines set = { .ctx = ctx }; 1693 unsigned int num_engines, n; 1694 u64 extensions; 1695 int err; 1696 1697 if (!args->size) { /* switch back to legacy user_ring_map */ 1698 if (!i915_gem_context_user_engines(ctx)) 1699 return 0; 1700 1701 set.engines = default_engines(ctx); 1702 if (IS_ERR(set.engines)) 1703 return PTR_ERR(set.engines); 1704 1705 goto replace; 1706 } 1707 1708 BUILD_BUG_ON(!IS_ALIGNED(sizeof(*user), sizeof(*user->engines))); 1709 if (args->size < sizeof(*user) || 1710 !IS_ALIGNED(args->size, sizeof(*user->engines))) { 1711 drm_dbg(&i915->drm, "Invalid size for engine array: %d\n", 1712 args->size); 1713 return -EINVAL; 1714 } 1715 1716 /* 1717 * Note that I915_EXEC_RING_MASK limits execbuf to only using the 1718 * first 64 engines defined here. 1719 */ 1720 num_engines = (args->size - sizeof(*user)) / sizeof(*user->engines); 1721 set.engines = alloc_engines(num_engines); 1722 if (!set.engines) 1723 return -ENOMEM; 1724 1725 for (n = 0; n < num_engines; n++) { 1726 struct i915_engine_class_instance ci; 1727 struct intel_engine_cs *engine; 1728 struct intel_context *ce; 1729 1730 if (copy_from_user(&ci, &user->engines[n], sizeof(ci))) { 1731 __free_engines(set.engines, n); 1732 return -EFAULT; 1733 } 1734 1735 if (ci.engine_class == (u16)I915_ENGINE_CLASS_INVALID && 1736 ci.engine_instance == (u16)I915_ENGINE_CLASS_INVALID_NONE) { 1737 set.engines->engines[n] = NULL; 1738 continue; 1739 } 1740 1741 engine = intel_engine_lookup_user(ctx->i915, 1742 ci.engine_class, 1743 ci.engine_instance); 1744 if (!engine) { 1745 drm_dbg(&i915->drm, 1746 "Invalid engine[%d]: { class:%d, instance:%d }\n", 1747 n, ci.engine_class, ci.engine_instance); 1748 __free_engines(set.engines, n); 1749 return -ENOENT; 1750 } 1751 1752 ce = intel_context_create(engine); 1753 if (IS_ERR(ce)) { 1754 __free_engines(set.engines, n); 1755 return PTR_ERR(ce); 1756 } 1757 1758 intel_context_set_gem(ce, ctx); 1759 1760 set.engines->engines[n] = ce; 1761 } 1762 set.engines->num_engines = num_engines; 1763 1764 err = -EFAULT; 1765 if (!get_user(extensions, &user->extensions)) 1766 err = i915_user_extensions(u64_to_user_ptr(extensions), 1767 set_engines__extensions, 1768 ARRAY_SIZE(set_engines__extensions), 1769 &set); 1770 if (err) { 1771 free_engines(set.engines); 1772 return err; 1773 } 1774 1775 replace: 1776 mutex_lock(&ctx->engines_mutex); 1777 if (i915_gem_context_is_closed(ctx)) { 1778 mutex_unlock(&ctx->engines_mutex); 1779 free_engines(set.engines); 1780 return -ENOENT; 1781 } 1782 if (args->size) 1783 i915_gem_context_set_user_engines(ctx); 1784 else 1785 i915_gem_context_clear_user_engines(ctx); 1786 set.engines = rcu_replace_pointer(ctx->engines, set.engines, 1); 1787 mutex_unlock(&ctx->engines_mutex); 1788 1789 /* Keep track of old engine sets for kill_context() */ 1790 engines_idle_release(ctx, set.engines); 1791 1792 return 0; 1793 } 1794 1795 static int 1796 get_engines(struct i915_gem_context *ctx, 1797 struct drm_i915_gem_context_param *args) 1798 { 1799 struct i915_context_param_engines __user *user; 1800 struct i915_gem_engines *e; 1801 size_t n, count, size; 1802 bool user_engines; 1803 int err = 0; 1804 1805 e = __context_engines_await(ctx, &user_engines); 1806 if (!e) 1807 return -ENOENT; 1808 1809 if (!user_engines) { 1810 i915_sw_fence_complete(&e->fence); 1811 args->size = 0; 1812 return 0; 1813 } 1814 1815 count = e->num_engines; 1816 1817 /* Be paranoid in case we have an impedance mismatch */ 1818 if (!check_struct_size(user, engines, count, &size)) { 1819 err = -EINVAL; 1820 goto err_free; 1821 } 1822 if (overflows_type(size, args->size)) { 1823 err = -EINVAL; 1824 goto err_free; 1825 } 1826 1827 if (!args->size) { 1828 args->size = size; 1829 goto err_free; 1830 } 1831 1832 if (args->size < size) { 1833 err = -EINVAL; 1834 goto err_free; 1835 } 1836 1837 user = u64_to_user_ptr(args->value); 1838 if (put_user(0, &user->extensions)) { 1839 err = -EFAULT; 1840 goto err_free; 1841 } 1842 1843 for (n = 0; n < count; n++) { 1844 struct i915_engine_class_instance ci = { 1845 .engine_class = I915_ENGINE_CLASS_INVALID, 1846 .engine_instance = I915_ENGINE_CLASS_INVALID_NONE, 1847 }; 1848 1849 if (e->engines[n]) { 1850 ci.engine_class = e->engines[n]->engine->uabi_class; 1851 ci.engine_instance = e->engines[n]->engine->uabi_instance; 1852 } 1853 1854 if (copy_to_user(&user->engines[n], &ci, sizeof(ci))) { 1855 err = -EFAULT; 1856 goto err_free; 1857 } 1858 } 1859 1860 args->size = size; 1861 1862 err_free: 1863 i915_sw_fence_complete(&e->fence); 1864 return err; 1865 } 1866 1867 static int 1868 set_persistence(struct i915_gem_context *ctx, 1869 const struct drm_i915_gem_context_param *args) 1870 { 1871 if (args->size) 1872 return -EINVAL; 1873 1874 return __context_set_persistence(ctx, args->value); 1875 } 1876 1877 static int __apply_priority(struct intel_context *ce, void *arg) 1878 { 1879 struct i915_gem_context *ctx = arg; 1880 1881 if (!intel_engine_has_timeslices(ce->engine)) 1882 return 0; 1883 1884 if (ctx->sched.priority >= I915_PRIORITY_NORMAL) 1885 intel_context_set_use_semaphores(ce); 1886 else 1887 intel_context_clear_use_semaphores(ce); 1888 1889 return 0; 1890 } 1891 1892 static int set_priority(struct i915_gem_context *ctx, 1893 const struct drm_i915_gem_context_param *args) 1894 { 1895 s64 priority = args->value; 1896 1897 if (args->size) 1898 return -EINVAL; 1899 1900 if (!(ctx->i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY)) 1901 return -ENODEV; 1902 1903 if (priority > I915_CONTEXT_MAX_USER_PRIORITY || 1904 priority < I915_CONTEXT_MIN_USER_PRIORITY) 1905 return -EINVAL; 1906 1907 if (priority > I915_CONTEXT_DEFAULT_PRIORITY && 1908 !capable(CAP_SYS_NICE)) 1909 return -EPERM; 1910 1911 ctx->sched.priority = priority; 1912 context_apply_all(ctx, __apply_priority, ctx); 1913 1914 return 0; 1915 } 1916 1917 static int ctx_setparam(struct drm_i915_file_private *fpriv, 1918 struct i915_gem_context *ctx, 1919 struct drm_i915_gem_context_param *args) 1920 { 1921 int ret = 0; 1922 1923 switch (args->param) { 1924 case I915_CONTEXT_PARAM_NO_ZEROMAP: 1925 if (args->size) 1926 ret = -EINVAL; 1927 else if (args->value) 1928 set_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); 1929 else 1930 clear_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); 1931 break; 1932 1933 case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 1934 if (args->size) 1935 ret = -EINVAL; 1936 else if (args->value) 1937 i915_gem_context_set_no_error_capture(ctx); 1938 else 1939 i915_gem_context_clear_no_error_capture(ctx); 1940 break; 1941 1942 case I915_CONTEXT_PARAM_BANNABLE: 1943 if (args->size) 1944 ret = -EINVAL; 1945 else if (!capable(CAP_SYS_ADMIN) && !args->value) 1946 ret = -EPERM; 1947 else if (args->value) 1948 i915_gem_context_set_bannable(ctx); 1949 else 1950 i915_gem_context_clear_bannable(ctx); 1951 break; 1952 1953 case I915_CONTEXT_PARAM_RECOVERABLE: 1954 if (args->size) 1955 ret = -EINVAL; 1956 else if (args->value) 1957 i915_gem_context_set_recoverable(ctx); 1958 else 1959 i915_gem_context_clear_recoverable(ctx); 1960 break; 1961 1962 case I915_CONTEXT_PARAM_PRIORITY: 1963 ret = set_priority(ctx, args); 1964 break; 1965 1966 case I915_CONTEXT_PARAM_SSEU: 1967 ret = set_sseu(ctx, args); 1968 break; 1969 1970 case I915_CONTEXT_PARAM_VM: 1971 ret = set_ppgtt(fpriv, ctx, args); 1972 break; 1973 1974 case I915_CONTEXT_PARAM_ENGINES: 1975 ret = set_engines(ctx, args); 1976 break; 1977 1978 case I915_CONTEXT_PARAM_PERSISTENCE: 1979 ret = set_persistence(ctx, args); 1980 break; 1981 1982 case I915_CONTEXT_PARAM_BAN_PERIOD: 1983 case I915_CONTEXT_PARAM_RINGSIZE: 1984 default: 1985 ret = -EINVAL; 1986 break; 1987 } 1988 1989 return ret; 1990 } 1991 1992 struct create_ext { 1993 struct i915_gem_context *ctx; 1994 struct drm_i915_file_private *fpriv; 1995 }; 1996 1997 static int create_setparam(struct i915_user_extension __user *ext, void *data) 1998 { 1999 struct drm_i915_gem_context_create_ext_setparam local; 2000 const struct create_ext *arg = data; 2001 2002 if (copy_from_user(&local, ext, sizeof(local))) 2003 return -EFAULT; 2004 2005 if (local.param.ctx_id) 2006 return -EINVAL; 2007 2008 return ctx_setparam(arg->fpriv, arg->ctx, &local.param); 2009 } 2010 2011 static int clone_engines(struct i915_gem_context *dst, 2012 struct i915_gem_context *src) 2013 { 2014 struct i915_gem_engines *clone, *e; 2015 bool user_engines; 2016 unsigned long n; 2017 2018 e = __context_engines_await(src, &user_engines); 2019 if (!e) 2020 return -ENOENT; 2021 2022 clone = alloc_engines(e->num_engines); 2023 if (!clone) 2024 goto err_unlock; 2025 2026 for (n = 0; n < e->num_engines; n++) { 2027 struct intel_engine_cs *engine; 2028 2029 if (!e->engines[n]) { 2030 clone->engines[n] = NULL; 2031 continue; 2032 } 2033 engine = e->engines[n]->engine; 2034 2035 /* 2036 * Virtual engines are singletons; they can only exist 2037 * inside a single context, because they embed their 2038 * HW context... As each virtual context implies a single 2039 * timeline (each engine can only dequeue a single request 2040 * at any time), it would be surprising for two contexts 2041 * to use the same engine. So let's create a copy of 2042 * the virtual engine instead. 2043 */ 2044 if (intel_engine_is_virtual(engine)) 2045 clone->engines[n] = 2046 intel_execlists_clone_virtual(engine); 2047 else 2048 clone->engines[n] = intel_context_create(engine); 2049 if (IS_ERR_OR_NULL(clone->engines[n])) { 2050 __free_engines(clone, n); 2051 goto err_unlock; 2052 } 2053 2054 intel_context_set_gem(clone->engines[n], dst); 2055 } 2056 clone->num_engines = n; 2057 i915_sw_fence_complete(&e->fence); 2058 2059 /* Serialised by constructor */ 2060 engines_idle_release(dst, rcu_replace_pointer(dst->engines, clone, 1)); 2061 if (user_engines) 2062 i915_gem_context_set_user_engines(dst); 2063 else 2064 i915_gem_context_clear_user_engines(dst); 2065 return 0; 2066 2067 err_unlock: 2068 i915_sw_fence_complete(&e->fence); 2069 return -ENOMEM; 2070 } 2071 2072 static int clone_flags(struct i915_gem_context *dst, 2073 struct i915_gem_context *src) 2074 { 2075 dst->user_flags = src->user_flags; 2076 return 0; 2077 } 2078 2079 static int clone_schedattr(struct i915_gem_context *dst, 2080 struct i915_gem_context *src) 2081 { 2082 dst->sched = src->sched; 2083 return 0; 2084 } 2085 2086 static int clone_sseu(struct i915_gem_context *dst, 2087 struct i915_gem_context *src) 2088 { 2089 struct i915_gem_engines *e = i915_gem_context_lock_engines(src); 2090 struct i915_gem_engines *clone; 2091 unsigned long n; 2092 int err; 2093 2094 /* no locking required; sole access under constructor*/ 2095 clone = __context_engines_static(dst); 2096 if (e->num_engines != clone->num_engines) { 2097 err = -EINVAL; 2098 goto unlock; 2099 } 2100 2101 for (n = 0; n < e->num_engines; n++) { 2102 struct intel_context *ce = e->engines[n]; 2103 2104 if (clone->engines[n]->engine->class != ce->engine->class) { 2105 /* Must have compatible engine maps! */ 2106 err = -EINVAL; 2107 goto unlock; 2108 } 2109 2110 /* serialises with set_sseu */ 2111 err = intel_context_lock_pinned(ce); 2112 if (err) 2113 goto unlock; 2114 2115 clone->engines[n]->sseu = ce->sseu; 2116 intel_context_unlock_pinned(ce); 2117 } 2118 2119 err = 0; 2120 unlock: 2121 i915_gem_context_unlock_engines(src); 2122 return err; 2123 } 2124 2125 static int clone_timeline(struct i915_gem_context *dst, 2126 struct i915_gem_context *src) 2127 { 2128 if (src->timeline) 2129 __assign_timeline(dst, src->timeline); 2130 2131 return 0; 2132 } 2133 2134 static int clone_vm(struct i915_gem_context *dst, 2135 struct i915_gem_context *src) 2136 { 2137 struct i915_address_space *vm; 2138 int err = 0; 2139 2140 if (!rcu_access_pointer(src->vm)) 2141 return 0; 2142 2143 rcu_read_lock(); 2144 vm = context_get_vm_rcu(src); 2145 rcu_read_unlock(); 2146 2147 if (!mutex_lock_interruptible(&dst->mutex)) { 2148 __assign_ppgtt(dst, vm); 2149 mutex_unlock(&dst->mutex); 2150 } else { 2151 err = -EINTR; 2152 } 2153 2154 i915_vm_put(vm); 2155 return err; 2156 } 2157 2158 static int create_clone(struct i915_user_extension __user *ext, void *data) 2159 { 2160 static int (* const fn[])(struct i915_gem_context *dst, 2161 struct i915_gem_context *src) = { 2162 #define MAP(x, y) [ilog2(I915_CONTEXT_CLONE_##x)] = y 2163 MAP(ENGINES, clone_engines), 2164 MAP(FLAGS, clone_flags), 2165 MAP(SCHEDATTR, clone_schedattr), 2166 MAP(SSEU, clone_sseu), 2167 MAP(TIMELINE, clone_timeline), 2168 MAP(VM, clone_vm), 2169 #undef MAP 2170 }; 2171 struct drm_i915_gem_context_create_ext_clone local; 2172 const struct create_ext *arg = data; 2173 struct i915_gem_context *dst = arg->ctx; 2174 struct i915_gem_context *src; 2175 int err, bit; 2176 2177 if (copy_from_user(&local, ext, sizeof(local))) 2178 return -EFAULT; 2179 2180 BUILD_BUG_ON(GENMASK(BITS_PER_TYPE(local.flags) - 1, ARRAY_SIZE(fn)) != 2181 I915_CONTEXT_CLONE_UNKNOWN); 2182 2183 if (local.flags & I915_CONTEXT_CLONE_UNKNOWN) 2184 return -EINVAL; 2185 2186 if (local.rsvd) 2187 return -EINVAL; 2188 2189 rcu_read_lock(); 2190 src = __i915_gem_context_lookup_rcu(arg->fpriv, local.clone_id); 2191 rcu_read_unlock(); 2192 if (!src) 2193 return -ENOENT; 2194 2195 GEM_BUG_ON(src == dst); 2196 2197 for (bit = 0; bit < ARRAY_SIZE(fn); bit++) { 2198 if (!(local.flags & BIT(bit))) 2199 continue; 2200 2201 err = fn[bit](dst, src); 2202 if (err) 2203 return err; 2204 } 2205 2206 return 0; 2207 } 2208 2209 static const i915_user_extension_fn create_extensions[] = { 2210 [I915_CONTEXT_CREATE_EXT_SETPARAM] = create_setparam, 2211 [I915_CONTEXT_CREATE_EXT_CLONE] = create_clone, 2212 }; 2213 2214 static bool client_is_banned(struct drm_i915_file_private *file_priv) 2215 { 2216 return atomic_read(&file_priv->ban_score) >= I915_CLIENT_SCORE_BANNED; 2217 } 2218 2219 int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, 2220 struct drm_file *file) 2221 { 2222 struct drm_i915_private *i915 = to_i915(dev); 2223 struct drm_i915_gem_context_create_ext *args = data; 2224 struct create_ext ext_data; 2225 int ret; 2226 u32 id; 2227 2228 if (!DRIVER_CAPS(i915)->has_logical_contexts) 2229 return -ENODEV; 2230 2231 if (args->flags & I915_CONTEXT_CREATE_FLAGS_UNKNOWN) 2232 return -EINVAL; 2233 2234 ret = intel_gt_terminally_wedged(&i915->gt); 2235 if (ret) 2236 return ret; 2237 2238 ext_data.fpriv = file->driver_priv; 2239 if (client_is_banned(ext_data.fpriv)) { 2240 drm_dbg(&i915->drm, 2241 "client %s[%d] banned from creating ctx\n", 2242 current->comm, task_pid_nr(current)); 2243 return -EIO; 2244 } 2245 2246 ext_data.ctx = i915_gem_create_context(i915, args->flags); 2247 if (IS_ERR(ext_data.ctx)) 2248 return PTR_ERR(ext_data.ctx); 2249 2250 if (args->flags & I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS) { 2251 ret = i915_user_extensions(u64_to_user_ptr(args->extensions), 2252 create_extensions, 2253 ARRAY_SIZE(create_extensions), 2254 &ext_data); 2255 if (ret) 2256 goto err_ctx; 2257 } 2258 2259 ret = gem_context_register(ext_data.ctx, ext_data.fpriv, &id); 2260 if (ret < 0) 2261 goto err_ctx; 2262 2263 args->ctx_id = id; 2264 drm_dbg(&i915->drm, "HW context %d created\n", args->ctx_id); 2265 2266 return 0; 2267 2268 err_ctx: 2269 context_close(ext_data.ctx); 2270 return ret; 2271 } 2272 2273 int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, 2274 struct drm_file *file) 2275 { 2276 struct drm_i915_gem_context_destroy *args = data; 2277 struct drm_i915_file_private *file_priv = file->driver_priv; 2278 struct i915_gem_context *ctx; 2279 2280 if (args->pad != 0) 2281 return -EINVAL; 2282 2283 if (!args->ctx_id) 2284 return -ENOENT; 2285 2286 ctx = xa_erase(&file_priv->context_xa, args->ctx_id); 2287 if (!ctx) 2288 return -ENOENT; 2289 2290 context_close(ctx); 2291 return 0; 2292 } 2293 2294 static int get_sseu(struct i915_gem_context *ctx, 2295 struct drm_i915_gem_context_param *args) 2296 { 2297 struct drm_i915_gem_context_param_sseu user_sseu; 2298 struct intel_context *ce; 2299 unsigned long lookup; 2300 int err; 2301 2302 if (args->size == 0) 2303 goto out; 2304 else if (args->size < sizeof(user_sseu)) 2305 return -EINVAL; 2306 2307 if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), 2308 sizeof(user_sseu))) 2309 return -EFAULT; 2310 2311 if (user_sseu.rsvd) 2312 return -EINVAL; 2313 2314 if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) 2315 return -EINVAL; 2316 2317 lookup = 0; 2318 if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) 2319 lookup |= LOOKUP_USER_INDEX; 2320 2321 ce = lookup_user_engine(ctx, lookup, &user_sseu.engine); 2322 if (IS_ERR(ce)) 2323 return PTR_ERR(ce); 2324 2325 err = intel_context_lock_pinned(ce); /* serialises with set_sseu */ 2326 if (err) { 2327 intel_context_put(ce); 2328 return err; 2329 } 2330 2331 user_sseu.slice_mask = ce->sseu.slice_mask; 2332 user_sseu.subslice_mask = ce->sseu.subslice_mask; 2333 user_sseu.min_eus_per_subslice = ce->sseu.min_eus_per_subslice; 2334 user_sseu.max_eus_per_subslice = ce->sseu.max_eus_per_subslice; 2335 2336 intel_context_unlock_pinned(ce); 2337 intel_context_put(ce); 2338 2339 if (copy_to_user(u64_to_user_ptr(args->value), &user_sseu, 2340 sizeof(user_sseu))) 2341 return -EFAULT; 2342 2343 out: 2344 args->size = sizeof(user_sseu); 2345 2346 return 0; 2347 } 2348 2349 int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, 2350 struct drm_file *file) 2351 { 2352 struct drm_i915_file_private *file_priv = file->driver_priv; 2353 struct drm_i915_gem_context_param *args = data; 2354 struct i915_gem_context *ctx; 2355 int ret = 0; 2356 2357 ctx = i915_gem_context_lookup(file_priv, args->ctx_id); 2358 if (!ctx) 2359 return -ENOENT; 2360 2361 switch (args->param) { 2362 case I915_CONTEXT_PARAM_NO_ZEROMAP: 2363 args->size = 0; 2364 args->value = test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); 2365 break; 2366 2367 case I915_CONTEXT_PARAM_GTT_SIZE: 2368 args->size = 0; 2369 rcu_read_lock(); 2370 if (rcu_access_pointer(ctx->vm)) 2371 args->value = rcu_dereference(ctx->vm)->total; 2372 else 2373 args->value = to_i915(dev)->ggtt.vm.total; 2374 rcu_read_unlock(); 2375 break; 2376 2377 case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 2378 args->size = 0; 2379 args->value = i915_gem_context_no_error_capture(ctx); 2380 break; 2381 2382 case I915_CONTEXT_PARAM_BANNABLE: 2383 args->size = 0; 2384 args->value = i915_gem_context_is_bannable(ctx); 2385 break; 2386 2387 case I915_CONTEXT_PARAM_RECOVERABLE: 2388 args->size = 0; 2389 args->value = i915_gem_context_is_recoverable(ctx); 2390 break; 2391 2392 case I915_CONTEXT_PARAM_PRIORITY: 2393 args->size = 0; 2394 args->value = ctx->sched.priority; 2395 break; 2396 2397 case I915_CONTEXT_PARAM_SSEU: 2398 ret = get_sseu(ctx, args); 2399 break; 2400 2401 case I915_CONTEXT_PARAM_VM: 2402 ret = get_ppgtt(file_priv, ctx, args); 2403 break; 2404 2405 case I915_CONTEXT_PARAM_ENGINES: 2406 ret = get_engines(ctx, args); 2407 break; 2408 2409 case I915_CONTEXT_PARAM_PERSISTENCE: 2410 args->size = 0; 2411 args->value = i915_gem_context_is_persistent(ctx); 2412 break; 2413 2414 case I915_CONTEXT_PARAM_BAN_PERIOD: 2415 case I915_CONTEXT_PARAM_RINGSIZE: 2416 default: 2417 ret = -EINVAL; 2418 break; 2419 } 2420 2421 i915_gem_context_put(ctx); 2422 return ret; 2423 } 2424 2425 int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, 2426 struct drm_file *file) 2427 { 2428 struct drm_i915_file_private *file_priv = file->driver_priv; 2429 struct drm_i915_gem_context_param *args = data; 2430 struct i915_gem_context *ctx; 2431 int ret; 2432 2433 ctx = i915_gem_context_lookup(file_priv, args->ctx_id); 2434 if (!ctx) 2435 return -ENOENT; 2436 2437 ret = ctx_setparam(file_priv, ctx, args); 2438 2439 i915_gem_context_put(ctx); 2440 return ret; 2441 } 2442 2443 int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, 2444 void *data, struct drm_file *file) 2445 { 2446 struct drm_i915_private *i915 = to_i915(dev); 2447 struct drm_i915_reset_stats *args = data; 2448 struct i915_gem_context *ctx; 2449 int ret; 2450 2451 if (args->flags || args->pad) 2452 return -EINVAL; 2453 2454 ret = -ENOENT; 2455 rcu_read_lock(); 2456 ctx = __i915_gem_context_lookup_rcu(file->driver_priv, args->ctx_id); 2457 if (!ctx) 2458 goto out; 2459 2460 /* 2461 * We opt for unserialised reads here. This may result in tearing 2462 * in the extremely unlikely event of a GPU hang on this context 2463 * as we are querying them. If we need that extra layer of protection, 2464 * we should wrap the hangstats with a seqlock. 2465 */ 2466 2467 if (capable(CAP_SYS_ADMIN)) 2468 args->reset_count = i915_reset_count(&i915->gpu_error); 2469 else 2470 args->reset_count = 0; 2471 2472 args->batch_active = atomic_read(&ctx->guilty_count); 2473 args->batch_pending = atomic_read(&ctx->active_count); 2474 2475 ret = 0; 2476 out: 2477 rcu_read_unlock(); 2478 return ret; 2479 } 2480 2481 /* GEM context-engines iterator: for_each_gem_engine() */ 2482 struct intel_context * 2483 i915_gem_engines_iter_next(struct i915_gem_engines_iter *it) 2484 { 2485 const struct i915_gem_engines *e = it->engines; 2486 struct intel_context *ctx; 2487 2488 if (unlikely(!e)) 2489 return NULL; 2490 2491 do { 2492 if (it->idx >= e->num_engines) 2493 return NULL; 2494 2495 ctx = e->engines[it->idx++]; 2496 } while (!ctx); 2497 2498 return ctx; 2499 } 2500 2501 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2502 #include "selftests/mock_context.c" 2503 #include "selftests/i915_gem_context.c" 2504 #endif 2505 2506 static void i915_global_gem_context_shrink(void) 2507 { 2508 kmem_cache_shrink(global.slab_luts); 2509 } 2510 2511 static void i915_global_gem_context_exit(void) 2512 { 2513 kmem_cache_destroy(global.slab_luts); 2514 } 2515 2516 static struct i915_global_gem_context global = { { 2517 .shrink = i915_global_gem_context_shrink, 2518 .exit = i915_global_gem_context_exit, 2519 } }; 2520 2521 int __init i915_global_gem_context_init(void) 2522 { 2523 global.slab_luts = KMEM_CACHE(i915_lut_handle, 0); 2524 if (!global.slab_luts) 2525 return -ENOMEM; 2526 2527 i915_global_register(&global.base); 2528 return 0; 2529 } 2530