1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2011-2012 Intel Corporation 5 */ 6 7 /* 8 * This file implements HW context support. On gen5+ a HW context consists of an 9 * opaque GPU object which is referenced at times of context saves and restores. 10 * With RC6 enabled, the context is also referenced as the GPU enters and exists 11 * from RC6 (GPU has it's own internal power context, except on gen5). Though 12 * something like a context does exist for the media ring, the code only 13 * supports contexts for the render ring. 14 * 15 * In software, there is a distinction between contexts created by the user, 16 * and the default HW context. The default HW context is used by GPU clients 17 * that do not request setup of their own hardware context. The default 18 * context's state is never restored to help prevent programming errors. This 19 * would happen if a client ran and piggy-backed off another clients GPU state. 20 * The default context only exists to give the GPU some offset to load as the 21 * current to invoke a save of the context we actually care about. In fact, the 22 * code could likely be constructed, albeit in a more complicated fashion, to 23 * never use the default context, though that limits the driver's ability to 24 * swap out, and/or destroy other contexts. 25 * 26 * All other contexts are created as a request by the GPU client. These contexts 27 * store GPU state, and thus allow GPU clients to not re-emit state (and 28 * potentially query certain state) at any time. The kernel driver makes 29 * certain that the appropriate commands are inserted. 30 * 31 * The context life cycle is semi-complicated in that context BOs may live 32 * longer than the context itself because of the way the hardware, and object 33 * tracking works. Below is a very crude representation of the state machine 34 * describing the context life. 35 * refcount pincount active 36 * S0: initial state 0 0 0 37 * S1: context created 1 0 0 38 * S2: context is currently running 2 1 X 39 * S3: GPU referenced, but not current 2 0 1 40 * S4: context is current, but destroyed 1 1 0 41 * S5: like S3, but destroyed 1 0 1 42 * 43 * The most common (but not all) transitions: 44 * S0->S1: client creates a context 45 * S1->S2: client submits execbuf with context 46 * S2->S3: other clients submits execbuf with context 47 * S3->S1: context object was retired 48 * S3->S2: clients submits another execbuf 49 * S2->S4: context destroy called with current context 50 * S3->S5->S0: destroy path 51 * S4->S5->S0: destroy path on current context 52 * 53 * There are two confusing terms used above: 54 * The "current context" means the context which is currently running on the 55 * GPU. The GPU has loaded its state already and has stored away the gtt 56 * offset of the BO. The GPU is not actively referencing the data at this 57 * offset, but it will on the next context switch. The only way to avoid this 58 * is to do a GPU reset. 59 * 60 * An "active context' is one which was previously the "current context" and is 61 * on the active list waiting for the next context switch to occur. Until this 62 * happens, the object must remain at the same gtt offset. It is therefore 63 * possible to destroy a context, but it is still active. 64 * 65 */ 66 67 #include <linux/log2.h> 68 #include <linux/nospec.h> 69 70 #include <drm/drm_cache.h> 71 #include <drm/drm_syncobj.h> 72 73 #include "gt/gen6_ppgtt.h" 74 #include "gt/intel_context.h" 75 #include "gt/intel_context_param.h" 76 #include "gt/intel_engine_heartbeat.h" 77 #include "gt/intel_engine_user.h" 78 #include "gt/intel_gpu_commands.h" 79 #include "gt/intel_ring.h" 80 81 #include "pxp/intel_pxp.h" 82 83 #include "i915_file_private.h" 84 #include "i915_gem_context.h" 85 #include "i915_trace.h" 86 #include "i915_user_extensions.h" 87 88 #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1 89 90 static struct kmem_cache *slab_luts; 91 92 struct i915_lut_handle *i915_lut_handle_alloc(void) 93 { 94 return kmem_cache_alloc(slab_luts, GFP_KERNEL); 95 } 96 97 void i915_lut_handle_free(struct i915_lut_handle *lut) 98 { 99 return kmem_cache_free(slab_luts, lut); 100 } 101 102 static void lut_close(struct i915_gem_context *ctx) 103 { 104 struct radix_tree_iter iter; 105 void __rcu **slot; 106 107 mutex_lock(&ctx->lut_mutex); 108 rcu_read_lock(); 109 radix_tree_for_each_slot(slot, &ctx->handles_vma, &iter, 0) { 110 struct i915_vma *vma = rcu_dereference_raw(*slot); 111 struct drm_i915_gem_object *obj = vma->obj; 112 struct i915_lut_handle *lut; 113 114 if (!kref_get_unless_zero(&obj->base.refcount)) 115 continue; 116 117 spin_lock(&obj->lut_lock); 118 list_for_each_entry(lut, &obj->lut_list, obj_link) { 119 if (lut->ctx != ctx) 120 continue; 121 122 if (lut->handle != iter.index) 123 continue; 124 125 list_del(&lut->obj_link); 126 break; 127 } 128 spin_unlock(&obj->lut_lock); 129 130 if (&lut->obj_link != &obj->lut_list) { 131 i915_lut_handle_free(lut); 132 radix_tree_iter_delete(&ctx->handles_vma, &iter, slot); 133 i915_vma_close(vma); 134 i915_gem_object_put(obj); 135 } 136 137 i915_gem_object_put(obj); 138 } 139 rcu_read_unlock(); 140 mutex_unlock(&ctx->lut_mutex); 141 } 142 143 static struct intel_context * 144 lookup_user_engine(struct i915_gem_context *ctx, 145 unsigned long flags, 146 const struct i915_engine_class_instance *ci) 147 #define LOOKUP_USER_INDEX BIT(0) 148 { 149 int idx; 150 151 if (!!(flags & LOOKUP_USER_INDEX) != i915_gem_context_user_engines(ctx)) 152 return ERR_PTR(-EINVAL); 153 154 if (!i915_gem_context_user_engines(ctx)) { 155 struct intel_engine_cs *engine; 156 157 engine = intel_engine_lookup_user(ctx->i915, 158 ci->engine_class, 159 ci->engine_instance); 160 if (!engine) 161 return ERR_PTR(-EINVAL); 162 163 idx = engine->legacy_idx; 164 } else { 165 idx = ci->engine_instance; 166 } 167 168 return i915_gem_context_get_engine(ctx, idx); 169 } 170 171 static int validate_priority(struct drm_i915_private *i915, 172 const struct drm_i915_gem_context_param *args) 173 { 174 s64 priority = args->value; 175 176 if (args->size) 177 return -EINVAL; 178 179 if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY)) 180 return -ENODEV; 181 182 if (priority > I915_CONTEXT_MAX_USER_PRIORITY || 183 priority < I915_CONTEXT_MIN_USER_PRIORITY) 184 return -EINVAL; 185 186 if (priority > I915_CONTEXT_DEFAULT_PRIORITY && 187 !capable(CAP_SYS_NICE)) 188 return -EPERM; 189 190 return 0; 191 } 192 193 static void proto_context_close(struct drm_i915_private *i915, 194 struct i915_gem_proto_context *pc) 195 { 196 int i; 197 198 if (pc->pxp_wakeref) 199 intel_runtime_pm_put(&i915->runtime_pm, pc->pxp_wakeref); 200 if (pc->vm) 201 i915_vm_put(pc->vm); 202 if (pc->user_engines) { 203 for (i = 0; i < pc->num_user_engines; i++) 204 kfree(pc->user_engines[i].siblings); 205 kfree(pc->user_engines); 206 } 207 kfree(pc); 208 } 209 210 static int proto_context_set_persistence(struct drm_i915_private *i915, 211 struct i915_gem_proto_context *pc, 212 bool persist) 213 { 214 if (persist) { 215 /* 216 * Only contexts that are short-lived [that will expire or be 217 * reset] are allowed to survive past termination. We require 218 * hangcheck to ensure that the persistent requests are healthy. 219 */ 220 if (!i915->params.enable_hangcheck) 221 return -EINVAL; 222 223 pc->user_flags |= BIT(UCONTEXT_PERSISTENCE); 224 } else { 225 /* To cancel a context we use "preempt-to-idle" */ 226 if (!(i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION)) 227 return -ENODEV; 228 229 /* 230 * If the cancel fails, we then need to reset, cleanly! 231 * 232 * If the per-engine reset fails, all hope is lost! We resort 233 * to a full GPU reset in that unlikely case, but realistically 234 * if the engine could not reset, the full reset does not fare 235 * much better. The damage has been done. 236 * 237 * However, if we cannot reset an engine by itself, we cannot 238 * cleanup a hanging persistent context without causing 239 * colateral damage, and we should not pretend we can by 240 * exposing the interface. 241 */ 242 if (!intel_has_reset_engine(to_gt(i915))) 243 return -ENODEV; 244 245 pc->user_flags &= ~BIT(UCONTEXT_PERSISTENCE); 246 } 247 248 return 0; 249 } 250 251 static int proto_context_set_protected(struct drm_i915_private *i915, 252 struct i915_gem_proto_context *pc, 253 bool protected) 254 { 255 int ret = 0; 256 257 if (!protected) { 258 pc->uses_protected_content = false; 259 } else if (!intel_pxp_is_enabled(&to_gt(i915)->pxp)) { 260 ret = -ENODEV; 261 } else if ((pc->user_flags & BIT(UCONTEXT_RECOVERABLE)) || 262 !(pc->user_flags & BIT(UCONTEXT_BANNABLE))) { 263 ret = -EPERM; 264 } else { 265 pc->uses_protected_content = true; 266 267 /* 268 * protected context usage requires the PXP session to be up, 269 * which in turn requires the device to be active. 270 */ 271 pc->pxp_wakeref = intel_runtime_pm_get(&i915->runtime_pm); 272 273 if (!intel_pxp_is_active(&to_gt(i915)->pxp)) 274 ret = intel_pxp_start(&to_gt(i915)->pxp); 275 } 276 277 return ret; 278 } 279 280 static struct i915_gem_proto_context * 281 proto_context_create(struct drm_i915_private *i915, unsigned int flags) 282 { 283 struct i915_gem_proto_context *pc, *err; 284 285 pc = kzalloc(sizeof(*pc), GFP_KERNEL); 286 if (!pc) 287 return ERR_PTR(-ENOMEM); 288 289 pc->num_user_engines = -1; 290 pc->user_engines = NULL; 291 pc->user_flags = BIT(UCONTEXT_BANNABLE) | 292 BIT(UCONTEXT_RECOVERABLE); 293 if (i915->params.enable_hangcheck) 294 pc->user_flags |= BIT(UCONTEXT_PERSISTENCE); 295 pc->sched.priority = I915_PRIORITY_NORMAL; 296 297 if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE) { 298 if (!HAS_EXECLISTS(i915)) { 299 err = ERR_PTR(-EINVAL); 300 goto proto_close; 301 } 302 pc->single_timeline = true; 303 } 304 305 return pc; 306 307 proto_close: 308 proto_context_close(i915, pc); 309 return err; 310 } 311 312 static int proto_context_register_locked(struct drm_i915_file_private *fpriv, 313 struct i915_gem_proto_context *pc, 314 u32 *id) 315 { 316 int ret; 317 void *old; 318 319 lockdep_assert_held(&fpriv->proto_context_lock); 320 321 ret = xa_alloc(&fpriv->context_xa, id, NULL, xa_limit_32b, GFP_KERNEL); 322 if (ret) 323 return ret; 324 325 old = xa_store(&fpriv->proto_context_xa, *id, pc, GFP_KERNEL); 326 if (xa_is_err(old)) { 327 xa_erase(&fpriv->context_xa, *id); 328 return xa_err(old); 329 } 330 WARN_ON(old); 331 332 return 0; 333 } 334 335 static int proto_context_register(struct drm_i915_file_private *fpriv, 336 struct i915_gem_proto_context *pc, 337 u32 *id) 338 { 339 int ret; 340 341 mutex_lock(&fpriv->proto_context_lock); 342 ret = proto_context_register_locked(fpriv, pc, id); 343 mutex_unlock(&fpriv->proto_context_lock); 344 345 return ret; 346 } 347 348 static struct i915_address_space * 349 i915_gem_vm_lookup(struct drm_i915_file_private *file_priv, u32 id) 350 { 351 struct i915_address_space *vm; 352 353 xa_lock(&file_priv->vm_xa); 354 vm = xa_load(&file_priv->vm_xa, id); 355 if (vm) 356 kref_get(&vm->ref); 357 xa_unlock(&file_priv->vm_xa); 358 359 return vm; 360 } 361 362 static int set_proto_ctx_vm(struct drm_i915_file_private *fpriv, 363 struct i915_gem_proto_context *pc, 364 const struct drm_i915_gem_context_param *args) 365 { 366 struct drm_i915_private *i915 = fpriv->dev_priv; 367 struct i915_address_space *vm; 368 369 if (args->size) 370 return -EINVAL; 371 372 if (!HAS_FULL_PPGTT(i915)) 373 return -ENODEV; 374 375 if (upper_32_bits(args->value)) 376 return -ENOENT; 377 378 vm = i915_gem_vm_lookup(fpriv, args->value); 379 if (!vm) 380 return -ENOENT; 381 382 if (pc->vm) 383 i915_vm_put(pc->vm); 384 pc->vm = vm; 385 386 return 0; 387 } 388 389 struct set_proto_ctx_engines { 390 struct drm_i915_private *i915; 391 unsigned num_engines; 392 struct i915_gem_proto_engine *engines; 393 }; 394 395 static int 396 set_proto_ctx_engines_balance(struct i915_user_extension __user *base, 397 void *data) 398 { 399 struct i915_context_engines_load_balance __user *ext = 400 container_of_user(base, typeof(*ext), base); 401 const struct set_proto_ctx_engines *set = data; 402 struct drm_i915_private *i915 = set->i915; 403 struct intel_engine_cs **siblings; 404 u16 num_siblings, idx; 405 unsigned int n; 406 int err; 407 408 if (!HAS_EXECLISTS(i915)) 409 return -ENODEV; 410 411 if (get_user(idx, &ext->engine_index)) 412 return -EFAULT; 413 414 if (idx >= set->num_engines) { 415 drm_dbg(&i915->drm, "Invalid placement value, %d >= %d\n", 416 idx, set->num_engines); 417 return -EINVAL; 418 } 419 420 idx = array_index_nospec(idx, set->num_engines); 421 if (set->engines[idx].type != I915_GEM_ENGINE_TYPE_INVALID) { 422 drm_dbg(&i915->drm, 423 "Invalid placement[%d], already occupied\n", idx); 424 return -EEXIST; 425 } 426 427 if (get_user(num_siblings, &ext->num_siblings)) 428 return -EFAULT; 429 430 err = check_user_mbz(&ext->flags); 431 if (err) 432 return err; 433 434 err = check_user_mbz(&ext->mbz64); 435 if (err) 436 return err; 437 438 if (num_siblings == 0) 439 return 0; 440 441 siblings = kmalloc_array(num_siblings, sizeof(*siblings), GFP_KERNEL); 442 if (!siblings) 443 return -ENOMEM; 444 445 for (n = 0; n < num_siblings; n++) { 446 struct i915_engine_class_instance ci; 447 448 if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) { 449 err = -EFAULT; 450 goto err_siblings; 451 } 452 453 siblings[n] = intel_engine_lookup_user(i915, 454 ci.engine_class, 455 ci.engine_instance); 456 if (!siblings[n]) { 457 drm_dbg(&i915->drm, 458 "Invalid sibling[%d]: { class:%d, inst:%d }\n", 459 n, ci.engine_class, ci.engine_instance); 460 err = -EINVAL; 461 goto err_siblings; 462 } 463 } 464 465 if (num_siblings == 1) { 466 set->engines[idx].type = I915_GEM_ENGINE_TYPE_PHYSICAL; 467 set->engines[idx].engine = siblings[0]; 468 kfree(siblings); 469 } else { 470 set->engines[idx].type = I915_GEM_ENGINE_TYPE_BALANCED; 471 set->engines[idx].num_siblings = num_siblings; 472 set->engines[idx].siblings = siblings; 473 } 474 475 return 0; 476 477 err_siblings: 478 kfree(siblings); 479 480 return err; 481 } 482 483 static int 484 set_proto_ctx_engines_bond(struct i915_user_extension __user *base, void *data) 485 { 486 struct i915_context_engines_bond __user *ext = 487 container_of_user(base, typeof(*ext), base); 488 const struct set_proto_ctx_engines *set = data; 489 struct drm_i915_private *i915 = set->i915; 490 struct i915_engine_class_instance ci; 491 struct intel_engine_cs *master; 492 u16 idx, num_bonds; 493 int err, n; 494 495 if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915) && 496 !IS_ROCKETLAKE(i915) && !IS_ALDERLAKE_S(i915)) { 497 drm_dbg(&i915->drm, 498 "Bonding not supported on this platform\n"); 499 return -ENODEV; 500 } 501 502 if (get_user(idx, &ext->virtual_index)) 503 return -EFAULT; 504 505 if (idx >= set->num_engines) { 506 drm_dbg(&i915->drm, 507 "Invalid index for virtual engine: %d >= %d\n", 508 idx, set->num_engines); 509 return -EINVAL; 510 } 511 512 idx = array_index_nospec(idx, set->num_engines); 513 if (set->engines[idx].type == I915_GEM_ENGINE_TYPE_INVALID) { 514 drm_dbg(&i915->drm, "Invalid engine at %d\n", idx); 515 return -EINVAL; 516 } 517 518 if (set->engines[idx].type != I915_GEM_ENGINE_TYPE_PHYSICAL) { 519 drm_dbg(&i915->drm, 520 "Bonding with virtual engines not allowed\n"); 521 return -EINVAL; 522 } 523 524 err = check_user_mbz(&ext->flags); 525 if (err) 526 return err; 527 528 for (n = 0; n < ARRAY_SIZE(ext->mbz64); n++) { 529 err = check_user_mbz(&ext->mbz64[n]); 530 if (err) 531 return err; 532 } 533 534 if (copy_from_user(&ci, &ext->master, sizeof(ci))) 535 return -EFAULT; 536 537 master = intel_engine_lookup_user(i915, 538 ci.engine_class, 539 ci.engine_instance); 540 if (!master) { 541 drm_dbg(&i915->drm, 542 "Unrecognised master engine: { class:%u, instance:%u }\n", 543 ci.engine_class, ci.engine_instance); 544 return -EINVAL; 545 } 546 547 if (intel_engine_uses_guc(master)) { 548 DRM_DEBUG("bonding extension not supported with GuC submission"); 549 return -ENODEV; 550 } 551 552 if (get_user(num_bonds, &ext->num_bonds)) 553 return -EFAULT; 554 555 for (n = 0; n < num_bonds; n++) { 556 struct intel_engine_cs *bond; 557 558 if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) 559 return -EFAULT; 560 561 bond = intel_engine_lookup_user(i915, 562 ci.engine_class, 563 ci.engine_instance); 564 if (!bond) { 565 drm_dbg(&i915->drm, 566 "Unrecognised engine[%d] for bonding: { class:%d, instance: %d }\n", 567 n, ci.engine_class, ci.engine_instance); 568 return -EINVAL; 569 } 570 } 571 572 return 0; 573 } 574 575 static int 576 set_proto_ctx_engines_parallel_submit(struct i915_user_extension __user *base, 577 void *data) 578 { 579 struct i915_context_engines_parallel_submit __user *ext = 580 container_of_user(base, typeof(*ext), base); 581 const struct set_proto_ctx_engines *set = data; 582 struct drm_i915_private *i915 = set->i915; 583 struct i915_engine_class_instance prev_engine; 584 u64 flags; 585 int err = 0, n, i, j; 586 u16 slot, width, num_siblings; 587 struct intel_engine_cs **siblings = NULL; 588 intel_engine_mask_t prev_mask; 589 590 if (get_user(slot, &ext->engine_index)) 591 return -EFAULT; 592 593 if (get_user(width, &ext->width)) 594 return -EFAULT; 595 596 if (get_user(num_siblings, &ext->num_siblings)) 597 return -EFAULT; 598 599 if (!intel_uc_uses_guc_submission(&to_gt(i915)->uc) && 600 num_siblings != 1) { 601 drm_dbg(&i915->drm, "Only 1 sibling (%d) supported in non-GuC mode\n", 602 num_siblings); 603 return -EINVAL; 604 } 605 606 if (slot >= set->num_engines) { 607 drm_dbg(&i915->drm, "Invalid placement value, %d >= %d\n", 608 slot, set->num_engines); 609 return -EINVAL; 610 } 611 612 if (set->engines[slot].type != I915_GEM_ENGINE_TYPE_INVALID) { 613 drm_dbg(&i915->drm, 614 "Invalid placement[%d], already occupied\n", slot); 615 return -EINVAL; 616 } 617 618 if (get_user(flags, &ext->flags)) 619 return -EFAULT; 620 621 if (flags) { 622 drm_dbg(&i915->drm, "Unknown flags 0x%02llx", flags); 623 return -EINVAL; 624 } 625 626 for (n = 0; n < ARRAY_SIZE(ext->mbz64); n++) { 627 err = check_user_mbz(&ext->mbz64[n]); 628 if (err) 629 return err; 630 } 631 632 if (width < 2) { 633 drm_dbg(&i915->drm, "Width (%d) < 2\n", width); 634 return -EINVAL; 635 } 636 637 if (num_siblings < 1) { 638 drm_dbg(&i915->drm, "Number siblings (%d) < 1\n", 639 num_siblings); 640 return -EINVAL; 641 } 642 643 siblings = kmalloc_array(num_siblings * width, 644 sizeof(*siblings), 645 GFP_KERNEL); 646 if (!siblings) 647 return -ENOMEM; 648 649 /* Create contexts / engines */ 650 for (i = 0; i < width; ++i) { 651 intel_engine_mask_t current_mask = 0; 652 653 for (j = 0; j < num_siblings; ++j) { 654 struct i915_engine_class_instance ci; 655 656 n = i * num_siblings + j; 657 if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) { 658 err = -EFAULT; 659 goto out_err; 660 } 661 662 siblings[n] = 663 intel_engine_lookup_user(i915, ci.engine_class, 664 ci.engine_instance); 665 if (!siblings[n]) { 666 drm_dbg(&i915->drm, 667 "Invalid sibling[%d]: { class:%d, inst:%d }\n", 668 n, ci.engine_class, ci.engine_instance); 669 err = -EINVAL; 670 goto out_err; 671 } 672 673 /* 674 * We don't support breadcrumb handshake on these 675 * classes 676 */ 677 if (siblings[n]->class == RENDER_CLASS || 678 siblings[n]->class == COMPUTE_CLASS) { 679 err = -EINVAL; 680 goto out_err; 681 } 682 683 if (n) { 684 if (prev_engine.engine_class != 685 ci.engine_class) { 686 drm_dbg(&i915->drm, 687 "Mismatched class %d, %d\n", 688 prev_engine.engine_class, 689 ci.engine_class); 690 err = -EINVAL; 691 goto out_err; 692 } 693 } 694 695 prev_engine = ci; 696 current_mask |= siblings[n]->logical_mask; 697 } 698 699 if (i > 0) { 700 if (current_mask != prev_mask << 1) { 701 drm_dbg(&i915->drm, 702 "Non contiguous logical mask 0x%x, 0x%x\n", 703 prev_mask, current_mask); 704 err = -EINVAL; 705 goto out_err; 706 } 707 } 708 prev_mask = current_mask; 709 } 710 711 set->engines[slot].type = I915_GEM_ENGINE_TYPE_PARALLEL; 712 set->engines[slot].num_siblings = num_siblings; 713 set->engines[slot].width = width; 714 set->engines[slot].siblings = siblings; 715 716 return 0; 717 718 out_err: 719 kfree(siblings); 720 721 return err; 722 } 723 724 static const i915_user_extension_fn set_proto_ctx_engines_extensions[] = { 725 [I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE] = set_proto_ctx_engines_balance, 726 [I915_CONTEXT_ENGINES_EXT_BOND] = set_proto_ctx_engines_bond, 727 [I915_CONTEXT_ENGINES_EXT_PARALLEL_SUBMIT] = 728 set_proto_ctx_engines_parallel_submit, 729 }; 730 731 static int set_proto_ctx_engines(struct drm_i915_file_private *fpriv, 732 struct i915_gem_proto_context *pc, 733 const struct drm_i915_gem_context_param *args) 734 { 735 struct drm_i915_private *i915 = fpriv->dev_priv; 736 struct set_proto_ctx_engines set = { .i915 = i915 }; 737 struct i915_context_param_engines __user *user = 738 u64_to_user_ptr(args->value); 739 unsigned int n; 740 u64 extensions; 741 int err; 742 743 if (pc->num_user_engines >= 0) { 744 drm_dbg(&i915->drm, "Cannot set engines twice"); 745 return -EINVAL; 746 } 747 748 if (args->size < sizeof(*user) || 749 !IS_ALIGNED(args->size - sizeof(*user), sizeof(*user->engines))) { 750 drm_dbg(&i915->drm, "Invalid size for engine array: %d\n", 751 args->size); 752 return -EINVAL; 753 } 754 755 set.num_engines = (args->size - sizeof(*user)) / sizeof(*user->engines); 756 /* RING_MASK has no shift so we can use it directly here */ 757 if (set.num_engines > I915_EXEC_RING_MASK + 1) 758 return -EINVAL; 759 760 set.engines = kmalloc_array(set.num_engines, sizeof(*set.engines), GFP_KERNEL); 761 if (!set.engines) 762 return -ENOMEM; 763 764 for (n = 0; n < set.num_engines; n++) { 765 struct i915_engine_class_instance ci; 766 struct intel_engine_cs *engine; 767 768 if (copy_from_user(&ci, &user->engines[n], sizeof(ci))) { 769 kfree(set.engines); 770 return -EFAULT; 771 } 772 773 memset(&set.engines[n], 0, sizeof(set.engines[n])); 774 775 if (ci.engine_class == (u16)I915_ENGINE_CLASS_INVALID && 776 ci.engine_instance == (u16)I915_ENGINE_CLASS_INVALID_NONE) 777 continue; 778 779 engine = intel_engine_lookup_user(i915, 780 ci.engine_class, 781 ci.engine_instance); 782 if (!engine) { 783 drm_dbg(&i915->drm, 784 "Invalid engine[%d]: { class:%d, instance:%d }\n", 785 n, ci.engine_class, ci.engine_instance); 786 kfree(set.engines); 787 return -ENOENT; 788 } 789 790 set.engines[n].type = I915_GEM_ENGINE_TYPE_PHYSICAL; 791 set.engines[n].engine = engine; 792 } 793 794 err = -EFAULT; 795 if (!get_user(extensions, &user->extensions)) 796 err = i915_user_extensions(u64_to_user_ptr(extensions), 797 set_proto_ctx_engines_extensions, 798 ARRAY_SIZE(set_proto_ctx_engines_extensions), 799 &set); 800 if (err) { 801 kfree(set.engines); 802 return err; 803 } 804 805 pc->num_user_engines = set.num_engines; 806 pc->user_engines = set.engines; 807 808 return 0; 809 } 810 811 static int set_proto_ctx_sseu(struct drm_i915_file_private *fpriv, 812 struct i915_gem_proto_context *pc, 813 struct drm_i915_gem_context_param *args) 814 { 815 struct drm_i915_private *i915 = fpriv->dev_priv; 816 struct drm_i915_gem_context_param_sseu user_sseu; 817 struct intel_sseu *sseu; 818 int ret; 819 820 if (args->size < sizeof(user_sseu)) 821 return -EINVAL; 822 823 if (GRAPHICS_VER(i915) != 11) 824 return -ENODEV; 825 826 if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), 827 sizeof(user_sseu))) 828 return -EFAULT; 829 830 if (user_sseu.rsvd) 831 return -EINVAL; 832 833 if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) 834 return -EINVAL; 835 836 if (!!(user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) != (pc->num_user_engines >= 0)) 837 return -EINVAL; 838 839 if (pc->num_user_engines >= 0) { 840 int idx = user_sseu.engine.engine_instance; 841 struct i915_gem_proto_engine *pe; 842 843 if (idx >= pc->num_user_engines) 844 return -EINVAL; 845 846 pe = &pc->user_engines[idx]; 847 848 /* Only render engine supports RPCS configuration. */ 849 if (pe->engine->class != RENDER_CLASS) 850 return -EINVAL; 851 852 sseu = &pe->sseu; 853 } else { 854 /* Only render engine supports RPCS configuration. */ 855 if (user_sseu.engine.engine_class != I915_ENGINE_CLASS_RENDER) 856 return -EINVAL; 857 858 /* There is only one render engine */ 859 if (user_sseu.engine.engine_instance != 0) 860 return -EINVAL; 861 862 sseu = &pc->legacy_rcs_sseu; 863 } 864 865 ret = i915_gem_user_to_context_sseu(to_gt(i915), &user_sseu, sseu); 866 if (ret) 867 return ret; 868 869 args->size = sizeof(user_sseu); 870 871 return 0; 872 } 873 874 static int set_proto_ctx_param(struct drm_i915_file_private *fpriv, 875 struct i915_gem_proto_context *pc, 876 struct drm_i915_gem_context_param *args) 877 { 878 int ret = 0; 879 880 switch (args->param) { 881 case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 882 if (args->size) 883 ret = -EINVAL; 884 else if (args->value) 885 pc->user_flags |= BIT(UCONTEXT_NO_ERROR_CAPTURE); 886 else 887 pc->user_flags &= ~BIT(UCONTEXT_NO_ERROR_CAPTURE); 888 break; 889 890 case I915_CONTEXT_PARAM_BANNABLE: 891 if (args->size) 892 ret = -EINVAL; 893 else if (!capable(CAP_SYS_ADMIN) && !args->value) 894 ret = -EPERM; 895 else if (args->value) 896 pc->user_flags |= BIT(UCONTEXT_BANNABLE); 897 else if (pc->uses_protected_content) 898 ret = -EPERM; 899 else 900 pc->user_flags &= ~BIT(UCONTEXT_BANNABLE); 901 break; 902 903 case I915_CONTEXT_PARAM_RECOVERABLE: 904 if (args->size) 905 ret = -EINVAL; 906 else if (!args->value) 907 pc->user_flags &= ~BIT(UCONTEXT_RECOVERABLE); 908 else if (pc->uses_protected_content) 909 ret = -EPERM; 910 else 911 pc->user_flags |= BIT(UCONTEXT_RECOVERABLE); 912 break; 913 914 case I915_CONTEXT_PARAM_PRIORITY: 915 ret = validate_priority(fpriv->dev_priv, args); 916 if (!ret) 917 pc->sched.priority = args->value; 918 break; 919 920 case I915_CONTEXT_PARAM_SSEU: 921 ret = set_proto_ctx_sseu(fpriv, pc, args); 922 break; 923 924 case I915_CONTEXT_PARAM_VM: 925 ret = set_proto_ctx_vm(fpriv, pc, args); 926 break; 927 928 case I915_CONTEXT_PARAM_ENGINES: 929 ret = set_proto_ctx_engines(fpriv, pc, args); 930 break; 931 932 case I915_CONTEXT_PARAM_PERSISTENCE: 933 if (args->size) 934 ret = -EINVAL; 935 ret = proto_context_set_persistence(fpriv->dev_priv, pc, 936 args->value); 937 break; 938 939 case I915_CONTEXT_PARAM_PROTECTED_CONTENT: 940 ret = proto_context_set_protected(fpriv->dev_priv, pc, 941 args->value); 942 break; 943 944 case I915_CONTEXT_PARAM_NO_ZEROMAP: 945 case I915_CONTEXT_PARAM_BAN_PERIOD: 946 case I915_CONTEXT_PARAM_RINGSIZE: 947 default: 948 ret = -EINVAL; 949 break; 950 } 951 952 return ret; 953 } 954 955 static int intel_context_set_gem(struct intel_context *ce, 956 struct i915_gem_context *ctx, 957 struct intel_sseu sseu) 958 { 959 int ret = 0; 960 961 GEM_BUG_ON(rcu_access_pointer(ce->gem_context)); 962 RCU_INIT_POINTER(ce->gem_context, ctx); 963 964 GEM_BUG_ON(intel_context_is_pinned(ce)); 965 ce->ring_size = SZ_16K; 966 967 i915_vm_put(ce->vm); 968 ce->vm = i915_gem_context_get_eb_vm(ctx); 969 970 if (ctx->sched.priority >= I915_PRIORITY_NORMAL && 971 intel_engine_has_timeslices(ce->engine) && 972 intel_engine_has_semaphores(ce->engine)) 973 __set_bit(CONTEXT_USE_SEMAPHORES, &ce->flags); 974 975 if (CONFIG_DRM_I915_REQUEST_TIMEOUT && 976 ctx->i915->params.request_timeout_ms) { 977 unsigned int timeout_ms = ctx->i915->params.request_timeout_ms; 978 979 intel_context_set_watchdog_us(ce, (u64)timeout_ms * 1000); 980 } 981 982 /* A valid SSEU has no zero fields */ 983 if (sseu.slice_mask && !WARN_ON(ce->engine->class != RENDER_CLASS)) 984 ret = intel_context_reconfigure_sseu(ce, sseu); 985 986 return ret; 987 } 988 989 static void __unpin_engines(struct i915_gem_engines *e, unsigned int count) 990 { 991 while (count--) { 992 struct intel_context *ce = e->engines[count], *child; 993 994 if (!ce || !test_bit(CONTEXT_PERMA_PIN, &ce->flags)) 995 continue; 996 997 for_each_child(ce, child) 998 intel_context_unpin(child); 999 intel_context_unpin(ce); 1000 } 1001 } 1002 1003 static void unpin_engines(struct i915_gem_engines *e) 1004 { 1005 __unpin_engines(e, e->num_engines); 1006 } 1007 1008 static void __free_engines(struct i915_gem_engines *e, unsigned int count) 1009 { 1010 while (count--) { 1011 if (!e->engines[count]) 1012 continue; 1013 1014 intel_context_put(e->engines[count]); 1015 } 1016 kfree(e); 1017 } 1018 1019 static void free_engines(struct i915_gem_engines *e) 1020 { 1021 __free_engines(e, e->num_engines); 1022 } 1023 1024 static void free_engines_rcu(struct rcu_head *rcu) 1025 { 1026 struct i915_gem_engines *engines = 1027 container_of(rcu, struct i915_gem_engines, rcu); 1028 1029 i915_sw_fence_fini(&engines->fence); 1030 free_engines(engines); 1031 } 1032 1033 static int 1034 engines_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) 1035 { 1036 struct i915_gem_engines *engines = 1037 container_of(fence, typeof(*engines), fence); 1038 1039 switch (state) { 1040 case FENCE_COMPLETE: 1041 if (!list_empty(&engines->link)) { 1042 struct i915_gem_context *ctx = engines->ctx; 1043 unsigned long flags; 1044 1045 spin_lock_irqsave(&ctx->stale.lock, flags); 1046 list_del(&engines->link); 1047 spin_unlock_irqrestore(&ctx->stale.lock, flags); 1048 } 1049 i915_gem_context_put(engines->ctx); 1050 break; 1051 1052 case FENCE_FREE: 1053 init_rcu_head(&engines->rcu); 1054 call_rcu(&engines->rcu, free_engines_rcu); 1055 break; 1056 } 1057 1058 return NOTIFY_DONE; 1059 } 1060 1061 static struct i915_gem_engines *alloc_engines(unsigned int count) 1062 { 1063 struct i915_gem_engines *e; 1064 1065 e = kzalloc(struct_size(e, engines, count), GFP_KERNEL); 1066 if (!e) 1067 return NULL; 1068 1069 i915_sw_fence_init(&e->fence, engines_notify); 1070 return e; 1071 } 1072 1073 static struct i915_gem_engines *default_engines(struct i915_gem_context *ctx, 1074 struct intel_sseu rcs_sseu) 1075 { 1076 const struct intel_gt *gt = to_gt(ctx->i915); 1077 struct intel_engine_cs *engine; 1078 struct i915_gem_engines *e, *err; 1079 enum intel_engine_id id; 1080 1081 e = alloc_engines(I915_NUM_ENGINES); 1082 if (!e) 1083 return ERR_PTR(-ENOMEM); 1084 1085 for_each_engine(engine, gt, id) { 1086 struct intel_context *ce; 1087 struct intel_sseu sseu = {}; 1088 int ret; 1089 1090 if (engine->legacy_idx == INVALID_ENGINE) 1091 continue; 1092 1093 GEM_BUG_ON(engine->legacy_idx >= I915_NUM_ENGINES); 1094 GEM_BUG_ON(e->engines[engine->legacy_idx]); 1095 1096 ce = intel_context_create(engine); 1097 if (IS_ERR(ce)) { 1098 err = ERR_CAST(ce); 1099 goto free_engines; 1100 } 1101 1102 e->engines[engine->legacy_idx] = ce; 1103 e->num_engines = max(e->num_engines, engine->legacy_idx + 1); 1104 1105 if (engine->class == RENDER_CLASS) 1106 sseu = rcs_sseu; 1107 1108 ret = intel_context_set_gem(ce, ctx, sseu); 1109 if (ret) { 1110 err = ERR_PTR(ret); 1111 goto free_engines; 1112 } 1113 1114 } 1115 1116 return e; 1117 1118 free_engines: 1119 free_engines(e); 1120 return err; 1121 } 1122 1123 static int perma_pin_contexts(struct intel_context *ce) 1124 { 1125 struct intel_context *child; 1126 int i = 0, j = 0, ret; 1127 1128 GEM_BUG_ON(!intel_context_is_parent(ce)); 1129 1130 ret = intel_context_pin(ce); 1131 if (unlikely(ret)) 1132 return ret; 1133 1134 for_each_child(ce, child) { 1135 ret = intel_context_pin(child); 1136 if (unlikely(ret)) 1137 goto unwind; 1138 ++i; 1139 } 1140 1141 set_bit(CONTEXT_PERMA_PIN, &ce->flags); 1142 1143 return 0; 1144 1145 unwind: 1146 intel_context_unpin(ce); 1147 for_each_child(ce, child) { 1148 if (j++ < i) 1149 intel_context_unpin(child); 1150 else 1151 break; 1152 } 1153 1154 return ret; 1155 } 1156 1157 static struct i915_gem_engines *user_engines(struct i915_gem_context *ctx, 1158 unsigned int num_engines, 1159 struct i915_gem_proto_engine *pe) 1160 { 1161 struct i915_gem_engines *e, *err; 1162 unsigned int n; 1163 1164 e = alloc_engines(num_engines); 1165 if (!e) 1166 return ERR_PTR(-ENOMEM); 1167 e->num_engines = num_engines; 1168 1169 for (n = 0; n < num_engines; n++) { 1170 struct intel_context *ce, *child; 1171 int ret; 1172 1173 switch (pe[n].type) { 1174 case I915_GEM_ENGINE_TYPE_PHYSICAL: 1175 ce = intel_context_create(pe[n].engine); 1176 break; 1177 1178 case I915_GEM_ENGINE_TYPE_BALANCED: 1179 ce = intel_engine_create_virtual(pe[n].siblings, 1180 pe[n].num_siblings, 0); 1181 break; 1182 1183 case I915_GEM_ENGINE_TYPE_PARALLEL: 1184 ce = intel_engine_create_parallel(pe[n].siblings, 1185 pe[n].num_siblings, 1186 pe[n].width); 1187 break; 1188 1189 case I915_GEM_ENGINE_TYPE_INVALID: 1190 default: 1191 GEM_WARN_ON(pe[n].type != I915_GEM_ENGINE_TYPE_INVALID); 1192 continue; 1193 } 1194 1195 if (IS_ERR(ce)) { 1196 err = ERR_CAST(ce); 1197 goto free_engines; 1198 } 1199 1200 e->engines[n] = ce; 1201 1202 ret = intel_context_set_gem(ce, ctx, pe->sseu); 1203 if (ret) { 1204 err = ERR_PTR(ret); 1205 goto free_engines; 1206 } 1207 for_each_child(ce, child) { 1208 ret = intel_context_set_gem(child, ctx, pe->sseu); 1209 if (ret) { 1210 err = ERR_PTR(ret); 1211 goto free_engines; 1212 } 1213 } 1214 1215 /* 1216 * XXX: Must be done after calling intel_context_set_gem as that 1217 * function changes the ring size. The ring is allocated when 1218 * the context is pinned. If the ring size is changed after 1219 * allocation we have a mismatch of the ring size and will cause 1220 * the context to hang. Presumably with a bit of reordering we 1221 * could move the perma-pin step to the backend function 1222 * intel_engine_create_parallel. 1223 */ 1224 if (pe[n].type == I915_GEM_ENGINE_TYPE_PARALLEL) { 1225 ret = perma_pin_contexts(ce); 1226 if (ret) { 1227 err = ERR_PTR(ret); 1228 goto free_engines; 1229 } 1230 } 1231 } 1232 1233 return e; 1234 1235 free_engines: 1236 free_engines(e); 1237 return err; 1238 } 1239 1240 static void i915_gem_context_release_work(struct work_struct *work) 1241 { 1242 struct i915_gem_context *ctx = container_of(work, typeof(*ctx), 1243 release_work); 1244 struct i915_address_space *vm; 1245 1246 trace_i915_context_free(ctx); 1247 GEM_BUG_ON(!i915_gem_context_is_closed(ctx)); 1248 1249 if (ctx->syncobj) 1250 drm_syncobj_put(ctx->syncobj); 1251 1252 vm = ctx->vm; 1253 if (vm) 1254 i915_vm_put(vm); 1255 1256 if (ctx->pxp_wakeref) 1257 intel_runtime_pm_put(&ctx->i915->runtime_pm, ctx->pxp_wakeref); 1258 1259 mutex_destroy(&ctx->engines_mutex); 1260 mutex_destroy(&ctx->lut_mutex); 1261 1262 put_pid(ctx->pid); 1263 mutex_destroy(&ctx->mutex); 1264 1265 kfree_rcu(ctx, rcu); 1266 } 1267 1268 void i915_gem_context_release(struct kref *ref) 1269 { 1270 struct i915_gem_context *ctx = container_of(ref, typeof(*ctx), ref); 1271 1272 queue_work(ctx->i915->wq, &ctx->release_work); 1273 } 1274 1275 static inline struct i915_gem_engines * 1276 __context_engines_static(const struct i915_gem_context *ctx) 1277 { 1278 return rcu_dereference_protected(ctx->engines, true); 1279 } 1280 1281 static void __reset_context(struct i915_gem_context *ctx, 1282 struct intel_engine_cs *engine) 1283 { 1284 intel_gt_handle_error(engine->gt, engine->mask, 0, 1285 "context closure in %s", ctx->name); 1286 } 1287 1288 static bool __cancel_engine(struct intel_engine_cs *engine) 1289 { 1290 /* 1291 * Send a "high priority pulse" down the engine to cause the 1292 * current request to be momentarily preempted. (If it fails to 1293 * be preempted, it will be reset). As we have marked our context 1294 * as banned, any incomplete request, including any running, will 1295 * be skipped following the preemption. 1296 * 1297 * If there is no hangchecking (one of the reasons why we try to 1298 * cancel the context) and no forced preemption, there may be no 1299 * means by which we reset the GPU and evict the persistent hog. 1300 * Ergo if we are unable to inject a preemptive pulse that can 1301 * kill the banned context, we fallback to doing a local reset 1302 * instead. 1303 */ 1304 return intel_engine_pulse(engine) == 0; 1305 } 1306 1307 static struct intel_engine_cs *active_engine(struct intel_context *ce) 1308 { 1309 struct intel_engine_cs *engine = NULL; 1310 struct i915_request *rq; 1311 1312 if (intel_context_has_inflight(ce)) 1313 return intel_context_inflight(ce); 1314 1315 if (!ce->timeline) 1316 return NULL; 1317 1318 /* 1319 * rq->link is only SLAB_TYPESAFE_BY_RCU, we need to hold a reference 1320 * to the request to prevent it being transferred to a new timeline 1321 * (and onto a new timeline->requests list). 1322 */ 1323 rcu_read_lock(); 1324 list_for_each_entry_reverse(rq, &ce->timeline->requests, link) { 1325 bool found; 1326 1327 /* timeline is already completed upto this point? */ 1328 if (!i915_request_get_rcu(rq)) 1329 break; 1330 1331 /* Check with the backend if the request is inflight */ 1332 found = true; 1333 if (likely(rcu_access_pointer(rq->timeline) == ce->timeline)) 1334 found = i915_request_active_engine(rq, &engine); 1335 1336 i915_request_put(rq); 1337 if (found) 1338 break; 1339 } 1340 rcu_read_unlock(); 1341 1342 return engine; 1343 } 1344 1345 static void kill_engines(struct i915_gem_engines *engines, bool ban) 1346 { 1347 struct i915_gem_engines_iter it; 1348 struct intel_context *ce; 1349 1350 /* 1351 * Map the user's engine back to the actual engines; one virtual 1352 * engine will be mapped to multiple engines, and using ctx->engine[] 1353 * the same engine may be have multiple instances in the user's map. 1354 * However, we only care about pending requests, so only include 1355 * engines on which there are incomplete requests. 1356 */ 1357 for_each_gem_engine(ce, engines, it) { 1358 struct intel_engine_cs *engine; 1359 1360 if (ban && intel_context_ban(ce, NULL)) 1361 continue; 1362 1363 /* 1364 * Check the current active state of this context; if we 1365 * are currently executing on the GPU we need to evict 1366 * ourselves. On the other hand, if we haven't yet been 1367 * submitted to the GPU or if everything is complete, 1368 * we have nothing to do. 1369 */ 1370 engine = active_engine(ce); 1371 1372 /* First attempt to gracefully cancel the context */ 1373 if (engine && !__cancel_engine(engine) && ban) 1374 /* 1375 * If we are unable to send a preemptive pulse to bump 1376 * the context from the GPU, we have to resort to a full 1377 * reset. We hope the collateral damage is worth it. 1378 */ 1379 __reset_context(engines->ctx, engine); 1380 } 1381 } 1382 1383 static void kill_context(struct i915_gem_context *ctx) 1384 { 1385 bool ban = (!i915_gem_context_is_persistent(ctx) || 1386 !ctx->i915->params.enable_hangcheck); 1387 struct i915_gem_engines *pos, *next; 1388 1389 spin_lock_irq(&ctx->stale.lock); 1390 GEM_BUG_ON(!i915_gem_context_is_closed(ctx)); 1391 list_for_each_entry_safe(pos, next, &ctx->stale.engines, link) { 1392 if (!i915_sw_fence_await(&pos->fence)) { 1393 list_del_init(&pos->link); 1394 continue; 1395 } 1396 1397 spin_unlock_irq(&ctx->stale.lock); 1398 1399 kill_engines(pos, ban); 1400 1401 spin_lock_irq(&ctx->stale.lock); 1402 GEM_BUG_ON(i915_sw_fence_signaled(&pos->fence)); 1403 list_safe_reset_next(pos, next, link); 1404 list_del_init(&pos->link); /* decouple from FENCE_COMPLETE */ 1405 1406 i915_sw_fence_complete(&pos->fence); 1407 } 1408 spin_unlock_irq(&ctx->stale.lock); 1409 } 1410 1411 static void engines_idle_release(struct i915_gem_context *ctx, 1412 struct i915_gem_engines *engines) 1413 { 1414 struct i915_gem_engines_iter it; 1415 struct intel_context *ce; 1416 1417 INIT_LIST_HEAD(&engines->link); 1418 1419 engines->ctx = i915_gem_context_get(ctx); 1420 1421 for_each_gem_engine(ce, engines, it) { 1422 int err; 1423 1424 /* serialises with execbuf */ 1425 set_bit(CONTEXT_CLOSED_BIT, &ce->flags); 1426 if (!intel_context_pin_if_active(ce)) 1427 continue; 1428 1429 /* Wait until context is finally scheduled out and retired */ 1430 err = i915_sw_fence_await_active(&engines->fence, 1431 &ce->active, 1432 I915_ACTIVE_AWAIT_BARRIER); 1433 intel_context_unpin(ce); 1434 if (err) 1435 goto kill; 1436 } 1437 1438 spin_lock_irq(&ctx->stale.lock); 1439 if (!i915_gem_context_is_closed(ctx)) 1440 list_add_tail(&engines->link, &ctx->stale.engines); 1441 spin_unlock_irq(&ctx->stale.lock); 1442 1443 kill: 1444 if (list_empty(&engines->link)) /* raced, already closed */ 1445 kill_engines(engines, true); 1446 1447 i915_sw_fence_commit(&engines->fence); 1448 } 1449 1450 static void set_closed_name(struct i915_gem_context *ctx) 1451 { 1452 char *s; 1453 1454 /* Replace '[]' with '<>' to indicate closed in debug prints */ 1455 1456 s = strrchr(ctx->name, '['); 1457 if (!s) 1458 return; 1459 1460 *s = '<'; 1461 1462 s = strchr(s + 1, ']'); 1463 if (s) 1464 *s = '>'; 1465 } 1466 1467 static void context_close(struct i915_gem_context *ctx) 1468 { 1469 struct i915_address_space *vm; 1470 1471 /* Flush any concurrent set_engines() */ 1472 mutex_lock(&ctx->engines_mutex); 1473 unpin_engines(__context_engines_static(ctx)); 1474 engines_idle_release(ctx, rcu_replace_pointer(ctx->engines, NULL, 1)); 1475 i915_gem_context_set_closed(ctx); 1476 mutex_unlock(&ctx->engines_mutex); 1477 1478 mutex_lock(&ctx->mutex); 1479 1480 set_closed_name(ctx); 1481 1482 vm = ctx->vm; 1483 if (vm) { 1484 /* i915_vm_close drops the final reference, which is a bit too 1485 * early and could result in surprises with concurrent 1486 * operations racing with thist ctx close. Keep a full reference 1487 * until the end. 1488 */ 1489 i915_vm_get(vm); 1490 i915_vm_close(vm); 1491 } 1492 1493 ctx->file_priv = ERR_PTR(-EBADF); 1494 1495 /* 1496 * The LUT uses the VMA as a backpointer to unref the object, 1497 * so we need to clear the LUT before we close all the VMA (inside 1498 * the ppgtt). 1499 */ 1500 lut_close(ctx); 1501 1502 spin_lock(&ctx->i915->gem.contexts.lock); 1503 list_del(&ctx->link); 1504 spin_unlock(&ctx->i915->gem.contexts.lock); 1505 1506 mutex_unlock(&ctx->mutex); 1507 1508 /* 1509 * If the user has disabled hangchecking, we can not be sure that 1510 * the batches will ever complete after the context is closed, 1511 * keeping the context and all resources pinned forever. So in this 1512 * case we opt to forcibly kill off all remaining requests on 1513 * context close. 1514 */ 1515 kill_context(ctx); 1516 1517 i915_gem_context_put(ctx); 1518 } 1519 1520 static int __context_set_persistence(struct i915_gem_context *ctx, bool state) 1521 { 1522 if (i915_gem_context_is_persistent(ctx) == state) 1523 return 0; 1524 1525 if (state) { 1526 /* 1527 * Only contexts that are short-lived [that will expire or be 1528 * reset] are allowed to survive past termination. We require 1529 * hangcheck to ensure that the persistent requests are healthy. 1530 */ 1531 if (!ctx->i915->params.enable_hangcheck) 1532 return -EINVAL; 1533 1534 i915_gem_context_set_persistence(ctx); 1535 } else { 1536 /* To cancel a context we use "preempt-to-idle" */ 1537 if (!(ctx->i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION)) 1538 return -ENODEV; 1539 1540 /* 1541 * If the cancel fails, we then need to reset, cleanly! 1542 * 1543 * If the per-engine reset fails, all hope is lost! We resort 1544 * to a full GPU reset in that unlikely case, but realistically 1545 * if the engine could not reset, the full reset does not fare 1546 * much better. The damage has been done. 1547 * 1548 * However, if we cannot reset an engine by itself, we cannot 1549 * cleanup a hanging persistent context without causing 1550 * colateral damage, and we should not pretend we can by 1551 * exposing the interface. 1552 */ 1553 if (!intel_has_reset_engine(to_gt(ctx->i915))) 1554 return -ENODEV; 1555 1556 i915_gem_context_clear_persistence(ctx); 1557 } 1558 1559 return 0; 1560 } 1561 1562 static struct i915_gem_context * 1563 i915_gem_create_context(struct drm_i915_private *i915, 1564 const struct i915_gem_proto_context *pc) 1565 { 1566 struct i915_gem_context *ctx; 1567 struct i915_address_space *vm = NULL; 1568 struct i915_gem_engines *e; 1569 int err; 1570 int i; 1571 1572 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 1573 if (!ctx) 1574 return ERR_PTR(-ENOMEM); 1575 1576 kref_init(&ctx->ref); 1577 ctx->i915 = i915; 1578 ctx->sched = pc->sched; 1579 mutex_init(&ctx->mutex); 1580 INIT_LIST_HEAD(&ctx->link); 1581 INIT_WORK(&ctx->release_work, i915_gem_context_release_work); 1582 1583 spin_lock_init(&ctx->stale.lock); 1584 INIT_LIST_HEAD(&ctx->stale.engines); 1585 1586 if (pc->vm) { 1587 vm = i915_vm_get(pc->vm); 1588 } else if (HAS_FULL_PPGTT(i915)) { 1589 struct i915_ppgtt *ppgtt; 1590 1591 ppgtt = i915_ppgtt_create(to_gt(i915), 0); 1592 if (IS_ERR(ppgtt)) { 1593 drm_dbg(&i915->drm, "PPGTT setup failed (%ld)\n", 1594 PTR_ERR(ppgtt)); 1595 err = PTR_ERR(ppgtt); 1596 goto err_ctx; 1597 } 1598 vm = &ppgtt->vm; 1599 } 1600 if (vm) { 1601 ctx->vm = i915_vm_open(vm); 1602 1603 /* i915_vm_open() takes a reference */ 1604 i915_vm_put(vm); 1605 } 1606 1607 mutex_init(&ctx->engines_mutex); 1608 if (pc->num_user_engines >= 0) { 1609 i915_gem_context_set_user_engines(ctx); 1610 e = user_engines(ctx, pc->num_user_engines, pc->user_engines); 1611 } else { 1612 i915_gem_context_clear_user_engines(ctx); 1613 e = default_engines(ctx, pc->legacy_rcs_sseu); 1614 } 1615 if (IS_ERR(e)) { 1616 err = PTR_ERR(e); 1617 goto err_vm; 1618 } 1619 RCU_INIT_POINTER(ctx->engines, e); 1620 1621 INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL); 1622 mutex_init(&ctx->lut_mutex); 1623 1624 /* NB: Mark all slices as needing a remap so that when the context first 1625 * loads it will restore whatever remap state already exists. If there 1626 * is no remap info, it will be a NOP. */ 1627 ctx->remap_slice = ALL_L3_SLICES(i915); 1628 1629 ctx->user_flags = pc->user_flags; 1630 1631 for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++) 1632 ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES; 1633 1634 if (pc->single_timeline) { 1635 err = drm_syncobj_create(&ctx->syncobj, 1636 DRM_SYNCOBJ_CREATE_SIGNALED, 1637 NULL); 1638 if (err) 1639 goto err_engines; 1640 } 1641 1642 if (pc->uses_protected_content) { 1643 ctx->pxp_wakeref = intel_runtime_pm_get(&i915->runtime_pm); 1644 ctx->uses_protected_content = true; 1645 } 1646 1647 trace_i915_context_create(ctx); 1648 1649 return ctx; 1650 1651 err_engines: 1652 free_engines(e); 1653 err_vm: 1654 if (ctx->vm) 1655 i915_vm_close(ctx->vm); 1656 err_ctx: 1657 kfree(ctx); 1658 return ERR_PTR(err); 1659 } 1660 1661 static void init_contexts(struct i915_gem_contexts *gc) 1662 { 1663 spin_lock_init(&gc->lock); 1664 INIT_LIST_HEAD(&gc->list); 1665 } 1666 1667 void i915_gem_init__contexts(struct drm_i915_private *i915) 1668 { 1669 init_contexts(&i915->gem.contexts); 1670 } 1671 1672 static void gem_context_register(struct i915_gem_context *ctx, 1673 struct drm_i915_file_private *fpriv, 1674 u32 id) 1675 { 1676 struct drm_i915_private *i915 = ctx->i915; 1677 void *old; 1678 1679 ctx->file_priv = fpriv; 1680 1681 ctx->pid = get_task_pid(current, PIDTYPE_PID); 1682 snprintf(ctx->name, sizeof(ctx->name), "%s[%d]", 1683 current->comm, pid_nr(ctx->pid)); 1684 1685 /* And finally expose ourselves to userspace via the idr */ 1686 old = xa_store(&fpriv->context_xa, id, ctx, GFP_KERNEL); 1687 WARN_ON(old); 1688 1689 spin_lock(&i915->gem.contexts.lock); 1690 list_add_tail(&ctx->link, &i915->gem.contexts.list); 1691 spin_unlock(&i915->gem.contexts.lock); 1692 } 1693 1694 int i915_gem_context_open(struct drm_i915_private *i915, 1695 struct drm_file *file) 1696 { 1697 struct drm_i915_file_private *file_priv = file->driver_priv; 1698 struct i915_gem_proto_context *pc; 1699 struct i915_gem_context *ctx; 1700 int err; 1701 1702 mutex_init(&file_priv->proto_context_lock); 1703 xa_init_flags(&file_priv->proto_context_xa, XA_FLAGS_ALLOC); 1704 1705 /* 0 reserved for the default context */ 1706 xa_init_flags(&file_priv->context_xa, XA_FLAGS_ALLOC1); 1707 1708 /* 0 reserved for invalid/unassigned ppgtt */ 1709 xa_init_flags(&file_priv->vm_xa, XA_FLAGS_ALLOC1); 1710 1711 pc = proto_context_create(i915, 0); 1712 if (IS_ERR(pc)) { 1713 err = PTR_ERR(pc); 1714 goto err; 1715 } 1716 1717 ctx = i915_gem_create_context(i915, pc); 1718 proto_context_close(i915, pc); 1719 if (IS_ERR(ctx)) { 1720 err = PTR_ERR(ctx); 1721 goto err; 1722 } 1723 1724 gem_context_register(ctx, file_priv, 0); 1725 1726 return 0; 1727 1728 err: 1729 xa_destroy(&file_priv->vm_xa); 1730 xa_destroy(&file_priv->context_xa); 1731 xa_destroy(&file_priv->proto_context_xa); 1732 mutex_destroy(&file_priv->proto_context_lock); 1733 return err; 1734 } 1735 1736 void i915_gem_context_close(struct drm_file *file) 1737 { 1738 struct drm_i915_file_private *file_priv = file->driver_priv; 1739 struct i915_gem_proto_context *pc; 1740 struct i915_address_space *vm; 1741 struct i915_gem_context *ctx; 1742 unsigned long idx; 1743 1744 xa_for_each(&file_priv->proto_context_xa, idx, pc) 1745 proto_context_close(file_priv->dev_priv, pc); 1746 xa_destroy(&file_priv->proto_context_xa); 1747 mutex_destroy(&file_priv->proto_context_lock); 1748 1749 xa_for_each(&file_priv->context_xa, idx, ctx) 1750 context_close(ctx); 1751 xa_destroy(&file_priv->context_xa); 1752 1753 xa_for_each(&file_priv->vm_xa, idx, vm) 1754 i915_vm_put(vm); 1755 xa_destroy(&file_priv->vm_xa); 1756 } 1757 1758 int i915_gem_vm_create_ioctl(struct drm_device *dev, void *data, 1759 struct drm_file *file) 1760 { 1761 struct drm_i915_private *i915 = to_i915(dev); 1762 struct drm_i915_gem_vm_control *args = data; 1763 struct drm_i915_file_private *file_priv = file->driver_priv; 1764 struct i915_ppgtt *ppgtt; 1765 u32 id; 1766 int err; 1767 1768 if (!HAS_FULL_PPGTT(i915)) 1769 return -ENODEV; 1770 1771 if (args->flags) 1772 return -EINVAL; 1773 1774 ppgtt = i915_ppgtt_create(to_gt(i915), 0); 1775 if (IS_ERR(ppgtt)) 1776 return PTR_ERR(ppgtt); 1777 1778 if (args->extensions) { 1779 err = i915_user_extensions(u64_to_user_ptr(args->extensions), 1780 NULL, 0, 1781 ppgtt); 1782 if (err) 1783 goto err_put; 1784 } 1785 1786 err = xa_alloc(&file_priv->vm_xa, &id, &ppgtt->vm, 1787 xa_limit_32b, GFP_KERNEL); 1788 if (err) 1789 goto err_put; 1790 1791 GEM_BUG_ON(id == 0); /* reserved for invalid/unassigned ppgtt */ 1792 args->vm_id = id; 1793 return 0; 1794 1795 err_put: 1796 i915_vm_put(&ppgtt->vm); 1797 return err; 1798 } 1799 1800 int i915_gem_vm_destroy_ioctl(struct drm_device *dev, void *data, 1801 struct drm_file *file) 1802 { 1803 struct drm_i915_file_private *file_priv = file->driver_priv; 1804 struct drm_i915_gem_vm_control *args = data; 1805 struct i915_address_space *vm; 1806 1807 if (args->flags) 1808 return -EINVAL; 1809 1810 if (args->extensions) 1811 return -EINVAL; 1812 1813 vm = xa_erase(&file_priv->vm_xa, args->vm_id); 1814 if (!vm) 1815 return -ENOENT; 1816 1817 i915_vm_put(vm); 1818 return 0; 1819 } 1820 1821 static int get_ppgtt(struct drm_i915_file_private *file_priv, 1822 struct i915_gem_context *ctx, 1823 struct drm_i915_gem_context_param *args) 1824 { 1825 struct i915_address_space *vm; 1826 int err; 1827 u32 id; 1828 1829 if (!i915_gem_context_has_full_ppgtt(ctx)) 1830 return -ENODEV; 1831 1832 vm = ctx->vm; 1833 GEM_BUG_ON(!vm); 1834 1835 err = xa_alloc(&file_priv->vm_xa, &id, vm, xa_limit_32b, GFP_KERNEL); 1836 if (err) 1837 return err; 1838 1839 i915_vm_open(vm); 1840 1841 GEM_BUG_ON(id == 0); /* reserved for invalid/unassigned ppgtt */ 1842 args->value = id; 1843 args->size = 0; 1844 1845 return err; 1846 } 1847 1848 int 1849 i915_gem_user_to_context_sseu(struct intel_gt *gt, 1850 const struct drm_i915_gem_context_param_sseu *user, 1851 struct intel_sseu *context) 1852 { 1853 const struct sseu_dev_info *device = >->info.sseu; 1854 struct drm_i915_private *i915 = gt->i915; 1855 1856 /* No zeros in any field. */ 1857 if (!user->slice_mask || !user->subslice_mask || 1858 !user->min_eus_per_subslice || !user->max_eus_per_subslice) 1859 return -EINVAL; 1860 1861 /* Max > min. */ 1862 if (user->max_eus_per_subslice < user->min_eus_per_subslice) 1863 return -EINVAL; 1864 1865 /* 1866 * Some future proofing on the types since the uAPI is wider than the 1867 * current internal implementation. 1868 */ 1869 if (overflows_type(user->slice_mask, context->slice_mask) || 1870 overflows_type(user->subslice_mask, context->subslice_mask) || 1871 overflows_type(user->min_eus_per_subslice, 1872 context->min_eus_per_subslice) || 1873 overflows_type(user->max_eus_per_subslice, 1874 context->max_eus_per_subslice)) 1875 return -EINVAL; 1876 1877 /* Check validity against hardware. */ 1878 if (user->slice_mask & ~device->slice_mask) 1879 return -EINVAL; 1880 1881 if (user->subslice_mask & ~device->subslice_mask[0]) 1882 return -EINVAL; 1883 1884 if (user->max_eus_per_subslice > device->max_eus_per_subslice) 1885 return -EINVAL; 1886 1887 context->slice_mask = user->slice_mask; 1888 context->subslice_mask = user->subslice_mask; 1889 context->min_eus_per_subslice = user->min_eus_per_subslice; 1890 context->max_eus_per_subslice = user->max_eus_per_subslice; 1891 1892 /* Part specific restrictions. */ 1893 if (GRAPHICS_VER(i915) == 11) { 1894 unsigned int hw_s = hweight8(device->slice_mask); 1895 unsigned int hw_ss_per_s = hweight8(device->subslice_mask[0]); 1896 unsigned int req_s = hweight8(context->slice_mask); 1897 unsigned int req_ss = hweight8(context->subslice_mask); 1898 1899 /* 1900 * Only full subslice enablement is possible if more than one 1901 * slice is turned on. 1902 */ 1903 if (req_s > 1 && req_ss != hw_ss_per_s) 1904 return -EINVAL; 1905 1906 /* 1907 * If more than four (SScount bitfield limit) subslices are 1908 * requested then the number has to be even. 1909 */ 1910 if (req_ss > 4 && (req_ss & 1)) 1911 return -EINVAL; 1912 1913 /* 1914 * If only one slice is enabled and subslice count is below the 1915 * device full enablement, it must be at most half of the all 1916 * available subslices. 1917 */ 1918 if (req_s == 1 && req_ss < hw_ss_per_s && 1919 req_ss > (hw_ss_per_s / 2)) 1920 return -EINVAL; 1921 1922 /* ABI restriction - VME use case only. */ 1923 1924 /* All slices or one slice only. */ 1925 if (req_s != 1 && req_s != hw_s) 1926 return -EINVAL; 1927 1928 /* 1929 * Half subslices or full enablement only when one slice is 1930 * enabled. 1931 */ 1932 if (req_s == 1 && 1933 (req_ss != hw_ss_per_s && req_ss != (hw_ss_per_s / 2))) 1934 return -EINVAL; 1935 1936 /* No EU configuration changes. */ 1937 if ((user->min_eus_per_subslice != 1938 device->max_eus_per_subslice) || 1939 (user->max_eus_per_subslice != 1940 device->max_eus_per_subslice)) 1941 return -EINVAL; 1942 } 1943 1944 return 0; 1945 } 1946 1947 static int set_sseu(struct i915_gem_context *ctx, 1948 struct drm_i915_gem_context_param *args) 1949 { 1950 struct drm_i915_private *i915 = ctx->i915; 1951 struct drm_i915_gem_context_param_sseu user_sseu; 1952 struct intel_context *ce; 1953 struct intel_sseu sseu; 1954 unsigned long lookup; 1955 int ret; 1956 1957 if (args->size < sizeof(user_sseu)) 1958 return -EINVAL; 1959 1960 if (GRAPHICS_VER(i915) != 11) 1961 return -ENODEV; 1962 1963 if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), 1964 sizeof(user_sseu))) 1965 return -EFAULT; 1966 1967 if (user_sseu.rsvd) 1968 return -EINVAL; 1969 1970 if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) 1971 return -EINVAL; 1972 1973 lookup = 0; 1974 if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) 1975 lookup |= LOOKUP_USER_INDEX; 1976 1977 ce = lookup_user_engine(ctx, lookup, &user_sseu.engine); 1978 if (IS_ERR(ce)) 1979 return PTR_ERR(ce); 1980 1981 /* Only render engine supports RPCS configuration. */ 1982 if (ce->engine->class != RENDER_CLASS) { 1983 ret = -ENODEV; 1984 goto out_ce; 1985 } 1986 1987 ret = i915_gem_user_to_context_sseu(ce->engine->gt, &user_sseu, &sseu); 1988 if (ret) 1989 goto out_ce; 1990 1991 ret = intel_context_reconfigure_sseu(ce, sseu); 1992 if (ret) 1993 goto out_ce; 1994 1995 args->size = sizeof(user_sseu); 1996 1997 out_ce: 1998 intel_context_put(ce); 1999 return ret; 2000 } 2001 2002 static int 2003 set_persistence(struct i915_gem_context *ctx, 2004 const struct drm_i915_gem_context_param *args) 2005 { 2006 if (args->size) 2007 return -EINVAL; 2008 2009 return __context_set_persistence(ctx, args->value); 2010 } 2011 2012 static int set_priority(struct i915_gem_context *ctx, 2013 const struct drm_i915_gem_context_param *args) 2014 { 2015 struct i915_gem_engines_iter it; 2016 struct intel_context *ce; 2017 int err; 2018 2019 err = validate_priority(ctx->i915, args); 2020 if (err) 2021 return err; 2022 2023 ctx->sched.priority = args->value; 2024 2025 for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 2026 if (!intel_engine_has_timeslices(ce->engine)) 2027 continue; 2028 2029 if (ctx->sched.priority >= I915_PRIORITY_NORMAL && 2030 intel_engine_has_semaphores(ce->engine)) 2031 intel_context_set_use_semaphores(ce); 2032 else 2033 intel_context_clear_use_semaphores(ce); 2034 } 2035 i915_gem_context_unlock_engines(ctx); 2036 2037 return 0; 2038 } 2039 2040 static int get_protected(struct i915_gem_context *ctx, 2041 struct drm_i915_gem_context_param *args) 2042 { 2043 args->size = 0; 2044 args->value = i915_gem_context_uses_protected_content(ctx); 2045 2046 return 0; 2047 } 2048 2049 static int ctx_setparam(struct drm_i915_file_private *fpriv, 2050 struct i915_gem_context *ctx, 2051 struct drm_i915_gem_context_param *args) 2052 { 2053 int ret = 0; 2054 2055 switch (args->param) { 2056 case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 2057 if (args->size) 2058 ret = -EINVAL; 2059 else if (args->value) 2060 i915_gem_context_set_no_error_capture(ctx); 2061 else 2062 i915_gem_context_clear_no_error_capture(ctx); 2063 break; 2064 2065 case I915_CONTEXT_PARAM_BANNABLE: 2066 if (args->size) 2067 ret = -EINVAL; 2068 else if (!capable(CAP_SYS_ADMIN) && !args->value) 2069 ret = -EPERM; 2070 else if (args->value) 2071 i915_gem_context_set_bannable(ctx); 2072 else if (i915_gem_context_uses_protected_content(ctx)) 2073 ret = -EPERM; /* can't clear this for protected contexts */ 2074 else 2075 i915_gem_context_clear_bannable(ctx); 2076 break; 2077 2078 case I915_CONTEXT_PARAM_RECOVERABLE: 2079 if (args->size) 2080 ret = -EINVAL; 2081 else if (!args->value) 2082 i915_gem_context_clear_recoverable(ctx); 2083 else if (i915_gem_context_uses_protected_content(ctx)) 2084 ret = -EPERM; /* can't set this for protected contexts */ 2085 else 2086 i915_gem_context_set_recoverable(ctx); 2087 break; 2088 2089 case I915_CONTEXT_PARAM_PRIORITY: 2090 ret = set_priority(ctx, args); 2091 break; 2092 2093 case I915_CONTEXT_PARAM_SSEU: 2094 ret = set_sseu(ctx, args); 2095 break; 2096 2097 case I915_CONTEXT_PARAM_PERSISTENCE: 2098 ret = set_persistence(ctx, args); 2099 break; 2100 2101 case I915_CONTEXT_PARAM_PROTECTED_CONTENT: 2102 case I915_CONTEXT_PARAM_NO_ZEROMAP: 2103 case I915_CONTEXT_PARAM_BAN_PERIOD: 2104 case I915_CONTEXT_PARAM_RINGSIZE: 2105 case I915_CONTEXT_PARAM_VM: 2106 case I915_CONTEXT_PARAM_ENGINES: 2107 default: 2108 ret = -EINVAL; 2109 break; 2110 } 2111 2112 return ret; 2113 } 2114 2115 struct create_ext { 2116 struct i915_gem_proto_context *pc; 2117 struct drm_i915_file_private *fpriv; 2118 }; 2119 2120 static int create_setparam(struct i915_user_extension __user *ext, void *data) 2121 { 2122 struct drm_i915_gem_context_create_ext_setparam local; 2123 const struct create_ext *arg = data; 2124 2125 if (copy_from_user(&local, ext, sizeof(local))) 2126 return -EFAULT; 2127 2128 if (local.param.ctx_id) 2129 return -EINVAL; 2130 2131 return set_proto_ctx_param(arg->fpriv, arg->pc, &local.param); 2132 } 2133 2134 static int invalid_ext(struct i915_user_extension __user *ext, void *data) 2135 { 2136 return -EINVAL; 2137 } 2138 2139 static const i915_user_extension_fn create_extensions[] = { 2140 [I915_CONTEXT_CREATE_EXT_SETPARAM] = create_setparam, 2141 [I915_CONTEXT_CREATE_EXT_CLONE] = invalid_ext, 2142 }; 2143 2144 static bool client_is_banned(struct drm_i915_file_private *file_priv) 2145 { 2146 return atomic_read(&file_priv->ban_score) >= I915_CLIENT_SCORE_BANNED; 2147 } 2148 2149 static inline struct i915_gem_context * 2150 __context_lookup(struct drm_i915_file_private *file_priv, u32 id) 2151 { 2152 struct i915_gem_context *ctx; 2153 2154 rcu_read_lock(); 2155 ctx = xa_load(&file_priv->context_xa, id); 2156 if (ctx && !kref_get_unless_zero(&ctx->ref)) 2157 ctx = NULL; 2158 rcu_read_unlock(); 2159 2160 return ctx; 2161 } 2162 2163 static struct i915_gem_context * 2164 finalize_create_context_locked(struct drm_i915_file_private *file_priv, 2165 struct i915_gem_proto_context *pc, u32 id) 2166 { 2167 struct i915_gem_context *ctx; 2168 void *old; 2169 2170 lockdep_assert_held(&file_priv->proto_context_lock); 2171 2172 ctx = i915_gem_create_context(file_priv->dev_priv, pc); 2173 if (IS_ERR(ctx)) 2174 return ctx; 2175 2176 gem_context_register(ctx, file_priv, id); 2177 2178 old = xa_erase(&file_priv->proto_context_xa, id); 2179 GEM_BUG_ON(old != pc); 2180 proto_context_close(file_priv->dev_priv, pc); 2181 2182 /* One for the xarray and one for the caller */ 2183 return i915_gem_context_get(ctx); 2184 } 2185 2186 struct i915_gem_context * 2187 i915_gem_context_lookup(struct drm_i915_file_private *file_priv, u32 id) 2188 { 2189 struct i915_gem_proto_context *pc; 2190 struct i915_gem_context *ctx; 2191 2192 ctx = __context_lookup(file_priv, id); 2193 if (ctx) 2194 return ctx; 2195 2196 mutex_lock(&file_priv->proto_context_lock); 2197 /* Try one more time under the lock */ 2198 ctx = __context_lookup(file_priv, id); 2199 if (!ctx) { 2200 pc = xa_load(&file_priv->proto_context_xa, id); 2201 if (!pc) 2202 ctx = ERR_PTR(-ENOENT); 2203 else 2204 ctx = finalize_create_context_locked(file_priv, pc, id); 2205 } 2206 mutex_unlock(&file_priv->proto_context_lock); 2207 2208 return ctx; 2209 } 2210 2211 int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, 2212 struct drm_file *file) 2213 { 2214 struct drm_i915_private *i915 = to_i915(dev); 2215 struct drm_i915_gem_context_create_ext *args = data; 2216 struct create_ext ext_data; 2217 int ret; 2218 u32 id; 2219 2220 if (!DRIVER_CAPS(i915)->has_logical_contexts) 2221 return -ENODEV; 2222 2223 if (args->flags & I915_CONTEXT_CREATE_FLAGS_UNKNOWN) 2224 return -EINVAL; 2225 2226 ret = intel_gt_terminally_wedged(to_gt(i915)); 2227 if (ret) 2228 return ret; 2229 2230 ext_data.fpriv = file->driver_priv; 2231 if (client_is_banned(ext_data.fpriv)) { 2232 drm_dbg(&i915->drm, 2233 "client %s[%d] banned from creating ctx\n", 2234 current->comm, task_pid_nr(current)); 2235 return -EIO; 2236 } 2237 2238 ext_data.pc = proto_context_create(i915, args->flags); 2239 if (IS_ERR(ext_data.pc)) 2240 return PTR_ERR(ext_data.pc); 2241 2242 if (args->flags & I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS) { 2243 ret = i915_user_extensions(u64_to_user_ptr(args->extensions), 2244 create_extensions, 2245 ARRAY_SIZE(create_extensions), 2246 &ext_data); 2247 if (ret) 2248 goto err_pc; 2249 } 2250 2251 if (GRAPHICS_VER(i915) > 12) { 2252 struct i915_gem_context *ctx; 2253 2254 /* Get ourselves a context ID */ 2255 ret = xa_alloc(&ext_data.fpriv->context_xa, &id, NULL, 2256 xa_limit_32b, GFP_KERNEL); 2257 if (ret) 2258 goto err_pc; 2259 2260 ctx = i915_gem_create_context(i915, ext_data.pc); 2261 if (IS_ERR(ctx)) { 2262 ret = PTR_ERR(ctx); 2263 goto err_pc; 2264 } 2265 2266 proto_context_close(i915, ext_data.pc); 2267 gem_context_register(ctx, ext_data.fpriv, id); 2268 } else { 2269 ret = proto_context_register(ext_data.fpriv, ext_data.pc, &id); 2270 if (ret < 0) 2271 goto err_pc; 2272 } 2273 2274 args->ctx_id = id; 2275 drm_dbg(&i915->drm, "HW context %d created\n", args->ctx_id); 2276 2277 return 0; 2278 2279 err_pc: 2280 proto_context_close(i915, ext_data.pc); 2281 return ret; 2282 } 2283 2284 int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, 2285 struct drm_file *file) 2286 { 2287 struct drm_i915_gem_context_destroy *args = data; 2288 struct drm_i915_file_private *file_priv = file->driver_priv; 2289 struct i915_gem_proto_context *pc; 2290 struct i915_gem_context *ctx; 2291 2292 if (args->pad != 0) 2293 return -EINVAL; 2294 2295 if (!args->ctx_id) 2296 return -ENOENT; 2297 2298 /* We need to hold the proto-context lock here to prevent races 2299 * with finalize_create_context_locked(). 2300 */ 2301 mutex_lock(&file_priv->proto_context_lock); 2302 ctx = xa_erase(&file_priv->context_xa, args->ctx_id); 2303 pc = xa_erase(&file_priv->proto_context_xa, args->ctx_id); 2304 mutex_unlock(&file_priv->proto_context_lock); 2305 2306 if (!ctx && !pc) 2307 return -ENOENT; 2308 GEM_WARN_ON(ctx && pc); 2309 2310 if (pc) 2311 proto_context_close(file_priv->dev_priv, pc); 2312 2313 if (ctx) 2314 context_close(ctx); 2315 2316 return 0; 2317 } 2318 2319 static int get_sseu(struct i915_gem_context *ctx, 2320 struct drm_i915_gem_context_param *args) 2321 { 2322 struct drm_i915_gem_context_param_sseu user_sseu; 2323 struct intel_context *ce; 2324 unsigned long lookup; 2325 int err; 2326 2327 if (args->size == 0) 2328 goto out; 2329 else if (args->size < sizeof(user_sseu)) 2330 return -EINVAL; 2331 2332 if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), 2333 sizeof(user_sseu))) 2334 return -EFAULT; 2335 2336 if (user_sseu.rsvd) 2337 return -EINVAL; 2338 2339 if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) 2340 return -EINVAL; 2341 2342 lookup = 0; 2343 if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) 2344 lookup |= LOOKUP_USER_INDEX; 2345 2346 ce = lookup_user_engine(ctx, lookup, &user_sseu.engine); 2347 if (IS_ERR(ce)) 2348 return PTR_ERR(ce); 2349 2350 err = intel_context_lock_pinned(ce); /* serialises with set_sseu */ 2351 if (err) { 2352 intel_context_put(ce); 2353 return err; 2354 } 2355 2356 user_sseu.slice_mask = ce->sseu.slice_mask; 2357 user_sseu.subslice_mask = ce->sseu.subslice_mask; 2358 user_sseu.min_eus_per_subslice = ce->sseu.min_eus_per_subslice; 2359 user_sseu.max_eus_per_subslice = ce->sseu.max_eus_per_subslice; 2360 2361 intel_context_unlock_pinned(ce); 2362 intel_context_put(ce); 2363 2364 if (copy_to_user(u64_to_user_ptr(args->value), &user_sseu, 2365 sizeof(user_sseu))) 2366 return -EFAULT; 2367 2368 out: 2369 args->size = sizeof(user_sseu); 2370 2371 return 0; 2372 } 2373 2374 int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, 2375 struct drm_file *file) 2376 { 2377 struct drm_i915_file_private *file_priv = file->driver_priv; 2378 struct drm_i915_gem_context_param *args = data; 2379 struct i915_gem_context *ctx; 2380 struct i915_address_space *vm; 2381 int ret = 0; 2382 2383 ctx = i915_gem_context_lookup(file_priv, args->ctx_id); 2384 if (IS_ERR(ctx)) 2385 return PTR_ERR(ctx); 2386 2387 switch (args->param) { 2388 case I915_CONTEXT_PARAM_GTT_SIZE: 2389 args->size = 0; 2390 vm = i915_gem_context_get_eb_vm(ctx); 2391 args->value = vm->total; 2392 i915_vm_put(vm); 2393 2394 break; 2395 2396 case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 2397 args->size = 0; 2398 args->value = i915_gem_context_no_error_capture(ctx); 2399 break; 2400 2401 case I915_CONTEXT_PARAM_BANNABLE: 2402 args->size = 0; 2403 args->value = i915_gem_context_is_bannable(ctx); 2404 break; 2405 2406 case I915_CONTEXT_PARAM_RECOVERABLE: 2407 args->size = 0; 2408 args->value = i915_gem_context_is_recoverable(ctx); 2409 break; 2410 2411 case I915_CONTEXT_PARAM_PRIORITY: 2412 args->size = 0; 2413 args->value = ctx->sched.priority; 2414 break; 2415 2416 case I915_CONTEXT_PARAM_SSEU: 2417 ret = get_sseu(ctx, args); 2418 break; 2419 2420 case I915_CONTEXT_PARAM_VM: 2421 ret = get_ppgtt(file_priv, ctx, args); 2422 break; 2423 2424 case I915_CONTEXT_PARAM_PERSISTENCE: 2425 args->size = 0; 2426 args->value = i915_gem_context_is_persistent(ctx); 2427 break; 2428 2429 case I915_CONTEXT_PARAM_PROTECTED_CONTENT: 2430 ret = get_protected(ctx, args); 2431 break; 2432 2433 case I915_CONTEXT_PARAM_NO_ZEROMAP: 2434 case I915_CONTEXT_PARAM_BAN_PERIOD: 2435 case I915_CONTEXT_PARAM_ENGINES: 2436 case I915_CONTEXT_PARAM_RINGSIZE: 2437 default: 2438 ret = -EINVAL; 2439 break; 2440 } 2441 2442 i915_gem_context_put(ctx); 2443 return ret; 2444 } 2445 2446 int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, 2447 struct drm_file *file) 2448 { 2449 struct drm_i915_file_private *file_priv = file->driver_priv; 2450 struct drm_i915_gem_context_param *args = data; 2451 struct i915_gem_proto_context *pc; 2452 struct i915_gem_context *ctx; 2453 int ret = 0; 2454 2455 mutex_lock(&file_priv->proto_context_lock); 2456 ctx = __context_lookup(file_priv, args->ctx_id); 2457 if (!ctx) { 2458 pc = xa_load(&file_priv->proto_context_xa, args->ctx_id); 2459 if (pc) { 2460 /* Contexts should be finalized inside 2461 * GEM_CONTEXT_CREATE starting with graphics 2462 * version 13. 2463 */ 2464 WARN_ON(GRAPHICS_VER(file_priv->dev_priv) > 12); 2465 ret = set_proto_ctx_param(file_priv, pc, args); 2466 } else { 2467 ret = -ENOENT; 2468 } 2469 } 2470 mutex_unlock(&file_priv->proto_context_lock); 2471 2472 if (ctx) { 2473 ret = ctx_setparam(file_priv, ctx, args); 2474 i915_gem_context_put(ctx); 2475 } 2476 2477 return ret; 2478 } 2479 2480 int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, 2481 void *data, struct drm_file *file) 2482 { 2483 struct drm_i915_private *i915 = to_i915(dev); 2484 struct drm_i915_reset_stats *args = data; 2485 struct i915_gem_context *ctx; 2486 2487 if (args->flags || args->pad) 2488 return -EINVAL; 2489 2490 ctx = i915_gem_context_lookup(file->driver_priv, args->ctx_id); 2491 if (IS_ERR(ctx)) 2492 return PTR_ERR(ctx); 2493 2494 /* 2495 * We opt for unserialised reads here. This may result in tearing 2496 * in the extremely unlikely event of a GPU hang on this context 2497 * as we are querying them. If we need that extra layer of protection, 2498 * we should wrap the hangstats with a seqlock. 2499 */ 2500 2501 if (capable(CAP_SYS_ADMIN)) 2502 args->reset_count = i915_reset_count(&i915->gpu_error); 2503 else 2504 args->reset_count = 0; 2505 2506 args->batch_active = atomic_read(&ctx->guilty_count); 2507 args->batch_pending = atomic_read(&ctx->active_count); 2508 2509 i915_gem_context_put(ctx); 2510 return 0; 2511 } 2512 2513 /* GEM context-engines iterator: for_each_gem_engine() */ 2514 struct intel_context * 2515 i915_gem_engines_iter_next(struct i915_gem_engines_iter *it) 2516 { 2517 const struct i915_gem_engines *e = it->engines; 2518 struct intel_context *ctx; 2519 2520 if (unlikely(!e)) 2521 return NULL; 2522 2523 do { 2524 if (it->idx >= e->num_engines) 2525 return NULL; 2526 2527 ctx = e->engines[it->idx++]; 2528 } while (!ctx); 2529 2530 return ctx; 2531 } 2532 2533 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 2534 #include "selftests/mock_context.c" 2535 #include "selftests/i915_gem_context.c" 2536 #endif 2537 2538 void i915_gem_context_module_exit(void) 2539 { 2540 kmem_cache_destroy(slab_luts); 2541 } 2542 2543 int __init i915_gem_context_module_init(void) 2544 { 2545 slab_luts = KMEM_CACHE(i915_lut_handle, 0); 2546 if (!slab_luts) 2547 return -ENOMEM; 2548 2549 return 0; 2550 } 2551