110be98a7SChris Wilson /* 210be98a7SChris Wilson * SPDX-License-Identifier: MIT 310be98a7SChris Wilson * 410be98a7SChris Wilson * Copyright © 2011-2012 Intel Corporation 510be98a7SChris Wilson */ 610be98a7SChris Wilson 710be98a7SChris Wilson /* 810be98a7SChris Wilson * This file implements HW context support. On gen5+ a HW context consists of an 910be98a7SChris Wilson * opaque GPU object which is referenced at times of context saves and restores. 1010be98a7SChris Wilson * With RC6 enabled, the context is also referenced as the GPU enters and exists 1110be98a7SChris Wilson * from RC6 (GPU has it's own internal power context, except on gen5). Though 1210be98a7SChris Wilson * something like a context does exist for the media ring, the code only 1310be98a7SChris Wilson * supports contexts for the render ring. 1410be98a7SChris Wilson * 1510be98a7SChris Wilson * In software, there is a distinction between contexts created by the user, 1610be98a7SChris Wilson * and the default HW context. The default HW context is used by GPU clients 1710be98a7SChris Wilson * that do not request setup of their own hardware context. The default 1810be98a7SChris Wilson * context's state is never restored to help prevent programming errors. This 1910be98a7SChris Wilson * would happen if a client ran and piggy-backed off another clients GPU state. 2010be98a7SChris Wilson * The default context only exists to give the GPU some offset to load as the 2110be98a7SChris Wilson * current to invoke a save of the context we actually care about. In fact, the 2210be98a7SChris Wilson * code could likely be constructed, albeit in a more complicated fashion, to 2310be98a7SChris Wilson * never use the default context, though that limits the driver's ability to 2410be98a7SChris Wilson * swap out, and/or destroy other contexts. 2510be98a7SChris Wilson * 2610be98a7SChris Wilson * All other contexts are created as a request by the GPU client. These contexts 2710be98a7SChris Wilson * store GPU state, and thus allow GPU clients to not re-emit state (and 2810be98a7SChris Wilson * potentially query certain state) at any time. The kernel driver makes 2910be98a7SChris Wilson * certain that the appropriate commands are inserted. 3010be98a7SChris Wilson * 3110be98a7SChris Wilson * The context life cycle is semi-complicated in that context BOs may live 3210be98a7SChris Wilson * longer than the context itself because of the way the hardware, and object 3310be98a7SChris Wilson * tracking works. Below is a very crude representation of the state machine 3410be98a7SChris Wilson * describing the context life. 3510be98a7SChris Wilson * refcount pincount active 3610be98a7SChris Wilson * S0: initial state 0 0 0 3710be98a7SChris Wilson * S1: context created 1 0 0 3810be98a7SChris Wilson * S2: context is currently running 2 1 X 3910be98a7SChris Wilson * S3: GPU referenced, but not current 2 0 1 4010be98a7SChris Wilson * S4: context is current, but destroyed 1 1 0 4110be98a7SChris Wilson * S5: like S3, but destroyed 1 0 1 4210be98a7SChris Wilson * 4310be98a7SChris Wilson * The most common (but not all) transitions: 4410be98a7SChris Wilson * S0->S1: client creates a context 4510be98a7SChris Wilson * S1->S2: client submits execbuf with context 4610be98a7SChris Wilson * S2->S3: other clients submits execbuf with context 4710be98a7SChris Wilson * S3->S1: context object was retired 4810be98a7SChris Wilson * S3->S2: clients submits another execbuf 4910be98a7SChris Wilson * S2->S4: context destroy called with current context 5010be98a7SChris Wilson * S3->S5->S0: destroy path 5110be98a7SChris Wilson * S4->S5->S0: destroy path on current context 5210be98a7SChris Wilson * 5310be98a7SChris Wilson * There are two confusing terms used above: 5410be98a7SChris Wilson * The "current context" means the context which is currently running on the 5510be98a7SChris Wilson * GPU. The GPU has loaded its state already and has stored away the gtt 5610be98a7SChris Wilson * offset of the BO. The GPU is not actively referencing the data at this 5710be98a7SChris Wilson * offset, but it will on the next context switch. The only way to avoid this 5810be98a7SChris Wilson * is to do a GPU reset. 5910be98a7SChris Wilson * 6010be98a7SChris Wilson * An "active context' is one which was previously the "current context" and is 6110be98a7SChris Wilson * on the active list waiting for the next context switch to occur. Until this 6210be98a7SChris Wilson * happens, the object must remain at the same gtt offset. It is therefore 6310be98a7SChris Wilson * possible to destroy a context, but it is still active. 6410be98a7SChris Wilson * 6510be98a7SChris Wilson */ 6610be98a7SChris Wilson 6710be98a7SChris Wilson #include <linux/log2.h> 6810be98a7SChris Wilson #include <linux/nospec.h> 6910be98a7SChris Wilson 702c86e55dSMatthew Auld #include "gt/gen6_ppgtt.h" 719f3ccd40SChris Wilson #include "gt/intel_context.h" 7288be76cdSChris Wilson #include "gt/intel_context_param.h" 732e0986a5SChris Wilson #include "gt/intel_engine_heartbeat.h" 74750e76b4SChris Wilson #include "gt/intel_engine_user.h" 752871ea85SChris Wilson #include "gt/intel_ring.h" 7610be98a7SChris Wilson 7710be98a7SChris Wilson #include "i915_gem_context.h" 7810be98a7SChris Wilson #include "i915_globals.h" 7910be98a7SChris Wilson #include "i915_trace.h" 8010be98a7SChris Wilson #include "i915_user_extensions.h" 8110be98a7SChris Wilson 8210be98a7SChris Wilson #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1 8310be98a7SChris Wilson 8410be98a7SChris Wilson static struct i915_global_gem_context { 8510be98a7SChris Wilson struct i915_global base; 8610be98a7SChris Wilson struct kmem_cache *slab_luts; 8710be98a7SChris Wilson } global; 8810be98a7SChris Wilson 8910be98a7SChris Wilson struct i915_lut_handle *i915_lut_handle_alloc(void) 9010be98a7SChris Wilson { 9110be98a7SChris Wilson return kmem_cache_alloc(global.slab_luts, GFP_KERNEL); 9210be98a7SChris Wilson } 9310be98a7SChris Wilson 9410be98a7SChris Wilson void i915_lut_handle_free(struct i915_lut_handle *lut) 9510be98a7SChris Wilson { 9610be98a7SChris Wilson return kmem_cache_free(global.slab_luts, lut); 9710be98a7SChris Wilson } 9810be98a7SChris Wilson 9910be98a7SChris Wilson static void lut_close(struct i915_gem_context *ctx) 10010be98a7SChris Wilson { 10110be98a7SChris Wilson struct radix_tree_iter iter; 10210be98a7SChris Wilson void __rcu **slot; 10310be98a7SChris Wilson 104f7ce8639SChris Wilson mutex_lock(&ctx->lut_mutex); 10510be98a7SChris Wilson rcu_read_lock(); 10610be98a7SChris Wilson radix_tree_for_each_slot(slot, &ctx->handles_vma, &iter, 0) { 10710be98a7SChris Wilson struct i915_vma *vma = rcu_dereference_raw(*slot); 108155ab883SChris Wilson struct drm_i915_gem_object *obj = vma->obj; 109155ab883SChris Wilson struct i915_lut_handle *lut; 11010be98a7SChris Wilson 111155ab883SChris Wilson if (!kref_get_unless_zero(&obj->base.refcount)) 112155ab883SChris Wilson continue; 113155ab883SChris Wilson 114096a42ddSChris Wilson spin_lock(&obj->lut_lock); 115155ab883SChris Wilson list_for_each_entry(lut, &obj->lut_list, obj_link) { 116155ab883SChris Wilson if (lut->ctx != ctx) 117155ab883SChris Wilson continue; 118155ab883SChris Wilson 119155ab883SChris Wilson if (lut->handle != iter.index) 120155ab883SChris Wilson continue; 121155ab883SChris Wilson 122155ab883SChris Wilson list_del(&lut->obj_link); 123155ab883SChris Wilson break; 124155ab883SChris Wilson } 125096a42ddSChris Wilson spin_unlock(&obj->lut_lock); 126155ab883SChris Wilson 127155ab883SChris Wilson if (&lut->obj_link != &obj->lut_list) { 128155ab883SChris Wilson i915_lut_handle_free(lut); 12910be98a7SChris Wilson radix_tree_iter_delete(&ctx->handles_vma, &iter, slot); 130155ab883SChris Wilson i915_vma_close(vma); 131155ab883SChris Wilson i915_gem_object_put(obj); 132155ab883SChris Wilson } 13310be98a7SChris Wilson 134155ab883SChris Wilson i915_gem_object_put(obj); 13510be98a7SChris Wilson } 13610be98a7SChris Wilson rcu_read_unlock(); 137f7ce8639SChris Wilson mutex_unlock(&ctx->lut_mutex); 13810be98a7SChris Wilson } 13910be98a7SChris Wilson 14010be98a7SChris Wilson static struct intel_context * 14110be98a7SChris Wilson lookup_user_engine(struct i915_gem_context *ctx, 14210be98a7SChris Wilson unsigned long flags, 14310be98a7SChris Wilson const struct i915_engine_class_instance *ci) 14410be98a7SChris Wilson #define LOOKUP_USER_INDEX BIT(0) 14510be98a7SChris Wilson { 14610be98a7SChris Wilson int idx; 14710be98a7SChris Wilson 14810be98a7SChris Wilson if (!!(flags & LOOKUP_USER_INDEX) != i915_gem_context_user_engines(ctx)) 14910be98a7SChris Wilson return ERR_PTR(-EINVAL); 15010be98a7SChris Wilson 15110be98a7SChris Wilson if (!i915_gem_context_user_engines(ctx)) { 15210be98a7SChris Wilson struct intel_engine_cs *engine; 15310be98a7SChris Wilson 15410be98a7SChris Wilson engine = intel_engine_lookup_user(ctx->i915, 15510be98a7SChris Wilson ci->engine_class, 15610be98a7SChris Wilson ci->engine_instance); 15710be98a7SChris Wilson if (!engine) 15810be98a7SChris Wilson return ERR_PTR(-EINVAL); 15910be98a7SChris Wilson 160f1c4d157SChris Wilson idx = engine->legacy_idx; 16110be98a7SChris Wilson } else { 16210be98a7SChris Wilson idx = ci->engine_instance; 16310be98a7SChris Wilson } 16410be98a7SChris Wilson 16510be98a7SChris Wilson return i915_gem_context_get_engine(ctx, idx); 16610be98a7SChris Wilson } 16710be98a7SChris Wilson 16827dbae8fSChris Wilson static struct i915_address_space * 16927dbae8fSChris Wilson context_get_vm_rcu(struct i915_gem_context *ctx) 17027dbae8fSChris Wilson { 17127dbae8fSChris Wilson GEM_BUG_ON(!rcu_access_pointer(ctx->vm)); 17227dbae8fSChris Wilson 17327dbae8fSChris Wilson do { 17427dbae8fSChris Wilson struct i915_address_space *vm; 17527dbae8fSChris Wilson 17627dbae8fSChris Wilson /* 17727dbae8fSChris Wilson * We do not allow downgrading from full-ppgtt [to a shared 17827dbae8fSChris Wilson * global gtt], so ctx->vm cannot become NULL. 17927dbae8fSChris Wilson */ 18027dbae8fSChris Wilson vm = rcu_dereference(ctx->vm); 18127dbae8fSChris Wilson if (!kref_get_unless_zero(&vm->ref)) 18227dbae8fSChris Wilson continue; 18327dbae8fSChris Wilson 18427dbae8fSChris Wilson /* 18527dbae8fSChris Wilson * This ppgtt may have be reallocated between 18627dbae8fSChris Wilson * the read and the kref, and reassigned to a third 18727dbae8fSChris Wilson * context. In order to avoid inadvertent sharing 18827dbae8fSChris Wilson * of this ppgtt with that third context (and not 18927dbae8fSChris Wilson * src), we have to confirm that we have the same 19027dbae8fSChris Wilson * ppgtt after passing through the strong memory 19127dbae8fSChris Wilson * barrier implied by a successful 19227dbae8fSChris Wilson * kref_get_unless_zero(). 19327dbae8fSChris Wilson * 19427dbae8fSChris Wilson * Once we have acquired the current ppgtt of ctx, 19527dbae8fSChris Wilson * we no longer care if it is released from ctx, as 19627dbae8fSChris Wilson * it cannot be reallocated elsewhere. 19727dbae8fSChris Wilson */ 19827dbae8fSChris Wilson 19927dbae8fSChris Wilson if (vm == rcu_access_pointer(ctx->vm)) 20027dbae8fSChris Wilson return rcu_pointer_handoff(vm); 20127dbae8fSChris Wilson 20227dbae8fSChris Wilson i915_vm_put(vm); 20327dbae8fSChris Wilson } while (1); 20427dbae8fSChris Wilson } 20527dbae8fSChris Wilson 206e6ba7648SChris Wilson static void intel_context_set_gem(struct intel_context *ce, 207e6ba7648SChris Wilson struct i915_gem_context *ctx) 208e6ba7648SChris Wilson { 2096a8679c0SChris Wilson GEM_BUG_ON(rcu_access_pointer(ce->gem_context)); 2106a8679c0SChris Wilson RCU_INIT_POINTER(ce->gem_context, ctx); 211e6ba7648SChris Wilson 212e6ba7648SChris Wilson if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) 213e6ba7648SChris Wilson ce->ring = __intel_context_ring_size(SZ_16K); 214e6ba7648SChris Wilson 215e6ba7648SChris Wilson if (rcu_access_pointer(ctx->vm)) { 216e6ba7648SChris Wilson struct i915_address_space *vm; 217e6ba7648SChris Wilson 218e6ba7648SChris Wilson rcu_read_lock(); 219e6ba7648SChris Wilson vm = context_get_vm_rcu(ctx); /* hmm */ 220e6ba7648SChris Wilson rcu_read_unlock(); 221e6ba7648SChris Wilson 222e6ba7648SChris Wilson i915_vm_put(ce->vm); 223e6ba7648SChris Wilson ce->vm = vm; 224e6ba7648SChris Wilson } 225e6ba7648SChris Wilson 226e6ba7648SChris Wilson GEM_BUG_ON(ce->timeline); 227e6ba7648SChris Wilson if (ctx->timeline) 228e6ba7648SChris Wilson ce->timeline = intel_timeline_get(ctx->timeline); 229e6ba7648SChris Wilson 230e6ba7648SChris Wilson if (ctx->sched.priority >= I915_PRIORITY_NORMAL && 2310eb670aaSChris Wilson intel_engine_has_timeslices(ce->engine)) 232e6ba7648SChris Wilson __set_bit(CONTEXT_USE_SEMAPHORES, &ce->flags); 233e6ba7648SChris Wilson } 234e6ba7648SChris Wilson 23510be98a7SChris Wilson static void __free_engines(struct i915_gem_engines *e, unsigned int count) 23610be98a7SChris Wilson { 23710be98a7SChris Wilson while (count--) { 23810be98a7SChris Wilson if (!e->engines[count]) 23910be98a7SChris Wilson continue; 24010be98a7SChris Wilson 24110be98a7SChris Wilson intel_context_put(e->engines[count]); 24210be98a7SChris Wilson } 24310be98a7SChris Wilson kfree(e); 24410be98a7SChris Wilson } 24510be98a7SChris Wilson 24610be98a7SChris Wilson static void free_engines(struct i915_gem_engines *e) 24710be98a7SChris Wilson { 24810be98a7SChris Wilson __free_engines(e, e->num_engines); 24910be98a7SChris Wilson } 25010be98a7SChris Wilson 251155ab883SChris Wilson static void free_engines_rcu(struct rcu_head *rcu) 25210be98a7SChris Wilson { 253130a95e9SChris Wilson struct i915_gem_engines *engines = 254130a95e9SChris Wilson container_of(rcu, struct i915_gem_engines, rcu); 255130a95e9SChris Wilson 256130a95e9SChris Wilson i915_sw_fence_fini(&engines->fence); 257130a95e9SChris Wilson free_engines(engines); 25810be98a7SChris Wilson } 25910be98a7SChris Wilson 26070c96e39SChris Wilson static int __i915_sw_fence_call 26170c96e39SChris Wilson engines_notify(struct i915_sw_fence *fence, enum i915_sw_fence_notify state) 26270c96e39SChris Wilson { 26370c96e39SChris Wilson struct i915_gem_engines *engines = 26470c96e39SChris Wilson container_of(fence, typeof(*engines), fence); 26570c96e39SChris Wilson 26670c96e39SChris Wilson switch (state) { 26770c96e39SChris Wilson case FENCE_COMPLETE: 26870c96e39SChris Wilson if (!list_empty(&engines->link)) { 26970c96e39SChris Wilson struct i915_gem_context *ctx = engines->ctx; 27070c96e39SChris Wilson unsigned long flags; 27170c96e39SChris Wilson 27270c96e39SChris Wilson spin_lock_irqsave(&ctx->stale.lock, flags); 27370c96e39SChris Wilson list_del(&engines->link); 27470c96e39SChris Wilson spin_unlock_irqrestore(&ctx->stale.lock, flags); 27570c96e39SChris Wilson } 27670c96e39SChris Wilson i915_gem_context_put(engines->ctx); 27770c96e39SChris Wilson break; 27870c96e39SChris Wilson 27970c96e39SChris Wilson case FENCE_FREE: 28070c96e39SChris Wilson init_rcu_head(&engines->rcu); 28170c96e39SChris Wilson call_rcu(&engines->rcu, free_engines_rcu); 28270c96e39SChris Wilson break; 28370c96e39SChris Wilson } 28470c96e39SChris Wilson 28570c96e39SChris Wilson return NOTIFY_DONE; 28670c96e39SChris Wilson } 28770c96e39SChris Wilson 28870c96e39SChris Wilson static struct i915_gem_engines *alloc_engines(unsigned int count) 28970c96e39SChris Wilson { 29070c96e39SChris Wilson struct i915_gem_engines *e; 29170c96e39SChris Wilson 29270c96e39SChris Wilson e = kzalloc(struct_size(e, engines, count), GFP_KERNEL); 29370c96e39SChris Wilson if (!e) 29470c96e39SChris Wilson return NULL; 29570c96e39SChris Wilson 29670c96e39SChris Wilson i915_sw_fence_init(&e->fence, engines_notify); 29770c96e39SChris Wilson return e; 29870c96e39SChris Wilson } 29970c96e39SChris Wilson 30010be98a7SChris Wilson static struct i915_gem_engines *default_engines(struct i915_gem_context *ctx) 30110be98a7SChris Wilson { 302f1c4d157SChris Wilson const struct intel_gt *gt = &ctx->i915->gt; 30310be98a7SChris Wilson struct intel_engine_cs *engine; 30410be98a7SChris Wilson struct i915_gem_engines *e; 30510be98a7SChris Wilson enum intel_engine_id id; 30610be98a7SChris Wilson 30770c96e39SChris Wilson e = alloc_engines(I915_NUM_ENGINES); 30810be98a7SChris Wilson if (!e) 30910be98a7SChris Wilson return ERR_PTR(-ENOMEM); 31010be98a7SChris Wilson 311f1c4d157SChris Wilson for_each_engine(engine, gt, id) { 31210be98a7SChris Wilson struct intel_context *ce; 31310be98a7SChris Wilson 314a50134b1STvrtko Ursulin if (engine->legacy_idx == INVALID_ENGINE) 315a50134b1STvrtko Ursulin continue; 316a50134b1STvrtko Ursulin 317a50134b1STvrtko Ursulin GEM_BUG_ON(engine->legacy_idx >= I915_NUM_ENGINES); 318a50134b1STvrtko Ursulin GEM_BUG_ON(e->engines[engine->legacy_idx]); 319a50134b1STvrtko Ursulin 320e6ba7648SChris Wilson ce = intel_context_create(engine); 32110be98a7SChris Wilson if (IS_ERR(ce)) { 322a50134b1STvrtko Ursulin __free_engines(e, e->num_engines + 1); 32310be98a7SChris Wilson return ERR_CAST(ce); 32410be98a7SChris Wilson } 32510be98a7SChris Wilson 326e6ba7648SChris Wilson intel_context_set_gem(ce, ctx); 327e6ba7648SChris Wilson 328a50134b1STvrtko Ursulin e->engines[engine->legacy_idx] = ce; 329a50134b1STvrtko Ursulin e->num_engines = max(e->num_engines, engine->legacy_idx); 33010be98a7SChris Wilson } 331a50134b1STvrtko Ursulin e->num_engines++; 33210be98a7SChris Wilson 33310be98a7SChris Wilson return e; 33410be98a7SChris Wilson } 33510be98a7SChris Wilson 33610be98a7SChris Wilson static void i915_gem_context_free(struct i915_gem_context *ctx) 33710be98a7SChris Wilson { 33810be98a7SChris Wilson GEM_BUG_ON(!i915_gem_context_is_closed(ctx)); 33910be98a7SChris Wilson 340a4e7ccdaSChris Wilson spin_lock(&ctx->i915->gem.contexts.lock); 341a4e7ccdaSChris Wilson list_del(&ctx->link); 342a4e7ccdaSChris Wilson spin_unlock(&ctx->i915->gem.contexts.lock); 343a4e7ccdaSChris Wilson 34410be98a7SChris Wilson mutex_destroy(&ctx->engines_mutex); 345f7ce8639SChris Wilson mutex_destroy(&ctx->lut_mutex); 34610be98a7SChris Wilson 34710be98a7SChris Wilson if (ctx->timeline) 348f0c02c1bSTvrtko Ursulin intel_timeline_put(ctx->timeline); 34910be98a7SChris Wilson 35010be98a7SChris Wilson put_pid(ctx->pid); 35110be98a7SChris Wilson mutex_destroy(&ctx->mutex); 35210be98a7SChris Wilson 35310be98a7SChris Wilson kfree_rcu(ctx, rcu); 35410be98a7SChris Wilson } 35510be98a7SChris Wilson 356a4e7ccdaSChris Wilson static void contexts_free_all(struct llist_node *list) 35710be98a7SChris Wilson { 35810be98a7SChris Wilson struct i915_gem_context *ctx, *cn; 35910be98a7SChris Wilson 360a4e7ccdaSChris Wilson llist_for_each_entry_safe(ctx, cn, list, free_link) 36110be98a7SChris Wilson i915_gem_context_free(ctx); 36210be98a7SChris Wilson } 36310be98a7SChris Wilson 364a4e7ccdaSChris Wilson static void contexts_flush_free(struct i915_gem_contexts *gc) 36510be98a7SChris Wilson { 366a4e7ccdaSChris Wilson contexts_free_all(llist_del_all(&gc->free_list)); 36710be98a7SChris Wilson } 36810be98a7SChris Wilson 36910be98a7SChris Wilson static void contexts_free_worker(struct work_struct *work) 37010be98a7SChris Wilson { 371a4e7ccdaSChris Wilson struct i915_gem_contexts *gc = 372a4e7ccdaSChris Wilson container_of(work, typeof(*gc), free_work); 37310be98a7SChris Wilson 374a4e7ccdaSChris Wilson contexts_flush_free(gc); 37510be98a7SChris Wilson } 37610be98a7SChris Wilson 37710be98a7SChris Wilson void i915_gem_context_release(struct kref *ref) 37810be98a7SChris Wilson { 37910be98a7SChris Wilson struct i915_gem_context *ctx = container_of(ref, typeof(*ctx), ref); 380a4e7ccdaSChris Wilson struct i915_gem_contexts *gc = &ctx->i915->gem.contexts; 38110be98a7SChris Wilson 38210be98a7SChris Wilson trace_i915_context_free(ctx); 383a4e7ccdaSChris Wilson if (llist_add(&ctx->free_link, &gc->free_list)) 384a4e7ccdaSChris Wilson schedule_work(&gc->free_work); 38510be98a7SChris Wilson } 38610be98a7SChris Wilson 3872e0986a5SChris Wilson static inline struct i915_gem_engines * 3882e0986a5SChris Wilson __context_engines_static(const struct i915_gem_context *ctx) 3892e0986a5SChris Wilson { 3902e0986a5SChris Wilson return rcu_dereference_protected(ctx->engines, true); 3912e0986a5SChris Wilson } 3922e0986a5SChris Wilson 3932e0986a5SChris Wilson static bool __reset_engine(struct intel_engine_cs *engine) 3942e0986a5SChris Wilson { 3952e0986a5SChris Wilson struct intel_gt *gt = engine->gt; 3962e0986a5SChris Wilson bool success = false; 3972e0986a5SChris Wilson 3982e0986a5SChris Wilson if (!intel_has_reset_engine(gt)) 3992e0986a5SChris Wilson return false; 4002e0986a5SChris Wilson 4012e0986a5SChris Wilson if (!test_and_set_bit(I915_RESET_ENGINE + engine->id, 4022e0986a5SChris Wilson >->reset.flags)) { 4032e0986a5SChris Wilson success = intel_engine_reset(engine, NULL) == 0; 4042e0986a5SChris Wilson clear_and_wake_up_bit(I915_RESET_ENGINE + engine->id, 4052e0986a5SChris Wilson >->reset.flags); 4062e0986a5SChris Wilson } 4072e0986a5SChris Wilson 4082e0986a5SChris Wilson return success; 4092e0986a5SChris Wilson } 4102e0986a5SChris Wilson 4112e0986a5SChris Wilson static void __reset_context(struct i915_gem_context *ctx, 4122e0986a5SChris Wilson struct intel_engine_cs *engine) 4132e0986a5SChris Wilson { 4142e0986a5SChris Wilson intel_gt_handle_error(engine->gt, engine->mask, 0, 4152e0986a5SChris Wilson "context closure in %s", ctx->name); 4162e0986a5SChris Wilson } 4172e0986a5SChris Wilson 4182e0986a5SChris Wilson static bool __cancel_engine(struct intel_engine_cs *engine) 4192e0986a5SChris Wilson { 4202e0986a5SChris Wilson /* 4212e0986a5SChris Wilson * Send a "high priority pulse" down the engine to cause the 4222e0986a5SChris Wilson * current request to be momentarily preempted. (If it fails to 4232e0986a5SChris Wilson * be preempted, it will be reset). As we have marked our context 4242e0986a5SChris Wilson * as banned, any incomplete request, including any running, will 4252e0986a5SChris Wilson * be skipped following the preemption. 4262e0986a5SChris Wilson * 4272e0986a5SChris Wilson * If there is no hangchecking (one of the reasons why we try to 4282e0986a5SChris Wilson * cancel the context) and no forced preemption, there may be no 4292e0986a5SChris Wilson * means by which we reset the GPU and evict the persistent hog. 4302e0986a5SChris Wilson * Ergo if we are unable to inject a preemptive pulse that can 4312e0986a5SChris Wilson * kill the banned context, we fallback to doing a local reset 4322e0986a5SChris Wilson * instead. 4332e0986a5SChris Wilson */ 434babaab2fSChris Wilson if (IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT) && 435babaab2fSChris Wilson !intel_engine_pulse(engine)) 4362e0986a5SChris Wilson return true; 4372e0986a5SChris Wilson 4382e0986a5SChris Wilson /* If we are unable to send a pulse, try resetting this engine. */ 4392e0986a5SChris Wilson return __reset_engine(engine); 4402e0986a5SChris Wilson } 4412e0986a5SChris Wilson 442736e785fSChris Wilson static bool 443736e785fSChris Wilson __active_engine(struct i915_request *rq, struct intel_engine_cs **active) 4442e0986a5SChris Wilson { 4452e0986a5SChris Wilson struct intel_engine_cs *engine, *locked; 446736e785fSChris Wilson bool ret = false; 4472e0986a5SChris Wilson 4482e0986a5SChris Wilson /* 4492e0986a5SChris Wilson * Serialise with __i915_request_submit() so that it sees 4502e0986a5SChris Wilson * is-banned?, or we know the request is already inflight. 451736e785fSChris Wilson * 452736e785fSChris Wilson * Note that rq->engine is unstable, and so we double 453736e785fSChris Wilson * check that we have acquired the lock on the final engine. 4542e0986a5SChris Wilson */ 4552e0986a5SChris Wilson locked = READ_ONCE(rq->engine); 4562e0986a5SChris Wilson spin_lock_irq(&locked->active.lock); 4572e0986a5SChris Wilson while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) { 4582e0986a5SChris Wilson spin_unlock(&locked->active.lock); 4592e0986a5SChris Wilson locked = engine; 460736e785fSChris Wilson spin_lock(&locked->active.lock); 4612e0986a5SChris Wilson } 4622e0986a5SChris Wilson 463736e785fSChris Wilson if (!i915_request_completed(rq)) { 4646e17ae73SChris Wilson if (i915_request_is_active(rq) && rq->fence.error != -EIO) 465736e785fSChris Wilson *active = locked; 466736e785fSChris Wilson ret = true; 467736e785fSChris Wilson } 4682e0986a5SChris Wilson 4692e0986a5SChris Wilson spin_unlock_irq(&locked->active.lock); 4702e0986a5SChris Wilson 471736e785fSChris Wilson return ret; 4722e0986a5SChris Wilson } 4732e0986a5SChris Wilson 4744a317415SChris Wilson static struct intel_engine_cs *active_engine(struct intel_context *ce) 4754a317415SChris Wilson { 4764a317415SChris Wilson struct intel_engine_cs *engine = NULL; 4774a317415SChris Wilson struct i915_request *rq; 4784a317415SChris Wilson 4794a317415SChris Wilson if (!ce->timeline) 4804a317415SChris Wilson return NULL; 4814a317415SChris Wilson 482736e785fSChris Wilson rcu_read_lock(); 483736e785fSChris Wilson list_for_each_entry_rcu(rq, &ce->timeline->requests, link) { 484736e785fSChris Wilson if (i915_request_is_active(rq) && i915_request_completed(rq)) 485736e785fSChris Wilson continue; 4864a317415SChris Wilson 4874a317415SChris Wilson /* Check with the backend if the request is inflight */ 488736e785fSChris Wilson if (__active_engine(rq, &engine)) 4894a317415SChris Wilson break; 4904a317415SChris Wilson } 491736e785fSChris Wilson rcu_read_unlock(); 4924a317415SChris Wilson 4934a317415SChris Wilson return engine; 4944a317415SChris Wilson } 4954a317415SChris Wilson 49642fb60deSChris Wilson static void kill_engines(struct i915_gem_engines *engines) 4972e0986a5SChris Wilson { 4982e0986a5SChris Wilson struct i915_gem_engines_iter it; 4992e0986a5SChris Wilson struct intel_context *ce; 5002e0986a5SChris Wilson 5012e0986a5SChris Wilson /* 5022e0986a5SChris Wilson * Map the user's engine back to the actual engines; one virtual 5032e0986a5SChris Wilson * engine will be mapped to multiple engines, and using ctx->engine[] 5042e0986a5SChris Wilson * the same engine may be have multiple instances in the user's map. 5052e0986a5SChris Wilson * However, we only care about pending requests, so only include 5062e0986a5SChris Wilson * engines on which there are incomplete requests. 5072e0986a5SChris Wilson */ 50842fb60deSChris Wilson for_each_gem_engine(ce, engines, it) { 5092e0986a5SChris Wilson struct intel_engine_cs *engine; 5102e0986a5SChris Wilson 5119f3ccd40SChris Wilson if (intel_context_set_banned(ce)) 5129f3ccd40SChris Wilson continue; 5139f3ccd40SChris Wilson 5144a317415SChris Wilson /* 5154a317415SChris Wilson * Check the current active state of this context; if we 5164a317415SChris Wilson * are currently executing on the GPU we need to evict 5174a317415SChris Wilson * ourselves. On the other hand, if we haven't yet been 5184a317415SChris Wilson * submitted to the GPU or if everything is complete, 5194a317415SChris Wilson * we have nothing to do. 5204a317415SChris Wilson */ 5214a317415SChris Wilson engine = active_engine(ce); 5222e0986a5SChris Wilson 5232e0986a5SChris Wilson /* First attempt to gracefully cancel the context */ 5242e0986a5SChris Wilson if (engine && !__cancel_engine(engine)) 5252e0986a5SChris Wilson /* 5262e0986a5SChris Wilson * If we are unable to send a preemptive pulse to bump 5272e0986a5SChris Wilson * the context from the GPU, we have to resort to a full 5282e0986a5SChris Wilson * reset. We hope the collateral damage is worth it. 5292e0986a5SChris Wilson */ 53042fb60deSChris Wilson __reset_context(engines->ctx, engine); 5312e0986a5SChris Wilson } 5322e0986a5SChris Wilson } 5332e0986a5SChris Wilson 53442fb60deSChris Wilson static void kill_stale_engines(struct i915_gem_context *ctx) 53542fb60deSChris Wilson { 53642fb60deSChris Wilson struct i915_gem_engines *pos, *next; 53742fb60deSChris Wilson 538130a95e9SChris Wilson spin_lock_irq(&ctx->stale.lock); 539130a95e9SChris Wilson GEM_BUG_ON(!i915_gem_context_is_closed(ctx)); 54042fb60deSChris Wilson list_for_each_entry_safe(pos, next, &ctx->stale.engines, link) { 541130a95e9SChris Wilson if (!i915_sw_fence_await(&pos->fence)) { 542130a95e9SChris Wilson list_del_init(&pos->link); 54342fb60deSChris Wilson continue; 544130a95e9SChris Wilson } 54542fb60deSChris Wilson 546130a95e9SChris Wilson spin_unlock_irq(&ctx->stale.lock); 54742fb60deSChris Wilson 54842fb60deSChris Wilson kill_engines(pos); 54942fb60deSChris Wilson 550130a95e9SChris Wilson spin_lock_irq(&ctx->stale.lock); 551130a95e9SChris Wilson GEM_BUG_ON(i915_sw_fence_signaled(&pos->fence)); 55242fb60deSChris Wilson list_safe_reset_next(pos, next, link); 55342fb60deSChris Wilson list_del_init(&pos->link); /* decouple from FENCE_COMPLETE */ 55442fb60deSChris Wilson 55542fb60deSChris Wilson i915_sw_fence_complete(&pos->fence); 55642fb60deSChris Wilson } 557130a95e9SChris Wilson spin_unlock_irq(&ctx->stale.lock); 55842fb60deSChris Wilson } 55942fb60deSChris Wilson 56042fb60deSChris Wilson static void kill_context(struct i915_gem_context *ctx) 56142fb60deSChris Wilson { 56242fb60deSChris Wilson kill_stale_engines(ctx); 563130a95e9SChris Wilson } 564130a95e9SChris Wilson 565130a95e9SChris Wilson static void engines_idle_release(struct i915_gem_context *ctx, 566130a95e9SChris Wilson struct i915_gem_engines *engines) 567130a95e9SChris Wilson { 568130a95e9SChris Wilson struct i915_gem_engines_iter it; 569130a95e9SChris Wilson struct intel_context *ce; 570130a95e9SChris Wilson 571130a95e9SChris Wilson INIT_LIST_HEAD(&engines->link); 572130a95e9SChris Wilson 573130a95e9SChris Wilson engines->ctx = i915_gem_context_get(ctx); 574130a95e9SChris Wilson 575130a95e9SChris Wilson for_each_gem_engine(ce, engines, it) { 576e6829625SChris Wilson int err; 577130a95e9SChris Wilson 578130a95e9SChris Wilson /* serialises with execbuf */ 579207e4a71SChris Wilson set_bit(CONTEXT_CLOSED_BIT, &ce->flags); 580130a95e9SChris Wilson if (!intel_context_pin_if_active(ce)) 581130a95e9SChris Wilson continue; 582130a95e9SChris Wilson 583e6829625SChris Wilson /* Wait until context is finally scheduled out and retired */ 584e6829625SChris Wilson err = i915_sw_fence_await_active(&engines->fence, 585e6829625SChris Wilson &ce->active, 586e6829625SChris Wilson I915_ACTIVE_AWAIT_BARRIER); 587130a95e9SChris Wilson intel_context_unpin(ce); 588e6829625SChris Wilson if (err) 589130a95e9SChris Wilson goto kill; 590130a95e9SChris Wilson } 591130a95e9SChris Wilson 592130a95e9SChris Wilson spin_lock_irq(&ctx->stale.lock); 593130a95e9SChris Wilson if (!i915_gem_context_is_closed(ctx)) 594130a95e9SChris Wilson list_add_tail(&engines->link, &ctx->stale.engines); 595130a95e9SChris Wilson spin_unlock_irq(&ctx->stale.lock); 596130a95e9SChris Wilson 597130a95e9SChris Wilson kill: 598130a95e9SChris Wilson if (list_empty(&engines->link)) /* raced, already closed */ 599130a95e9SChris Wilson kill_engines(engines); 600130a95e9SChris Wilson 601130a95e9SChris Wilson i915_sw_fence_commit(&engines->fence); 60242fb60deSChris Wilson } 60342fb60deSChris Wilson 604267c0126SChris Wilson static void set_closed_name(struct i915_gem_context *ctx) 605267c0126SChris Wilson { 606267c0126SChris Wilson char *s; 607267c0126SChris Wilson 608267c0126SChris Wilson /* Replace '[]' with '<>' to indicate closed in debug prints */ 609267c0126SChris Wilson 610267c0126SChris Wilson s = strrchr(ctx->name, '['); 611267c0126SChris Wilson if (!s) 612267c0126SChris Wilson return; 613267c0126SChris Wilson 614267c0126SChris Wilson *s = '<'; 615267c0126SChris Wilson 616267c0126SChris Wilson s = strchr(s + 1, ']'); 617267c0126SChris Wilson if (s) 618267c0126SChris Wilson *s = '>'; 619267c0126SChris Wilson } 620267c0126SChris Wilson 62110be98a7SChris Wilson static void context_close(struct i915_gem_context *ctx) 62210be98a7SChris Wilson { 623a4e7ccdaSChris Wilson struct i915_address_space *vm; 624a4e7ccdaSChris Wilson 625130a95e9SChris Wilson /* Flush any concurrent set_engines() */ 626130a95e9SChris Wilson mutex_lock(&ctx->engines_mutex); 627130a95e9SChris Wilson engines_idle_release(ctx, rcu_replace_pointer(ctx->engines, NULL, 1)); 6282850748eSChris Wilson i915_gem_context_set_closed(ctx); 629130a95e9SChris Wilson mutex_unlock(&ctx->engines_mutex); 6302850748eSChris Wilson 631155ab883SChris Wilson mutex_lock(&ctx->mutex); 632155ab883SChris Wilson 633130a95e9SChris Wilson set_closed_name(ctx); 634130a95e9SChris Wilson 635a4e7ccdaSChris Wilson vm = i915_gem_context_vm(ctx); 636a4e7ccdaSChris Wilson if (vm) 637a4e7ccdaSChris Wilson i915_vm_close(vm); 638a4e7ccdaSChris Wilson 639155ab883SChris Wilson ctx->file_priv = ERR_PTR(-EBADF); 64010be98a7SChris Wilson 64110be98a7SChris Wilson /* 64210be98a7SChris Wilson * The LUT uses the VMA as a backpointer to unref the object, 64310be98a7SChris Wilson * so we need to clear the LUT before we close all the VMA (inside 64410be98a7SChris Wilson * the ppgtt). 64510be98a7SChris Wilson */ 64610be98a7SChris Wilson lut_close(ctx); 64710be98a7SChris Wilson 648155ab883SChris Wilson mutex_unlock(&ctx->mutex); 6492e0986a5SChris Wilson 6502e0986a5SChris Wilson /* 6512e0986a5SChris Wilson * If the user has disabled hangchecking, we can not be sure that 6522e0986a5SChris Wilson * the batches will ever complete after the context is closed, 6532e0986a5SChris Wilson * keeping the context and all resources pinned forever. So in this 6542e0986a5SChris Wilson * case we opt to forcibly kill off all remaining requests on 6552e0986a5SChris Wilson * context close. 6562e0986a5SChris Wilson */ 657a0e04715SChris Wilson if (!i915_gem_context_is_persistent(ctx) || 6588a25c4beSJani Nikula !ctx->i915->params.enable_hangcheck) 6592e0986a5SChris Wilson kill_context(ctx); 6602e0986a5SChris Wilson 66110be98a7SChris Wilson i915_gem_context_put(ctx); 66210be98a7SChris Wilson } 66310be98a7SChris Wilson 664a0e04715SChris Wilson static int __context_set_persistence(struct i915_gem_context *ctx, bool state) 665a0e04715SChris Wilson { 666a0e04715SChris Wilson if (i915_gem_context_is_persistent(ctx) == state) 667a0e04715SChris Wilson return 0; 668a0e04715SChris Wilson 669a0e04715SChris Wilson if (state) { 670a0e04715SChris Wilson /* 671a0e04715SChris Wilson * Only contexts that are short-lived [that will expire or be 672a0e04715SChris Wilson * reset] are allowed to survive past termination. We require 673a0e04715SChris Wilson * hangcheck to ensure that the persistent requests are healthy. 674a0e04715SChris Wilson */ 6758a25c4beSJani Nikula if (!ctx->i915->params.enable_hangcheck) 676a0e04715SChris Wilson return -EINVAL; 677a0e04715SChris Wilson 678a0e04715SChris Wilson i915_gem_context_set_persistence(ctx); 679a0e04715SChris Wilson } else { 680a0e04715SChris Wilson /* To cancel a context we use "preempt-to-idle" */ 681a0e04715SChris Wilson if (!(ctx->i915->caps.scheduler & I915_SCHEDULER_CAP_PREEMPTION)) 682a0e04715SChris Wilson return -ENODEV; 683a0e04715SChris Wilson 684d1b9b5f1SChris Wilson /* 685d1b9b5f1SChris Wilson * If the cancel fails, we then need to reset, cleanly! 686d1b9b5f1SChris Wilson * 687d1b9b5f1SChris Wilson * If the per-engine reset fails, all hope is lost! We resort 688d1b9b5f1SChris Wilson * to a full GPU reset in that unlikely case, but realistically 689d1b9b5f1SChris Wilson * if the engine could not reset, the full reset does not fare 690d1b9b5f1SChris Wilson * much better. The damage has been done. 691d1b9b5f1SChris Wilson * 692d1b9b5f1SChris Wilson * However, if we cannot reset an engine by itself, we cannot 693d1b9b5f1SChris Wilson * cleanup a hanging persistent context without causing 694d1b9b5f1SChris Wilson * colateral damage, and we should not pretend we can by 695d1b9b5f1SChris Wilson * exposing the interface. 696d1b9b5f1SChris Wilson */ 697d1b9b5f1SChris Wilson if (!intel_has_reset_engine(&ctx->i915->gt)) 698d1b9b5f1SChris Wilson return -ENODEV; 699d1b9b5f1SChris Wilson 700a0e04715SChris Wilson i915_gem_context_clear_persistence(ctx); 701a0e04715SChris Wilson } 702a0e04715SChris Wilson 703a0e04715SChris Wilson return 0; 704a0e04715SChris Wilson } 705a0e04715SChris Wilson 70610be98a7SChris Wilson static struct i915_gem_context * 707e568ac38SChris Wilson __create_context(struct drm_i915_private *i915) 70810be98a7SChris Wilson { 70910be98a7SChris Wilson struct i915_gem_context *ctx; 71010be98a7SChris Wilson struct i915_gem_engines *e; 71110be98a7SChris Wilson int err; 71210be98a7SChris Wilson int i; 71310be98a7SChris Wilson 71410be98a7SChris Wilson ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 71510be98a7SChris Wilson if (!ctx) 71610be98a7SChris Wilson return ERR_PTR(-ENOMEM); 71710be98a7SChris Wilson 71810be98a7SChris Wilson kref_init(&ctx->ref); 719e568ac38SChris Wilson ctx->i915 = i915; 72010be98a7SChris Wilson ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_NORMAL); 72110be98a7SChris Wilson mutex_init(&ctx->mutex); 722eb4dedaeSChris Wilson INIT_LIST_HEAD(&ctx->link); 72310be98a7SChris Wilson 72442fb60deSChris Wilson spin_lock_init(&ctx->stale.lock); 72542fb60deSChris Wilson INIT_LIST_HEAD(&ctx->stale.engines); 72642fb60deSChris Wilson 72710be98a7SChris Wilson mutex_init(&ctx->engines_mutex); 72810be98a7SChris Wilson e = default_engines(ctx); 72910be98a7SChris Wilson if (IS_ERR(e)) { 73010be98a7SChris Wilson err = PTR_ERR(e); 73110be98a7SChris Wilson goto err_free; 73210be98a7SChris Wilson } 73310be98a7SChris Wilson RCU_INIT_POINTER(ctx->engines, e); 73410be98a7SChris Wilson 73510be98a7SChris Wilson INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL); 736f7ce8639SChris Wilson mutex_init(&ctx->lut_mutex); 73710be98a7SChris Wilson 73810be98a7SChris Wilson /* NB: Mark all slices as needing a remap so that when the context first 73910be98a7SChris Wilson * loads it will restore whatever remap state already exists. If there 74010be98a7SChris Wilson * is no remap info, it will be a NOP. */ 741e568ac38SChris Wilson ctx->remap_slice = ALL_L3_SLICES(i915); 74210be98a7SChris Wilson 74310be98a7SChris Wilson i915_gem_context_set_bannable(ctx); 74410be98a7SChris Wilson i915_gem_context_set_recoverable(ctx); 745a0e04715SChris Wilson __context_set_persistence(ctx, true /* cgroup hook? */); 74610be98a7SChris Wilson 74710be98a7SChris Wilson for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++) 74810be98a7SChris Wilson ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES; 74910be98a7SChris Wilson 75010be98a7SChris Wilson return ctx; 75110be98a7SChris Wilson 75210be98a7SChris Wilson err_free: 75310be98a7SChris Wilson kfree(ctx); 75410be98a7SChris Wilson return ERR_PTR(err); 75510be98a7SChris Wilson } 75610be98a7SChris Wilson 75789ff76bfSChris Wilson static inline struct i915_gem_engines * 75889ff76bfSChris Wilson __context_engines_await(const struct i915_gem_context *ctx) 75989ff76bfSChris Wilson { 76089ff76bfSChris Wilson struct i915_gem_engines *engines; 76189ff76bfSChris Wilson 76289ff76bfSChris Wilson rcu_read_lock(); 76389ff76bfSChris Wilson do { 76489ff76bfSChris Wilson engines = rcu_dereference(ctx->engines); 76589ff76bfSChris Wilson GEM_BUG_ON(!engines); 76689ff76bfSChris Wilson 76789ff76bfSChris Wilson if (unlikely(!i915_sw_fence_await(&engines->fence))) 76889ff76bfSChris Wilson continue; 76989ff76bfSChris Wilson 77089ff76bfSChris Wilson if (likely(engines == rcu_access_pointer(ctx->engines))) 77189ff76bfSChris Wilson break; 77289ff76bfSChris Wilson 77389ff76bfSChris Wilson i915_sw_fence_complete(&engines->fence); 77489ff76bfSChris Wilson } while (1); 77589ff76bfSChris Wilson rcu_read_unlock(); 77689ff76bfSChris Wilson 77789ff76bfSChris Wilson return engines; 77889ff76bfSChris Wilson } 77989ff76bfSChris Wilson 78088be76cdSChris Wilson static int 78148ae397bSChris Wilson context_apply_all(struct i915_gem_context *ctx, 78288be76cdSChris Wilson int (*fn)(struct intel_context *ce, void *data), 78348ae397bSChris Wilson void *data) 78448ae397bSChris Wilson { 78548ae397bSChris Wilson struct i915_gem_engines_iter it; 78689ff76bfSChris Wilson struct i915_gem_engines *e; 78748ae397bSChris Wilson struct intel_context *ce; 78888be76cdSChris Wilson int err = 0; 78948ae397bSChris Wilson 79089ff76bfSChris Wilson e = __context_engines_await(ctx); 79189ff76bfSChris Wilson for_each_gem_engine(ce, e, it) { 79288be76cdSChris Wilson err = fn(ce, data); 79388be76cdSChris Wilson if (err) 79488be76cdSChris Wilson break; 79588be76cdSChris Wilson } 79689ff76bfSChris Wilson i915_sw_fence_complete(&e->fence); 79788be76cdSChris Wilson 79888be76cdSChris Wilson return err; 79948ae397bSChris Wilson } 80048ae397bSChris Wilson 80188be76cdSChris Wilson static int __apply_ppgtt(struct intel_context *ce, void *vm) 80248ae397bSChris Wilson { 80348ae397bSChris Wilson i915_vm_put(ce->vm); 80448ae397bSChris Wilson ce->vm = i915_vm_get(vm); 80588be76cdSChris Wilson return 0; 80648ae397bSChris Wilson } 80748ae397bSChris Wilson 808e568ac38SChris Wilson static struct i915_address_space * 809e568ac38SChris Wilson __set_ppgtt(struct i915_gem_context *ctx, struct i915_address_space *vm) 81010be98a7SChris Wilson { 81189ff76bfSChris Wilson struct i915_address_space *old; 81210be98a7SChris Wilson 81389ff76bfSChris Wilson old = rcu_replace_pointer(ctx->vm, 81489ff76bfSChris Wilson i915_vm_open(vm), 81589ff76bfSChris Wilson lockdep_is_held(&ctx->mutex)); 816a1c9ca22SChris Wilson GEM_BUG_ON(old && i915_vm_is_4lvl(vm) != i915_vm_is_4lvl(old)); 817a1c9ca22SChris Wilson 81848ae397bSChris Wilson context_apply_all(ctx, __apply_ppgtt, vm); 819f5d974f9SChris Wilson 82010be98a7SChris Wilson return old; 82110be98a7SChris Wilson } 82210be98a7SChris Wilson 82310be98a7SChris Wilson static void __assign_ppgtt(struct i915_gem_context *ctx, 824e568ac38SChris Wilson struct i915_address_space *vm) 82510be98a7SChris Wilson { 826a4e7ccdaSChris Wilson if (vm == rcu_access_pointer(ctx->vm)) 82710be98a7SChris Wilson return; 82810be98a7SChris Wilson 829e568ac38SChris Wilson vm = __set_ppgtt(ctx, vm); 830e568ac38SChris Wilson if (vm) 8312850748eSChris Wilson i915_vm_close(vm); 83210be98a7SChris Wilson } 83310be98a7SChris Wilson 83475d0a7f3SChris Wilson static void __set_timeline(struct intel_timeline **dst, 83575d0a7f3SChris Wilson struct intel_timeline *src) 83675d0a7f3SChris Wilson { 83775d0a7f3SChris Wilson struct intel_timeline *old = *dst; 83875d0a7f3SChris Wilson 83975d0a7f3SChris Wilson *dst = src ? intel_timeline_get(src) : NULL; 84075d0a7f3SChris Wilson 84175d0a7f3SChris Wilson if (old) 84275d0a7f3SChris Wilson intel_timeline_put(old); 84375d0a7f3SChris Wilson } 84475d0a7f3SChris Wilson 84588be76cdSChris Wilson static int __apply_timeline(struct intel_context *ce, void *timeline) 84675d0a7f3SChris Wilson { 84775d0a7f3SChris Wilson __set_timeline(&ce->timeline, timeline); 84888be76cdSChris Wilson return 0; 84975d0a7f3SChris Wilson } 85075d0a7f3SChris Wilson 85175d0a7f3SChris Wilson static void __assign_timeline(struct i915_gem_context *ctx, 85275d0a7f3SChris Wilson struct intel_timeline *timeline) 85375d0a7f3SChris Wilson { 85475d0a7f3SChris Wilson __set_timeline(&ctx->timeline, timeline); 85575d0a7f3SChris Wilson context_apply_all(ctx, __apply_timeline, timeline); 85675d0a7f3SChris Wilson } 85775d0a7f3SChris Wilson 85810be98a7SChris Wilson static struct i915_gem_context * 859a4e7ccdaSChris Wilson i915_gem_create_context(struct drm_i915_private *i915, unsigned int flags) 86010be98a7SChris Wilson { 86110be98a7SChris Wilson struct i915_gem_context *ctx; 86210be98a7SChris Wilson 86310be98a7SChris Wilson if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE && 864a4e7ccdaSChris Wilson !HAS_EXECLISTS(i915)) 86510be98a7SChris Wilson return ERR_PTR(-EINVAL); 86610be98a7SChris Wilson 867a4e7ccdaSChris Wilson /* Reap the stale contexts */ 868a4e7ccdaSChris Wilson contexts_flush_free(&i915->gem.contexts); 86910be98a7SChris Wilson 870a4e7ccdaSChris Wilson ctx = __create_context(i915); 87110be98a7SChris Wilson if (IS_ERR(ctx)) 87210be98a7SChris Wilson return ctx; 87310be98a7SChris Wilson 874a4e7ccdaSChris Wilson if (HAS_FULL_PPGTT(i915)) { 875ab53497bSChris Wilson struct i915_ppgtt *ppgtt; 87610be98a7SChris Wilson 8772c86e55dSMatthew Auld ppgtt = i915_ppgtt_create(&i915->gt); 87810be98a7SChris Wilson if (IS_ERR(ppgtt)) { 879baa89ba3SWambui Karuga drm_dbg(&i915->drm, "PPGTT setup failed (%ld)\n", 88010be98a7SChris Wilson PTR_ERR(ppgtt)); 88110be98a7SChris Wilson context_close(ctx); 88210be98a7SChris Wilson return ERR_CAST(ppgtt); 88310be98a7SChris Wilson } 88410be98a7SChris Wilson 885a4e7ccdaSChris Wilson mutex_lock(&ctx->mutex); 886e568ac38SChris Wilson __assign_ppgtt(ctx, &ppgtt->vm); 887a4e7ccdaSChris Wilson mutex_unlock(&ctx->mutex); 888a4e7ccdaSChris Wilson 889e568ac38SChris Wilson i915_vm_put(&ppgtt->vm); 89010be98a7SChris Wilson } 89110be98a7SChris Wilson 89210be98a7SChris Wilson if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE) { 893f0c02c1bSTvrtko Ursulin struct intel_timeline *timeline; 89410be98a7SChris Wilson 895d1bf5dd8SChris Wilson timeline = intel_timeline_create(&i915->gt); 89610be98a7SChris Wilson if (IS_ERR(timeline)) { 89710be98a7SChris Wilson context_close(ctx); 89810be98a7SChris Wilson return ERR_CAST(timeline); 89910be98a7SChris Wilson } 90010be98a7SChris Wilson 90175d0a7f3SChris Wilson __assign_timeline(ctx, timeline); 90275d0a7f3SChris Wilson intel_timeline_put(timeline); 90310be98a7SChris Wilson } 90410be98a7SChris Wilson 90510be98a7SChris Wilson trace_i915_context_create(ctx); 90610be98a7SChris Wilson 90710be98a7SChris Wilson return ctx; 90810be98a7SChris Wilson } 90910be98a7SChris Wilson 910a4e7ccdaSChris Wilson static void init_contexts(struct i915_gem_contexts *gc) 91110be98a7SChris Wilson { 912a4e7ccdaSChris Wilson spin_lock_init(&gc->lock); 913a4e7ccdaSChris Wilson INIT_LIST_HEAD(&gc->list); 91410be98a7SChris Wilson 915a4e7ccdaSChris Wilson INIT_WORK(&gc->free_work, contexts_free_worker); 916a4e7ccdaSChris Wilson init_llist_head(&gc->free_list); 91710be98a7SChris Wilson } 91810be98a7SChris Wilson 919e6ba7648SChris Wilson void i915_gem_init__contexts(struct drm_i915_private *i915) 92010be98a7SChris Wilson { 921a4e7ccdaSChris Wilson init_contexts(&i915->gem.contexts); 922baa89ba3SWambui Karuga drm_dbg(&i915->drm, "%s context support initialized\n", 923a4e7ccdaSChris Wilson DRIVER_CAPS(i915)->has_logical_contexts ? 92410be98a7SChris Wilson "logical" : "fake"); 92510be98a7SChris Wilson } 92610be98a7SChris Wilson 927a4e7ccdaSChris Wilson void i915_gem_driver_release__contexts(struct drm_i915_private *i915) 92810be98a7SChris Wilson { 9295f00cac9SChris Wilson flush_work(&i915->gem.contexts.free_work); 93022ca8a45SChris Wilson rcu_barrier(); /* and flush the left over RCU frees */ 93110be98a7SChris Wilson } 93210be98a7SChris Wilson 93310be98a7SChris Wilson static int gem_context_register(struct i915_gem_context *ctx, 934c100777cSTvrtko Ursulin struct drm_i915_file_private *fpriv, 935c100777cSTvrtko Ursulin u32 *id) 93610be98a7SChris Wilson { 937eb4dedaeSChris Wilson struct drm_i915_private *i915 = ctx->i915; 938a4e7ccdaSChris Wilson struct i915_address_space *vm; 93910be98a7SChris Wilson int ret; 94010be98a7SChris Wilson 94110be98a7SChris Wilson ctx->file_priv = fpriv; 942a4e7ccdaSChris Wilson 943a4e7ccdaSChris Wilson mutex_lock(&ctx->mutex); 944a4e7ccdaSChris Wilson vm = i915_gem_context_vm(ctx); 945a4e7ccdaSChris Wilson if (vm) 946a4e7ccdaSChris Wilson WRITE_ONCE(vm->file, fpriv); /* XXX */ 947a4e7ccdaSChris Wilson mutex_unlock(&ctx->mutex); 94810be98a7SChris Wilson 94910be98a7SChris Wilson ctx->pid = get_task_pid(current, PIDTYPE_PID); 950fc4f125dSChris Wilson snprintf(ctx->name, sizeof(ctx->name), "%s[%d]", 95110be98a7SChris Wilson current->comm, pid_nr(ctx->pid)); 95210be98a7SChris Wilson 95310be98a7SChris Wilson /* And finally expose ourselves to userspace via the idr */ 954c100777cSTvrtko Ursulin ret = xa_alloc(&fpriv->context_xa, id, ctx, xa_limit_32b, GFP_KERNEL); 955c100777cSTvrtko Ursulin if (ret) 956eb4dedaeSChris Wilson goto err_pid; 957c100777cSTvrtko Ursulin 958eb4dedaeSChris Wilson spin_lock(&i915->gem.contexts.lock); 959eb4dedaeSChris Wilson list_add_tail(&ctx->link, &i915->gem.contexts.list); 960eb4dedaeSChris Wilson spin_unlock(&i915->gem.contexts.lock); 961eb4dedaeSChris Wilson 962eb4dedaeSChris Wilson return 0; 963eb4dedaeSChris Wilson 964eb4dedaeSChris Wilson err_pid: 965eb4dedaeSChris Wilson put_pid(fetch_and_zero(&ctx->pid)); 96610be98a7SChris Wilson return ret; 96710be98a7SChris Wilson } 96810be98a7SChris Wilson 96910be98a7SChris Wilson int i915_gem_context_open(struct drm_i915_private *i915, 97010be98a7SChris Wilson struct drm_file *file) 97110be98a7SChris Wilson { 97210be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 97310be98a7SChris Wilson struct i915_gem_context *ctx; 97410be98a7SChris Wilson int err; 975c100777cSTvrtko Ursulin u32 id; 97610be98a7SChris Wilson 977c100777cSTvrtko Ursulin xa_init_flags(&file_priv->context_xa, XA_FLAGS_ALLOC); 978c100777cSTvrtko Ursulin 9795dbd2b7bSChris Wilson /* 0 reserved for invalid/unassigned ppgtt */ 9805dbd2b7bSChris Wilson xa_init_flags(&file_priv->vm_xa, XA_FLAGS_ALLOC1); 98110be98a7SChris Wilson 98210be98a7SChris Wilson ctx = i915_gem_create_context(i915, 0); 98310be98a7SChris Wilson if (IS_ERR(ctx)) { 98410be98a7SChris Wilson err = PTR_ERR(ctx); 98510be98a7SChris Wilson goto err; 98610be98a7SChris Wilson } 98710be98a7SChris Wilson 988c100777cSTvrtko Ursulin err = gem_context_register(ctx, file_priv, &id); 98910be98a7SChris Wilson if (err < 0) 99010be98a7SChris Wilson goto err_ctx; 99110be98a7SChris Wilson 992c100777cSTvrtko Ursulin GEM_BUG_ON(id); 99310be98a7SChris Wilson return 0; 99410be98a7SChris Wilson 99510be98a7SChris Wilson err_ctx: 99610be98a7SChris Wilson context_close(ctx); 99710be98a7SChris Wilson err: 9985dbd2b7bSChris Wilson xa_destroy(&file_priv->vm_xa); 999c100777cSTvrtko Ursulin xa_destroy(&file_priv->context_xa); 100010be98a7SChris Wilson return err; 100110be98a7SChris Wilson } 100210be98a7SChris Wilson 100310be98a7SChris Wilson void i915_gem_context_close(struct drm_file *file) 100410be98a7SChris Wilson { 100510be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 1006a4e7ccdaSChris Wilson struct drm_i915_private *i915 = file_priv->dev_priv; 10075dbd2b7bSChris Wilson struct i915_address_space *vm; 1008c100777cSTvrtko Ursulin struct i915_gem_context *ctx; 1009c100777cSTvrtko Ursulin unsigned long idx; 101010be98a7SChris Wilson 1011c100777cSTvrtko Ursulin xa_for_each(&file_priv->context_xa, idx, ctx) 1012c100777cSTvrtko Ursulin context_close(ctx); 1013c100777cSTvrtko Ursulin xa_destroy(&file_priv->context_xa); 101410be98a7SChris Wilson 10155dbd2b7bSChris Wilson xa_for_each(&file_priv->vm_xa, idx, vm) 10165dbd2b7bSChris Wilson i915_vm_put(vm); 10175dbd2b7bSChris Wilson xa_destroy(&file_priv->vm_xa); 1018a4e7ccdaSChris Wilson 1019a4e7ccdaSChris Wilson contexts_flush_free(&i915->gem.contexts); 102010be98a7SChris Wilson } 102110be98a7SChris Wilson 102210be98a7SChris Wilson int i915_gem_vm_create_ioctl(struct drm_device *dev, void *data, 102310be98a7SChris Wilson struct drm_file *file) 102410be98a7SChris Wilson { 102510be98a7SChris Wilson struct drm_i915_private *i915 = to_i915(dev); 102610be98a7SChris Wilson struct drm_i915_gem_vm_control *args = data; 102710be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 1028ab53497bSChris Wilson struct i915_ppgtt *ppgtt; 10295dbd2b7bSChris Wilson u32 id; 103010be98a7SChris Wilson int err; 103110be98a7SChris Wilson 103210be98a7SChris Wilson if (!HAS_FULL_PPGTT(i915)) 103310be98a7SChris Wilson return -ENODEV; 103410be98a7SChris Wilson 103510be98a7SChris Wilson if (args->flags) 103610be98a7SChris Wilson return -EINVAL; 103710be98a7SChris Wilson 10382c86e55dSMatthew Auld ppgtt = i915_ppgtt_create(&i915->gt); 103910be98a7SChris Wilson if (IS_ERR(ppgtt)) 104010be98a7SChris Wilson return PTR_ERR(ppgtt); 104110be98a7SChris Wilson 104210be98a7SChris Wilson ppgtt->vm.file = file_priv; 104310be98a7SChris Wilson 104410be98a7SChris Wilson if (args->extensions) { 104510be98a7SChris Wilson err = i915_user_extensions(u64_to_user_ptr(args->extensions), 104610be98a7SChris Wilson NULL, 0, 104710be98a7SChris Wilson ppgtt); 104810be98a7SChris Wilson if (err) 104910be98a7SChris Wilson goto err_put; 105010be98a7SChris Wilson } 105110be98a7SChris Wilson 10525dbd2b7bSChris Wilson err = xa_alloc(&file_priv->vm_xa, &id, &ppgtt->vm, 10535dbd2b7bSChris Wilson xa_limit_32b, GFP_KERNEL); 105410be98a7SChris Wilson if (err) 105510be98a7SChris Wilson goto err_put; 105610be98a7SChris Wilson 10575dbd2b7bSChris Wilson GEM_BUG_ON(id == 0); /* reserved for invalid/unassigned ppgtt */ 10585dbd2b7bSChris Wilson args->vm_id = id; 105910be98a7SChris Wilson return 0; 106010be98a7SChris Wilson 106110be98a7SChris Wilson err_put: 1062e568ac38SChris Wilson i915_vm_put(&ppgtt->vm); 106310be98a7SChris Wilson return err; 106410be98a7SChris Wilson } 106510be98a7SChris Wilson 106610be98a7SChris Wilson int i915_gem_vm_destroy_ioctl(struct drm_device *dev, void *data, 106710be98a7SChris Wilson struct drm_file *file) 106810be98a7SChris Wilson { 106910be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 107010be98a7SChris Wilson struct drm_i915_gem_vm_control *args = data; 1071e568ac38SChris Wilson struct i915_address_space *vm; 107210be98a7SChris Wilson 107310be98a7SChris Wilson if (args->flags) 107410be98a7SChris Wilson return -EINVAL; 107510be98a7SChris Wilson 107610be98a7SChris Wilson if (args->extensions) 107710be98a7SChris Wilson return -EINVAL; 107810be98a7SChris Wilson 10795dbd2b7bSChris Wilson vm = xa_erase(&file_priv->vm_xa, args->vm_id); 1080e568ac38SChris Wilson if (!vm) 108110be98a7SChris Wilson return -ENOENT; 108210be98a7SChris Wilson 1083e568ac38SChris Wilson i915_vm_put(vm); 108410be98a7SChris Wilson return 0; 108510be98a7SChris Wilson } 108610be98a7SChris Wilson 108710be98a7SChris Wilson struct context_barrier_task { 108810be98a7SChris Wilson struct i915_active base; 108910be98a7SChris Wilson void (*task)(void *data); 109010be98a7SChris Wilson void *data; 109110be98a7SChris Wilson }; 109210be98a7SChris Wilson 1093274cbf20SChris Wilson __i915_active_call 109410be98a7SChris Wilson static void cb_retire(struct i915_active *base) 109510be98a7SChris Wilson { 109610be98a7SChris Wilson struct context_barrier_task *cb = container_of(base, typeof(*cb), base); 109710be98a7SChris Wilson 109810be98a7SChris Wilson if (cb->task) 109910be98a7SChris Wilson cb->task(cb->data); 110010be98a7SChris Wilson 110110be98a7SChris Wilson i915_active_fini(&cb->base); 110210be98a7SChris Wilson kfree(cb); 110310be98a7SChris Wilson } 110410be98a7SChris Wilson 110510be98a7SChris Wilson I915_SELFTEST_DECLARE(static intel_engine_mask_t context_barrier_inject_fault); 110610be98a7SChris Wilson static int context_barrier_task(struct i915_gem_context *ctx, 110710be98a7SChris Wilson intel_engine_mask_t engines, 11081fe2d6f9SChris Wilson bool (*skip)(struct intel_context *ce, void *data), 110999f08d67SMaarten Lankhorst int (*pin)(struct intel_context *ce, struct i915_gem_ww_ctx *ww, void *data), 111010be98a7SChris Wilson int (*emit)(struct i915_request *rq, void *data), 111110be98a7SChris Wilson void (*task)(void *data), 111210be98a7SChris Wilson void *data) 111310be98a7SChris Wilson { 111410be98a7SChris Wilson struct context_barrier_task *cb; 111510be98a7SChris Wilson struct i915_gem_engines_iter it; 111670c96e39SChris Wilson struct i915_gem_engines *e; 111799f08d67SMaarten Lankhorst struct i915_gem_ww_ctx ww; 111810be98a7SChris Wilson struct intel_context *ce; 111910be98a7SChris Wilson int err = 0; 112010be98a7SChris Wilson 112110be98a7SChris Wilson GEM_BUG_ON(!task); 112210be98a7SChris Wilson 112310be98a7SChris Wilson cb = kmalloc(sizeof(*cb), GFP_KERNEL); 112410be98a7SChris Wilson if (!cb) 112510be98a7SChris Wilson return -ENOMEM; 112610be98a7SChris Wilson 1127b1e3177bSChris Wilson i915_active_init(&cb->base, NULL, cb_retire); 112812c255b5SChris Wilson err = i915_active_acquire(&cb->base); 112912c255b5SChris Wilson if (err) { 113012c255b5SChris Wilson kfree(cb); 113112c255b5SChris Wilson return err; 113212c255b5SChris Wilson } 113310be98a7SChris Wilson 113470c96e39SChris Wilson e = __context_engines_await(ctx); 113570c96e39SChris Wilson if (!e) { 113670c96e39SChris Wilson i915_active_release(&cb->base); 113770c96e39SChris Wilson return -ENOENT; 113870c96e39SChris Wilson } 113970c96e39SChris Wilson 114070c96e39SChris Wilson for_each_gem_engine(ce, e, it) { 114110be98a7SChris Wilson struct i915_request *rq; 114210be98a7SChris Wilson 114310be98a7SChris Wilson if (I915_SELFTEST_ONLY(context_barrier_inject_fault & 114410be98a7SChris Wilson ce->engine->mask)) { 114510be98a7SChris Wilson err = -ENXIO; 114610be98a7SChris Wilson break; 114710be98a7SChris Wilson } 114810be98a7SChris Wilson 11491fe2d6f9SChris Wilson if (!(ce->engine->mask & engines)) 11501fe2d6f9SChris Wilson continue; 11511fe2d6f9SChris Wilson 11521fe2d6f9SChris Wilson if (skip && skip(ce, data)) 115310be98a7SChris Wilson continue; 115410be98a7SChris Wilson 115599f08d67SMaarten Lankhorst i915_gem_ww_ctx_init(&ww, true); 115699f08d67SMaarten Lankhorst retry: 115747b08693SMaarten Lankhorst err = intel_context_pin_ww(ce, &ww); 115899f08d67SMaarten Lankhorst if (err) 115999f08d67SMaarten Lankhorst goto err; 116099f08d67SMaarten Lankhorst 116199f08d67SMaarten Lankhorst if (pin) 116299f08d67SMaarten Lankhorst err = pin(ce, &ww, data); 116399f08d67SMaarten Lankhorst if (err) 116499f08d67SMaarten Lankhorst goto err_unpin; 116599f08d67SMaarten Lankhorst 116699f08d67SMaarten Lankhorst rq = i915_request_create(ce); 116710be98a7SChris Wilson if (IS_ERR(rq)) { 116810be98a7SChris Wilson err = PTR_ERR(rq); 116999f08d67SMaarten Lankhorst goto err_unpin; 117010be98a7SChris Wilson } 117110be98a7SChris Wilson 117210be98a7SChris Wilson err = 0; 117310be98a7SChris Wilson if (emit) 117410be98a7SChris Wilson err = emit(rq, data); 117510be98a7SChris Wilson if (err == 0) 1176d19d71fcSChris Wilson err = i915_active_add_request(&cb->base, rq); 117710be98a7SChris Wilson 117810be98a7SChris Wilson i915_request_add(rq); 117999f08d67SMaarten Lankhorst err_unpin: 118099f08d67SMaarten Lankhorst intel_context_unpin(ce); 118199f08d67SMaarten Lankhorst err: 118299f08d67SMaarten Lankhorst if (err == -EDEADLK) { 118399f08d67SMaarten Lankhorst err = i915_gem_ww_ctx_backoff(&ww); 118499f08d67SMaarten Lankhorst if (!err) 118599f08d67SMaarten Lankhorst goto retry; 118699f08d67SMaarten Lankhorst } 118799f08d67SMaarten Lankhorst i915_gem_ww_ctx_fini(&ww); 118899f08d67SMaarten Lankhorst 118910be98a7SChris Wilson if (err) 119010be98a7SChris Wilson break; 119110be98a7SChris Wilson } 119270c96e39SChris Wilson i915_sw_fence_complete(&e->fence); 119310be98a7SChris Wilson 119410be98a7SChris Wilson cb->task = err ? NULL : task; /* caller needs to unwind instead */ 119510be98a7SChris Wilson cb->data = data; 119610be98a7SChris Wilson 119710be98a7SChris Wilson i915_active_release(&cb->base); 119810be98a7SChris Wilson 119910be98a7SChris Wilson return err; 120010be98a7SChris Wilson } 120110be98a7SChris Wilson 120210be98a7SChris Wilson static int get_ppgtt(struct drm_i915_file_private *file_priv, 120310be98a7SChris Wilson struct i915_gem_context *ctx, 120410be98a7SChris Wilson struct drm_i915_gem_context_param *args) 120510be98a7SChris Wilson { 1206e568ac38SChris Wilson struct i915_address_space *vm; 12075dbd2b7bSChris Wilson int err; 12085dbd2b7bSChris Wilson u32 id; 120910be98a7SChris Wilson 1210a4e7ccdaSChris Wilson if (!rcu_access_pointer(ctx->vm)) 121110be98a7SChris Wilson return -ENODEV; 121210be98a7SChris Wilson 1213a4e7ccdaSChris Wilson rcu_read_lock(); 121427dbae8fSChris Wilson vm = context_get_vm_rcu(ctx); 1215a4e7ccdaSChris Wilson rcu_read_unlock(); 121690211ea4SChris Wilson if (!vm) 121790211ea4SChris Wilson return -ENODEV; 121890211ea4SChris Wilson 121990211ea4SChris Wilson err = xa_alloc(&file_priv->vm_xa, &id, vm, xa_limit_32b, GFP_KERNEL); 12205dbd2b7bSChris Wilson if (err) 122110be98a7SChris Wilson goto err_put; 122210be98a7SChris Wilson 12232850748eSChris Wilson i915_vm_open(vm); 122410be98a7SChris Wilson 12255dbd2b7bSChris Wilson GEM_BUG_ON(id == 0); /* reserved for invalid/unassigned ppgtt */ 12265dbd2b7bSChris Wilson args->value = id; 122710be98a7SChris Wilson args->size = 0; 122810be98a7SChris Wilson 122910be98a7SChris Wilson err_put: 1230e568ac38SChris Wilson i915_vm_put(vm); 12315dbd2b7bSChris Wilson return err; 123210be98a7SChris Wilson } 123310be98a7SChris Wilson 123410be98a7SChris Wilson static void set_ppgtt_barrier(void *data) 123510be98a7SChris Wilson { 1236e568ac38SChris Wilson struct i915_address_space *old = data; 123710be98a7SChris Wilson 1238e568ac38SChris Wilson if (INTEL_GEN(old->i915) < 8) 1239e568ac38SChris Wilson gen6_ppgtt_unpin_all(i915_vm_to_ppgtt(old)); 124010be98a7SChris Wilson 12412850748eSChris Wilson i915_vm_close(old); 124210be98a7SChris Wilson } 124310be98a7SChris Wilson 124499f08d67SMaarten Lankhorst static int pin_ppgtt_update(struct intel_context *ce, struct i915_gem_ww_ctx *ww, void *data) 124599f08d67SMaarten Lankhorst { 124699f08d67SMaarten Lankhorst struct i915_address_space *vm = ce->vm; 124799f08d67SMaarten Lankhorst 124899f08d67SMaarten Lankhorst if (!HAS_LOGICAL_RING_CONTEXTS(vm->i915)) 124999f08d67SMaarten Lankhorst /* ppGTT is not part of the legacy context image */ 125047b08693SMaarten Lankhorst return gen6_ppgtt_pin(i915_vm_to_ppgtt(vm), ww); 125199f08d67SMaarten Lankhorst 125299f08d67SMaarten Lankhorst return 0; 125399f08d67SMaarten Lankhorst } 125499f08d67SMaarten Lankhorst 125510be98a7SChris Wilson static int emit_ppgtt_update(struct i915_request *rq, void *data) 125610be98a7SChris Wilson { 12579f3ccd40SChris Wilson struct i915_address_space *vm = rq->context->vm; 125810be98a7SChris Wilson struct intel_engine_cs *engine = rq->engine; 125910be98a7SChris Wilson u32 base = engine->mmio_base; 126010be98a7SChris Wilson u32 *cs; 126110be98a7SChris Wilson int i; 126210be98a7SChris Wilson 1263e568ac38SChris Wilson if (i915_vm_is_4lvl(vm)) { 1264ab53497bSChris Wilson struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1265b5b7bef9SMika Kuoppala const dma_addr_t pd_daddr = px_dma(ppgtt->pd); 126610be98a7SChris Wilson 126710be98a7SChris Wilson cs = intel_ring_begin(rq, 6); 126810be98a7SChris Wilson if (IS_ERR(cs)) 126910be98a7SChris Wilson return PTR_ERR(cs); 127010be98a7SChris Wilson 127110be98a7SChris Wilson *cs++ = MI_LOAD_REGISTER_IMM(2); 127210be98a7SChris Wilson 127310be98a7SChris Wilson *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 0)); 127410be98a7SChris Wilson *cs++ = upper_32_bits(pd_daddr); 127510be98a7SChris Wilson *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 0)); 127610be98a7SChris Wilson *cs++ = lower_32_bits(pd_daddr); 127710be98a7SChris Wilson 127810be98a7SChris Wilson *cs++ = MI_NOOP; 127910be98a7SChris Wilson intel_ring_advance(rq, cs); 128010be98a7SChris Wilson } else if (HAS_LOGICAL_RING_CONTEXTS(engine->i915)) { 1281ab53497bSChris Wilson struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1282191797a8SChris Wilson int err; 1283191797a8SChris Wilson 1284191797a8SChris Wilson /* Magic required to prevent forcewake errors! */ 1285191797a8SChris Wilson err = engine->emit_flush(rq, EMIT_INVALIDATE); 1286191797a8SChris Wilson if (err) 1287191797a8SChris Wilson return err; 1288e568ac38SChris Wilson 128910be98a7SChris Wilson cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2); 129010be98a7SChris Wilson if (IS_ERR(cs)) 129110be98a7SChris Wilson return PTR_ERR(cs); 129210be98a7SChris Wilson 1293191797a8SChris Wilson *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED; 129410be98a7SChris Wilson for (i = GEN8_3LVL_PDPES; i--; ) { 129510be98a7SChris Wilson const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); 129610be98a7SChris Wilson 129710be98a7SChris Wilson *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i)); 129810be98a7SChris Wilson *cs++ = upper_32_bits(pd_daddr); 129910be98a7SChris Wilson *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i)); 130010be98a7SChris Wilson *cs++ = lower_32_bits(pd_daddr); 130110be98a7SChris Wilson } 130210be98a7SChris Wilson *cs++ = MI_NOOP; 130310be98a7SChris Wilson intel_ring_advance(rq, cs); 130410be98a7SChris Wilson } 130510be98a7SChris Wilson 130610be98a7SChris Wilson return 0; 130710be98a7SChris Wilson } 130810be98a7SChris Wilson 13091fe2d6f9SChris Wilson static bool skip_ppgtt_update(struct intel_context *ce, void *data) 13101fe2d6f9SChris Wilson { 13111fe2d6f9SChris Wilson if (HAS_LOGICAL_RING_CONTEXTS(ce->engine->i915)) 131299f08d67SMaarten Lankhorst return !ce->state; 131399f08d67SMaarten Lankhorst else 131499f08d67SMaarten Lankhorst return !atomic_read(&ce->pin_count); 13151fe2d6f9SChris Wilson } 13161fe2d6f9SChris Wilson 131710be98a7SChris Wilson static int set_ppgtt(struct drm_i915_file_private *file_priv, 131810be98a7SChris Wilson struct i915_gem_context *ctx, 131910be98a7SChris Wilson struct drm_i915_gem_context_param *args) 132010be98a7SChris Wilson { 1321e568ac38SChris Wilson struct i915_address_space *vm, *old; 132210be98a7SChris Wilson int err; 132310be98a7SChris Wilson 132410be98a7SChris Wilson if (args->size) 132510be98a7SChris Wilson return -EINVAL; 132610be98a7SChris Wilson 1327a4e7ccdaSChris Wilson if (!rcu_access_pointer(ctx->vm)) 132810be98a7SChris Wilson return -ENODEV; 132910be98a7SChris Wilson 133010be98a7SChris Wilson if (upper_32_bits(args->value)) 133110be98a7SChris Wilson return -ENOENT; 133210be98a7SChris Wilson 1333aabbe344SChris Wilson rcu_read_lock(); 13345dbd2b7bSChris Wilson vm = xa_load(&file_priv->vm_xa, args->value); 1335aabbe344SChris Wilson if (vm && !kref_get_unless_zero(&vm->ref)) 1336aabbe344SChris Wilson vm = NULL; 1337aabbe344SChris Wilson rcu_read_unlock(); 1338e568ac38SChris Wilson if (!vm) 133910be98a7SChris Wilson return -ENOENT; 134010be98a7SChris Wilson 1341a4e7ccdaSChris Wilson err = mutex_lock_interruptible(&ctx->mutex); 134210be98a7SChris Wilson if (err) 134310be98a7SChris Wilson goto out; 134410be98a7SChris Wilson 1345a4e7ccdaSChris Wilson if (i915_gem_context_is_closed(ctx)) { 1346a4e7ccdaSChris Wilson err = -ENOENT; 1347feba2b81SChris Wilson goto unlock; 1348a4e7ccdaSChris Wilson } 1349a4e7ccdaSChris Wilson 1350a4e7ccdaSChris Wilson if (vm == rcu_access_pointer(ctx->vm)) 135110be98a7SChris Wilson goto unlock; 135210be98a7SChris Wilson 1353f7ce8639SChris Wilson old = __set_ppgtt(ctx, vm); 1354f7ce8639SChris Wilson 135510be98a7SChris Wilson /* Teardown the existing obj:vma cache, it will have to be rebuilt. */ 135610be98a7SChris Wilson lut_close(ctx); 135710be98a7SChris Wilson 135810be98a7SChris Wilson /* 135910be98a7SChris Wilson * We need to flush any requests using the current ppgtt before 136010be98a7SChris Wilson * we release it as the requests do not hold a reference themselves, 136110be98a7SChris Wilson * only indirectly through the context. 136210be98a7SChris Wilson */ 136310be98a7SChris Wilson err = context_barrier_task(ctx, ALL_ENGINES, 13641fe2d6f9SChris Wilson skip_ppgtt_update, 136599f08d67SMaarten Lankhorst pin_ppgtt_update, 136610be98a7SChris Wilson emit_ppgtt_update, 136710be98a7SChris Wilson set_ppgtt_barrier, 136810be98a7SChris Wilson old); 136910be98a7SChris Wilson if (err) { 13702850748eSChris Wilson i915_vm_close(__set_ppgtt(ctx, old)); 13712850748eSChris Wilson i915_vm_close(old); 1372f7ce8639SChris Wilson lut_close(ctx); /* force a rebuild of the old obj:vma cache */ 137310be98a7SChris Wilson } 137410be98a7SChris Wilson 137510be98a7SChris Wilson unlock: 1376a4e7ccdaSChris Wilson mutex_unlock(&ctx->mutex); 137710be98a7SChris Wilson out: 1378e568ac38SChris Wilson i915_vm_put(vm); 137910be98a7SChris Wilson return err; 138010be98a7SChris Wilson } 138110be98a7SChris Wilson 138288be76cdSChris Wilson static int __apply_ringsize(struct intel_context *ce, void *sz) 138388be76cdSChris Wilson { 138488be76cdSChris Wilson return intel_context_set_ring_size(ce, (unsigned long)sz); 138588be76cdSChris Wilson } 138688be76cdSChris Wilson 138788be76cdSChris Wilson static int set_ringsize(struct i915_gem_context *ctx, 138888be76cdSChris Wilson struct drm_i915_gem_context_param *args) 138988be76cdSChris Wilson { 139088be76cdSChris Wilson if (!HAS_LOGICAL_RING_CONTEXTS(ctx->i915)) 139188be76cdSChris Wilson return -ENODEV; 139288be76cdSChris Wilson 139388be76cdSChris Wilson if (args->size) 139488be76cdSChris Wilson return -EINVAL; 139588be76cdSChris Wilson 139688be76cdSChris Wilson if (!IS_ALIGNED(args->value, I915_GTT_PAGE_SIZE)) 139788be76cdSChris Wilson return -EINVAL; 139888be76cdSChris Wilson 139988be76cdSChris Wilson if (args->value < I915_GTT_PAGE_SIZE) 140088be76cdSChris Wilson return -EINVAL; 140188be76cdSChris Wilson 140288be76cdSChris Wilson if (args->value > 128 * I915_GTT_PAGE_SIZE) 140388be76cdSChris Wilson return -EINVAL; 140488be76cdSChris Wilson 140588be76cdSChris Wilson return context_apply_all(ctx, 140688be76cdSChris Wilson __apply_ringsize, 140788be76cdSChris Wilson __intel_context_ring_size(args->value)); 140888be76cdSChris Wilson } 140988be76cdSChris Wilson 141088be76cdSChris Wilson static int __get_ringsize(struct intel_context *ce, void *arg) 141188be76cdSChris Wilson { 141288be76cdSChris Wilson long sz; 141388be76cdSChris Wilson 141488be76cdSChris Wilson sz = intel_context_get_ring_size(ce); 141588be76cdSChris Wilson GEM_BUG_ON(sz > INT_MAX); 141688be76cdSChris Wilson 141788be76cdSChris Wilson return sz; /* stop on first engine */ 141888be76cdSChris Wilson } 141988be76cdSChris Wilson 142088be76cdSChris Wilson static int get_ringsize(struct i915_gem_context *ctx, 142188be76cdSChris Wilson struct drm_i915_gem_context_param *args) 142288be76cdSChris Wilson { 142388be76cdSChris Wilson int sz; 142488be76cdSChris Wilson 142588be76cdSChris Wilson if (!HAS_LOGICAL_RING_CONTEXTS(ctx->i915)) 142688be76cdSChris Wilson return -ENODEV; 142788be76cdSChris Wilson 142888be76cdSChris Wilson if (args->size) 142988be76cdSChris Wilson return -EINVAL; 143088be76cdSChris Wilson 143188be76cdSChris Wilson sz = context_apply_all(ctx, __get_ringsize, NULL); 143288be76cdSChris Wilson if (sz < 0) 143388be76cdSChris Wilson return sz; 143488be76cdSChris Wilson 143588be76cdSChris Wilson args->value = sz; 143688be76cdSChris Wilson return 0; 143788be76cdSChris Wilson } 143888be76cdSChris Wilson 143911ecbdddSLionel Landwerlin int 14400b6613c6SVenkata Sandeep Dhanalakota i915_gem_user_to_context_sseu(struct intel_gt *gt, 144110be98a7SChris Wilson const struct drm_i915_gem_context_param_sseu *user, 144210be98a7SChris Wilson struct intel_sseu *context) 144310be98a7SChris Wilson { 14440b6613c6SVenkata Sandeep Dhanalakota const struct sseu_dev_info *device = >->info.sseu; 14450b6613c6SVenkata Sandeep Dhanalakota struct drm_i915_private *i915 = gt->i915; 144610be98a7SChris Wilson 144710be98a7SChris Wilson /* No zeros in any field. */ 144810be98a7SChris Wilson if (!user->slice_mask || !user->subslice_mask || 144910be98a7SChris Wilson !user->min_eus_per_subslice || !user->max_eus_per_subslice) 145010be98a7SChris Wilson return -EINVAL; 145110be98a7SChris Wilson 145210be98a7SChris Wilson /* Max > min. */ 145310be98a7SChris Wilson if (user->max_eus_per_subslice < user->min_eus_per_subslice) 145410be98a7SChris Wilson return -EINVAL; 145510be98a7SChris Wilson 145610be98a7SChris Wilson /* 145710be98a7SChris Wilson * Some future proofing on the types since the uAPI is wider than the 145810be98a7SChris Wilson * current internal implementation. 145910be98a7SChris Wilson */ 146010be98a7SChris Wilson if (overflows_type(user->slice_mask, context->slice_mask) || 146110be98a7SChris Wilson overflows_type(user->subslice_mask, context->subslice_mask) || 146210be98a7SChris Wilson overflows_type(user->min_eus_per_subslice, 146310be98a7SChris Wilson context->min_eus_per_subslice) || 146410be98a7SChris Wilson overflows_type(user->max_eus_per_subslice, 146510be98a7SChris Wilson context->max_eus_per_subslice)) 146610be98a7SChris Wilson return -EINVAL; 146710be98a7SChris Wilson 146810be98a7SChris Wilson /* Check validity against hardware. */ 146910be98a7SChris Wilson if (user->slice_mask & ~device->slice_mask) 147010be98a7SChris Wilson return -EINVAL; 147110be98a7SChris Wilson 147210be98a7SChris Wilson if (user->subslice_mask & ~device->subslice_mask[0]) 147310be98a7SChris Wilson return -EINVAL; 147410be98a7SChris Wilson 147510be98a7SChris Wilson if (user->max_eus_per_subslice > device->max_eus_per_subslice) 147610be98a7SChris Wilson return -EINVAL; 147710be98a7SChris Wilson 147810be98a7SChris Wilson context->slice_mask = user->slice_mask; 147910be98a7SChris Wilson context->subslice_mask = user->subslice_mask; 148010be98a7SChris Wilson context->min_eus_per_subslice = user->min_eus_per_subslice; 148110be98a7SChris Wilson context->max_eus_per_subslice = user->max_eus_per_subslice; 148210be98a7SChris Wilson 148310be98a7SChris Wilson /* Part specific restrictions. */ 148410be98a7SChris Wilson if (IS_GEN(i915, 11)) { 148510be98a7SChris Wilson unsigned int hw_s = hweight8(device->slice_mask); 148610be98a7SChris Wilson unsigned int hw_ss_per_s = hweight8(device->subslice_mask[0]); 148710be98a7SChris Wilson unsigned int req_s = hweight8(context->slice_mask); 148810be98a7SChris Wilson unsigned int req_ss = hweight8(context->subslice_mask); 148910be98a7SChris Wilson 149010be98a7SChris Wilson /* 149110be98a7SChris Wilson * Only full subslice enablement is possible if more than one 149210be98a7SChris Wilson * slice is turned on. 149310be98a7SChris Wilson */ 149410be98a7SChris Wilson if (req_s > 1 && req_ss != hw_ss_per_s) 149510be98a7SChris Wilson return -EINVAL; 149610be98a7SChris Wilson 149710be98a7SChris Wilson /* 149810be98a7SChris Wilson * If more than four (SScount bitfield limit) subslices are 149910be98a7SChris Wilson * requested then the number has to be even. 150010be98a7SChris Wilson */ 150110be98a7SChris Wilson if (req_ss > 4 && (req_ss & 1)) 150210be98a7SChris Wilson return -EINVAL; 150310be98a7SChris Wilson 150410be98a7SChris Wilson /* 150510be98a7SChris Wilson * If only one slice is enabled and subslice count is below the 150610be98a7SChris Wilson * device full enablement, it must be at most half of the all 150710be98a7SChris Wilson * available subslices. 150810be98a7SChris Wilson */ 150910be98a7SChris Wilson if (req_s == 1 && req_ss < hw_ss_per_s && 151010be98a7SChris Wilson req_ss > (hw_ss_per_s / 2)) 151110be98a7SChris Wilson return -EINVAL; 151210be98a7SChris Wilson 151310be98a7SChris Wilson /* ABI restriction - VME use case only. */ 151410be98a7SChris Wilson 151510be98a7SChris Wilson /* All slices or one slice only. */ 151610be98a7SChris Wilson if (req_s != 1 && req_s != hw_s) 151710be98a7SChris Wilson return -EINVAL; 151810be98a7SChris Wilson 151910be98a7SChris Wilson /* 152010be98a7SChris Wilson * Half subslices or full enablement only when one slice is 152110be98a7SChris Wilson * enabled. 152210be98a7SChris Wilson */ 152310be98a7SChris Wilson if (req_s == 1 && 152410be98a7SChris Wilson (req_ss != hw_ss_per_s && req_ss != (hw_ss_per_s / 2))) 152510be98a7SChris Wilson return -EINVAL; 152610be98a7SChris Wilson 152710be98a7SChris Wilson /* No EU configuration changes. */ 152810be98a7SChris Wilson if ((user->min_eus_per_subslice != 152910be98a7SChris Wilson device->max_eus_per_subslice) || 153010be98a7SChris Wilson (user->max_eus_per_subslice != 153110be98a7SChris Wilson device->max_eus_per_subslice)) 153210be98a7SChris Wilson return -EINVAL; 153310be98a7SChris Wilson } 153410be98a7SChris Wilson 153510be98a7SChris Wilson return 0; 153610be98a7SChris Wilson } 153710be98a7SChris Wilson 153810be98a7SChris Wilson static int set_sseu(struct i915_gem_context *ctx, 153910be98a7SChris Wilson struct drm_i915_gem_context_param *args) 154010be98a7SChris Wilson { 154110be98a7SChris Wilson struct drm_i915_private *i915 = ctx->i915; 154210be98a7SChris Wilson struct drm_i915_gem_context_param_sseu user_sseu; 154310be98a7SChris Wilson struct intel_context *ce; 154410be98a7SChris Wilson struct intel_sseu sseu; 154510be98a7SChris Wilson unsigned long lookup; 154610be98a7SChris Wilson int ret; 154710be98a7SChris Wilson 154810be98a7SChris Wilson if (args->size < sizeof(user_sseu)) 154910be98a7SChris Wilson return -EINVAL; 155010be98a7SChris Wilson 155110be98a7SChris Wilson if (!IS_GEN(i915, 11)) 155210be98a7SChris Wilson return -ENODEV; 155310be98a7SChris Wilson 155410be98a7SChris Wilson if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), 155510be98a7SChris Wilson sizeof(user_sseu))) 155610be98a7SChris Wilson return -EFAULT; 155710be98a7SChris Wilson 155810be98a7SChris Wilson if (user_sseu.rsvd) 155910be98a7SChris Wilson return -EINVAL; 156010be98a7SChris Wilson 156110be98a7SChris Wilson if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) 156210be98a7SChris Wilson return -EINVAL; 156310be98a7SChris Wilson 156410be98a7SChris Wilson lookup = 0; 156510be98a7SChris Wilson if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) 156610be98a7SChris Wilson lookup |= LOOKUP_USER_INDEX; 156710be98a7SChris Wilson 156810be98a7SChris Wilson ce = lookup_user_engine(ctx, lookup, &user_sseu.engine); 156910be98a7SChris Wilson if (IS_ERR(ce)) 157010be98a7SChris Wilson return PTR_ERR(ce); 157110be98a7SChris Wilson 157210be98a7SChris Wilson /* Only render engine supports RPCS configuration. */ 157310be98a7SChris Wilson if (ce->engine->class != RENDER_CLASS) { 157410be98a7SChris Wilson ret = -ENODEV; 157510be98a7SChris Wilson goto out_ce; 157610be98a7SChris Wilson } 157710be98a7SChris Wilson 15780b6613c6SVenkata Sandeep Dhanalakota ret = i915_gem_user_to_context_sseu(ce->engine->gt, &user_sseu, &sseu); 157910be98a7SChris Wilson if (ret) 158010be98a7SChris Wilson goto out_ce; 158110be98a7SChris Wilson 158210be98a7SChris Wilson ret = intel_context_reconfigure_sseu(ce, sseu); 158310be98a7SChris Wilson if (ret) 158410be98a7SChris Wilson goto out_ce; 158510be98a7SChris Wilson 158610be98a7SChris Wilson args->size = sizeof(user_sseu); 158710be98a7SChris Wilson 158810be98a7SChris Wilson out_ce: 158910be98a7SChris Wilson intel_context_put(ce); 159010be98a7SChris Wilson return ret; 159110be98a7SChris Wilson } 159210be98a7SChris Wilson 159310be98a7SChris Wilson struct set_engines { 159410be98a7SChris Wilson struct i915_gem_context *ctx; 159510be98a7SChris Wilson struct i915_gem_engines *engines; 159610be98a7SChris Wilson }; 159710be98a7SChris Wilson 159810be98a7SChris Wilson static int 159910be98a7SChris Wilson set_engines__load_balance(struct i915_user_extension __user *base, void *data) 160010be98a7SChris Wilson { 160110be98a7SChris Wilson struct i915_context_engines_load_balance __user *ext = 160210be98a7SChris Wilson container_of_user(base, typeof(*ext), base); 160310be98a7SChris Wilson const struct set_engines *set = data; 1604d0bf4582SWambui Karuga struct drm_i915_private *i915 = set->ctx->i915; 160510be98a7SChris Wilson struct intel_engine_cs *stack[16]; 160610be98a7SChris Wilson struct intel_engine_cs **siblings; 160710be98a7SChris Wilson struct intel_context *ce; 160810be98a7SChris Wilson u16 num_siblings, idx; 160910be98a7SChris Wilson unsigned int n; 161010be98a7SChris Wilson int err; 161110be98a7SChris Wilson 1612d0bf4582SWambui Karuga if (!HAS_EXECLISTS(i915)) 161310be98a7SChris Wilson return -ENODEV; 161410be98a7SChris Wilson 1615065273f7SDaniele Ceraolo Spurio if (intel_uc_uses_guc_submission(&i915->gt.uc)) 161610be98a7SChris Wilson return -ENODEV; /* not implement yet */ 161710be98a7SChris Wilson 161810be98a7SChris Wilson if (get_user(idx, &ext->engine_index)) 161910be98a7SChris Wilson return -EFAULT; 162010be98a7SChris Wilson 162110be98a7SChris Wilson if (idx >= set->engines->num_engines) { 1622d0bf4582SWambui Karuga drm_dbg(&i915->drm, "Invalid placement value, %d >= %d\n", 162310be98a7SChris Wilson idx, set->engines->num_engines); 162410be98a7SChris Wilson return -EINVAL; 162510be98a7SChris Wilson } 162610be98a7SChris Wilson 162710be98a7SChris Wilson idx = array_index_nospec(idx, set->engines->num_engines); 162810be98a7SChris Wilson if (set->engines->engines[idx]) { 1629d0bf4582SWambui Karuga drm_dbg(&i915->drm, 1630d0bf4582SWambui Karuga "Invalid placement[%d], already occupied\n", idx); 163110be98a7SChris Wilson return -EEXIST; 163210be98a7SChris Wilson } 163310be98a7SChris Wilson 163410be98a7SChris Wilson if (get_user(num_siblings, &ext->num_siblings)) 163510be98a7SChris Wilson return -EFAULT; 163610be98a7SChris Wilson 163710be98a7SChris Wilson err = check_user_mbz(&ext->flags); 163810be98a7SChris Wilson if (err) 163910be98a7SChris Wilson return err; 164010be98a7SChris Wilson 164110be98a7SChris Wilson err = check_user_mbz(&ext->mbz64); 164210be98a7SChris Wilson if (err) 164310be98a7SChris Wilson return err; 164410be98a7SChris Wilson 164510be98a7SChris Wilson siblings = stack; 164610be98a7SChris Wilson if (num_siblings > ARRAY_SIZE(stack)) { 164710be98a7SChris Wilson siblings = kmalloc_array(num_siblings, 164810be98a7SChris Wilson sizeof(*siblings), 164910be98a7SChris Wilson GFP_KERNEL); 165010be98a7SChris Wilson if (!siblings) 165110be98a7SChris Wilson return -ENOMEM; 165210be98a7SChris Wilson } 165310be98a7SChris Wilson 165410be98a7SChris Wilson for (n = 0; n < num_siblings; n++) { 165510be98a7SChris Wilson struct i915_engine_class_instance ci; 165610be98a7SChris Wilson 165710be98a7SChris Wilson if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) { 165810be98a7SChris Wilson err = -EFAULT; 165910be98a7SChris Wilson goto out_siblings; 166010be98a7SChris Wilson } 166110be98a7SChris Wilson 1662d0bf4582SWambui Karuga siblings[n] = intel_engine_lookup_user(i915, 166310be98a7SChris Wilson ci.engine_class, 166410be98a7SChris Wilson ci.engine_instance); 166510be98a7SChris Wilson if (!siblings[n]) { 1666d0bf4582SWambui Karuga drm_dbg(&i915->drm, 1667d0bf4582SWambui Karuga "Invalid sibling[%d]: { class:%d, inst:%d }\n", 166810be98a7SChris Wilson n, ci.engine_class, ci.engine_instance); 166910be98a7SChris Wilson err = -EINVAL; 167010be98a7SChris Wilson goto out_siblings; 167110be98a7SChris Wilson } 167210be98a7SChris Wilson } 167310be98a7SChris Wilson 1674e6ba7648SChris Wilson ce = intel_execlists_create_virtual(siblings, n); 167510be98a7SChris Wilson if (IS_ERR(ce)) { 167610be98a7SChris Wilson err = PTR_ERR(ce); 167710be98a7SChris Wilson goto out_siblings; 167810be98a7SChris Wilson } 167910be98a7SChris Wilson 1680e6ba7648SChris Wilson intel_context_set_gem(ce, set->ctx); 1681e6ba7648SChris Wilson 168210be98a7SChris Wilson if (cmpxchg(&set->engines->engines[idx], NULL, ce)) { 168310be98a7SChris Wilson intel_context_put(ce); 168410be98a7SChris Wilson err = -EEXIST; 168510be98a7SChris Wilson goto out_siblings; 168610be98a7SChris Wilson } 168710be98a7SChris Wilson 168810be98a7SChris Wilson out_siblings: 168910be98a7SChris Wilson if (siblings != stack) 169010be98a7SChris Wilson kfree(siblings); 169110be98a7SChris Wilson 169210be98a7SChris Wilson return err; 169310be98a7SChris Wilson } 169410be98a7SChris Wilson 169510be98a7SChris Wilson static int 169610be98a7SChris Wilson set_engines__bond(struct i915_user_extension __user *base, void *data) 169710be98a7SChris Wilson { 169810be98a7SChris Wilson struct i915_context_engines_bond __user *ext = 169910be98a7SChris Wilson container_of_user(base, typeof(*ext), base); 170010be98a7SChris Wilson const struct set_engines *set = data; 1701d0bf4582SWambui Karuga struct drm_i915_private *i915 = set->ctx->i915; 170210be98a7SChris Wilson struct i915_engine_class_instance ci; 170310be98a7SChris Wilson struct intel_engine_cs *virtual; 170410be98a7SChris Wilson struct intel_engine_cs *master; 170510be98a7SChris Wilson u16 idx, num_bonds; 170610be98a7SChris Wilson int err, n; 170710be98a7SChris Wilson 170810be98a7SChris Wilson if (get_user(idx, &ext->virtual_index)) 170910be98a7SChris Wilson return -EFAULT; 171010be98a7SChris Wilson 171110be98a7SChris Wilson if (idx >= set->engines->num_engines) { 1712d0bf4582SWambui Karuga drm_dbg(&i915->drm, 1713d0bf4582SWambui Karuga "Invalid index for virtual engine: %d >= %d\n", 171410be98a7SChris Wilson idx, set->engines->num_engines); 171510be98a7SChris Wilson return -EINVAL; 171610be98a7SChris Wilson } 171710be98a7SChris Wilson 171810be98a7SChris Wilson idx = array_index_nospec(idx, set->engines->num_engines); 171910be98a7SChris Wilson if (!set->engines->engines[idx]) { 1720d0bf4582SWambui Karuga drm_dbg(&i915->drm, "Invalid engine at %d\n", idx); 172110be98a7SChris Wilson return -EINVAL; 172210be98a7SChris Wilson } 172310be98a7SChris Wilson virtual = set->engines->engines[idx]->engine; 172410be98a7SChris Wilson 172510be98a7SChris Wilson err = check_user_mbz(&ext->flags); 172610be98a7SChris Wilson if (err) 172710be98a7SChris Wilson return err; 172810be98a7SChris Wilson 172910be98a7SChris Wilson for (n = 0; n < ARRAY_SIZE(ext->mbz64); n++) { 173010be98a7SChris Wilson err = check_user_mbz(&ext->mbz64[n]); 173110be98a7SChris Wilson if (err) 173210be98a7SChris Wilson return err; 173310be98a7SChris Wilson } 173410be98a7SChris Wilson 173510be98a7SChris Wilson if (copy_from_user(&ci, &ext->master, sizeof(ci))) 173610be98a7SChris Wilson return -EFAULT; 173710be98a7SChris Wilson 1738d0bf4582SWambui Karuga master = intel_engine_lookup_user(i915, 173910be98a7SChris Wilson ci.engine_class, ci.engine_instance); 174010be98a7SChris Wilson if (!master) { 1741d0bf4582SWambui Karuga drm_dbg(&i915->drm, 1742d0bf4582SWambui Karuga "Unrecognised master engine: { class:%u, instance:%u }\n", 174310be98a7SChris Wilson ci.engine_class, ci.engine_instance); 174410be98a7SChris Wilson return -EINVAL; 174510be98a7SChris Wilson } 174610be98a7SChris Wilson 174710be98a7SChris Wilson if (get_user(num_bonds, &ext->num_bonds)) 174810be98a7SChris Wilson return -EFAULT; 174910be98a7SChris Wilson 175010be98a7SChris Wilson for (n = 0; n < num_bonds; n++) { 175110be98a7SChris Wilson struct intel_engine_cs *bond; 175210be98a7SChris Wilson 175310be98a7SChris Wilson if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) 175410be98a7SChris Wilson return -EFAULT; 175510be98a7SChris Wilson 1756d0bf4582SWambui Karuga bond = intel_engine_lookup_user(i915, 175710be98a7SChris Wilson ci.engine_class, 175810be98a7SChris Wilson ci.engine_instance); 175910be98a7SChris Wilson if (!bond) { 1760d0bf4582SWambui Karuga drm_dbg(&i915->drm, 1761d0bf4582SWambui Karuga "Unrecognised engine[%d] for bonding: { class:%d, instance: %d }\n", 176210be98a7SChris Wilson n, ci.engine_class, ci.engine_instance); 176310be98a7SChris Wilson return -EINVAL; 176410be98a7SChris Wilson } 176510be98a7SChris Wilson 176610be98a7SChris Wilson /* 176710be98a7SChris Wilson * A non-virtual engine has no siblings to choose between; and 176810be98a7SChris Wilson * a submit fence will always be directed to the one engine. 176910be98a7SChris Wilson */ 177010be98a7SChris Wilson if (intel_engine_is_virtual(virtual)) { 177110be98a7SChris Wilson err = intel_virtual_engine_attach_bond(virtual, 177210be98a7SChris Wilson master, 177310be98a7SChris Wilson bond); 177410be98a7SChris Wilson if (err) 177510be98a7SChris Wilson return err; 177610be98a7SChris Wilson } 177710be98a7SChris Wilson } 177810be98a7SChris Wilson 177910be98a7SChris Wilson return 0; 178010be98a7SChris Wilson } 178110be98a7SChris Wilson 178210be98a7SChris Wilson static const i915_user_extension_fn set_engines__extensions[] = { 178310be98a7SChris Wilson [I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE] = set_engines__load_balance, 178410be98a7SChris Wilson [I915_CONTEXT_ENGINES_EXT_BOND] = set_engines__bond, 178510be98a7SChris Wilson }; 178610be98a7SChris Wilson 178710be98a7SChris Wilson static int 178810be98a7SChris Wilson set_engines(struct i915_gem_context *ctx, 178910be98a7SChris Wilson const struct drm_i915_gem_context_param *args) 179010be98a7SChris Wilson { 1791baa89ba3SWambui Karuga struct drm_i915_private *i915 = ctx->i915; 179210be98a7SChris Wilson struct i915_context_param_engines __user *user = 179310be98a7SChris Wilson u64_to_user_ptr(args->value); 179410be98a7SChris Wilson struct set_engines set = { .ctx = ctx }; 179510be98a7SChris Wilson unsigned int num_engines, n; 179610be98a7SChris Wilson u64 extensions; 179710be98a7SChris Wilson int err; 179810be98a7SChris Wilson 179910be98a7SChris Wilson if (!args->size) { /* switch back to legacy user_ring_map */ 180010be98a7SChris Wilson if (!i915_gem_context_user_engines(ctx)) 180110be98a7SChris Wilson return 0; 180210be98a7SChris Wilson 180310be98a7SChris Wilson set.engines = default_engines(ctx); 180410be98a7SChris Wilson if (IS_ERR(set.engines)) 180510be98a7SChris Wilson return PTR_ERR(set.engines); 180610be98a7SChris Wilson 180710be98a7SChris Wilson goto replace; 180810be98a7SChris Wilson } 180910be98a7SChris Wilson 181010be98a7SChris Wilson BUILD_BUG_ON(!IS_ALIGNED(sizeof(*user), sizeof(*user->engines))); 181110be98a7SChris Wilson if (args->size < sizeof(*user) || 181210be98a7SChris Wilson !IS_ALIGNED(args->size, sizeof(*user->engines))) { 1813baa89ba3SWambui Karuga drm_dbg(&i915->drm, "Invalid size for engine array: %d\n", 181410be98a7SChris Wilson args->size); 181510be98a7SChris Wilson return -EINVAL; 181610be98a7SChris Wilson } 181710be98a7SChris Wilson 181810be98a7SChris Wilson /* 181910be98a7SChris Wilson * Note that I915_EXEC_RING_MASK limits execbuf to only using the 182010be98a7SChris Wilson * first 64 engines defined here. 182110be98a7SChris Wilson */ 182210be98a7SChris Wilson num_engines = (args->size - sizeof(*user)) / sizeof(*user->engines); 182370c96e39SChris Wilson set.engines = alloc_engines(num_engines); 182410be98a7SChris Wilson if (!set.engines) 182510be98a7SChris Wilson return -ENOMEM; 182610be98a7SChris Wilson 182710be98a7SChris Wilson for (n = 0; n < num_engines; n++) { 182810be98a7SChris Wilson struct i915_engine_class_instance ci; 182910be98a7SChris Wilson struct intel_engine_cs *engine; 183048ae397bSChris Wilson struct intel_context *ce; 183110be98a7SChris Wilson 183210be98a7SChris Wilson if (copy_from_user(&ci, &user->engines[n], sizeof(ci))) { 183310be98a7SChris Wilson __free_engines(set.engines, n); 183410be98a7SChris Wilson return -EFAULT; 183510be98a7SChris Wilson } 183610be98a7SChris Wilson 183710be98a7SChris Wilson if (ci.engine_class == (u16)I915_ENGINE_CLASS_INVALID && 183810be98a7SChris Wilson ci.engine_instance == (u16)I915_ENGINE_CLASS_INVALID_NONE) { 183910be98a7SChris Wilson set.engines->engines[n] = NULL; 184010be98a7SChris Wilson continue; 184110be98a7SChris Wilson } 184210be98a7SChris Wilson 184310be98a7SChris Wilson engine = intel_engine_lookup_user(ctx->i915, 184410be98a7SChris Wilson ci.engine_class, 184510be98a7SChris Wilson ci.engine_instance); 184610be98a7SChris Wilson if (!engine) { 1847baa89ba3SWambui Karuga drm_dbg(&i915->drm, 1848baa89ba3SWambui Karuga "Invalid engine[%d]: { class:%d, instance:%d }\n", 184910be98a7SChris Wilson n, ci.engine_class, ci.engine_instance); 185010be98a7SChris Wilson __free_engines(set.engines, n); 185110be98a7SChris Wilson return -ENOENT; 185210be98a7SChris Wilson } 185310be98a7SChris Wilson 1854e6ba7648SChris Wilson ce = intel_context_create(engine); 185548ae397bSChris Wilson if (IS_ERR(ce)) { 185610be98a7SChris Wilson __free_engines(set.engines, n); 185748ae397bSChris Wilson return PTR_ERR(ce); 185810be98a7SChris Wilson } 185948ae397bSChris Wilson 1860e6ba7648SChris Wilson intel_context_set_gem(ce, ctx); 1861e6ba7648SChris Wilson 186248ae397bSChris Wilson set.engines->engines[n] = ce; 186310be98a7SChris Wilson } 186410be98a7SChris Wilson set.engines->num_engines = num_engines; 186510be98a7SChris Wilson 186610be98a7SChris Wilson err = -EFAULT; 186710be98a7SChris Wilson if (!get_user(extensions, &user->extensions)) 186810be98a7SChris Wilson err = i915_user_extensions(u64_to_user_ptr(extensions), 186910be98a7SChris Wilson set_engines__extensions, 187010be98a7SChris Wilson ARRAY_SIZE(set_engines__extensions), 187110be98a7SChris Wilson &set); 187210be98a7SChris Wilson if (err) { 187310be98a7SChris Wilson free_engines(set.engines); 187410be98a7SChris Wilson return err; 187510be98a7SChris Wilson } 187610be98a7SChris Wilson 187710be98a7SChris Wilson replace: 187810be98a7SChris Wilson mutex_lock(&ctx->engines_mutex); 1879130a95e9SChris Wilson if (i915_gem_context_is_closed(ctx)) { 1880130a95e9SChris Wilson mutex_unlock(&ctx->engines_mutex); 1881130a95e9SChris Wilson free_engines(set.engines); 1882130a95e9SChris Wilson return -ENOENT; 1883130a95e9SChris Wilson } 188410be98a7SChris Wilson if (args->size) 188510be98a7SChris Wilson i915_gem_context_set_user_engines(ctx); 188610be98a7SChris Wilson else 188710be98a7SChris Wilson i915_gem_context_clear_user_engines(ctx); 18881feace5dSPaul E. McKenney set.engines = rcu_replace_pointer(ctx->engines, set.engines, 1); 188910be98a7SChris Wilson mutex_unlock(&ctx->engines_mutex); 189010be98a7SChris Wilson 189142fb60deSChris Wilson /* Keep track of old engine sets for kill_context() */ 1892130a95e9SChris Wilson engines_idle_release(ctx, set.engines); 189310be98a7SChris Wilson 189410be98a7SChris Wilson return 0; 189510be98a7SChris Wilson } 189610be98a7SChris Wilson 189710be98a7SChris Wilson static struct i915_gem_engines * 189810be98a7SChris Wilson __copy_engines(struct i915_gem_engines *e) 189910be98a7SChris Wilson { 190010be98a7SChris Wilson struct i915_gem_engines *copy; 190110be98a7SChris Wilson unsigned int n; 190210be98a7SChris Wilson 190370c96e39SChris Wilson copy = alloc_engines(e->num_engines); 190410be98a7SChris Wilson if (!copy) 190510be98a7SChris Wilson return ERR_PTR(-ENOMEM); 190610be98a7SChris Wilson 190710be98a7SChris Wilson for (n = 0; n < e->num_engines; n++) { 190810be98a7SChris Wilson if (e->engines[n]) 190910be98a7SChris Wilson copy->engines[n] = intel_context_get(e->engines[n]); 191010be98a7SChris Wilson else 191110be98a7SChris Wilson copy->engines[n] = NULL; 191210be98a7SChris Wilson } 191310be98a7SChris Wilson copy->num_engines = n; 191410be98a7SChris Wilson 191510be98a7SChris Wilson return copy; 191610be98a7SChris Wilson } 191710be98a7SChris Wilson 191810be98a7SChris Wilson static int 191910be98a7SChris Wilson get_engines(struct i915_gem_context *ctx, 192010be98a7SChris Wilson struct drm_i915_gem_context_param *args) 192110be98a7SChris Wilson { 192210be98a7SChris Wilson struct i915_context_param_engines __user *user; 192310be98a7SChris Wilson struct i915_gem_engines *e; 192410be98a7SChris Wilson size_t n, count, size; 192510be98a7SChris Wilson int err = 0; 192610be98a7SChris Wilson 192710be98a7SChris Wilson err = mutex_lock_interruptible(&ctx->engines_mutex); 192810be98a7SChris Wilson if (err) 192910be98a7SChris Wilson return err; 193010be98a7SChris Wilson 193110be98a7SChris Wilson e = NULL; 193210be98a7SChris Wilson if (i915_gem_context_user_engines(ctx)) 193310be98a7SChris Wilson e = __copy_engines(i915_gem_context_engines(ctx)); 193410be98a7SChris Wilson mutex_unlock(&ctx->engines_mutex); 193510be98a7SChris Wilson if (IS_ERR_OR_NULL(e)) { 193610be98a7SChris Wilson args->size = 0; 193710be98a7SChris Wilson return PTR_ERR_OR_ZERO(e); 193810be98a7SChris Wilson } 193910be98a7SChris Wilson 194010be98a7SChris Wilson count = e->num_engines; 194110be98a7SChris Wilson 194210be98a7SChris Wilson /* Be paranoid in case we have an impedance mismatch */ 194310be98a7SChris Wilson if (!check_struct_size(user, engines, count, &size)) { 194410be98a7SChris Wilson err = -EINVAL; 194510be98a7SChris Wilson goto err_free; 194610be98a7SChris Wilson } 194710be98a7SChris Wilson if (overflows_type(size, args->size)) { 194810be98a7SChris Wilson err = -EINVAL; 194910be98a7SChris Wilson goto err_free; 195010be98a7SChris Wilson } 195110be98a7SChris Wilson 195210be98a7SChris Wilson if (!args->size) { 195310be98a7SChris Wilson args->size = size; 195410be98a7SChris Wilson goto err_free; 195510be98a7SChris Wilson } 195610be98a7SChris Wilson 195710be98a7SChris Wilson if (args->size < size) { 195810be98a7SChris Wilson err = -EINVAL; 195910be98a7SChris Wilson goto err_free; 196010be98a7SChris Wilson } 196110be98a7SChris Wilson 196210be98a7SChris Wilson user = u64_to_user_ptr(args->value); 196310be98a7SChris Wilson if (put_user(0, &user->extensions)) { 196410be98a7SChris Wilson err = -EFAULT; 196510be98a7SChris Wilson goto err_free; 196610be98a7SChris Wilson } 196710be98a7SChris Wilson 196810be98a7SChris Wilson for (n = 0; n < count; n++) { 196910be98a7SChris Wilson struct i915_engine_class_instance ci = { 197010be98a7SChris Wilson .engine_class = I915_ENGINE_CLASS_INVALID, 197110be98a7SChris Wilson .engine_instance = I915_ENGINE_CLASS_INVALID_NONE, 197210be98a7SChris Wilson }; 197310be98a7SChris Wilson 197410be98a7SChris Wilson if (e->engines[n]) { 197510be98a7SChris Wilson ci.engine_class = e->engines[n]->engine->uabi_class; 1976750e76b4SChris Wilson ci.engine_instance = e->engines[n]->engine->uabi_instance; 197710be98a7SChris Wilson } 197810be98a7SChris Wilson 197910be98a7SChris Wilson if (copy_to_user(&user->engines[n], &ci, sizeof(ci))) { 198010be98a7SChris Wilson err = -EFAULT; 198110be98a7SChris Wilson goto err_free; 198210be98a7SChris Wilson } 198310be98a7SChris Wilson } 198410be98a7SChris Wilson 198510be98a7SChris Wilson args->size = size; 198610be98a7SChris Wilson 198710be98a7SChris Wilson err_free: 1988155ab883SChris Wilson free_engines(e); 198910be98a7SChris Wilson return err; 199010be98a7SChris Wilson } 199110be98a7SChris Wilson 1992a0e04715SChris Wilson static int 1993a0e04715SChris Wilson set_persistence(struct i915_gem_context *ctx, 1994a0e04715SChris Wilson const struct drm_i915_gem_context_param *args) 1995a0e04715SChris Wilson { 1996a0e04715SChris Wilson if (args->size) 1997a0e04715SChris Wilson return -EINVAL; 1998a0e04715SChris Wilson 1999a0e04715SChris Wilson return __context_set_persistence(ctx, args->value); 2000a0e04715SChris Wilson } 2001a0e04715SChris Wilson 200288be76cdSChris Wilson static int __apply_priority(struct intel_context *ce, void *arg) 20030f100b70SChris Wilson { 20040f100b70SChris Wilson struct i915_gem_context *ctx = arg; 20050f100b70SChris Wilson 20060eb670aaSChris Wilson if (!intel_engine_has_timeslices(ce->engine)) 200788be76cdSChris Wilson return 0; 20080f100b70SChris Wilson 20090f100b70SChris Wilson if (ctx->sched.priority >= I915_PRIORITY_NORMAL) 20100f100b70SChris Wilson intel_context_set_use_semaphores(ce); 20110f100b70SChris Wilson else 20120f100b70SChris Wilson intel_context_clear_use_semaphores(ce); 201388be76cdSChris Wilson 201488be76cdSChris Wilson return 0; 20150f100b70SChris Wilson } 20160f100b70SChris Wilson 20170f100b70SChris Wilson static int set_priority(struct i915_gem_context *ctx, 20180f100b70SChris Wilson const struct drm_i915_gem_context_param *args) 20190f100b70SChris Wilson { 20200f100b70SChris Wilson s64 priority = args->value; 20210f100b70SChris Wilson 20220f100b70SChris Wilson if (args->size) 20230f100b70SChris Wilson return -EINVAL; 20240f100b70SChris Wilson 20250f100b70SChris Wilson if (!(ctx->i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY)) 20260f100b70SChris Wilson return -ENODEV; 20270f100b70SChris Wilson 20280f100b70SChris Wilson if (priority > I915_CONTEXT_MAX_USER_PRIORITY || 20290f100b70SChris Wilson priority < I915_CONTEXT_MIN_USER_PRIORITY) 20300f100b70SChris Wilson return -EINVAL; 20310f100b70SChris Wilson 20320f100b70SChris Wilson if (priority > I915_CONTEXT_DEFAULT_PRIORITY && 20330f100b70SChris Wilson !capable(CAP_SYS_NICE)) 20340f100b70SChris Wilson return -EPERM; 20350f100b70SChris Wilson 20360f100b70SChris Wilson ctx->sched.priority = I915_USER_PRIORITY(priority); 20370f100b70SChris Wilson context_apply_all(ctx, __apply_priority, ctx); 20380f100b70SChris Wilson 20390f100b70SChris Wilson return 0; 20400f100b70SChris Wilson } 20410f100b70SChris Wilson 204210be98a7SChris Wilson static int ctx_setparam(struct drm_i915_file_private *fpriv, 204310be98a7SChris Wilson struct i915_gem_context *ctx, 204410be98a7SChris Wilson struct drm_i915_gem_context_param *args) 204510be98a7SChris Wilson { 204610be98a7SChris Wilson int ret = 0; 204710be98a7SChris Wilson 204810be98a7SChris Wilson switch (args->param) { 204910be98a7SChris Wilson case I915_CONTEXT_PARAM_NO_ZEROMAP: 205010be98a7SChris Wilson if (args->size) 205110be98a7SChris Wilson ret = -EINVAL; 205210be98a7SChris Wilson else if (args->value) 205310be98a7SChris Wilson set_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); 205410be98a7SChris Wilson else 205510be98a7SChris Wilson clear_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); 205610be98a7SChris Wilson break; 205710be98a7SChris Wilson 205810be98a7SChris Wilson case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 205910be98a7SChris Wilson if (args->size) 206010be98a7SChris Wilson ret = -EINVAL; 206110be98a7SChris Wilson else if (args->value) 206210be98a7SChris Wilson i915_gem_context_set_no_error_capture(ctx); 206310be98a7SChris Wilson else 206410be98a7SChris Wilson i915_gem_context_clear_no_error_capture(ctx); 206510be98a7SChris Wilson break; 206610be98a7SChris Wilson 206710be98a7SChris Wilson case I915_CONTEXT_PARAM_BANNABLE: 206810be98a7SChris Wilson if (args->size) 206910be98a7SChris Wilson ret = -EINVAL; 207010be98a7SChris Wilson else if (!capable(CAP_SYS_ADMIN) && !args->value) 207110be98a7SChris Wilson ret = -EPERM; 207210be98a7SChris Wilson else if (args->value) 207310be98a7SChris Wilson i915_gem_context_set_bannable(ctx); 207410be98a7SChris Wilson else 207510be98a7SChris Wilson i915_gem_context_clear_bannable(ctx); 207610be98a7SChris Wilson break; 207710be98a7SChris Wilson 207810be98a7SChris Wilson case I915_CONTEXT_PARAM_RECOVERABLE: 207910be98a7SChris Wilson if (args->size) 208010be98a7SChris Wilson ret = -EINVAL; 208110be98a7SChris Wilson else if (args->value) 208210be98a7SChris Wilson i915_gem_context_set_recoverable(ctx); 208310be98a7SChris Wilson else 208410be98a7SChris Wilson i915_gem_context_clear_recoverable(ctx); 208510be98a7SChris Wilson break; 208610be98a7SChris Wilson 208710be98a7SChris Wilson case I915_CONTEXT_PARAM_PRIORITY: 20880f100b70SChris Wilson ret = set_priority(ctx, args); 208910be98a7SChris Wilson break; 209010be98a7SChris Wilson 209110be98a7SChris Wilson case I915_CONTEXT_PARAM_SSEU: 209210be98a7SChris Wilson ret = set_sseu(ctx, args); 209310be98a7SChris Wilson break; 209410be98a7SChris Wilson 209510be98a7SChris Wilson case I915_CONTEXT_PARAM_VM: 209610be98a7SChris Wilson ret = set_ppgtt(fpriv, ctx, args); 209710be98a7SChris Wilson break; 209810be98a7SChris Wilson 209910be98a7SChris Wilson case I915_CONTEXT_PARAM_ENGINES: 210010be98a7SChris Wilson ret = set_engines(ctx, args); 210110be98a7SChris Wilson break; 210210be98a7SChris Wilson 2103a0e04715SChris Wilson case I915_CONTEXT_PARAM_PERSISTENCE: 2104a0e04715SChris Wilson ret = set_persistence(ctx, args); 2105a0e04715SChris Wilson break; 2106a0e04715SChris Wilson 210788be76cdSChris Wilson case I915_CONTEXT_PARAM_RINGSIZE: 210888be76cdSChris Wilson ret = set_ringsize(ctx, args); 210988be76cdSChris Wilson break; 211088be76cdSChris Wilson 211110be98a7SChris Wilson case I915_CONTEXT_PARAM_BAN_PERIOD: 211210be98a7SChris Wilson default: 211310be98a7SChris Wilson ret = -EINVAL; 211410be98a7SChris Wilson break; 211510be98a7SChris Wilson } 211610be98a7SChris Wilson 211710be98a7SChris Wilson return ret; 211810be98a7SChris Wilson } 211910be98a7SChris Wilson 212010be98a7SChris Wilson struct create_ext { 212110be98a7SChris Wilson struct i915_gem_context *ctx; 212210be98a7SChris Wilson struct drm_i915_file_private *fpriv; 212310be98a7SChris Wilson }; 212410be98a7SChris Wilson 212510be98a7SChris Wilson static int create_setparam(struct i915_user_extension __user *ext, void *data) 212610be98a7SChris Wilson { 212710be98a7SChris Wilson struct drm_i915_gem_context_create_ext_setparam local; 212810be98a7SChris Wilson const struct create_ext *arg = data; 212910be98a7SChris Wilson 213010be98a7SChris Wilson if (copy_from_user(&local, ext, sizeof(local))) 213110be98a7SChris Wilson return -EFAULT; 213210be98a7SChris Wilson 213310be98a7SChris Wilson if (local.param.ctx_id) 213410be98a7SChris Wilson return -EINVAL; 213510be98a7SChris Wilson 213610be98a7SChris Wilson return ctx_setparam(arg->fpriv, arg->ctx, &local.param); 213710be98a7SChris Wilson } 213810be98a7SChris Wilson 213988be76cdSChris Wilson static int copy_ring_size(struct intel_context *dst, 214088be76cdSChris Wilson struct intel_context *src) 214188be76cdSChris Wilson { 214288be76cdSChris Wilson long sz; 214388be76cdSChris Wilson 214488be76cdSChris Wilson sz = intel_context_get_ring_size(src); 214588be76cdSChris Wilson if (sz < 0) 214688be76cdSChris Wilson return sz; 214788be76cdSChris Wilson 214888be76cdSChris Wilson return intel_context_set_ring_size(dst, sz); 214988be76cdSChris Wilson } 215088be76cdSChris Wilson 215110be98a7SChris Wilson static int clone_engines(struct i915_gem_context *dst, 215210be98a7SChris Wilson struct i915_gem_context *src) 215310be98a7SChris Wilson { 215410be98a7SChris Wilson struct i915_gem_engines *e = i915_gem_context_lock_engines(src); 215510be98a7SChris Wilson struct i915_gem_engines *clone; 215610be98a7SChris Wilson bool user_engines; 215710be98a7SChris Wilson unsigned long n; 215810be98a7SChris Wilson 215970c96e39SChris Wilson clone = alloc_engines(e->num_engines); 216010be98a7SChris Wilson if (!clone) 216110be98a7SChris Wilson goto err_unlock; 216210be98a7SChris Wilson 216310be98a7SChris Wilson for (n = 0; n < e->num_engines; n++) { 216410be98a7SChris Wilson struct intel_engine_cs *engine; 216510be98a7SChris Wilson 216610be98a7SChris Wilson if (!e->engines[n]) { 216710be98a7SChris Wilson clone->engines[n] = NULL; 216810be98a7SChris Wilson continue; 216910be98a7SChris Wilson } 217010be98a7SChris Wilson engine = e->engines[n]->engine; 217110be98a7SChris Wilson 217210be98a7SChris Wilson /* 217310be98a7SChris Wilson * Virtual engines are singletons; they can only exist 217410be98a7SChris Wilson * inside a single context, because they embed their 217510be98a7SChris Wilson * HW context... As each virtual context implies a single 217610be98a7SChris Wilson * timeline (each engine can only dequeue a single request 217710be98a7SChris Wilson * at any time), it would be surprising for two contexts 217810be98a7SChris Wilson * to use the same engine. So let's create a copy of 217910be98a7SChris Wilson * the virtual engine instead. 218010be98a7SChris Wilson */ 218110be98a7SChris Wilson if (intel_engine_is_virtual(engine)) 218210be98a7SChris Wilson clone->engines[n] = 2183e6ba7648SChris Wilson intel_execlists_clone_virtual(engine); 218410be98a7SChris Wilson else 2185e6ba7648SChris Wilson clone->engines[n] = intel_context_create(engine); 218610be98a7SChris Wilson if (IS_ERR_OR_NULL(clone->engines[n])) { 218710be98a7SChris Wilson __free_engines(clone, n); 218810be98a7SChris Wilson goto err_unlock; 218910be98a7SChris Wilson } 2190e6ba7648SChris Wilson 2191e6ba7648SChris Wilson intel_context_set_gem(clone->engines[n], dst); 219288be76cdSChris Wilson 219388be76cdSChris Wilson /* Copy across the preferred ringsize */ 219488be76cdSChris Wilson if (copy_ring_size(clone->engines[n], e->engines[n])) { 219588be76cdSChris Wilson __free_engines(clone, n + 1); 219688be76cdSChris Wilson goto err_unlock; 219788be76cdSChris Wilson } 219810be98a7SChris Wilson } 219910be98a7SChris Wilson clone->num_engines = n; 220010be98a7SChris Wilson 220110be98a7SChris Wilson user_engines = i915_gem_context_user_engines(src); 220210be98a7SChris Wilson i915_gem_context_unlock_engines(src); 220310be98a7SChris Wilson 2204d96bb620SChris Wilson /* Serialised by constructor */ 2205130a95e9SChris Wilson engines_idle_release(dst, rcu_replace_pointer(dst->engines, clone, 1)); 220610be98a7SChris Wilson if (user_engines) 220710be98a7SChris Wilson i915_gem_context_set_user_engines(dst); 220810be98a7SChris Wilson else 220910be98a7SChris Wilson i915_gem_context_clear_user_engines(dst); 221010be98a7SChris Wilson return 0; 221110be98a7SChris Wilson 221210be98a7SChris Wilson err_unlock: 221310be98a7SChris Wilson i915_gem_context_unlock_engines(src); 221410be98a7SChris Wilson return -ENOMEM; 221510be98a7SChris Wilson } 221610be98a7SChris Wilson 221710be98a7SChris Wilson static int clone_flags(struct i915_gem_context *dst, 221810be98a7SChris Wilson struct i915_gem_context *src) 221910be98a7SChris Wilson { 222010be98a7SChris Wilson dst->user_flags = src->user_flags; 222110be98a7SChris Wilson return 0; 222210be98a7SChris Wilson } 222310be98a7SChris Wilson 222410be98a7SChris Wilson static int clone_schedattr(struct i915_gem_context *dst, 222510be98a7SChris Wilson struct i915_gem_context *src) 222610be98a7SChris Wilson { 222710be98a7SChris Wilson dst->sched = src->sched; 222810be98a7SChris Wilson return 0; 222910be98a7SChris Wilson } 223010be98a7SChris Wilson 223110be98a7SChris Wilson static int clone_sseu(struct i915_gem_context *dst, 223210be98a7SChris Wilson struct i915_gem_context *src) 223310be98a7SChris Wilson { 223410be98a7SChris Wilson struct i915_gem_engines *e = i915_gem_context_lock_engines(src); 223510be98a7SChris Wilson struct i915_gem_engines *clone; 223610be98a7SChris Wilson unsigned long n; 223710be98a7SChris Wilson int err; 223810be98a7SChris Wilson 2239d96bb620SChris Wilson /* no locking required; sole access under constructor*/ 2240d96bb620SChris Wilson clone = __context_engines_static(dst); 224110be98a7SChris Wilson if (e->num_engines != clone->num_engines) { 224210be98a7SChris Wilson err = -EINVAL; 224310be98a7SChris Wilson goto unlock; 224410be98a7SChris Wilson } 224510be98a7SChris Wilson 224610be98a7SChris Wilson for (n = 0; n < e->num_engines; n++) { 224710be98a7SChris Wilson struct intel_context *ce = e->engines[n]; 224810be98a7SChris Wilson 224910be98a7SChris Wilson if (clone->engines[n]->engine->class != ce->engine->class) { 225010be98a7SChris Wilson /* Must have compatible engine maps! */ 225110be98a7SChris Wilson err = -EINVAL; 225210be98a7SChris Wilson goto unlock; 225310be98a7SChris Wilson } 225410be98a7SChris Wilson 225510be98a7SChris Wilson /* serialises with set_sseu */ 225610be98a7SChris Wilson err = intel_context_lock_pinned(ce); 225710be98a7SChris Wilson if (err) 225810be98a7SChris Wilson goto unlock; 225910be98a7SChris Wilson 226010be98a7SChris Wilson clone->engines[n]->sseu = ce->sseu; 226110be98a7SChris Wilson intel_context_unlock_pinned(ce); 226210be98a7SChris Wilson } 226310be98a7SChris Wilson 226410be98a7SChris Wilson err = 0; 226510be98a7SChris Wilson unlock: 226610be98a7SChris Wilson i915_gem_context_unlock_engines(src); 226710be98a7SChris Wilson return err; 226810be98a7SChris Wilson } 226910be98a7SChris Wilson 227010be98a7SChris Wilson static int clone_timeline(struct i915_gem_context *dst, 227110be98a7SChris Wilson struct i915_gem_context *src) 227210be98a7SChris Wilson { 227375d0a7f3SChris Wilson if (src->timeline) 227475d0a7f3SChris Wilson __assign_timeline(dst, src->timeline); 227510be98a7SChris Wilson 227610be98a7SChris Wilson return 0; 227710be98a7SChris Wilson } 227810be98a7SChris Wilson 227910be98a7SChris Wilson static int clone_vm(struct i915_gem_context *dst, 228010be98a7SChris Wilson struct i915_gem_context *src) 228110be98a7SChris Wilson { 2282e568ac38SChris Wilson struct i915_address_space *vm; 2283a4e7ccdaSChris Wilson int err = 0; 228410be98a7SChris Wilson 228527dbae8fSChris Wilson if (!rcu_access_pointer(src->vm)) 228627dbae8fSChris Wilson return 0; 228727dbae8fSChris Wilson 228810be98a7SChris Wilson rcu_read_lock(); 228927dbae8fSChris Wilson vm = context_get_vm_rcu(src); 229010be98a7SChris Wilson rcu_read_unlock(); 229110be98a7SChris Wilson 2292a4e7ccdaSChris Wilson if (!mutex_lock_interruptible(&dst->mutex)) { 2293e568ac38SChris Wilson __assign_ppgtt(dst, vm); 2294a4e7ccdaSChris Wilson mutex_unlock(&dst->mutex); 2295a4e7ccdaSChris Wilson } else { 2296a4e7ccdaSChris Wilson err = -EINTR; 2297a4e7ccdaSChris Wilson } 229810be98a7SChris Wilson 229927dbae8fSChris Wilson i915_vm_put(vm); 2300a4e7ccdaSChris Wilson return err; 230110be98a7SChris Wilson } 230210be98a7SChris Wilson 230310be98a7SChris Wilson static int create_clone(struct i915_user_extension __user *ext, void *data) 230410be98a7SChris Wilson { 230510be98a7SChris Wilson static int (* const fn[])(struct i915_gem_context *dst, 230610be98a7SChris Wilson struct i915_gem_context *src) = { 230710be98a7SChris Wilson #define MAP(x, y) [ilog2(I915_CONTEXT_CLONE_##x)] = y 230810be98a7SChris Wilson MAP(ENGINES, clone_engines), 230910be98a7SChris Wilson MAP(FLAGS, clone_flags), 231010be98a7SChris Wilson MAP(SCHEDATTR, clone_schedattr), 231110be98a7SChris Wilson MAP(SSEU, clone_sseu), 231210be98a7SChris Wilson MAP(TIMELINE, clone_timeline), 231310be98a7SChris Wilson MAP(VM, clone_vm), 231410be98a7SChris Wilson #undef MAP 231510be98a7SChris Wilson }; 231610be98a7SChris Wilson struct drm_i915_gem_context_create_ext_clone local; 231710be98a7SChris Wilson const struct create_ext *arg = data; 231810be98a7SChris Wilson struct i915_gem_context *dst = arg->ctx; 231910be98a7SChris Wilson struct i915_gem_context *src; 232010be98a7SChris Wilson int err, bit; 232110be98a7SChris Wilson 232210be98a7SChris Wilson if (copy_from_user(&local, ext, sizeof(local))) 232310be98a7SChris Wilson return -EFAULT; 232410be98a7SChris Wilson 232510be98a7SChris Wilson BUILD_BUG_ON(GENMASK(BITS_PER_TYPE(local.flags) - 1, ARRAY_SIZE(fn)) != 232610be98a7SChris Wilson I915_CONTEXT_CLONE_UNKNOWN); 232710be98a7SChris Wilson 232810be98a7SChris Wilson if (local.flags & I915_CONTEXT_CLONE_UNKNOWN) 232910be98a7SChris Wilson return -EINVAL; 233010be98a7SChris Wilson 233110be98a7SChris Wilson if (local.rsvd) 233210be98a7SChris Wilson return -EINVAL; 233310be98a7SChris Wilson 233410be98a7SChris Wilson rcu_read_lock(); 233510be98a7SChris Wilson src = __i915_gem_context_lookup_rcu(arg->fpriv, local.clone_id); 233610be98a7SChris Wilson rcu_read_unlock(); 233710be98a7SChris Wilson if (!src) 233810be98a7SChris Wilson return -ENOENT; 233910be98a7SChris Wilson 234010be98a7SChris Wilson GEM_BUG_ON(src == dst); 234110be98a7SChris Wilson 234210be98a7SChris Wilson for (bit = 0; bit < ARRAY_SIZE(fn); bit++) { 234310be98a7SChris Wilson if (!(local.flags & BIT(bit))) 234410be98a7SChris Wilson continue; 234510be98a7SChris Wilson 234610be98a7SChris Wilson err = fn[bit](dst, src); 234710be98a7SChris Wilson if (err) 234810be98a7SChris Wilson return err; 234910be98a7SChris Wilson } 235010be98a7SChris Wilson 235110be98a7SChris Wilson return 0; 235210be98a7SChris Wilson } 235310be98a7SChris Wilson 235410be98a7SChris Wilson static const i915_user_extension_fn create_extensions[] = { 235510be98a7SChris Wilson [I915_CONTEXT_CREATE_EXT_SETPARAM] = create_setparam, 235610be98a7SChris Wilson [I915_CONTEXT_CREATE_EXT_CLONE] = create_clone, 235710be98a7SChris Wilson }; 235810be98a7SChris Wilson 235910be98a7SChris Wilson static bool client_is_banned(struct drm_i915_file_private *file_priv) 236010be98a7SChris Wilson { 236110be98a7SChris Wilson return atomic_read(&file_priv->ban_score) >= I915_CLIENT_SCORE_BANNED; 236210be98a7SChris Wilson } 236310be98a7SChris Wilson 236410be98a7SChris Wilson int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, 236510be98a7SChris Wilson struct drm_file *file) 236610be98a7SChris Wilson { 236710be98a7SChris Wilson struct drm_i915_private *i915 = to_i915(dev); 236810be98a7SChris Wilson struct drm_i915_gem_context_create_ext *args = data; 236910be98a7SChris Wilson struct create_ext ext_data; 237010be98a7SChris Wilson int ret; 2371c100777cSTvrtko Ursulin u32 id; 237210be98a7SChris Wilson 237310be98a7SChris Wilson if (!DRIVER_CAPS(i915)->has_logical_contexts) 237410be98a7SChris Wilson return -ENODEV; 237510be98a7SChris Wilson 237610be98a7SChris Wilson if (args->flags & I915_CONTEXT_CREATE_FLAGS_UNKNOWN) 237710be98a7SChris Wilson return -EINVAL; 237810be98a7SChris Wilson 2379cb823ed9SChris Wilson ret = intel_gt_terminally_wedged(&i915->gt); 238010be98a7SChris Wilson if (ret) 238110be98a7SChris Wilson return ret; 238210be98a7SChris Wilson 238310be98a7SChris Wilson ext_data.fpriv = file->driver_priv; 238410be98a7SChris Wilson if (client_is_banned(ext_data.fpriv)) { 2385baa89ba3SWambui Karuga drm_dbg(&i915->drm, 2386baa89ba3SWambui Karuga "client %s[%d] banned from creating ctx\n", 2387ba16a48aSTvrtko Ursulin current->comm, task_pid_nr(current)); 238810be98a7SChris Wilson return -EIO; 238910be98a7SChris Wilson } 239010be98a7SChris Wilson 239110be98a7SChris Wilson ext_data.ctx = i915_gem_create_context(i915, args->flags); 239210be98a7SChris Wilson if (IS_ERR(ext_data.ctx)) 239310be98a7SChris Wilson return PTR_ERR(ext_data.ctx); 239410be98a7SChris Wilson 239510be98a7SChris Wilson if (args->flags & I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS) { 239610be98a7SChris Wilson ret = i915_user_extensions(u64_to_user_ptr(args->extensions), 239710be98a7SChris Wilson create_extensions, 239810be98a7SChris Wilson ARRAY_SIZE(create_extensions), 239910be98a7SChris Wilson &ext_data); 240010be98a7SChris Wilson if (ret) 240110be98a7SChris Wilson goto err_ctx; 240210be98a7SChris Wilson } 240310be98a7SChris Wilson 2404c100777cSTvrtko Ursulin ret = gem_context_register(ext_data.ctx, ext_data.fpriv, &id); 240510be98a7SChris Wilson if (ret < 0) 240610be98a7SChris Wilson goto err_ctx; 240710be98a7SChris Wilson 2408c100777cSTvrtko Ursulin args->ctx_id = id; 2409baa89ba3SWambui Karuga drm_dbg(&i915->drm, "HW context %d created\n", args->ctx_id); 241010be98a7SChris Wilson 241110be98a7SChris Wilson return 0; 241210be98a7SChris Wilson 241310be98a7SChris Wilson err_ctx: 241410be98a7SChris Wilson context_close(ext_data.ctx); 241510be98a7SChris Wilson return ret; 241610be98a7SChris Wilson } 241710be98a7SChris Wilson 241810be98a7SChris Wilson int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, 241910be98a7SChris Wilson struct drm_file *file) 242010be98a7SChris Wilson { 242110be98a7SChris Wilson struct drm_i915_gem_context_destroy *args = data; 242210be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 242310be98a7SChris Wilson struct i915_gem_context *ctx; 242410be98a7SChris Wilson 242510be98a7SChris Wilson if (args->pad != 0) 242610be98a7SChris Wilson return -EINVAL; 242710be98a7SChris Wilson 242810be98a7SChris Wilson if (!args->ctx_id) 242910be98a7SChris Wilson return -ENOENT; 243010be98a7SChris Wilson 2431c100777cSTvrtko Ursulin ctx = xa_erase(&file_priv->context_xa, args->ctx_id); 243210be98a7SChris Wilson if (!ctx) 243310be98a7SChris Wilson return -ENOENT; 243410be98a7SChris Wilson 243510be98a7SChris Wilson context_close(ctx); 243610be98a7SChris Wilson return 0; 243710be98a7SChris Wilson } 243810be98a7SChris Wilson 243910be98a7SChris Wilson static int get_sseu(struct i915_gem_context *ctx, 244010be98a7SChris Wilson struct drm_i915_gem_context_param *args) 244110be98a7SChris Wilson { 244210be98a7SChris Wilson struct drm_i915_gem_context_param_sseu user_sseu; 244310be98a7SChris Wilson struct intel_context *ce; 244410be98a7SChris Wilson unsigned long lookup; 244510be98a7SChris Wilson int err; 244610be98a7SChris Wilson 244710be98a7SChris Wilson if (args->size == 0) 244810be98a7SChris Wilson goto out; 244910be98a7SChris Wilson else if (args->size < sizeof(user_sseu)) 245010be98a7SChris Wilson return -EINVAL; 245110be98a7SChris Wilson 245210be98a7SChris Wilson if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), 245310be98a7SChris Wilson sizeof(user_sseu))) 245410be98a7SChris Wilson return -EFAULT; 245510be98a7SChris Wilson 245610be98a7SChris Wilson if (user_sseu.rsvd) 245710be98a7SChris Wilson return -EINVAL; 245810be98a7SChris Wilson 245910be98a7SChris Wilson if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) 246010be98a7SChris Wilson return -EINVAL; 246110be98a7SChris Wilson 246210be98a7SChris Wilson lookup = 0; 246310be98a7SChris Wilson if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) 246410be98a7SChris Wilson lookup |= LOOKUP_USER_INDEX; 246510be98a7SChris Wilson 246610be98a7SChris Wilson ce = lookup_user_engine(ctx, lookup, &user_sseu.engine); 246710be98a7SChris Wilson if (IS_ERR(ce)) 246810be98a7SChris Wilson return PTR_ERR(ce); 246910be98a7SChris Wilson 247010be98a7SChris Wilson err = intel_context_lock_pinned(ce); /* serialises with set_sseu */ 247110be98a7SChris Wilson if (err) { 247210be98a7SChris Wilson intel_context_put(ce); 247310be98a7SChris Wilson return err; 247410be98a7SChris Wilson } 247510be98a7SChris Wilson 247610be98a7SChris Wilson user_sseu.slice_mask = ce->sseu.slice_mask; 247710be98a7SChris Wilson user_sseu.subslice_mask = ce->sseu.subslice_mask; 247810be98a7SChris Wilson user_sseu.min_eus_per_subslice = ce->sseu.min_eus_per_subslice; 247910be98a7SChris Wilson user_sseu.max_eus_per_subslice = ce->sseu.max_eus_per_subslice; 248010be98a7SChris Wilson 248110be98a7SChris Wilson intel_context_unlock_pinned(ce); 248210be98a7SChris Wilson intel_context_put(ce); 248310be98a7SChris Wilson 248410be98a7SChris Wilson if (copy_to_user(u64_to_user_ptr(args->value), &user_sseu, 248510be98a7SChris Wilson sizeof(user_sseu))) 248610be98a7SChris Wilson return -EFAULT; 248710be98a7SChris Wilson 248810be98a7SChris Wilson out: 248910be98a7SChris Wilson args->size = sizeof(user_sseu); 249010be98a7SChris Wilson 249110be98a7SChris Wilson return 0; 249210be98a7SChris Wilson } 249310be98a7SChris Wilson 249410be98a7SChris Wilson int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, 249510be98a7SChris Wilson struct drm_file *file) 249610be98a7SChris Wilson { 249710be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 249810be98a7SChris Wilson struct drm_i915_gem_context_param *args = data; 249910be98a7SChris Wilson struct i915_gem_context *ctx; 250010be98a7SChris Wilson int ret = 0; 250110be98a7SChris Wilson 250210be98a7SChris Wilson ctx = i915_gem_context_lookup(file_priv, args->ctx_id); 250310be98a7SChris Wilson if (!ctx) 250410be98a7SChris Wilson return -ENOENT; 250510be98a7SChris Wilson 250610be98a7SChris Wilson switch (args->param) { 250710be98a7SChris Wilson case I915_CONTEXT_PARAM_NO_ZEROMAP: 250810be98a7SChris Wilson args->size = 0; 250910be98a7SChris Wilson args->value = test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); 251010be98a7SChris Wilson break; 251110be98a7SChris Wilson 251210be98a7SChris Wilson case I915_CONTEXT_PARAM_GTT_SIZE: 251310be98a7SChris Wilson args->size = 0; 2514a4e7ccdaSChris Wilson rcu_read_lock(); 2515a4e7ccdaSChris Wilson if (rcu_access_pointer(ctx->vm)) 2516a4e7ccdaSChris Wilson args->value = rcu_dereference(ctx->vm)->total; 251710be98a7SChris Wilson else 251810be98a7SChris Wilson args->value = to_i915(dev)->ggtt.vm.total; 2519a4e7ccdaSChris Wilson rcu_read_unlock(); 252010be98a7SChris Wilson break; 252110be98a7SChris Wilson 252210be98a7SChris Wilson case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 252310be98a7SChris Wilson args->size = 0; 252410be98a7SChris Wilson args->value = i915_gem_context_no_error_capture(ctx); 252510be98a7SChris Wilson break; 252610be98a7SChris Wilson 252710be98a7SChris Wilson case I915_CONTEXT_PARAM_BANNABLE: 252810be98a7SChris Wilson args->size = 0; 252910be98a7SChris Wilson args->value = i915_gem_context_is_bannable(ctx); 253010be98a7SChris Wilson break; 253110be98a7SChris Wilson 253210be98a7SChris Wilson case I915_CONTEXT_PARAM_RECOVERABLE: 253310be98a7SChris Wilson args->size = 0; 253410be98a7SChris Wilson args->value = i915_gem_context_is_recoverable(ctx); 253510be98a7SChris Wilson break; 253610be98a7SChris Wilson 253710be98a7SChris Wilson case I915_CONTEXT_PARAM_PRIORITY: 253810be98a7SChris Wilson args->size = 0; 253910be98a7SChris Wilson args->value = ctx->sched.priority >> I915_USER_PRIORITY_SHIFT; 254010be98a7SChris Wilson break; 254110be98a7SChris Wilson 254210be98a7SChris Wilson case I915_CONTEXT_PARAM_SSEU: 254310be98a7SChris Wilson ret = get_sseu(ctx, args); 254410be98a7SChris Wilson break; 254510be98a7SChris Wilson 254610be98a7SChris Wilson case I915_CONTEXT_PARAM_VM: 254710be98a7SChris Wilson ret = get_ppgtt(file_priv, ctx, args); 254810be98a7SChris Wilson break; 254910be98a7SChris Wilson 255010be98a7SChris Wilson case I915_CONTEXT_PARAM_ENGINES: 255110be98a7SChris Wilson ret = get_engines(ctx, args); 255210be98a7SChris Wilson break; 255310be98a7SChris Wilson 2554a0e04715SChris Wilson case I915_CONTEXT_PARAM_PERSISTENCE: 2555a0e04715SChris Wilson args->size = 0; 2556a0e04715SChris Wilson args->value = i915_gem_context_is_persistent(ctx); 2557a0e04715SChris Wilson break; 2558a0e04715SChris Wilson 255988be76cdSChris Wilson case I915_CONTEXT_PARAM_RINGSIZE: 256088be76cdSChris Wilson ret = get_ringsize(ctx, args); 256188be76cdSChris Wilson break; 256288be76cdSChris Wilson 256310be98a7SChris Wilson case I915_CONTEXT_PARAM_BAN_PERIOD: 256410be98a7SChris Wilson default: 256510be98a7SChris Wilson ret = -EINVAL; 256610be98a7SChris Wilson break; 256710be98a7SChris Wilson } 256810be98a7SChris Wilson 256910be98a7SChris Wilson i915_gem_context_put(ctx); 257010be98a7SChris Wilson return ret; 257110be98a7SChris Wilson } 257210be98a7SChris Wilson 257310be98a7SChris Wilson int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, 257410be98a7SChris Wilson struct drm_file *file) 257510be98a7SChris Wilson { 257610be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 257710be98a7SChris Wilson struct drm_i915_gem_context_param *args = data; 257810be98a7SChris Wilson struct i915_gem_context *ctx; 257910be98a7SChris Wilson int ret; 258010be98a7SChris Wilson 258110be98a7SChris Wilson ctx = i915_gem_context_lookup(file_priv, args->ctx_id); 258210be98a7SChris Wilson if (!ctx) 258310be98a7SChris Wilson return -ENOENT; 258410be98a7SChris Wilson 258510be98a7SChris Wilson ret = ctx_setparam(file_priv, ctx, args); 258610be98a7SChris Wilson 258710be98a7SChris Wilson i915_gem_context_put(ctx); 258810be98a7SChris Wilson return ret; 258910be98a7SChris Wilson } 259010be98a7SChris Wilson 259110be98a7SChris Wilson int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, 259210be98a7SChris Wilson void *data, struct drm_file *file) 259310be98a7SChris Wilson { 2594a4e7ccdaSChris Wilson struct drm_i915_private *i915 = to_i915(dev); 259510be98a7SChris Wilson struct drm_i915_reset_stats *args = data; 259610be98a7SChris Wilson struct i915_gem_context *ctx; 259710be98a7SChris Wilson int ret; 259810be98a7SChris Wilson 259910be98a7SChris Wilson if (args->flags || args->pad) 260010be98a7SChris Wilson return -EINVAL; 260110be98a7SChris Wilson 260210be98a7SChris Wilson ret = -ENOENT; 260310be98a7SChris Wilson rcu_read_lock(); 260410be98a7SChris Wilson ctx = __i915_gem_context_lookup_rcu(file->driver_priv, args->ctx_id); 260510be98a7SChris Wilson if (!ctx) 260610be98a7SChris Wilson goto out; 260710be98a7SChris Wilson 260810be98a7SChris Wilson /* 260910be98a7SChris Wilson * We opt for unserialised reads here. This may result in tearing 261010be98a7SChris Wilson * in the extremely unlikely event of a GPU hang on this context 261110be98a7SChris Wilson * as we are querying them. If we need that extra layer of protection, 261210be98a7SChris Wilson * we should wrap the hangstats with a seqlock. 261310be98a7SChris Wilson */ 261410be98a7SChris Wilson 261510be98a7SChris Wilson if (capable(CAP_SYS_ADMIN)) 2616a4e7ccdaSChris Wilson args->reset_count = i915_reset_count(&i915->gpu_error); 261710be98a7SChris Wilson else 261810be98a7SChris Wilson args->reset_count = 0; 261910be98a7SChris Wilson 262010be98a7SChris Wilson args->batch_active = atomic_read(&ctx->guilty_count); 262110be98a7SChris Wilson args->batch_pending = atomic_read(&ctx->active_count); 262210be98a7SChris Wilson 262310be98a7SChris Wilson ret = 0; 262410be98a7SChris Wilson out: 262510be98a7SChris Wilson rcu_read_unlock(); 262610be98a7SChris Wilson return ret; 262710be98a7SChris Wilson } 262810be98a7SChris Wilson 262910be98a7SChris Wilson /* GEM context-engines iterator: for_each_gem_engine() */ 263010be98a7SChris Wilson struct intel_context * 263110be98a7SChris Wilson i915_gem_engines_iter_next(struct i915_gem_engines_iter *it) 263210be98a7SChris Wilson { 263310be98a7SChris Wilson const struct i915_gem_engines *e = it->engines; 263410be98a7SChris Wilson struct intel_context *ctx; 263510be98a7SChris Wilson 2636130a95e9SChris Wilson if (unlikely(!e)) 2637130a95e9SChris Wilson return NULL; 2638130a95e9SChris Wilson 263910be98a7SChris Wilson do { 264010be98a7SChris Wilson if (it->idx >= e->num_engines) 264110be98a7SChris Wilson return NULL; 264210be98a7SChris Wilson 264310be98a7SChris Wilson ctx = e->engines[it->idx++]; 264410be98a7SChris Wilson } while (!ctx); 264510be98a7SChris Wilson 264610be98a7SChris Wilson return ctx; 264710be98a7SChris Wilson } 264810be98a7SChris Wilson 264910be98a7SChris Wilson #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 265010be98a7SChris Wilson #include "selftests/mock_context.c" 265110be98a7SChris Wilson #include "selftests/i915_gem_context.c" 265210be98a7SChris Wilson #endif 265310be98a7SChris Wilson 265410be98a7SChris Wilson static void i915_global_gem_context_shrink(void) 265510be98a7SChris Wilson { 265610be98a7SChris Wilson kmem_cache_shrink(global.slab_luts); 265710be98a7SChris Wilson } 265810be98a7SChris Wilson 265910be98a7SChris Wilson static void i915_global_gem_context_exit(void) 266010be98a7SChris Wilson { 266110be98a7SChris Wilson kmem_cache_destroy(global.slab_luts); 266210be98a7SChris Wilson } 266310be98a7SChris Wilson 266410be98a7SChris Wilson static struct i915_global_gem_context global = { { 266510be98a7SChris Wilson .shrink = i915_global_gem_context_shrink, 266610be98a7SChris Wilson .exit = i915_global_gem_context_exit, 266710be98a7SChris Wilson } }; 266810be98a7SChris Wilson 266910be98a7SChris Wilson int __init i915_global_gem_context_init(void) 267010be98a7SChris Wilson { 267110be98a7SChris Wilson global.slab_luts = KMEM_CACHE(i915_lut_handle, 0); 267210be98a7SChris Wilson if (!global.slab_luts) 267310be98a7SChris Wilson return -ENOMEM; 267410be98a7SChris Wilson 267510be98a7SChris Wilson i915_global_register(&global.base); 267610be98a7SChris Wilson return 0; 267710be98a7SChris Wilson } 2678