110be98a7SChris Wilson /* 210be98a7SChris Wilson * SPDX-License-Identifier: MIT 310be98a7SChris Wilson * 410be98a7SChris Wilson * Copyright © 2011-2012 Intel Corporation 510be98a7SChris Wilson */ 610be98a7SChris Wilson 710be98a7SChris Wilson /* 810be98a7SChris Wilson * This file implements HW context support. On gen5+ a HW context consists of an 910be98a7SChris Wilson * opaque GPU object which is referenced at times of context saves and restores. 1010be98a7SChris Wilson * With RC6 enabled, the context is also referenced as the GPU enters and exists 1110be98a7SChris Wilson * from RC6 (GPU has it's own internal power context, except on gen5). Though 1210be98a7SChris Wilson * something like a context does exist for the media ring, the code only 1310be98a7SChris Wilson * supports contexts for the render ring. 1410be98a7SChris Wilson * 1510be98a7SChris Wilson * In software, there is a distinction between contexts created by the user, 1610be98a7SChris Wilson * and the default HW context. The default HW context is used by GPU clients 1710be98a7SChris Wilson * that do not request setup of their own hardware context. The default 1810be98a7SChris Wilson * context's state is never restored to help prevent programming errors. This 1910be98a7SChris Wilson * would happen if a client ran and piggy-backed off another clients GPU state. 2010be98a7SChris Wilson * The default context only exists to give the GPU some offset to load as the 2110be98a7SChris Wilson * current to invoke a save of the context we actually care about. In fact, the 2210be98a7SChris Wilson * code could likely be constructed, albeit in a more complicated fashion, to 2310be98a7SChris Wilson * never use the default context, though that limits the driver's ability to 2410be98a7SChris Wilson * swap out, and/or destroy other contexts. 2510be98a7SChris Wilson * 2610be98a7SChris Wilson * All other contexts are created as a request by the GPU client. These contexts 2710be98a7SChris Wilson * store GPU state, and thus allow GPU clients to not re-emit state (and 2810be98a7SChris Wilson * potentially query certain state) at any time. The kernel driver makes 2910be98a7SChris Wilson * certain that the appropriate commands are inserted. 3010be98a7SChris Wilson * 3110be98a7SChris Wilson * The context life cycle is semi-complicated in that context BOs may live 3210be98a7SChris Wilson * longer than the context itself because of the way the hardware, and object 3310be98a7SChris Wilson * tracking works. Below is a very crude representation of the state machine 3410be98a7SChris Wilson * describing the context life. 3510be98a7SChris Wilson * refcount pincount active 3610be98a7SChris Wilson * S0: initial state 0 0 0 3710be98a7SChris Wilson * S1: context created 1 0 0 3810be98a7SChris Wilson * S2: context is currently running 2 1 X 3910be98a7SChris Wilson * S3: GPU referenced, but not current 2 0 1 4010be98a7SChris Wilson * S4: context is current, but destroyed 1 1 0 4110be98a7SChris Wilson * S5: like S3, but destroyed 1 0 1 4210be98a7SChris Wilson * 4310be98a7SChris Wilson * The most common (but not all) transitions: 4410be98a7SChris Wilson * S0->S1: client creates a context 4510be98a7SChris Wilson * S1->S2: client submits execbuf with context 4610be98a7SChris Wilson * S2->S3: other clients submits execbuf with context 4710be98a7SChris Wilson * S3->S1: context object was retired 4810be98a7SChris Wilson * S3->S2: clients submits another execbuf 4910be98a7SChris Wilson * S2->S4: context destroy called with current context 5010be98a7SChris Wilson * S3->S5->S0: destroy path 5110be98a7SChris Wilson * S4->S5->S0: destroy path on current context 5210be98a7SChris Wilson * 5310be98a7SChris Wilson * There are two confusing terms used above: 5410be98a7SChris Wilson * The "current context" means the context which is currently running on the 5510be98a7SChris Wilson * GPU. The GPU has loaded its state already and has stored away the gtt 5610be98a7SChris Wilson * offset of the BO. The GPU is not actively referencing the data at this 5710be98a7SChris Wilson * offset, but it will on the next context switch. The only way to avoid this 5810be98a7SChris Wilson * is to do a GPU reset. 5910be98a7SChris Wilson * 6010be98a7SChris Wilson * An "active context' is one which was previously the "current context" and is 6110be98a7SChris Wilson * on the active list waiting for the next context switch to occur. Until this 6210be98a7SChris Wilson * happens, the object must remain at the same gtt offset. It is therefore 6310be98a7SChris Wilson * possible to destroy a context, but it is still active. 6410be98a7SChris Wilson * 6510be98a7SChris Wilson */ 6610be98a7SChris Wilson 6710be98a7SChris Wilson #include <linux/log2.h> 6810be98a7SChris Wilson #include <linux/nospec.h> 6910be98a7SChris Wilson 7010be98a7SChris Wilson #include <drm/i915_drm.h> 7110be98a7SChris Wilson 7210be98a7SChris Wilson #include "gt/intel_lrc_reg.h" 73750e76b4SChris Wilson #include "gt/intel_engine_user.h" 7410be98a7SChris Wilson 7510be98a7SChris Wilson #include "i915_gem_context.h" 7610be98a7SChris Wilson #include "i915_globals.h" 7710be98a7SChris Wilson #include "i915_trace.h" 7810be98a7SChris Wilson #include "i915_user_extensions.h" 7910be98a7SChris Wilson 8010be98a7SChris Wilson #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1 8110be98a7SChris Wilson 8210be98a7SChris Wilson static struct i915_global_gem_context { 8310be98a7SChris Wilson struct i915_global base; 8410be98a7SChris Wilson struct kmem_cache *slab_luts; 8510be98a7SChris Wilson } global; 8610be98a7SChris Wilson 8710be98a7SChris Wilson struct i915_lut_handle *i915_lut_handle_alloc(void) 8810be98a7SChris Wilson { 8910be98a7SChris Wilson return kmem_cache_alloc(global.slab_luts, GFP_KERNEL); 9010be98a7SChris Wilson } 9110be98a7SChris Wilson 9210be98a7SChris Wilson void i915_lut_handle_free(struct i915_lut_handle *lut) 9310be98a7SChris Wilson { 9410be98a7SChris Wilson return kmem_cache_free(global.slab_luts, lut); 9510be98a7SChris Wilson } 9610be98a7SChris Wilson 9710be98a7SChris Wilson static void lut_close(struct i915_gem_context *ctx) 9810be98a7SChris Wilson { 9910be98a7SChris Wilson struct radix_tree_iter iter; 10010be98a7SChris Wilson void __rcu **slot; 10110be98a7SChris Wilson 102155ab883SChris Wilson lockdep_assert_held(&ctx->mutex); 10310be98a7SChris Wilson 10410be98a7SChris Wilson rcu_read_lock(); 10510be98a7SChris Wilson radix_tree_for_each_slot(slot, &ctx->handles_vma, &iter, 0) { 10610be98a7SChris Wilson struct i915_vma *vma = rcu_dereference_raw(*slot); 107155ab883SChris Wilson struct drm_i915_gem_object *obj = vma->obj; 108155ab883SChris Wilson struct i915_lut_handle *lut; 10910be98a7SChris Wilson 110155ab883SChris Wilson if (!kref_get_unless_zero(&obj->base.refcount)) 111155ab883SChris Wilson continue; 112155ab883SChris Wilson 113155ab883SChris Wilson rcu_read_unlock(); 114155ab883SChris Wilson i915_gem_object_lock(obj); 115155ab883SChris Wilson list_for_each_entry(lut, &obj->lut_list, obj_link) { 116155ab883SChris Wilson if (lut->ctx != ctx) 117155ab883SChris Wilson continue; 118155ab883SChris Wilson 119155ab883SChris Wilson if (lut->handle != iter.index) 120155ab883SChris Wilson continue; 121155ab883SChris Wilson 122155ab883SChris Wilson list_del(&lut->obj_link); 123155ab883SChris Wilson break; 124155ab883SChris Wilson } 125155ab883SChris Wilson i915_gem_object_unlock(obj); 126155ab883SChris Wilson rcu_read_lock(); 127155ab883SChris Wilson 128155ab883SChris Wilson if (&lut->obj_link != &obj->lut_list) { 129155ab883SChris Wilson i915_lut_handle_free(lut); 13010be98a7SChris Wilson radix_tree_iter_delete(&ctx->handles_vma, &iter, slot); 131155ab883SChris Wilson if (atomic_dec_and_test(&vma->open_count) && 132155ab883SChris Wilson !i915_vma_is_ggtt(vma)) 133155ab883SChris Wilson i915_vma_close(vma); 134155ab883SChris Wilson i915_gem_object_put(obj); 135155ab883SChris Wilson } 13610be98a7SChris Wilson 137155ab883SChris Wilson i915_gem_object_put(obj); 13810be98a7SChris Wilson } 13910be98a7SChris Wilson rcu_read_unlock(); 14010be98a7SChris Wilson } 14110be98a7SChris Wilson 14210be98a7SChris Wilson static struct intel_context * 14310be98a7SChris Wilson lookup_user_engine(struct i915_gem_context *ctx, 14410be98a7SChris Wilson unsigned long flags, 14510be98a7SChris Wilson const struct i915_engine_class_instance *ci) 14610be98a7SChris Wilson #define LOOKUP_USER_INDEX BIT(0) 14710be98a7SChris Wilson { 14810be98a7SChris Wilson int idx; 14910be98a7SChris Wilson 15010be98a7SChris Wilson if (!!(flags & LOOKUP_USER_INDEX) != i915_gem_context_user_engines(ctx)) 15110be98a7SChris Wilson return ERR_PTR(-EINVAL); 15210be98a7SChris Wilson 15310be98a7SChris Wilson if (!i915_gem_context_user_engines(ctx)) { 15410be98a7SChris Wilson struct intel_engine_cs *engine; 15510be98a7SChris Wilson 15610be98a7SChris Wilson engine = intel_engine_lookup_user(ctx->i915, 15710be98a7SChris Wilson ci->engine_class, 15810be98a7SChris Wilson ci->engine_instance); 15910be98a7SChris Wilson if (!engine) 16010be98a7SChris Wilson return ERR_PTR(-EINVAL); 16110be98a7SChris Wilson 162f1c4d157SChris Wilson idx = engine->legacy_idx; 16310be98a7SChris Wilson } else { 16410be98a7SChris Wilson idx = ci->engine_instance; 16510be98a7SChris Wilson } 16610be98a7SChris Wilson 16710be98a7SChris Wilson return i915_gem_context_get_engine(ctx, idx); 16810be98a7SChris Wilson } 16910be98a7SChris Wilson 17010be98a7SChris Wilson static inline int new_hw_id(struct drm_i915_private *i915, gfp_t gfp) 17110be98a7SChris Wilson { 17210be98a7SChris Wilson unsigned int max; 17310be98a7SChris Wilson 17410be98a7SChris Wilson lockdep_assert_held(&i915->contexts.mutex); 17510be98a7SChris Wilson 1766d26d9bbSDaniele Ceraolo Spurio if (INTEL_GEN(i915) >= 12) 1776d26d9bbSDaniele Ceraolo Spurio max = GEN12_MAX_CONTEXT_HW_ID; 1786d26d9bbSDaniele Ceraolo Spurio else if (INTEL_GEN(i915) >= 11) 17910be98a7SChris Wilson max = GEN11_MAX_CONTEXT_HW_ID; 18010be98a7SChris Wilson else if (USES_GUC_SUBMISSION(i915)) 18110be98a7SChris Wilson /* 18210be98a7SChris Wilson * When using GuC in proxy submission, GuC consumes the 18310be98a7SChris Wilson * highest bit in the context id to indicate proxy submission. 18410be98a7SChris Wilson */ 18510be98a7SChris Wilson max = MAX_GUC_CONTEXT_HW_ID; 18610be98a7SChris Wilson else 18710be98a7SChris Wilson max = MAX_CONTEXT_HW_ID; 18810be98a7SChris Wilson 18910be98a7SChris Wilson return ida_simple_get(&i915->contexts.hw_ida, 0, max, gfp); 19010be98a7SChris Wilson } 19110be98a7SChris Wilson 19210be98a7SChris Wilson static int steal_hw_id(struct drm_i915_private *i915) 19310be98a7SChris Wilson { 19410be98a7SChris Wilson struct i915_gem_context *ctx, *cn; 19510be98a7SChris Wilson LIST_HEAD(pinned); 19610be98a7SChris Wilson int id = -ENOSPC; 19710be98a7SChris Wilson 19810be98a7SChris Wilson lockdep_assert_held(&i915->contexts.mutex); 19910be98a7SChris Wilson 20010be98a7SChris Wilson list_for_each_entry_safe(ctx, cn, 20110be98a7SChris Wilson &i915->contexts.hw_id_list, hw_id_link) { 20210be98a7SChris Wilson if (atomic_read(&ctx->hw_id_pin_count)) { 20310be98a7SChris Wilson list_move_tail(&ctx->hw_id_link, &pinned); 20410be98a7SChris Wilson continue; 20510be98a7SChris Wilson } 20610be98a7SChris Wilson 20710be98a7SChris Wilson GEM_BUG_ON(!ctx->hw_id); /* perma-pinned kernel context */ 20810be98a7SChris Wilson list_del_init(&ctx->hw_id_link); 20910be98a7SChris Wilson id = ctx->hw_id; 21010be98a7SChris Wilson break; 21110be98a7SChris Wilson } 21210be98a7SChris Wilson 21310be98a7SChris Wilson /* 21410be98a7SChris Wilson * Remember how far we got up on the last repossesion scan, so the 21510be98a7SChris Wilson * list is kept in a "least recently scanned" order. 21610be98a7SChris Wilson */ 21710be98a7SChris Wilson list_splice_tail(&pinned, &i915->contexts.hw_id_list); 21810be98a7SChris Wilson return id; 21910be98a7SChris Wilson } 22010be98a7SChris Wilson 22110be98a7SChris Wilson static int assign_hw_id(struct drm_i915_private *i915, unsigned int *out) 22210be98a7SChris Wilson { 22310be98a7SChris Wilson int ret; 22410be98a7SChris Wilson 22510be98a7SChris Wilson lockdep_assert_held(&i915->contexts.mutex); 22610be98a7SChris Wilson 22710be98a7SChris Wilson /* 22810be98a7SChris Wilson * We prefer to steal/stall ourselves and our users over that of the 22910be98a7SChris Wilson * entire system. That may be a little unfair to our users, and 23010be98a7SChris Wilson * even hurt high priority clients. The choice is whether to oomkill 23110be98a7SChris Wilson * something else, or steal a context id. 23210be98a7SChris Wilson */ 23310be98a7SChris Wilson ret = new_hw_id(i915, GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); 23410be98a7SChris Wilson if (unlikely(ret < 0)) { 23510be98a7SChris Wilson ret = steal_hw_id(i915); 23610be98a7SChris Wilson if (ret < 0) /* once again for the correct errno code */ 23710be98a7SChris Wilson ret = new_hw_id(i915, GFP_KERNEL); 23810be98a7SChris Wilson if (ret < 0) 23910be98a7SChris Wilson return ret; 24010be98a7SChris Wilson } 24110be98a7SChris Wilson 24210be98a7SChris Wilson *out = ret; 24310be98a7SChris Wilson return 0; 24410be98a7SChris Wilson } 24510be98a7SChris Wilson 24610be98a7SChris Wilson static void release_hw_id(struct i915_gem_context *ctx) 24710be98a7SChris Wilson { 24810be98a7SChris Wilson struct drm_i915_private *i915 = ctx->i915; 24910be98a7SChris Wilson 25010be98a7SChris Wilson if (list_empty(&ctx->hw_id_link)) 25110be98a7SChris Wilson return; 25210be98a7SChris Wilson 25310be98a7SChris Wilson mutex_lock(&i915->contexts.mutex); 25410be98a7SChris Wilson if (!list_empty(&ctx->hw_id_link)) { 25510be98a7SChris Wilson ida_simple_remove(&i915->contexts.hw_ida, ctx->hw_id); 25610be98a7SChris Wilson list_del_init(&ctx->hw_id_link); 25710be98a7SChris Wilson } 25810be98a7SChris Wilson mutex_unlock(&i915->contexts.mutex); 25910be98a7SChris Wilson } 26010be98a7SChris Wilson 26110be98a7SChris Wilson static void __free_engines(struct i915_gem_engines *e, unsigned int count) 26210be98a7SChris Wilson { 26310be98a7SChris Wilson while (count--) { 26410be98a7SChris Wilson if (!e->engines[count]) 26510be98a7SChris Wilson continue; 26610be98a7SChris Wilson 26710be98a7SChris Wilson intel_context_put(e->engines[count]); 26810be98a7SChris Wilson } 26910be98a7SChris Wilson kfree(e); 27010be98a7SChris Wilson } 27110be98a7SChris Wilson 27210be98a7SChris Wilson static void free_engines(struct i915_gem_engines *e) 27310be98a7SChris Wilson { 27410be98a7SChris Wilson __free_engines(e, e->num_engines); 27510be98a7SChris Wilson } 27610be98a7SChris Wilson 277155ab883SChris Wilson static void free_engines_rcu(struct rcu_head *rcu) 27810be98a7SChris Wilson { 279155ab883SChris Wilson free_engines(container_of(rcu, struct i915_gem_engines, rcu)); 28010be98a7SChris Wilson } 28110be98a7SChris Wilson 28210be98a7SChris Wilson static struct i915_gem_engines *default_engines(struct i915_gem_context *ctx) 28310be98a7SChris Wilson { 284f1c4d157SChris Wilson const struct intel_gt *gt = &ctx->i915->gt; 28510be98a7SChris Wilson struct intel_engine_cs *engine; 28610be98a7SChris Wilson struct i915_gem_engines *e; 28710be98a7SChris Wilson enum intel_engine_id id; 28810be98a7SChris Wilson 28910be98a7SChris Wilson e = kzalloc(struct_size(e, engines, I915_NUM_ENGINES), GFP_KERNEL); 29010be98a7SChris Wilson if (!e) 29110be98a7SChris Wilson return ERR_PTR(-ENOMEM); 29210be98a7SChris Wilson 293155ab883SChris Wilson init_rcu_head(&e->rcu); 294f1c4d157SChris Wilson for_each_engine(engine, gt, id) { 29510be98a7SChris Wilson struct intel_context *ce; 29610be98a7SChris Wilson 29710be98a7SChris Wilson ce = intel_context_create(ctx, engine); 29810be98a7SChris Wilson if (IS_ERR(ce)) { 29910be98a7SChris Wilson __free_engines(e, id); 30010be98a7SChris Wilson return ERR_CAST(ce); 30110be98a7SChris Wilson } 30210be98a7SChris Wilson 30310be98a7SChris Wilson e->engines[id] = ce; 304f1c4d157SChris Wilson e->num_engines = id + 1; 30510be98a7SChris Wilson } 30610be98a7SChris Wilson 30710be98a7SChris Wilson return e; 30810be98a7SChris Wilson } 30910be98a7SChris Wilson 31010be98a7SChris Wilson static void i915_gem_context_free(struct i915_gem_context *ctx) 31110be98a7SChris Wilson { 31210be98a7SChris Wilson lockdep_assert_held(&ctx->i915->drm.struct_mutex); 31310be98a7SChris Wilson GEM_BUG_ON(!i915_gem_context_is_closed(ctx)); 31410be98a7SChris Wilson 31510be98a7SChris Wilson release_hw_id(ctx); 316e568ac38SChris Wilson if (ctx->vm) 317e568ac38SChris Wilson i915_vm_put(ctx->vm); 31810be98a7SChris Wilson 31910be98a7SChris Wilson free_engines(rcu_access_pointer(ctx->engines)); 32010be98a7SChris Wilson mutex_destroy(&ctx->engines_mutex); 32110be98a7SChris Wilson 32210be98a7SChris Wilson if (ctx->timeline) 323f0c02c1bSTvrtko Ursulin intel_timeline_put(ctx->timeline); 32410be98a7SChris Wilson 32510be98a7SChris Wilson kfree(ctx->name); 32610be98a7SChris Wilson put_pid(ctx->pid); 32710be98a7SChris Wilson 32810be98a7SChris Wilson list_del(&ctx->link); 32910be98a7SChris Wilson mutex_destroy(&ctx->mutex); 33010be98a7SChris Wilson 33110be98a7SChris Wilson kfree_rcu(ctx, rcu); 33210be98a7SChris Wilson } 33310be98a7SChris Wilson 33410be98a7SChris Wilson static void contexts_free(struct drm_i915_private *i915) 33510be98a7SChris Wilson { 33610be98a7SChris Wilson struct llist_node *freed = llist_del_all(&i915->contexts.free_list); 33710be98a7SChris Wilson struct i915_gem_context *ctx, *cn; 33810be98a7SChris Wilson 33910be98a7SChris Wilson lockdep_assert_held(&i915->drm.struct_mutex); 34010be98a7SChris Wilson 34110be98a7SChris Wilson llist_for_each_entry_safe(ctx, cn, freed, free_link) 34210be98a7SChris Wilson i915_gem_context_free(ctx); 34310be98a7SChris Wilson } 34410be98a7SChris Wilson 34510be98a7SChris Wilson static void contexts_free_first(struct drm_i915_private *i915) 34610be98a7SChris Wilson { 34710be98a7SChris Wilson struct i915_gem_context *ctx; 34810be98a7SChris Wilson struct llist_node *freed; 34910be98a7SChris Wilson 35010be98a7SChris Wilson lockdep_assert_held(&i915->drm.struct_mutex); 35110be98a7SChris Wilson 35210be98a7SChris Wilson freed = llist_del_first(&i915->contexts.free_list); 35310be98a7SChris Wilson if (!freed) 35410be98a7SChris Wilson return; 35510be98a7SChris Wilson 35610be98a7SChris Wilson ctx = container_of(freed, typeof(*ctx), free_link); 35710be98a7SChris Wilson i915_gem_context_free(ctx); 35810be98a7SChris Wilson } 35910be98a7SChris Wilson 36010be98a7SChris Wilson static void contexts_free_worker(struct work_struct *work) 36110be98a7SChris Wilson { 36210be98a7SChris Wilson struct drm_i915_private *i915 = 36310be98a7SChris Wilson container_of(work, typeof(*i915), contexts.free_work); 36410be98a7SChris Wilson 36510be98a7SChris Wilson mutex_lock(&i915->drm.struct_mutex); 36610be98a7SChris Wilson contexts_free(i915); 36710be98a7SChris Wilson mutex_unlock(&i915->drm.struct_mutex); 36810be98a7SChris Wilson } 36910be98a7SChris Wilson 37010be98a7SChris Wilson void i915_gem_context_release(struct kref *ref) 37110be98a7SChris Wilson { 37210be98a7SChris Wilson struct i915_gem_context *ctx = container_of(ref, typeof(*ctx), ref); 37310be98a7SChris Wilson struct drm_i915_private *i915 = ctx->i915; 37410be98a7SChris Wilson 37510be98a7SChris Wilson trace_i915_context_free(ctx); 37610be98a7SChris Wilson if (llist_add(&ctx->free_link, &i915->contexts.free_list)) 37710be98a7SChris Wilson queue_work(i915->wq, &i915->contexts.free_work); 37810be98a7SChris Wilson } 37910be98a7SChris Wilson 38010be98a7SChris Wilson static void context_close(struct i915_gem_context *ctx) 38110be98a7SChris Wilson { 382155ab883SChris Wilson mutex_lock(&ctx->mutex); 383155ab883SChris Wilson 38410be98a7SChris Wilson i915_gem_context_set_closed(ctx); 385155ab883SChris Wilson ctx->file_priv = ERR_PTR(-EBADF); 38610be98a7SChris Wilson 38710be98a7SChris Wilson /* 38810be98a7SChris Wilson * This context will never again be assinged to HW, so we can 38910be98a7SChris Wilson * reuse its ID for the next context. 39010be98a7SChris Wilson */ 39110be98a7SChris Wilson release_hw_id(ctx); 39210be98a7SChris Wilson 39310be98a7SChris Wilson /* 39410be98a7SChris Wilson * The LUT uses the VMA as a backpointer to unref the object, 39510be98a7SChris Wilson * so we need to clear the LUT before we close all the VMA (inside 39610be98a7SChris Wilson * the ppgtt). 39710be98a7SChris Wilson */ 39810be98a7SChris Wilson lut_close(ctx); 39910be98a7SChris Wilson 400155ab883SChris Wilson mutex_unlock(&ctx->mutex); 40110be98a7SChris Wilson i915_gem_context_put(ctx); 40210be98a7SChris Wilson } 40310be98a7SChris Wilson 40410be98a7SChris Wilson static struct i915_gem_context * 405e568ac38SChris Wilson __create_context(struct drm_i915_private *i915) 40610be98a7SChris Wilson { 40710be98a7SChris Wilson struct i915_gem_context *ctx; 40810be98a7SChris Wilson struct i915_gem_engines *e; 40910be98a7SChris Wilson int err; 41010be98a7SChris Wilson int i; 41110be98a7SChris Wilson 41210be98a7SChris Wilson ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 41310be98a7SChris Wilson if (!ctx) 41410be98a7SChris Wilson return ERR_PTR(-ENOMEM); 41510be98a7SChris Wilson 41610be98a7SChris Wilson kref_init(&ctx->ref); 417e568ac38SChris Wilson list_add_tail(&ctx->link, &i915->contexts.list); 418e568ac38SChris Wilson ctx->i915 = i915; 41910be98a7SChris Wilson ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_NORMAL); 42010be98a7SChris Wilson mutex_init(&ctx->mutex); 42110be98a7SChris Wilson 42210be98a7SChris Wilson mutex_init(&ctx->engines_mutex); 42310be98a7SChris Wilson e = default_engines(ctx); 42410be98a7SChris Wilson if (IS_ERR(e)) { 42510be98a7SChris Wilson err = PTR_ERR(e); 42610be98a7SChris Wilson goto err_free; 42710be98a7SChris Wilson } 42810be98a7SChris Wilson RCU_INIT_POINTER(ctx->engines, e); 42910be98a7SChris Wilson 43010be98a7SChris Wilson INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL); 43110be98a7SChris Wilson INIT_LIST_HEAD(&ctx->hw_id_link); 43210be98a7SChris Wilson 43310be98a7SChris Wilson /* NB: Mark all slices as needing a remap so that when the context first 43410be98a7SChris Wilson * loads it will restore whatever remap state already exists. If there 43510be98a7SChris Wilson * is no remap info, it will be a NOP. */ 436e568ac38SChris Wilson ctx->remap_slice = ALL_L3_SLICES(i915); 43710be98a7SChris Wilson 43810be98a7SChris Wilson i915_gem_context_set_bannable(ctx); 43910be98a7SChris Wilson i915_gem_context_set_recoverable(ctx); 44010be98a7SChris Wilson 44110be98a7SChris Wilson for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++) 44210be98a7SChris Wilson ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES; 44310be98a7SChris Wilson 44410be98a7SChris Wilson return ctx; 44510be98a7SChris Wilson 44610be98a7SChris Wilson err_free: 44710be98a7SChris Wilson kfree(ctx); 44810be98a7SChris Wilson return ERR_PTR(err); 44910be98a7SChris Wilson } 45010be98a7SChris Wilson 45148ae397bSChris Wilson static void 45248ae397bSChris Wilson context_apply_all(struct i915_gem_context *ctx, 45348ae397bSChris Wilson void (*fn)(struct intel_context *ce, void *data), 45448ae397bSChris Wilson void *data) 45548ae397bSChris Wilson { 45648ae397bSChris Wilson struct i915_gem_engines_iter it; 45748ae397bSChris Wilson struct intel_context *ce; 45848ae397bSChris Wilson 45948ae397bSChris Wilson for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) 46048ae397bSChris Wilson fn(ce, data); 46148ae397bSChris Wilson i915_gem_context_unlock_engines(ctx); 46248ae397bSChris Wilson } 46348ae397bSChris Wilson 46448ae397bSChris Wilson static void __apply_ppgtt(struct intel_context *ce, void *vm) 46548ae397bSChris Wilson { 46648ae397bSChris Wilson i915_vm_put(ce->vm); 46748ae397bSChris Wilson ce->vm = i915_vm_get(vm); 46848ae397bSChris Wilson } 46948ae397bSChris Wilson 470e568ac38SChris Wilson static struct i915_address_space * 471e568ac38SChris Wilson __set_ppgtt(struct i915_gem_context *ctx, struct i915_address_space *vm) 47210be98a7SChris Wilson { 473e568ac38SChris Wilson struct i915_address_space *old = ctx->vm; 47410be98a7SChris Wilson 475a1c9ca22SChris Wilson GEM_BUG_ON(old && i915_vm_is_4lvl(vm) != i915_vm_is_4lvl(old)); 476a1c9ca22SChris Wilson 477e568ac38SChris Wilson ctx->vm = i915_vm_get(vm); 47848ae397bSChris Wilson context_apply_all(ctx, __apply_ppgtt, vm); 479f5d974f9SChris Wilson 48010be98a7SChris Wilson return old; 48110be98a7SChris Wilson } 48210be98a7SChris Wilson 48310be98a7SChris Wilson static void __assign_ppgtt(struct i915_gem_context *ctx, 484e568ac38SChris Wilson struct i915_address_space *vm) 48510be98a7SChris Wilson { 486e568ac38SChris Wilson if (vm == ctx->vm) 48710be98a7SChris Wilson return; 48810be98a7SChris Wilson 489e568ac38SChris Wilson vm = __set_ppgtt(ctx, vm); 490e568ac38SChris Wilson if (vm) 491e568ac38SChris Wilson i915_vm_put(vm); 49210be98a7SChris Wilson } 49310be98a7SChris Wilson 49475d0a7f3SChris Wilson static void __set_timeline(struct intel_timeline **dst, 49575d0a7f3SChris Wilson struct intel_timeline *src) 49675d0a7f3SChris Wilson { 49775d0a7f3SChris Wilson struct intel_timeline *old = *dst; 49875d0a7f3SChris Wilson 49975d0a7f3SChris Wilson *dst = src ? intel_timeline_get(src) : NULL; 50075d0a7f3SChris Wilson 50175d0a7f3SChris Wilson if (old) 50275d0a7f3SChris Wilson intel_timeline_put(old); 50375d0a7f3SChris Wilson } 50475d0a7f3SChris Wilson 50575d0a7f3SChris Wilson static void __apply_timeline(struct intel_context *ce, void *timeline) 50675d0a7f3SChris Wilson { 50775d0a7f3SChris Wilson __set_timeline(&ce->timeline, timeline); 50875d0a7f3SChris Wilson } 50975d0a7f3SChris Wilson 51075d0a7f3SChris Wilson static void __assign_timeline(struct i915_gem_context *ctx, 51175d0a7f3SChris Wilson struct intel_timeline *timeline) 51275d0a7f3SChris Wilson { 51375d0a7f3SChris Wilson __set_timeline(&ctx->timeline, timeline); 51475d0a7f3SChris Wilson context_apply_all(ctx, __apply_timeline, timeline); 51575d0a7f3SChris Wilson } 51675d0a7f3SChris Wilson 51710be98a7SChris Wilson static struct i915_gem_context * 51810be98a7SChris Wilson i915_gem_create_context(struct drm_i915_private *dev_priv, unsigned int flags) 51910be98a7SChris Wilson { 52010be98a7SChris Wilson struct i915_gem_context *ctx; 52110be98a7SChris Wilson 52210be98a7SChris Wilson lockdep_assert_held(&dev_priv->drm.struct_mutex); 52310be98a7SChris Wilson 52410be98a7SChris Wilson if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE && 52510be98a7SChris Wilson !HAS_EXECLISTS(dev_priv)) 52610be98a7SChris Wilson return ERR_PTR(-EINVAL); 52710be98a7SChris Wilson 52810be98a7SChris Wilson /* Reap the most stale context */ 52910be98a7SChris Wilson contexts_free_first(dev_priv); 53010be98a7SChris Wilson 53110be98a7SChris Wilson ctx = __create_context(dev_priv); 53210be98a7SChris Wilson if (IS_ERR(ctx)) 53310be98a7SChris Wilson return ctx; 53410be98a7SChris Wilson 53510be98a7SChris Wilson if (HAS_FULL_PPGTT(dev_priv)) { 536ab53497bSChris Wilson struct i915_ppgtt *ppgtt; 53710be98a7SChris Wilson 53810be98a7SChris Wilson ppgtt = i915_ppgtt_create(dev_priv); 53910be98a7SChris Wilson if (IS_ERR(ppgtt)) { 54010be98a7SChris Wilson DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n", 54110be98a7SChris Wilson PTR_ERR(ppgtt)); 54210be98a7SChris Wilson context_close(ctx); 54310be98a7SChris Wilson return ERR_CAST(ppgtt); 54410be98a7SChris Wilson } 54510be98a7SChris Wilson 546e568ac38SChris Wilson __assign_ppgtt(ctx, &ppgtt->vm); 547e568ac38SChris Wilson i915_vm_put(&ppgtt->vm); 54810be98a7SChris Wilson } 54910be98a7SChris Wilson 55010be98a7SChris Wilson if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE) { 551f0c02c1bSTvrtko Ursulin struct intel_timeline *timeline; 55210be98a7SChris Wilson 553f0c02c1bSTvrtko Ursulin timeline = intel_timeline_create(&dev_priv->gt, NULL); 55410be98a7SChris Wilson if (IS_ERR(timeline)) { 55510be98a7SChris Wilson context_close(ctx); 55610be98a7SChris Wilson return ERR_CAST(timeline); 55710be98a7SChris Wilson } 55810be98a7SChris Wilson 55975d0a7f3SChris Wilson __assign_timeline(ctx, timeline); 56075d0a7f3SChris Wilson intel_timeline_put(timeline); 56110be98a7SChris Wilson } 56210be98a7SChris Wilson 56310be98a7SChris Wilson trace_i915_context_create(ctx); 56410be98a7SChris Wilson 56510be98a7SChris Wilson return ctx; 56610be98a7SChris Wilson } 56710be98a7SChris Wilson 56810be98a7SChris Wilson static void 56910be98a7SChris Wilson destroy_kernel_context(struct i915_gem_context **ctxp) 57010be98a7SChris Wilson { 57110be98a7SChris Wilson struct i915_gem_context *ctx; 57210be98a7SChris Wilson 57310be98a7SChris Wilson /* Keep the context ref so that we can free it immediately ourselves */ 57410be98a7SChris Wilson ctx = i915_gem_context_get(fetch_and_zero(ctxp)); 57510be98a7SChris Wilson GEM_BUG_ON(!i915_gem_context_is_kernel(ctx)); 57610be98a7SChris Wilson 57710be98a7SChris Wilson context_close(ctx); 57810be98a7SChris Wilson i915_gem_context_free(ctx); 57910be98a7SChris Wilson } 58010be98a7SChris Wilson 58110be98a7SChris Wilson struct i915_gem_context * 58210be98a7SChris Wilson i915_gem_context_create_kernel(struct drm_i915_private *i915, int prio) 58310be98a7SChris Wilson { 58410be98a7SChris Wilson struct i915_gem_context *ctx; 58510be98a7SChris Wilson int err; 58610be98a7SChris Wilson 58710be98a7SChris Wilson ctx = i915_gem_create_context(i915, 0); 58810be98a7SChris Wilson if (IS_ERR(ctx)) 58910be98a7SChris Wilson return ctx; 59010be98a7SChris Wilson 59110be98a7SChris Wilson err = i915_gem_context_pin_hw_id(ctx); 59210be98a7SChris Wilson if (err) { 59310be98a7SChris Wilson destroy_kernel_context(&ctx); 59410be98a7SChris Wilson return ERR_PTR(err); 59510be98a7SChris Wilson } 59610be98a7SChris Wilson 59710be98a7SChris Wilson i915_gem_context_clear_bannable(ctx); 59810be98a7SChris Wilson ctx->sched.priority = I915_USER_PRIORITY(prio); 59910be98a7SChris Wilson 60010be98a7SChris Wilson GEM_BUG_ON(!i915_gem_context_is_kernel(ctx)); 60110be98a7SChris Wilson 60210be98a7SChris Wilson return ctx; 60310be98a7SChris Wilson } 60410be98a7SChris Wilson 60510be98a7SChris Wilson static void init_contexts(struct drm_i915_private *i915) 60610be98a7SChris Wilson { 60710be98a7SChris Wilson mutex_init(&i915->contexts.mutex); 60810be98a7SChris Wilson INIT_LIST_HEAD(&i915->contexts.list); 60910be98a7SChris Wilson 61010be98a7SChris Wilson /* Using the simple ida interface, the max is limited by sizeof(int) */ 61110be98a7SChris Wilson BUILD_BUG_ON(MAX_CONTEXT_HW_ID > INT_MAX); 61210be98a7SChris Wilson BUILD_BUG_ON(GEN11_MAX_CONTEXT_HW_ID > INT_MAX); 61310be98a7SChris Wilson ida_init(&i915->contexts.hw_ida); 61410be98a7SChris Wilson INIT_LIST_HEAD(&i915->contexts.hw_id_list); 61510be98a7SChris Wilson 61610be98a7SChris Wilson INIT_WORK(&i915->contexts.free_work, contexts_free_worker); 61710be98a7SChris Wilson init_llist_head(&i915->contexts.free_list); 61810be98a7SChris Wilson } 61910be98a7SChris Wilson 62010be98a7SChris Wilson int i915_gem_contexts_init(struct drm_i915_private *dev_priv) 62110be98a7SChris Wilson { 62210be98a7SChris Wilson struct i915_gem_context *ctx; 62310be98a7SChris Wilson 62410be98a7SChris Wilson /* Reassure ourselves we are only called once */ 62510be98a7SChris Wilson GEM_BUG_ON(dev_priv->kernel_context); 62610be98a7SChris Wilson 62710be98a7SChris Wilson init_contexts(dev_priv); 62810be98a7SChris Wilson 62910be98a7SChris Wilson /* lowest priority; idle task */ 63010be98a7SChris Wilson ctx = i915_gem_context_create_kernel(dev_priv, I915_PRIORITY_MIN); 63110be98a7SChris Wilson if (IS_ERR(ctx)) { 63210be98a7SChris Wilson DRM_ERROR("Failed to create default global context\n"); 63310be98a7SChris Wilson return PTR_ERR(ctx); 63410be98a7SChris Wilson } 63510be98a7SChris Wilson /* 63610be98a7SChris Wilson * For easy recognisablity, we want the kernel context to be 0 and then 63710be98a7SChris Wilson * all user contexts will have non-zero hw_id. Kernel contexts are 63810be98a7SChris Wilson * permanently pinned, so that we never suffer a stall and can 63910be98a7SChris Wilson * use them from any allocation context (e.g. for evicting other 64010be98a7SChris Wilson * contexts and from inside the shrinker). 64110be98a7SChris Wilson */ 64210be98a7SChris Wilson GEM_BUG_ON(ctx->hw_id); 64310be98a7SChris Wilson GEM_BUG_ON(!atomic_read(&ctx->hw_id_pin_count)); 64410be98a7SChris Wilson dev_priv->kernel_context = ctx; 64510be98a7SChris Wilson 64610be98a7SChris Wilson DRM_DEBUG_DRIVER("%s context support initialized\n", 64710be98a7SChris Wilson DRIVER_CAPS(dev_priv)->has_logical_contexts ? 64810be98a7SChris Wilson "logical" : "fake"); 64910be98a7SChris Wilson return 0; 65010be98a7SChris Wilson } 65110be98a7SChris Wilson 65210be98a7SChris Wilson void i915_gem_contexts_fini(struct drm_i915_private *i915) 65310be98a7SChris Wilson { 65410be98a7SChris Wilson lockdep_assert_held(&i915->drm.struct_mutex); 65510be98a7SChris Wilson 65610be98a7SChris Wilson destroy_kernel_context(&i915->kernel_context); 65710be98a7SChris Wilson 65810be98a7SChris Wilson /* Must free all deferred contexts (via flush_workqueue) first */ 65910be98a7SChris Wilson GEM_BUG_ON(!list_empty(&i915->contexts.hw_id_list)); 66010be98a7SChris Wilson ida_destroy(&i915->contexts.hw_ida); 66110be98a7SChris Wilson } 66210be98a7SChris Wilson 66310be98a7SChris Wilson static int context_idr_cleanup(int id, void *p, void *data) 66410be98a7SChris Wilson { 66510be98a7SChris Wilson context_close(p); 66610be98a7SChris Wilson return 0; 66710be98a7SChris Wilson } 66810be98a7SChris Wilson 66910be98a7SChris Wilson static int vm_idr_cleanup(int id, void *p, void *data) 67010be98a7SChris Wilson { 671e568ac38SChris Wilson i915_vm_put(p); 67210be98a7SChris Wilson return 0; 67310be98a7SChris Wilson } 67410be98a7SChris Wilson 67510be98a7SChris Wilson static int gem_context_register(struct i915_gem_context *ctx, 67610be98a7SChris Wilson struct drm_i915_file_private *fpriv) 67710be98a7SChris Wilson { 67810be98a7SChris Wilson int ret; 67910be98a7SChris Wilson 68010be98a7SChris Wilson ctx->file_priv = fpriv; 681e568ac38SChris Wilson if (ctx->vm) 682e568ac38SChris Wilson ctx->vm->file = fpriv; 68310be98a7SChris Wilson 68410be98a7SChris Wilson ctx->pid = get_task_pid(current, PIDTYPE_PID); 68510be98a7SChris Wilson ctx->name = kasprintf(GFP_KERNEL, "%s[%d]", 68610be98a7SChris Wilson current->comm, pid_nr(ctx->pid)); 68710be98a7SChris Wilson if (!ctx->name) { 68810be98a7SChris Wilson ret = -ENOMEM; 68910be98a7SChris Wilson goto err_pid; 69010be98a7SChris Wilson } 69110be98a7SChris Wilson 69210be98a7SChris Wilson /* And finally expose ourselves to userspace via the idr */ 69310be98a7SChris Wilson mutex_lock(&fpriv->context_idr_lock); 69410be98a7SChris Wilson ret = idr_alloc(&fpriv->context_idr, ctx, 0, 0, GFP_KERNEL); 69510be98a7SChris Wilson mutex_unlock(&fpriv->context_idr_lock); 69610be98a7SChris Wilson if (ret >= 0) 69710be98a7SChris Wilson goto out; 69810be98a7SChris Wilson 69910be98a7SChris Wilson kfree(fetch_and_zero(&ctx->name)); 70010be98a7SChris Wilson err_pid: 70110be98a7SChris Wilson put_pid(fetch_and_zero(&ctx->pid)); 70210be98a7SChris Wilson out: 70310be98a7SChris Wilson return ret; 70410be98a7SChris Wilson } 70510be98a7SChris Wilson 70610be98a7SChris Wilson int i915_gem_context_open(struct drm_i915_private *i915, 70710be98a7SChris Wilson struct drm_file *file) 70810be98a7SChris Wilson { 70910be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 71010be98a7SChris Wilson struct i915_gem_context *ctx; 71110be98a7SChris Wilson int err; 71210be98a7SChris Wilson 71310be98a7SChris Wilson mutex_init(&file_priv->context_idr_lock); 71410be98a7SChris Wilson mutex_init(&file_priv->vm_idr_lock); 71510be98a7SChris Wilson 71610be98a7SChris Wilson idr_init(&file_priv->context_idr); 71710be98a7SChris Wilson idr_init_base(&file_priv->vm_idr, 1); 71810be98a7SChris Wilson 71910be98a7SChris Wilson mutex_lock(&i915->drm.struct_mutex); 72010be98a7SChris Wilson ctx = i915_gem_create_context(i915, 0); 72110be98a7SChris Wilson mutex_unlock(&i915->drm.struct_mutex); 72210be98a7SChris Wilson if (IS_ERR(ctx)) { 72310be98a7SChris Wilson err = PTR_ERR(ctx); 72410be98a7SChris Wilson goto err; 72510be98a7SChris Wilson } 72610be98a7SChris Wilson 72710be98a7SChris Wilson err = gem_context_register(ctx, file_priv); 72810be98a7SChris Wilson if (err < 0) 72910be98a7SChris Wilson goto err_ctx; 73010be98a7SChris Wilson 73110be98a7SChris Wilson GEM_BUG_ON(i915_gem_context_is_kernel(ctx)); 73210be98a7SChris Wilson GEM_BUG_ON(err > 0); 73310be98a7SChris Wilson 73410be98a7SChris Wilson return 0; 73510be98a7SChris Wilson 73610be98a7SChris Wilson err_ctx: 73710be98a7SChris Wilson context_close(ctx); 73810be98a7SChris Wilson err: 73910be98a7SChris Wilson idr_destroy(&file_priv->vm_idr); 74010be98a7SChris Wilson idr_destroy(&file_priv->context_idr); 74110be98a7SChris Wilson mutex_destroy(&file_priv->vm_idr_lock); 74210be98a7SChris Wilson mutex_destroy(&file_priv->context_idr_lock); 74310be98a7SChris Wilson return err; 74410be98a7SChris Wilson } 74510be98a7SChris Wilson 74610be98a7SChris Wilson void i915_gem_context_close(struct drm_file *file) 74710be98a7SChris Wilson { 74810be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 74910be98a7SChris Wilson 75010be98a7SChris Wilson idr_for_each(&file_priv->context_idr, context_idr_cleanup, NULL); 75110be98a7SChris Wilson idr_destroy(&file_priv->context_idr); 75210be98a7SChris Wilson mutex_destroy(&file_priv->context_idr_lock); 75310be98a7SChris Wilson 75410be98a7SChris Wilson idr_for_each(&file_priv->vm_idr, vm_idr_cleanup, NULL); 75510be98a7SChris Wilson idr_destroy(&file_priv->vm_idr); 75610be98a7SChris Wilson mutex_destroy(&file_priv->vm_idr_lock); 75710be98a7SChris Wilson } 75810be98a7SChris Wilson 75910be98a7SChris Wilson int i915_gem_vm_create_ioctl(struct drm_device *dev, void *data, 76010be98a7SChris Wilson struct drm_file *file) 76110be98a7SChris Wilson { 76210be98a7SChris Wilson struct drm_i915_private *i915 = to_i915(dev); 76310be98a7SChris Wilson struct drm_i915_gem_vm_control *args = data; 76410be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 765ab53497bSChris Wilson struct i915_ppgtt *ppgtt; 76610be98a7SChris Wilson int err; 76710be98a7SChris Wilson 76810be98a7SChris Wilson if (!HAS_FULL_PPGTT(i915)) 76910be98a7SChris Wilson return -ENODEV; 77010be98a7SChris Wilson 77110be98a7SChris Wilson if (args->flags) 77210be98a7SChris Wilson return -EINVAL; 77310be98a7SChris Wilson 77410be98a7SChris Wilson ppgtt = i915_ppgtt_create(i915); 77510be98a7SChris Wilson if (IS_ERR(ppgtt)) 77610be98a7SChris Wilson return PTR_ERR(ppgtt); 77710be98a7SChris Wilson 77810be98a7SChris Wilson ppgtt->vm.file = file_priv; 77910be98a7SChris Wilson 78010be98a7SChris Wilson if (args->extensions) { 78110be98a7SChris Wilson err = i915_user_extensions(u64_to_user_ptr(args->extensions), 78210be98a7SChris Wilson NULL, 0, 78310be98a7SChris Wilson ppgtt); 78410be98a7SChris Wilson if (err) 78510be98a7SChris Wilson goto err_put; 78610be98a7SChris Wilson } 78710be98a7SChris Wilson 78810be98a7SChris Wilson err = mutex_lock_interruptible(&file_priv->vm_idr_lock); 78910be98a7SChris Wilson if (err) 79010be98a7SChris Wilson goto err_put; 79110be98a7SChris Wilson 792e568ac38SChris Wilson err = idr_alloc(&file_priv->vm_idr, &ppgtt->vm, 0, 0, GFP_KERNEL); 79310be98a7SChris Wilson if (err < 0) 79410be98a7SChris Wilson goto err_unlock; 79510be98a7SChris Wilson 79610be98a7SChris Wilson GEM_BUG_ON(err == 0); /* reserved for invalid/unassigned ppgtt */ 79710be98a7SChris Wilson 79810be98a7SChris Wilson mutex_unlock(&file_priv->vm_idr_lock); 79910be98a7SChris Wilson 80010be98a7SChris Wilson args->vm_id = err; 80110be98a7SChris Wilson return 0; 80210be98a7SChris Wilson 80310be98a7SChris Wilson err_unlock: 80410be98a7SChris Wilson mutex_unlock(&file_priv->vm_idr_lock); 80510be98a7SChris Wilson err_put: 806e568ac38SChris Wilson i915_vm_put(&ppgtt->vm); 80710be98a7SChris Wilson return err; 80810be98a7SChris Wilson } 80910be98a7SChris Wilson 81010be98a7SChris Wilson int i915_gem_vm_destroy_ioctl(struct drm_device *dev, void *data, 81110be98a7SChris Wilson struct drm_file *file) 81210be98a7SChris Wilson { 81310be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 81410be98a7SChris Wilson struct drm_i915_gem_vm_control *args = data; 815e568ac38SChris Wilson struct i915_address_space *vm; 81610be98a7SChris Wilson int err; 81710be98a7SChris Wilson u32 id; 81810be98a7SChris Wilson 81910be98a7SChris Wilson if (args->flags) 82010be98a7SChris Wilson return -EINVAL; 82110be98a7SChris Wilson 82210be98a7SChris Wilson if (args->extensions) 82310be98a7SChris Wilson return -EINVAL; 82410be98a7SChris Wilson 82510be98a7SChris Wilson id = args->vm_id; 82610be98a7SChris Wilson if (!id) 82710be98a7SChris Wilson return -ENOENT; 82810be98a7SChris Wilson 82910be98a7SChris Wilson err = mutex_lock_interruptible(&file_priv->vm_idr_lock); 83010be98a7SChris Wilson if (err) 83110be98a7SChris Wilson return err; 83210be98a7SChris Wilson 833e568ac38SChris Wilson vm = idr_remove(&file_priv->vm_idr, id); 83410be98a7SChris Wilson 83510be98a7SChris Wilson mutex_unlock(&file_priv->vm_idr_lock); 836e568ac38SChris Wilson if (!vm) 83710be98a7SChris Wilson return -ENOENT; 83810be98a7SChris Wilson 839e568ac38SChris Wilson i915_vm_put(vm); 84010be98a7SChris Wilson return 0; 84110be98a7SChris Wilson } 84210be98a7SChris Wilson 84310be98a7SChris Wilson struct context_barrier_task { 84410be98a7SChris Wilson struct i915_active base; 84510be98a7SChris Wilson void (*task)(void *data); 84610be98a7SChris Wilson void *data; 84710be98a7SChris Wilson }; 84810be98a7SChris Wilson 84910be98a7SChris Wilson static void cb_retire(struct i915_active *base) 85010be98a7SChris Wilson { 85110be98a7SChris Wilson struct context_barrier_task *cb = container_of(base, typeof(*cb), base); 85210be98a7SChris Wilson 85310be98a7SChris Wilson if (cb->task) 85410be98a7SChris Wilson cb->task(cb->data); 85510be98a7SChris Wilson 85610be98a7SChris Wilson i915_active_fini(&cb->base); 85710be98a7SChris Wilson kfree(cb); 85810be98a7SChris Wilson } 85910be98a7SChris Wilson 86010be98a7SChris Wilson I915_SELFTEST_DECLARE(static intel_engine_mask_t context_barrier_inject_fault); 86110be98a7SChris Wilson static int context_barrier_task(struct i915_gem_context *ctx, 86210be98a7SChris Wilson intel_engine_mask_t engines, 8631fe2d6f9SChris Wilson bool (*skip)(struct intel_context *ce, void *data), 86410be98a7SChris Wilson int (*emit)(struct i915_request *rq, void *data), 86510be98a7SChris Wilson void (*task)(void *data), 86610be98a7SChris Wilson void *data) 86710be98a7SChris Wilson { 86810be98a7SChris Wilson struct drm_i915_private *i915 = ctx->i915; 86910be98a7SChris Wilson struct context_barrier_task *cb; 87010be98a7SChris Wilson struct i915_gem_engines_iter it; 87110be98a7SChris Wilson struct intel_context *ce; 87210be98a7SChris Wilson int err = 0; 87310be98a7SChris Wilson 87410be98a7SChris Wilson lockdep_assert_held(&i915->drm.struct_mutex); 87510be98a7SChris Wilson GEM_BUG_ON(!task); 87610be98a7SChris Wilson 87710be98a7SChris Wilson cb = kmalloc(sizeof(*cb), GFP_KERNEL); 87810be98a7SChris Wilson if (!cb) 87910be98a7SChris Wilson return -ENOMEM; 88010be98a7SChris Wilson 88112c255b5SChris Wilson i915_active_init(i915, &cb->base, NULL, cb_retire); 88212c255b5SChris Wilson err = i915_active_acquire(&cb->base); 88312c255b5SChris Wilson if (err) { 88412c255b5SChris Wilson kfree(cb); 88512c255b5SChris Wilson return err; 88612c255b5SChris Wilson } 88710be98a7SChris Wilson 88810be98a7SChris Wilson for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 88910be98a7SChris Wilson struct i915_request *rq; 89010be98a7SChris Wilson 89110be98a7SChris Wilson if (I915_SELFTEST_ONLY(context_barrier_inject_fault & 89210be98a7SChris Wilson ce->engine->mask)) { 89310be98a7SChris Wilson err = -ENXIO; 89410be98a7SChris Wilson break; 89510be98a7SChris Wilson } 89610be98a7SChris Wilson 8971fe2d6f9SChris Wilson if (!(ce->engine->mask & engines)) 8981fe2d6f9SChris Wilson continue; 8991fe2d6f9SChris Wilson 9001fe2d6f9SChris Wilson if (skip && skip(ce, data)) 90110be98a7SChris Wilson continue; 90210be98a7SChris Wilson 90310be98a7SChris Wilson rq = intel_context_create_request(ce); 90410be98a7SChris Wilson if (IS_ERR(rq)) { 90510be98a7SChris Wilson err = PTR_ERR(rq); 90610be98a7SChris Wilson break; 90710be98a7SChris Wilson } 90810be98a7SChris Wilson 90910be98a7SChris Wilson err = 0; 91010be98a7SChris Wilson if (emit) 91110be98a7SChris Wilson err = emit(rq, data); 91210be98a7SChris Wilson if (err == 0) 91325ffd4b1SChris Wilson err = i915_active_ref(&cb->base, rq->timeline, rq); 91410be98a7SChris Wilson 91510be98a7SChris Wilson i915_request_add(rq); 91610be98a7SChris Wilson if (err) 91710be98a7SChris Wilson break; 91810be98a7SChris Wilson } 91910be98a7SChris Wilson i915_gem_context_unlock_engines(ctx); 92010be98a7SChris Wilson 92110be98a7SChris Wilson cb->task = err ? NULL : task; /* caller needs to unwind instead */ 92210be98a7SChris Wilson cb->data = data; 92310be98a7SChris Wilson 92410be98a7SChris Wilson i915_active_release(&cb->base); 92510be98a7SChris Wilson 92610be98a7SChris Wilson return err; 92710be98a7SChris Wilson } 92810be98a7SChris Wilson 92910be98a7SChris Wilson static int get_ppgtt(struct drm_i915_file_private *file_priv, 93010be98a7SChris Wilson struct i915_gem_context *ctx, 93110be98a7SChris Wilson struct drm_i915_gem_context_param *args) 93210be98a7SChris Wilson { 933e568ac38SChris Wilson struct i915_address_space *vm; 93410be98a7SChris Wilson int ret; 93510be98a7SChris Wilson 936e568ac38SChris Wilson if (!ctx->vm) 93710be98a7SChris Wilson return -ENODEV; 93810be98a7SChris Wilson 93910be98a7SChris Wilson /* XXX rcu acquire? */ 94010be98a7SChris Wilson ret = mutex_lock_interruptible(&ctx->i915->drm.struct_mutex); 94110be98a7SChris Wilson if (ret) 94210be98a7SChris Wilson return ret; 94310be98a7SChris Wilson 944e568ac38SChris Wilson vm = i915_vm_get(ctx->vm); 94510be98a7SChris Wilson mutex_unlock(&ctx->i915->drm.struct_mutex); 94610be98a7SChris Wilson 94710be98a7SChris Wilson ret = mutex_lock_interruptible(&file_priv->vm_idr_lock); 94810be98a7SChris Wilson if (ret) 94910be98a7SChris Wilson goto err_put; 95010be98a7SChris Wilson 951e568ac38SChris Wilson ret = idr_alloc(&file_priv->vm_idr, vm, 0, 0, GFP_KERNEL); 95210be98a7SChris Wilson GEM_BUG_ON(!ret); 95310be98a7SChris Wilson if (ret < 0) 95410be98a7SChris Wilson goto err_unlock; 95510be98a7SChris Wilson 956e568ac38SChris Wilson i915_vm_get(vm); 95710be98a7SChris Wilson 95810be98a7SChris Wilson args->size = 0; 95910be98a7SChris Wilson args->value = ret; 96010be98a7SChris Wilson 96110be98a7SChris Wilson ret = 0; 96210be98a7SChris Wilson err_unlock: 96310be98a7SChris Wilson mutex_unlock(&file_priv->vm_idr_lock); 96410be98a7SChris Wilson err_put: 965e568ac38SChris Wilson i915_vm_put(vm); 96610be98a7SChris Wilson return ret; 96710be98a7SChris Wilson } 96810be98a7SChris Wilson 96910be98a7SChris Wilson static void set_ppgtt_barrier(void *data) 97010be98a7SChris Wilson { 971e568ac38SChris Wilson struct i915_address_space *old = data; 97210be98a7SChris Wilson 973e568ac38SChris Wilson if (INTEL_GEN(old->i915) < 8) 974e568ac38SChris Wilson gen6_ppgtt_unpin_all(i915_vm_to_ppgtt(old)); 97510be98a7SChris Wilson 976e568ac38SChris Wilson i915_vm_put(old); 97710be98a7SChris Wilson } 97810be98a7SChris Wilson 97910be98a7SChris Wilson static int emit_ppgtt_update(struct i915_request *rq, void *data) 98010be98a7SChris Wilson { 981f5d974f9SChris Wilson struct i915_address_space *vm = rq->hw_context->vm; 98210be98a7SChris Wilson struct intel_engine_cs *engine = rq->engine; 98310be98a7SChris Wilson u32 base = engine->mmio_base; 98410be98a7SChris Wilson u32 *cs; 98510be98a7SChris Wilson int i; 98610be98a7SChris Wilson 987e568ac38SChris Wilson if (i915_vm_is_4lvl(vm)) { 988ab53497bSChris Wilson struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 989b5b7bef9SMika Kuoppala const dma_addr_t pd_daddr = px_dma(ppgtt->pd); 99010be98a7SChris Wilson 99110be98a7SChris Wilson cs = intel_ring_begin(rq, 6); 99210be98a7SChris Wilson if (IS_ERR(cs)) 99310be98a7SChris Wilson return PTR_ERR(cs); 99410be98a7SChris Wilson 99510be98a7SChris Wilson *cs++ = MI_LOAD_REGISTER_IMM(2); 99610be98a7SChris Wilson 99710be98a7SChris Wilson *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 0)); 99810be98a7SChris Wilson *cs++ = upper_32_bits(pd_daddr); 99910be98a7SChris Wilson *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 0)); 100010be98a7SChris Wilson *cs++ = lower_32_bits(pd_daddr); 100110be98a7SChris Wilson 100210be98a7SChris Wilson *cs++ = MI_NOOP; 100310be98a7SChris Wilson intel_ring_advance(rq, cs); 100410be98a7SChris Wilson } else if (HAS_LOGICAL_RING_CONTEXTS(engine->i915)) { 1005ab53497bSChris Wilson struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1006e568ac38SChris Wilson 100710be98a7SChris Wilson cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2); 100810be98a7SChris Wilson if (IS_ERR(cs)) 100910be98a7SChris Wilson return PTR_ERR(cs); 101010be98a7SChris Wilson 101110be98a7SChris Wilson *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES); 101210be98a7SChris Wilson for (i = GEN8_3LVL_PDPES; i--; ) { 101310be98a7SChris Wilson const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); 101410be98a7SChris Wilson 101510be98a7SChris Wilson *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i)); 101610be98a7SChris Wilson *cs++ = upper_32_bits(pd_daddr); 101710be98a7SChris Wilson *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i)); 101810be98a7SChris Wilson *cs++ = lower_32_bits(pd_daddr); 101910be98a7SChris Wilson } 102010be98a7SChris Wilson *cs++ = MI_NOOP; 102110be98a7SChris Wilson intel_ring_advance(rq, cs); 102210be98a7SChris Wilson } else { 102310be98a7SChris Wilson /* ppGTT is not part of the legacy context image */ 1024e568ac38SChris Wilson gen6_ppgtt_pin(i915_vm_to_ppgtt(vm)); 102510be98a7SChris Wilson } 102610be98a7SChris Wilson 102710be98a7SChris Wilson return 0; 102810be98a7SChris Wilson } 102910be98a7SChris Wilson 10301fe2d6f9SChris Wilson static bool skip_ppgtt_update(struct intel_context *ce, void *data) 10311fe2d6f9SChris Wilson { 10321fe2d6f9SChris Wilson if (HAS_LOGICAL_RING_CONTEXTS(ce->engine->i915)) 10331fe2d6f9SChris Wilson return !ce->state; 10341fe2d6f9SChris Wilson else 10351fe2d6f9SChris Wilson return !atomic_read(&ce->pin_count); 10361fe2d6f9SChris Wilson } 10371fe2d6f9SChris Wilson 103810be98a7SChris Wilson static int set_ppgtt(struct drm_i915_file_private *file_priv, 103910be98a7SChris Wilson struct i915_gem_context *ctx, 104010be98a7SChris Wilson struct drm_i915_gem_context_param *args) 104110be98a7SChris Wilson { 1042e568ac38SChris Wilson struct i915_address_space *vm, *old; 104310be98a7SChris Wilson int err; 104410be98a7SChris Wilson 104510be98a7SChris Wilson if (args->size) 104610be98a7SChris Wilson return -EINVAL; 104710be98a7SChris Wilson 1048e568ac38SChris Wilson if (!ctx->vm) 104910be98a7SChris Wilson return -ENODEV; 105010be98a7SChris Wilson 105110be98a7SChris Wilson if (upper_32_bits(args->value)) 105210be98a7SChris Wilson return -ENOENT; 105310be98a7SChris Wilson 105410be98a7SChris Wilson err = mutex_lock_interruptible(&file_priv->vm_idr_lock); 105510be98a7SChris Wilson if (err) 105610be98a7SChris Wilson return err; 105710be98a7SChris Wilson 1058e568ac38SChris Wilson vm = idr_find(&file_priv->vm_idr, args->value); 1059e568ac38SChris Wilson if (vm) 1060e568ac38SChris Wilson i915_vm_get(vm); 106110be98a7SChris Wilson mutex_unlock(&file_priv->vm_idr_lock); 1062e568ac38SChris Wilson if (!vm) 106310be98a7SChris Wilson return -ENOENT; 106410be98a7SChris Wilson 106510be98a7SChris Wilson err = mutex_lock_interruptible(&ctx->i915->drm.struct_mutex); 106610be98a7SChris Wilson if (err) 106710be98a7SChris Wilson goto out; 106810be98a7SChris Wilson 1069e568ac38SChris Wilson if (vm == ctx->vm) 107010be98a7SChris Wilson goto unlock; 107110be98a7SChris Wilson 107210be98a7SChris Wilson /* Teardown the existing obj:vma cache, it will have to be rebuilt. */ 1073155ab883SChris Wilson mutex_lock(&ctx->mutex); 107410be98a7SChris Wilson lut_close(ctx); 1075155ab883SChris Wilson mutex_unlock(&ctx->mutex); 107610be98a7SChris Wilson 1077e568ac38SChris Wilson old = __set_ppgtt(ctx, vm); 107810be98a7SChris Wilson 107910be98a7SChris Wilson /* 108010be98a7SChris Wilson * We need to flush any requests using the current ppgtt before 108110be98a7SChris Wilson * we release it as the requests do not hold a reference themselves, 108210be98a7SChris Wilson * only indirectly through the context. 108310be98a7SChris Wilson */ 108410be98a7SChris Wilson err = context_barrier_task(ctx, ALL_ENGINES, 10851fe2d6f9SChris Wilson skip_ppgtt_update, 108610be98a7SChris Wilson emit_ppgtt_update, 108710be98a7SChris Wilson set_ppgtt_barrier, 108810be98a7SChris Wilson old); 108910be98a7SChris Wilson if (err) { 1090f5d974f9SChris Wilson i915_vm_put(__set_ppgtt(ctx, old)); 1091f5d974f9SChris Wilson i915_vm_put(old); 109210be98a7SChris Wilson } 109310be98a7SChris Wilson 109410be98a7SChris Wilson unlock: 109510be98a7SChris Wilson mutex_unlock(&ctx->i915->drm.struct_mutex); 109610be98a7SChris Wilson 109710be98a7SChris Wilson out: 1098e568ac38SChris Wilson i915_vm_put(vm); 109910be98a7SChris Wilson return err; 110010be98a7SChris Wilson } 110110be98a7SChris Wilson 110210be98a7SChris Wilson static int gen8_emit_rpcs_config(struct i915_request *rq, 110310be98a7SChris Wilson struct intel_context *ce, 110410be98a7SChris Wilson struct intel_sseu sseu) 110510be98a7SChris Wilson { 110610be98a7SChris Wilson u64 offset; 110710be98a7SChris Wilson u32 *cs; 110810be98a7SChris Wilson 110910be98a7SChris Wilson cs = intel_ring_begin(rq, 4); 111010be98a7SChris Wilson if (IS_ERR(cs)) 111110be98a7SChris Wilson return PTR_ERR(cs); 111210be98a7SChris Wilson 111310be98a7SChris Wilson offset = i915_ggtt_offset(ce->state) + 111410be98a7SChris Wilson LRC_STATE_PN * PAGE_SIZE + 111510be98a7SChris Wilson (CTX_R_PWR_CLK_STATE + 1) * 4; 111610be98a7SChris Wilson 111710be98a7SChris Wilson *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 111810be98a7SChris Wilson *cs++ = lower_32_bits(offset); 111910be98a7SChris Wilson *cs++ = upper_32_bits(offset); 112010be98a7SChris Wilson *cs++ = intel_sseu_make_rpcs(rq->i915, &sseu); 112110be98a7SChris Wilson 112210be98a7SChris Wilson intel_ring_advance(rq, cs); 112310be98a7SChris Wilson 112410be98a7SChris Wilson return 0; 112510be98a7SChris Wilson } 112610be98a7SChris Wilson 112710be98a7SChris Wilson static int 112810be98a7SChris Wilson gen8_modify_rpcs(struct intel_context *ce, struct intel_sseu sseu) 112910be98a7SChris Wilson { 113010be98a7SChris Wilson struct i915_request *rq; 113110be98a7SChris Wilson int ret; 113210be98a7SChris Wilson 113310be98a7SChris Wilson lockdep_assert_held(&ce->pin_mutex); 113410be98a7SChris Wilson 113510be98a7SChris Wilson /* 113610be98a7SChris Wilson * If the context is not idle, we have to submit an ordered request to 113710be98a7SChris Wilson * modify its context image via the kernel context (writing to our own 113810be98a7SChris Wilson * image, or into the registers directory, does not stick). Pristine 113910be98a7SChris Wilson * and idle contexts will be configured on pinning. 114010be98a7SChris Wilson */ 114110be98a7SChris Wilson if (!intel_context_is_pinned(ce)) 114210be98a7SChris Wilson return 0; 114310be98a7SChris Wilson 114410be98a7SChris Wilson rq = i915_request_create(ce->engine->kernel_context); 114510be98a7SChris Wilson if (IS_ERR(rq)) 114610be98a7SChris Wilson return PTR_ERR(rq); 114710be98a7SChris Wilson 1148a9877da2SChris Wilson /* Serialise with the remote context */ 1149a9877da2SChris Wilson ret = intel_context_prepare_remote_request(ce, rq); 1150a9877da2SChris Wilson if (ret == 0) 1151ce476c80SChris Wilson ret = gen8_emit_rpcs_config(rq, ce, sseu); 115210be98a7SChris Wilson 115310be98a7SChris Wilson i915_request_add(rq); 115410be98a7SChris Wilson return ret; 115510be98a7SChris Wilson } 115610be98a7SChris Wilson 115710be98a7SChris Wilson static int 115810be98a7SChris Wilson __intel_context_reconfigure_sseu(struct intel_context *ce, 115910be98a7SChris Wilson struct intel_sseu sseu) 116010be98a7SChris Wilson { 116110be98a7SChris Wilson int ret; 116210be98a7SChris Wilson 1163cb0c43f3SChris Wilson GEM_BUG_ON(INTEL_GEN(ce->engine->i915) < 8); 116410be98a7SChris Wilson 116510be98a7SChris Wilson ret = intel_context_lock_pinned(ce); 116610be98a7SChris Wilson if (ret) 116710be98a7SChris Wilson return ret; 116810be98a7SChris Wilson 116910be98a7SChris Wilson /* Nothing to do if unmodified. */ 117010be98a7SChris Wilson if (!memcmp(&ce->sseu, &sseu, sizeof(sseu))) 117110be98a7SChris Wilson goto unlock; 117210be98a7SChris Wilson 117310be98a7SChris Wilson ret = gen8_modify_rpcs(ce, sseu); 117410be98a7SChris Wilson if (!ret) 117510be98a7SChris Wilson ce->sseu = sseu; 117610be98a7SChris Wilson 117710be98a7SChris Wilson unlock: 117810be98a7SChris Wilson intel_context_unlock_pinned(ce); 117910be98a7SChris Wilson return ret; 118010be98a7SChris Wilson } 118110be98a7SChris Wilson 118210be98a7SChris Wilson static int 118310be98a7SChris Wilson intel_context_reconfigure_sseu(struct intel_context *ce, struct intel_sseu sseu) 118410be98a7SChris Wilson { 1185cb0c43f3SChris Wilson struct drm_i915_private *i915 = ce->engine->i915; 118610be98a7SChris Wilson int ret; 118710be98a7SChris Wilson 118810be98a7SChris Wilson ret = mutex_lock_interruptible(&i915->drm.struct_mutex); 118910be98a7SChris Wilson if (ret) 119010be98a7SChris Wilson return ret; 119110be98a7SChris Wilson 119210be98a7SChris Wilson ret = __intel_context_reconfigure_sseu(ce, sseu); 119310be98a7SChris Wilson 119410be98a7SChris Wilson mutex_unlock(&i915->drm.struct_mutex); 119510be98a7SChris Wilson 119610be98a7SChris Wilson return ret; 119710be98a7SChris Wilson } 119810be98a7SChris Wilson 119910be98a7SChris Wilson static int 120010be98a7SChris Wilson user_to_context_sseu(struct drm_i915_private *i915, 120110be98a7SChris Wilson const struct drm_i915_gem_context_param_sseu *user, 120210be98a7SChris Wilson struct intel_sseu *context) 120310be98a7SChris Wilson { 120410be98a7SChris Wilson const struct sseu_dev_info *device = &RUNTIME_INFO(i915)->sseu; 120510be98a7SChris Wilson 120610be98a7SChris Wilson /* No zeros in any field. */ 120710be98a7SChris Wilson if (!user->slice_mask || !user->subslice_mask || 120810be98a7SChris Wilson !user->min_eus_per_subslice || !user->max_eus_per_subslice) 120910be98a7SChris Wilson return -EINVAL; 121010be98a7SChris Wilson 121110be98a7SChris Wilson /* Max > min. */ 121210be98a7SChris Wilson if (user->max_eus_per_subslice < user->min_eus_per_subslice) 121310be98a7SChris Wilson return -EINVAL; 121410be98a7SChris Wilson 121510be98a7SChris Wilson /* 121610be98a7SChris Wilson * Some future proofing on the types since the uAPI is wider than the 121710be98a7SChris Wilson * current internal implementation. 121810be98a7SChris Wilson */ 121910be98a7SChris Wilson if (overflows_type(user->slice_mask, context->slice_mask) || 122010be98a7SChris Wilson overflows_type(user->subslice_mask, context->subslice_mask) || 122110be98a7SChris Wilson overflows_type(user->min_eus_per_subslice, 122210be98a7SChris Wilson context->min_eus_per_subslice) || 122310be98a7SChris Wilson overflows_type(user->max_eus_per_subslice, 122410be98a7SChris Wilson context->max_eus_per_subslice)) 122510be98a7SChris Wilson return -EINVAL; 122610be98a7SChris Wilson 122710be98a7SChris Wilson /* Check validity against hardware. */ 122810be98a7SChris Wilson if (user->slice_mask & ~device->slice_mask) 122910be98a7SChris Wilson return -EINVAL; 123010be98a7SChris Wilson 123110be98a7SChris Wilson if (user->subslice_mask & ~device->subslice_mask[0]) 123210be98a7SChris Wilson return -EINVAL; 123310be98a7SChris Wilson 123410be98a7SChris Wilson if (user->max_eus_per_subslice > device->max_eus_per_subslice) 123510be98a7SChris Wilson return -EINVAL; 123610be98a7SChris Wilson 123710be98a7SChris Wilson context->slice_mask = user->slice_mask; 123810be98a7SChris Wilson context->subslice_mask = user->subslice_mask; 123910be98a7SChris Wilson context->min_eus_per_subslice = user->min_eus_per_subslice; 124010be98a7SChris Wilson context->max_eus_per_subslice = user->max_eus_per_subslice; 124110be98a7SChris Wilson 124210be98a7SChris Wilson /* Part specific restrictions. */ 124310be98a7SChris Wilson if (IS_GEN(i915, 11)) { 124410be98a7SChris Wilson unsigned int hw_s = hweight8(device->slice_mask); 124510be98a7SChris Wilson unsigned int hw_ss_per_s = hweight8(device->subslice_mask[0]); 124610be98a7SChris Wilson unsigned int req_s = hweight8(context->slice_mask); 124710be98a7SChris Wilson unsigned int req_ss = hweight8(context->subslice_mask); 124810be98a7SChris Wilson 124910be98a7SChris Wilson /* 125010be98a7SChris Wilson * Only full subslice enablement is possible if more than one 125110be98a7SChris Wilson * slice is turned on. 125210be98a7SChris Wilson */ 125310be98a7SChris Wilson if (req_s > 1 && req_ss != hw_ss_per_s) 125410be98a7SChris Wilson return -EINVAL; 125510be98a7SChris Wilson 125610be98a7SChris Wilson /* 125710be98a7SChris Wilson * If more than four (SScount bitfield limit) subslices are 125810be98a7SChris Wilson * requested then the number has to be even. 125910be98a7SChris Wilson */ 126010be98a7SChris Wilson if (req_ss > 4 && (req_ss & 1)) 126110be98a7SChris Wilson return -EINVAL; 126210be98a7SChris Wilson 126310be98a7SChris Wilson /* 126410be98a7SChris Wilson * If only one slice is enabled and subslice count is below the 126510be98a7SChris Wilson * device full enablement, it must be at most half of the all 126610be98a7SChris Wilson * available subslices. 126710be98a7SChris Wilson */ 126810be98a7SChris Wilson if (req_s == 1 && req_ss < hw_ss_per_s && 126910be98a7SChris Wilson req_ss > (hw_ss_per_s / 2)) 127010be98a7SChris Wilson return -EINVAL; 127110be98a7SChris Wilson 127210be98a7SChris Wilson /* ABI restriction - VME use case only. */ 127310be98a7SChris Wilson 127410be98a7SChris Wilson /* All slices or one slice only. */ 127510be98a7SChris Wilson if (req_s != 1 && req_s != hw_s) 127610be98a7SChris Wilson return -EINVAL; 127710be98a7SChris Wilson 127810be98a7SChris Wilson /* 127910be98a7SChris Wilson * Half subslices or full enablement only when one slice is 128010be98a7SChris Wilson * enabled. 128110be98a7SChris Wilson */ 128210be98a7SChris Wilson if (req_s == 1 && 128310be98a7SChris Wilson (req_ss != hw_ss_per_s && req_ss != (hw_ss_per_s / 2))) 128410be98a7SChris Wilson return -EINVAL; 128510be98a7SChris Wilson 128610be98a7SChris Wilson /* No EU configuration changes. */ 128710be98a7SChris Wilson if ((user->min_eus_per_subslice != 128810be98a7SChris Wilson device->max_eus_per_subslice) || 128910be98a7SChris Wilson (user->max_eus_per_subslice != 129010be98a7SChris Wilson device->max_eus_per_subslice)) 129110be98a7SChris Wilson return -EINVAL; 129210be98a7SChris Wilson } 129310be98a7SChris Wilson 129410be98a7SChris Wilson return 0; 129510be98a7SChris Wilson } 129610be98a7SChris Wilson 129710be98a7SChris Wilson static int set_sseu(struct i915_gem_context *ctx, 129810be98a7SChris Wilson struct drm_i915_gem_context_param *args) 129910be98a7SChris Wilson { 130010be98a7SChris Wilson struct drm_i915_private *i915 = ctx->i915; 130110be98a7SChris Wilson struct drm_i915_gem_context_param_sseu user_sseu; 130210be98a7SChris Wilson struct intel_context *ce; 130310be98a7SChris Wilson struct intel_sseu sseu; 130410be98a7SChris Wilson unsigned long lookup; 130510be98a7SChris Wilson int ret; 130610be98a7SChris Wilson 130710be98a7SChris Wilson if (args->size < sizeof(user_sseu)) 130810be98a7SChris Wilson return -EINVAL; 130910be98a7SChris Wilson 131010be98a7SChris Wilson if (!IS_GEN(i915, 11)) 131110be98a7SChris Wilson return -ENODEV; 131210be98a7SChris Wilson 131310be98a7SChris Wilson if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), 131410be98a7SChris Wilson sizeof(user_sseu))) 131510be98a7SChris Wilson return -EFAULT; 131610be98a7SChris Wilson 131710be98a7SChris Wilson if (user_sseu.rsvd) 131810be98a7SChris Wilson return -EINVAL; 131910be98a7SChris Wilson 132010be98a7SChris Wilson if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) 132110be98a7SChris Wilson return -EINVAL; 132210be98a7SChris Wilson 132310be98a7SChris Wilson lookup = 0; 132410be98a7SChris Wilson if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) 132510be98a7SChris Wilson lookup |= LOOKUP_USER_INDEX; 132610be98a7SChris Wilson 132710be98a7SChris Wilson ce = lookup_user_engine(ctx, lookup, &user_sseu.engine); 132810be98a7SChris Wilson if (IS_ERR(ce)) 132910be98a7SChris Wilson return PTR_ERR(ce); 133010be98a7SChris Wilson 133110be98a7SChris Wilson /* Only render engine supports RPCS configuration. */ 133210be98a7SChris Wilson if (ce->engine->class != RENDER_CLASS) { 133310be98a7SChris Wilson ret = -ENODEV; 133410be98a7SChris Wilson goto out_ce; 133510be98a7SChris Wilson } 133610be98a7SChris Wilson 133710be98a7SChris Wilson ret = user_to_context_sseu(i915, &user_sseu, &sseu); 133810be98a7SChris Wilson if (ret) 133910be98a7SChris Wilson goto out_ce; 134010be98a7SChris Wilson 134110be98a7SChris Wilson ret = intel_context_reconfigure_sseu(ce, sseu); 134210be98a7SChris Wilson if (ret) 134310be98a7SChris Wilson goto out_ce; 134410be98a7SChris Wilson 134510be98a7SChris Wilson args->size = sizeof(user_sseu); 134610be98a7SChris Wilson 134710be98a7SChris Wilson out_ce: 134810be98a7SChris Wilson intel_context_put(ce); 134910be98a7SChris Wilson return ret; 135010be98a7SChris Wilson } 135110be98a7SChris Wilson 135210be98a7SChris Wilson struct set_engines { 135310be98a7SChris Wilson struct i915_gem_context *ctx; 135410be98a7SChris Wilson struct i915_gem_engines *engines; 135510be98a7SChris Wilson }; 135610be98a7SChris Wilson 135710be98a7SChris Wilson static int 135810be98a7SChris Wilson set_engines__load_balance(struct i915_user_extension __user *base, void *data) 135910be98a7SChris Wilson { 136010be98a7SChris Wilson struct i915_context_engines_load_balance __user *ext = 136110be98a7SChris Wilson container_of_user(base, typeof(*ext), base); 136210be98a7SChris Wilson const struct set_engines *set = data; 136310be98a7SChris Wilson struct intel_engine_cs *stack[16]; 136410be98a7SChris Wilson struct intel_engine_cs **siblings; 136510be98a7SChris Wilson struct intel_context *ce; 136610be98a7SChris Wilson u16 num_siblings, idx; 136710be98a7SChris Wilson unsigned int n; 136810be98a7SChris Wilson int err; 136910be98a7SChris Wilson 137010be98a7SChris Wilson if (!HAS_EXECLISTS(set->ctx->i915)) 137110be98a7SChris Wilson return -ENODEV; 137210be98a7SChris Wilson 137310be98a7SChris Wilson if (USES_GUC_SUBMISSION(set->ctx->i915)) 137410be98a7SChris Wilson return -ENODEV; /* not implement yet */ 137510be98a7SChris Wilson 137610be98a7SChris Wilson if (get_user(idx, &ext->engine_index)) 137710be98a7SChris Wilson return -EFAULT; 137810be98a7SChris Wilson 137910be98a7SChris Wilson if (idx >= set->engines->num_engines) { 138010be98a7SChris Wilson DRM_DEBUG("Invalid placement value, %d >= %d\n", 138110be98a7SChris Wilson idx, set->engines->num_engines); 138210be98a7SChris Wilson return -EINVAL; 138310be98a7SChris Wilson } 138410be98a7SChris Wilson 138510be98a7SChris Wilson idx = array_index_nospec(idx, set->engines->num_engines); 138610be98a7SChris Wilson if (set->engines->engines[idx]) { 138710be98a7SChris Wilson DRM_DEBUG("Invalid placement[%d], already occupied\n", idx); 138810be98a7SChris Wilson return -EEXIST; 138910be98a7SChris Wilson } 139010be98a7SChris Wilson 139110be98a7SChris Wilson if (get_user(num_siblings, &ext->num_siblings)) 139210be98a7SChris Wilson return -EFAULT; 139310be98a7SChris Wilson 139410be98a7SChris Wilson err = check_user_mbz(&ext->flags); 139510be98a7SChris Wilson if (err) 139610be98a7SChris Wilson return err; 139710be98a7SChris Wilson 139810be98a7SChris Wilson err = check_user_mbz(&ext->mbz64); 139910be98a7SChris Wilson if (err) 140010be98a7SChris Wilson return err; 140110be98a7SChris Wilson 140210be98a7SChris Wilson siblings = stack; 140310be98a7SChris Wilson if (num_siblings > ARRAY_SIZE(stack)) { 140410be98a7SChris Wilson siblings = kmalloc_array(num_siblings, 140510be98a7SChris Wilson sizeof(*siblings), 140610be98a7SChris Wilson GFP_KERNEL); 140710be98a7SChris Wilson if (!siblings) 140810be98a7SChris Wilson return -ENOMEM; 140910be98a7SChris Wilson } 141010be98a7SChris Wilson 141110be98a7SChris Wilson for (n = 0; n < num_siblings; n++) { 141210be98a7SChris Wilson struct i915_engine_class_instance ci; 141310be98a7SChris Wilson 141410be98a7SChris Wilson if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) { 141510be98a7SChris Wilson err = -EFAULT; 141610be98a7SChris Wilson goto out_siblings; 141710be98a7SChris Wilson } 141810be98a7SChris Wilson 141910be98a7SChris Wilson siblings[n] = intel_engine_lookup_user(set->ctx->i915, 142010be98a7SChris Wilson ci.engine_class, 142110be98a7SChris Wilson ci.engine_instance); 142210be98a7SChris Wilson if (!siblings[n]) { 142310be98a7SChris Wilson DRM_DEBUG("Invalid sibling[%d]: { class:%d, inst:%d }\n", 142410be98a7SChris Wilson n, ci.engine_class, ci.engine_instance); 142510be98a7SChris Wilson err = -EINVAL; 142610be98a7SChris Wilson goto out_siblings; 142710be98a7SChris Wilson } 142810be98a7SChris Wilson } 142910be98a7SChris Wilson 143010be98a7SChris Wilson ce = intel_execlists_create_virtual(set->ctx, siblings, n); 143110be98a7SChris Wilson if (IS_ERR(ce)) { 143210be98a7SChris Wilson err = PTR_ERR(ce); 143310be98a7SChris Wilson goto out_siblings; 143410be98a7SChris Wilson } 143510be98a7SChris Wilson 143610be98a7SChris Wilson if (cmpxchg(&set->engines->engines[idx], NULL, ce)) { 143710be98a7SChris Wilson intel_context_put(ce); 143810be98a7SChris Wilson err = -EEXIST; 143910be98a7SChris Wilson goto out_siblings; 144010be98a7SChris Wilson } 144110be98a7SChris Wilson 144210be98a7SChris Wilson out_siblings: 144310be98a7SChris Wilson if (siblings != stack) 144410be98a7SChris Wilson kfree(siblings); 144510be98a7SChris Wilson 144610be98a7SChris Wilson return err; 144710be98a7SChris Wilson } 144810be98a7SChris Wilson 144910be98a7SChris Wilson static int 145010be98a7SChris Wilson set_engines__bond(struct i915_user_extension __user *base, void *data) 145110be98a7SChris Wilson { 145210be98a7SChris Wilson struct i915_context_engines_bond __user *ext = 145310be98a7SChris Wilson container_of_user(base, typeof(*ext), base); 145410be98a7SChris Wilson const struct set_engines *set = data; 145510be98a7SChris Wilson struct i915_engine_class_instance ci; 145610be98a7SChris Wilson struct intel_engine_cs *virtual; 145710be98a7SChris Wilson struct intel_engine_cs *master; 145810be98a7SChris Wilson u16 idx, num_bonds; 145910be98a7SChris Wilson int err, n; 146010be98a7SChris Wilson 146110be98a7SChris Wilson if (get_user(idx, &ext->virtual_index)) 146210be98a7SChris Wilson return -EFAULT; 146310be98a7SChris Wilson 146410be98a7SChris Wilson if (idx >= set->engines->num_engines) { 146510be98a7SChris Wilson DRM_DEBUG("Invalid index for virtual engine: %d >= %d\n", 146610be98a7SChris Wilson idx, set->engines->num_engines); 146710be98a7SChris Wilson return -EINVAL; 146810be98a7SChris Wilson } 146910be98a7SChris Wilson 147010be98a7SChris Wilson idx = array_index_nospec(idx, set->engines->num_engines); 147110be98a7SChris Wilson if (!set->engines->engines[idx]) { 147210be98a7SChris Wilson DRM_DEBUG("Invalid engine at %d\n", idx); 147310be98a7SChris Wilson return -EINVAL; 147410be98a7SChris Wilson } 147510be98a7SChris Wilson virtual = set->engines->engines[idx]->engine; 147610be98a7SChris Wilson 147710be98a7SChris Wilson err = check_user_mbz(&ext->flags); 147810be98a7SChris Wilson if (err) 147910be98a7SChris Wilson return err; 148010be98a7SChris Wilson 148110be98a7SChris Wilson for (n = 0; n < ARRAY_SIZE(ext->mbz64); n++) { 148210be98a7SChris Wilson err = check_user_mbz(&ext->mbz64[n]); 148310be98a7SChris Wilson if (err) 148410be98a7SChris Wilson return err; 148510be98a7SChris Wilson } 148610be98a7SChris Wilson 148710be98a7SChris Wilson if (copy_from_user(&ci, &ext->master, sizeof(ci))) 148810be98a7SChris Wilson return -EFAULT; 148910be98a7SChris Wilson 149010be98a7SChris Wilson master = intel_engine_lookup_user(set->ctx->i915, 149110be98a7SChris Wilson ci.engine_class, ci.engine_instance); 149210be98a7SChris Wilson if (!master) { 149310be98a7SChris Wilson DRM_DEBUG("Unrecognised master engine: { class:%u, instance:%u }\n", 149410be98a7SChris Wilson ci.engine_class, ci.engine_instance); 149510be98a7SChris Wilson return -EINVAL; 149610be98a7SChris Wilson } 149710be98a7SChris Wilson 149810be98a7SChris Wilson if (get_user(num_bonds, &ext->num_bonds)) 149910be98a7SChris Wilson return -EFAULT; 150010be98a7SChris Wilson 150110be98a7SChris Wilson for (n = 0; n < num_bonds; n++) { 150210be98a7SChris Wilson struct intel_engine_cs *bond; 150310be98a7SChris Wilson 150410be98a7SChris Wilson if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) 150510be98a7SChris Wilson return -EFAULT; 150610be98a7SChris Wilson 150710be98a7SChris Wilson bond = intel_engine_lookup_user(set->ctx->i915, 150810be98a7SChris Wilson ci.engine_class, 150910be98a7SChris Wilson ci.engine_instance); 151010be98a7SChris Wilson if (!bond) { 151110be98a7SChris Wilson DRM_DEBUG("Unrecognised engine[%d] for bonding: { class:%d, instance: %d }\n", 151210be98a7SChris Wilson n, ci.engine_class, ci.engine_instance); 151310be98a7SChris Wilson return -EINVAL; 151410be98a7SChris Wilson } 151510be98a7SChris Wilson 151610be98a7SChris Wilson /* 151710be98a7SChris Wilson * A non-virtual engine has no siblings to choose between; and 151810be98a7SChris Wilson * a submit fence will always be directed to the one engine. 151910be98a7SChris Wilson */ 152010be98a7SChris Wilson if (intel_engine_is_virtual(virtual)) { 152110be98a7SChris Wilson err = intel_virtual_engine_attach_bond(virtual, 152210be98a7SChris Wilson master, 152310be98a7SChris Wilson bond); 152410be98a7SChris Wilson if (err) 152510be98a7SChris Wilson return err; 152610be98a7SChris Wilson } 152710be98a7SChris Wilson } 152810be98a7SChris Wilson 152910be98a7SChris Wilson return 0; 153010be98a7SChris Wilson } 153110be98a7SChris Wilson 153210be98a7SChris Wilson static const i915_user_extension_fn set_engines__extensions[] = { 153310be98a7SChris Wilson [I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE] = set_engines__load_balance, 153410be98a7SChris Wilson [I915_CONTEXT_ENGINES_EXT_BOND] = set_engines__bond, 153510be98a7SChris Wilson }; 153610be98a7SChris Wilson 153710be98a7SChris Wilson static int 153810be98a7SChris Wilson set_engines(struct i915_gem_context *ctx, 153910be98a7SChris Wilson const struct drm_i915_gem_context_param *args) 154010be98a7SChris Wilson { 154110be98a7SChris Wilson struct i915_context_param_engines __user *user = 154210be98a7SChris Wilson u64_to_user_ptr(args->value); 154310be98a7SChris Wilson struct set_engines set = { .ctx = ctx }; 154410be98a7SChris Wilson unsigned int num_engines, n; 154510be98a7SChris Wilson u64 extensions; 154610be98a7SChris Wilson int err; 154710be98a7SChris Wilson 154810be98a7SChris Wilson if (!args->size) { /* switch back to legacy user_ring_map */ 154910be98a7SChris Wilson if (!i915_gem_context_user_engines(ctx)) 155010be98a7SChris Wilson return 0; 155110be98a7SChris Wilson 155210be98a7SChris Wilson set.engines = default_engines(ctx); 155310be98a7SChris Wilson if (IS_ERR(set.engines)) 155410be98a7SChris Wilson return PTR_ERR(set.engines); 155510be98a7SChris Wilson 155610be98a7SChris Wilson goto replace; 155710be98a7SChris Wilson } 155810be98a7SChris Wilson 155910be98a7SChris Wilson BUILD_BUG_ON(!IS_ALIGNED(sizeof(*user), sizeof(*user->engines))); 156010be98a7SChris Wilson if (args->size < sizeof(*user) || 156110be98a7SChris Wilson !IS_ALIGNED(args->size, sizeof(*user->engines))) { 156210be98a7SChris Wilson DRM_DEBUG("Invalid size for engine array: %d\n", 156310be98a7SChris Wilson args->size); 156410be98a7SChris Wilson return -EINVAL; 156510be98a7SChris Wilson } 156610be98a7SChris Wilson 156710be98a7SChris Wilson /* 156810be98a7SChris Wilson * Note that I915_EXEC_RING_MASK limits execbuf to only using the 156910be98a7SChris Wilson * first 64 engines defined here. 157010be98a7SChris Wilson */ 157110be98a7SChris Wilson num_engines = (args->size - sizeof(*user)) / sizeof(*user->engines); 157210be98a7SChris Wilson 157310be98a7SChris Wilson set.engines = kmalloc(struct_size(set.engines, engines, num_engines), 157410be98a7SChris Wilson GFP_KERNEL); 157510be98a7SChris Wilson if (!set.engines) 157610be98a7SChris Wilson return -ENOMEM; 157710be98a7SChris Wilson 1578155ab883SChris Wilson init_rcu_head(&set.engines->rcu); 157910be98a7SChris Wilson for (n = 0; n < num_engines; n++) { 158010be98a7SChris Wilson struct i915_engine_class_instance ci; 158110be98a7SChris Wilson struct intel_engine_cs *engine; 158248ae397bSChris Wilson struct intel_context *ce; 158310be98a7SChris Wilson 158410be98a7SChris Wilson if (copy_from_user(&ci, &user->engines[n], sizeof(ci))) { 158510be98a7SChris Wilson __free_engines(set.engines, n); 158610be98a7SChris Wilson return -EFAULT; 158710be98a7SChris Wilson } 158810be98a7SChris Wilson 158910be98a7SChris Wilson if (ci.engine_class == (u16)I915_ENGINE_CLASS_INVALID && 159010be98a7SChris Wilson ci.engine_instance == (u16)I915_ENGINE_CLASS_INVALID_NONE) { 159110be98a7SChris Wilson set.engines->engines[n] = NULL; 159210be98a7SChris Wilson continue; 159310be98a7SChris Wilson } 159410be98a7SChris Wilson 159510be98a7SChris Wilson engine = intel_engine_lookup_user(ctx->i915, 159610be98a7SChris Wilson ci.engine_class, 159710be98a7SChris Wilson ci.engine_instance); 159810be98a7SChris Wilson if (!engine) { 159910be98a7SChris Wilson DRM_DEBUG("Invalid engine[%d]: { class:%d, instance:%d }\n", 160010be98a7SChris Wilson n, ci.engine_class, ci.engine_instance); 160110be98a7SChris Wilson __free_engines(set.engines, n); 160210be98a7SChris Wilson return -ENOENT; 160310be98a7SChris Wilson } 160410be98a7SChris Wilson 160548ae397bSChris Wilson ce = intel_context_create(ctx, engine); 160648ae397bSChris Wilson if (IS_ERR(ce)) { 160710be98a7SChris Wilson __free_engines(set.engines, n); 160848ae397bSChris Wilson return PTR_ERR(ce); 160910be98a7SChris Wilson } 161048ae397bSChris Wilson 161148ae397bSChris Wilson set.engines->engines[n] = ce; 161210be98a7SChris Wilson } 161310be98a7SChris Wilson set.engines->num_engines = num_engines; 161410be98a7SChris Wilson 161510be98a7SChris Wilson err = -EFAULT; 161610be98a7SChris Wilson if (!get_user(extensions, &user->extensions)) 161710be98a7SChris Wilson err = i915_user_extensions(u64_to_user_ptr(extensions), 161810be98a7SChris Wilson set_engines__extensions, 161910be98a7SChris Wilson ARRAY_SIZE(set_engines__extensions), 162010be98a7SChris Wilson &set); 162110be98a7SChris Wilson if (err) { 162210be98a7SChris Wilson free_engines(set.engines); 162310be98a7SChris Wilson return err; 162410be98a7SChris Wilson } 162510be98a7SChris Wilson 162610be98a7SChris Wilson replace: 162710be98a7SChris Wilson mutex_lock(&ctx->engines_mutex); 162810be98a7SChris Wilson if (args->size) 162910be98a7SChris Wilson i915_gem_context_set_user_engines(ctx); 163010be98a7SChris Wilson else 163110be98a7SChris Wilson i915_gem_context_clear_user_engines(ctx); 163210be98a7SChris Wilson rcu_swap_protected(ctx->engines, set.engines, 1); 163310be98a7SChris Wilson mutex_unlock(&ctx->engines_mutex); 163410be98a7SChris Wilson 1635155ab883SChris Wilson call_rcu(&set.engines->rcu, free_engines_rcu); 163610be98a7SChris Wilson 163710be98a7SChris Wilson return 0; 163810be98a7SChris Wilson } 163910be98a7SChris Wilson 164010be98a7SChris Wilson static struct i915_gem_engines * 164110be98a7SChris Wilson __copy_engines(struct i915_gem_engines *e) 164210be98a7SChris Wilson { 164310be98a7SChris Wilson struct i915_gem_engines *copy; 164410be98a7SChris Wilson unsigned int n; 164510be98a7SChris Wilson 164610be98a7SChris Wilson copy = kmalloc(struct_size(e, engines, e->num_engines), GFP_KERNEL); 164710be98a7SChris Wilson if (!copy) 164810be98a7SChris Wilson return ERR_PTR(-ENOMEM); 164910be98a7SChris Wilson 1650155ab883SChris Wilson init_rcu_head(©->rcu); 165110be98a7SChris Wilson for (n = 0; n < e->num_engines; n++) { 165210be98a7SChris Wilson if (e->engines[n]) 165310be98a7SChris Wilson copy->engines[n] = intel_context_get(e->engines[n]); 165410be98a7SChris Wilson else 165510be98a7SChris Wilson copy->engines[n] = NULL; 165610be98a7SChris Wilson } 165710be98a7SChris Wilson copy->num_engines = n; 165810be98a7SChris Wilson 165910be98a7SChris Wilson return copy; 166010be98a7SChris Wilson } 166110be98a7SChris Wilson 166210be98a7SChris Wilson static int 166310be98a7SChris Wilson get_engines(struct i915_gem_context *ctx, 166410be98a7SChris Wilson struct drm_i915_gem_context_param *args) 166510be98a7SChris Wilson { 166610be98a7SChris Wilson struct i915_context_param_engines __user *user; 166710be98a7SChris Wilson struct i915_gem_engines *e; 166810be98a7SChris Wilson size_t n, count, size; 166910be98a7SChris Wilson int err = 0; 167010be98a7SChris Wilson 167110be98a7SChris Wilson err = mutex_lock_interruptible(&ctx->engines_mutex); 167210be98a7SChris Wilson if (err) 167310be98a7SChris Wilson return err; 167410be98a7SChris Wilson 167510be98a7SChris Wilson e = NULL; 167610be98a7SChris Wilson if (i915_gem_context_user_engines(ctx)) 167710be98a7SChris Wilson e = __copy_engines(i915_gem_context_engines(ctx)); 167810be98a7SChris Wilson mutex_unlock(&ctx->engines_mutex); 167910be98a7SChris Wilson if (IS_ERR_OR_NULL(e)) { 168010be98a7SChris Wilson args->size = 0; 168110be98a7SChris Wilson return PTR_ERR_OR_ZERO(e); 168210be98a7SChris Wilson } 168310be98a7SChris Wilson 168410be98a7SChris Wilson count = e->num_engines; 168510be98a7SChris Wilson 168610be98a7SChris Wilson /* Be paranoid in case we have an impedance mismatch */ 168710be98a7SChris Wilson if (!check_struct_size(user, engines, count, &size)) { 168810be98a7SChris Wilson err = -EINVAL; 168910be98a7SChris Wilson goto err_free; 169010be98a7SChris Wilson } 169110be98a7SChris Wilson if (overflows_type(size, args->size)) { 169210be98a7SChris Wilson err = -EINVAL; 169310be98a7SChris Wilson goto err_free; 169410be98a7SChris Wilson } 169510be98a7SChris Wilson 169610be98a7SChris Wilson if (!args->size) { 169710be98a7SChris Wilson args->size = size; 169810be98a7SChris Wilson goto err_free; 169910be98a7SChris Wilson } 170010be98a7SChris Wilson 170110be98a7SChris Wilson if (args->size < size) { 170210be98a7SChris Wilson err = -EINVAL; 170310be98a7SChris Wilson goto err_free; 170410be98a7SChris Wilson } 170510be98a7SChris Wilson 170610be98a7SChris Wilson user = u64_to_user_ptr(args->value); 170710be98a7SChris Wilson if (!access_ok(user, size)) { 170810be98a7SChris Wilson err = -EFAULT; 170910be98a7SChris Wilson goto err_free; 171010be98a7SChris Wilson } 171110be98a7SChris Wilson 171210be98a7SChris Wilson if (put_user(0, &user->extensions)) { 171310be98a7SChris Wilson err = -EFAULT; 171410be98a7SChris Wilson goto err_free; 171510be98a7SChris Wilson } 171610be98a7SChris Wilson 171710be98a7SChris Wilson for (n = 0; n < count; n++) { 171810be98a7SChris Wilson struct i915_engine_class_instance ci = { 171910be98a7SChris Wilson .engine_class = I915_ENGINE_CLASS_INVALID, 172010be98a7SChris Wilson .engine_instance = I915_ENGINE_CLASS_INVALID_NONE, 172110be98a7SChris Wilson }; 172210be98a7SChris Wilson 172310be98a7SChris Wilson if (e->engines[n]) { 172410be98a7SChris Wilson ci.engine_class = e->engines[n]->engine->uabi_class; 1725750e76b4SChris Wilson ci.engine_instance = e->engines[n]->engine->uabi_instance; 172610be98a7SChris Wilson } 172710be98a7SChris Wilson 172810be98a7SChris Wilson if (copy_to_user(&user->engines[n], &ci, sizeof(ci))) { 172910be98a7SChris Wilson err = -EFAULT; 173010be98a7SChris Wilson goto err_free; 173110be98a7SChris Wilson } 173210be98a7SChris Wilson } 173310be98a7SChris Wilson 173410be98a7SChris Wilson args->size = size; 173510be98a7SChris Wilson 173610be98a7SChris Wilson err_free: 1737155ab883SChris Wilson free_engines(e); 173810be98a7SChris Wilson return err; 173910be98a7SChris Wilson } 174010be98a7SChris Wilson 174110be98a7SChris Wilson static int ctx_setparam(struct drm_i915_file_private *fpriv, 174210be98a7SChris Wilson struct i915_gem_context *ctx, 174310be98a7SChris Wilson struct drm_i915_gem_context_param *args) 174410be98a7SChris Wilson { 174510be98a7SChris Wilson int ret = 0; 174610be98a7SChris Wilson 174710be98a7SChris Wilson switch (args->param) { 174810be98a7SChris Wilson case I915_CONTEXT_PARAM_NO_ZEROMAP: 174910be98a7SChris Wilson if (args->size) 175010be98a7SChris Wilson ret = -EINVAL; 175110be98a7SChris Wilson else if (args->value) 175210be98a7SChris Wilson set_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); 175310be98a7SChris Wilson else 175410be98a7SChris Wilson clear_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); 175510be98a7SChris Wilson break; 175610be98a7SChris Wilson 175710be98a7SChris Wilson case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 175810be98a7SChris Wilson if (args->size) 175910be98a7SChris Wilson ret = -EINVAL; 176010be98a7SChris Wilson else if (args->value) 176110be98a7SChris Wilson i915_gem_context_set_no_error_capture(ctx); 176210be98a7SChris Wilson else 176310be98a7SChris Wilson i915_gem_context_clear_no_error_capture(ctx); 176410be98a7SChris Wilson break; 176510be98a7SChris Wilson 176610be98a7SChris Wilson case I915_CONTEXT_PARAM_BANNABLE: 176710be98a7SChris Wilson if (args->size) 176810be98a7SChris Wilson ret = -EINVAL; 176910be98a7SChris Wilson else if (!capable(CAP_SYS_ADMIN) && !args->value) 177010be98a7SChris Wilson ret = -EPERM; 177110be98a7SChris Wilson else if (args->value) 177210be98a7SChris Wilson i915_gem_context_set_bannable(ctx); 177310be98a7SChris Wilson else 177410be98a7SChris Wilson i915_gem_context_clear_bannable(ctx); 177510be98a7SChris Wilson break; 177610be98a7SChris Wilson 177710be98a7SChris Wilson case I915_CONTEXT_PARAM_RECOVERABLE: 177810be98a7SChris Wilson if (args->size) 177910be98a7SChris Wilson ret = -EINVAL; 178010be98a7SChris Wilson else if (args->value) 178110be98a7SChris Wilson i915_gem_context_set_recoverable(ctx); 178210be98a7SChris Wilson else 178310be98a7SChris Wilson i915_gem_context_clear_recoverable(ctx); 178410be98a7SChris Wilson break; 178510be98a7SChris Wilson 178610be98a7SChris Wilson case I915_CONTEXT_PARAM_PRIORITY: 178710be98a7SChris Wilson { 178810be98a7SChris Wilson s64 priority = args->value; 178910be98a7SChris Wilson 179010be98a7SChris Wilson if (args->size) 179110be98a7SChris Wilson ret = -EINVAL; 179210be98a7SChris Wilson else if (!(ctx->i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY)) 179310be98a7SChris Wilson ret = -ENODEV; 179410be98a7SChris Wilson else if (priority > I915_CONTEXT_MAX_USER_PRIORITY || 179510be98a7SChris Wilson priority < I915_CONTEXT_MIN_USER_PRIORITY) 179610be98a7SChris Wilson ret = -EINVAL; 179710be98a7SChris Wilson else if (priority > I915_CONTEXT_DEFAULT_PRIORITY && 179810be98a7SChris Wilson !capable(CAP_SYS_NICE)) 179910be98a7SChris Wilson ret = -EPERM; 180010be98a7SChris Wilson else 180110be98a7SChris Wilson ctx->sched.priority = 180210be98a7SChris Wilson I915_USER_PRIORITY(priority); 180310be98a7SChris Wilson } 180410be98a7SChris Wilson break; 180510be98a7SChris Wilson 180610be98a7SChris Wilson case I915_CONTEXT_PARAM_SSEU: 180710be98a7SChris Wilson ret = set_sseu(ctx, args); 180810be98a7SChris Wilson break; 180910be98a7SChris Wilson 181010be98a7SChris Wilson case I915_CONTEXT_PARAM_VM: 181110be98a7SChris Wilson ret = set_ppgtt(fpriv, ctx, args); 181210be98a7SChris Wilson break; 181310be98a7SChris Wilson 181410be98a7SChris Wilson case I915_CONTEXT_PARAM_ENGINES: 181510be98a7SChris Wilson ret = set_engines(ctx, args); 181610be98a7SChris Wilson break; 181710be98a7SChris Wilson 181810be98a7SChris Wilson case I915_CONTEXT_PARAM_BAN_PERIOD: 181910be98a7SChris Wilson default: 182010be98a7SChris Wilson ret = -EINVAL; 182110be98a7SChris Wilson break; 182210be98a7SChris Wilson } 182310be98a7SChris Wilson 182410be98a7SChris Wilson return ret; 182510be98a7SChris Wilson } 182610be98a7SChris Wilson 182710be98a7SChris Wilson struct create_ext { 182810be98a7SChris Wilson struct i915_gem_context *ctx; 182910be98a7SChris Wilson struct drm_i915_file_private *fpriv; 183010be98a7SChris Wilson }; 183110be98a7SChris Wilson 183210be98a7SChris Wilson static int create_setparam(struct i915_user_extension __user *ext, void *data) 183310be98a7SChris Wilson { 183410be98a7SChris Wilson struct drm_i915_gem_context_create_ext_setparam local; 183510be98a7SChris Wilson const struct create_ext *arg = data; 183610be98a7SChris Wilson 183710be98a7SChris Wilson if (copy_from_user(&local, ext, sizeof(local))) 183810be98a7SChris Wilson return -EFAULT; 183910be98a7SChris Wilson 184010be98a7SChris Wilson if (local.param.ctx_id) 184110be98a7SChris Wilson return -EINVAL; 184210be98a7SChris Wilson 184310be98a7SChris Wilson return ctx_setparam(arg->fpriv, arg->ctx, &local.param); 184410be98a7SChris Wilson } 184510be98a7SChris Wilson 184610be98a7SChris Wilson static int clone_engines(struct i915_gem_context *dst, 184710be98a7SChris Wilson struct i915_gem_context *src) 184810be98a7SChris Wilson { 184910be98a7SChris Wilson struct i915_gem_engines *e = i915_gem_context_lock_engines(src); 185010be98a7SChris Wilson struct i915_gem_engines *clone; 185110be98a7SChris Wilson bool user_engines; 185210be98a7SChris Wilson unsigned long n; 185310be98a7SChris Wilson 185410be98a7SChris Wilson clone = kmalloc(struct_size(e, engines, e->num_engines), GFP_KERNEL); 185510be98a7SChris Wilson if (!clone) 185610be98a7SChris Wilson goto err_unlock; 185710be98a7SChris Wilson 1858155ab883SChris Wilson init_rcu_head(&clone->rcu); 185910be98a7SChris Wilson for (n = 0; n < e->num_engines; n++) { 186010be98a7SChris Wilson struct intel_engine_cs *engine; 186110be98a7SChris Wilson 186210be98a7SChris Wilson if (!e->engines[n]) { 186310be98a7SChris Wilson clone->engines[n] = NULL; 186410be98a7SChris Wilson continue; 186510be98a7SChris Wilson } 186610be98a7SChris Wilson engine = e->engines[n]->engine; 186710be98a7SChris Wilson 186810be98a7SChris Wilson /* 186910be98a7SChris Wilson * Virtual engines are singletons; they can only exist 187010be98a7SChris Wilson * inside a single context, because they embed their 187110be98a7SChris Wilson * HW context... As each virtual context implies a single 187210be98a7SChris Wilson * timeline (each engine can only dequeue a single request 187310be98a7SChris Wilson * at any time), it would be surprising for two contexts 187410be98a7SChris Wilson * to use the same engine. So let's create a copy of 187510be98a7SChris Wilson * the virtual engine instead. 187610be98a7SChris Wilson */ 187710be98a7SChris Wilson if (intel_engine_is_virtual(engine)) 187810be98a7SChris Wilson clone->engines[n] = 187910be98a7SChris Wilson intel_execlists_clone_virtual(dst, engine); 188010be98a7SChris Wilson else 188110be98a7SChris Wilson clone->engines[n] = intel_context_create(dst, engine); 188210be98a7SChris Wilson if (IS_ERR_OR_NULL(clone->engines[n])) { 188310be98a7SChris Wilson __free_engines(clone, n); 188410be98a7SChris Wilson goto err_unlock; 188510be98a7SChris Wilson } 188610be98a7SChris Wilson } 188710be98a7SChris Wilson clone->num_engines = n; 188810be98a7SChris Wilson 188910be98a7SChris Wilson user_engines = i915_gem_context_user_engines(src); 189010be98a7SChris Wilson i915_gem_context_unlock_engines(src); 189110be98a7SChris Wilson 189210be98a7SChris Wilson free_engines(dst->engines); 189310be98a7SChris Wilson RCU_INIT_POINTER(dst->engines, clone); 189410be98a7SChris Wilson if (user_engines) 189510be98a7SChris Wilson i915_gem_context_set_user_engines(dst); 189610be98a7SChris Wilson else 189710be98a7SChris Wilson i915_gem_context_clear_user_engines(dst); 189810be98a7SChris Wilson return 0; 189910be98a7SChris Wilson 190010be98a7SChris Wilson err_unlock: 190110be98a7SChris Wilson i915_gem_context_unlock_engines(src); 190210be98a7SChris Wilson return -ENOMEM; 190310be98a7SChris Wilson } 190410be98a7SChris Wilson 190510be98a7SChris Wilson static int clone_flags(struct i915_gem_context *dst, 190610be98a7SChris Wilson struct i915_gem_context *src) 190710be98a7SChris Wilson { 190810be98a7SChris Wilson dst->user_flags = src->user_flags; 190910be98a7SChris Wilson return 0; 191010be98a7SChris Wilson } 191110be98a7SChris Wilson 191210be98a7SChris Wilson static int clone_schedattr(struct i915_gem_context *dst, 191310be98a7SChris Wilson struct i915_gem_context *src) 191410be98a7SChris Wilson { 191510be98a7SChris Wilson dst->sched = src->sched; 191610be98a7SChris Wilson return 0; 191710be98a7SChris Wilson } 191810be98a7SChris Wilson 191910be98a7SChris Wilson static int clone_sseu(struct i915_gem_context *dst, 192010be98a7SChris Wilson struct i915_gem_context *src) 192110be98a7SChris Wilson { 192210be98a7SChris Wilson struct i915_gem_engines *e = i915_gem_context_lock_engines(src); 192310be98a7SChris Wilson struct i915_gem_engines *clone; 192410be98a7SChris Wilson unsigned long n; 192510be98a7SChris Wilson int err; 192610be98a7SChris Wilson 192710be98a7SChris Wilson clone = dst->engines; /* no locking required; sole access */ 192810be98a7SChris Wilson if (e->num_engines != clone->num_engines) { 192910be98a7SChris Wilson err = -EINVAL; 193010be98a7SChris Wilson goto unlock; 193110be98a7SChris Wilson } 193210be98a7SChris Wilson 193310be98a7SChris Wilson for (n = 0; n < e->num_engines; n++) { 193410be98a7SChris Wilson struct intel_context *ce = e->engines[n]; 193510be98a7SChris Wilson 193610be98a7SChris Wilson if (clone->engines[n]->engine->class != ce->engine->class) { 193710be98a7SChris Wilson /* Must have compatible engine maps! */ 193810be98a7SChris Wilson err = -EINVAL; 193910be98a7SChris Wilson goto unlock; 194010be98a7SChris Wilson } 194110be98a7SChris Wilson 194210be98a7SChris Wilson /* serialises with set_sseu */ 194310be98a7SChris Wilson err = intel_context_lock_pinned(ce); 194410be98a7SChris Wilson if (err) 194510be98a7SChris Wilson goto unlock; 194610be98a7SChris Wilson 194710be98a7SChris Wilson clone->engines[n]->sseu = ce->sseu; 194810be98a7SChris Wilson intel_context_unlock_pinned(ce); 194910be98a7SChris Wilson } 195010be98a7SChris Wilson 195110be98a7SChris Wilson err = 0; 195210be98a7SChris Wilson unlock: 195310be98a7SChris Wilson i915_gem_context_unlock_engines(src); 195410be98a7SChris Wilson return err; 195510be98a7SChris Wilson } 195610be98a7SChris Wilson 195710be98a7SChris Wilson static int clone_timeline(struct i915_gem_context *dst, 195810be98a7SChris Wilson struct i915_gem_context *src) 195910be98a7SChris Wilson { 196075d0a7f3SChris Wilson if (src->timeline) 196175d0a7f3SChris Wilson __assign_timeline(dst, src->timeline); 196210be98a7SChris Wilson 196310be98a7SChris Wilson return 0; 196410be98a7SChris Wilson } 196510be98a7SChris Wilson 196610be98a7SChris Wilson static int clone_vm(struct i915_gem_context *dst, 196710be98a7SChris Wilson struct i915_gem_context *src) 196810be98a7SChris Wilson { 1969e568ac38SChris Wilson struct i915_address_space *vm; 197010be98a7SChris Wilson 197110be98a7SChris Wilson rcu_read_lock(); 197210be98a7SChris Wilson do { 1973e568ac38SChris Wilson vm = READ_ONCE(src->vm); 1974e568ac38SChris Wilson if (!vm) 197510be98a7SChris Wilson break; 197610be98a7SChris Wilson 1977e568ac38SChris Wilson if (!kref_get_unless_zero(&vm->ref)) 197810be98a7SChris Wilson continue; 197910be98a7SChris Wilson 198010be98a7SChris Wilson /* 198110be98a7SChris Wilson * This ppgtt may have be reallocated between 198210be98a7SChris Wilson * the read and the kref, and reassigned to a third 198310be98a7SChris Wilson * context. In order to avoid inadvertent sharing 198410be98a7SChris Wilson * of this ppgtt with that third context (and not 198510be98a7SChris Wilson * src), we have to confirm that we have the same 198610be98a7SChris Wilson * ppgtt after passing through the strong memory 198710be98a7SChris Wilson * barrier implied by a successful 198810be98a7SChris Wilson * kref_get_unless_zero(). 198910be98a7SChris Wilson * 199010be98a7SChris Wilson * Once we have acquired the current ppgtt of src, 199110be98a7SChris Wilson * we no longer care if it is released from src, as 199210be98a7SChris Wilson * it cannot be reallocated elsewhere. 199310be98a7SChris Wilson */ 199410be98a7SChris Wilson 1995e568ac38SChris Wilson if (vm == READ_ONCE(src->vm)) 199610be98a7SChris Wilson break; 199710be98a7SChris Wilson 1998e568ac38SChris Wilson i915_vm_put(vm); 199910be98a7SChris Wilson } while (1); 200010be98a7SChris Wilson rcu_read_unlock(); 200110be98a7SChris Wilson 2002e568ac38SChris Wilson if (vm) { 2003e568ac38SChris Wilson __assign_ppgtt(dst, vm); 2004e568ac38SChris Wilson i915_vm_put(vm); 200510be98a7SChris Wilson } 200610be98a7SChris Wilson 200710be98a7SChris Wilson return 0; 200810be98a7SChris Wilson } 200910be98a7SChris Wilson 201010be98a7SChris Wilson static int create_clone(struct i915_user_extension __user *ext, void *data) 201110be98a7SChris Wilson { 201210be98a7SChris Wilson static int (* const fn[])(struct i915_gem_context *dst, 201310be98a7SChris Wilson struct i915_gem_context *src) = { 201410be98a7SChris Wilson #define MAP(x, y) [ilog2(I915_CONTEXT_CLONE_##x)] = y 201510be98a7SChris Wilson MAP(ENGINES, clone_engines), 201610be98a7SChris Wilson MAP(FLAGS, clone_flags), 201710be98a7SChris Wilson MAP(SCHEDATTR, clone_schedattr), 201810be98a7SChris Wilson MAP(SSEU, clone_sseu), 201910be98a7SChris Wilson MAP(TIMELINE, clone_timeline), 202010be98a7SChris Wilson MAP(VM, clone_vm), 202110be98a7SChris Wilson #undef MAP 202210be98a7SChris Wilson }; 202310be98a7SChris Wilson struct drm_i915_gem_context_create_ext_clone local; 202410be98a7SChris Wilson const struct create_ext *arg = data; 202510be98a7SChris Wilson struct i915_gem_context *dst = arg->ctx; 202610be98a7SChris Wilson struct i915_gem_context *src; 202710be98a7SChris Wilson int err, bit; 202810be98a7SChris Wilson 202910be98a7SChris Wilson if (copy_from_user(&local, ext, sizeof(local))) 203010be98a7SChris Wilson return -EFAULT; 203110be98a7SChris Wilson 203210be98a7SChris Wilson BUILD_BUG_ON(GENMASK(BITS_PER_TYPE(local.flags) - 1, ARRAY_SIZE(fn)) != 203310be98a7SChris Wilson I915_CONTEXT_CLONE_UNKNOWN); 203410be98a7SChris Wilson 203510be98a7SChris Wilson if (local.flags & I915_CONTEXT_CLONE_UNKNOWN) 203610be98a7SChris Wilson return -EINVAL; 203710be98a7SChris Wilson 203810be98a7SChris Wilson if (local.rsvd) 203910be98a7SChris Wilson return -EINVAL; 204010be98a7SChris Wilson 204110be98a7SChris Wilson rcu_read_lock(); 204210be98a7SChris Wilson src = __i915_gem_context_lookup_rcu(arg->fpriv, local.clone_id); 204310be98a7SChris Wilson rcu_read_unlock(); 204410be98a7SChris Wilson if (!src) 204510be98a7SChris Wilson return -ENOENT; 204610be98a7SChris Wilson 204710be98a7SChris Wilson GEM_BUG_ON(src == dst); 204810be98a7SChris Wilson 204910be98a7SChris Wilson for (bit = 0; bit < ARRAY_SIZE(fn); bit++) { 205010be98a7SChris Wilson if (!(local.flags & BIT(bit))) 205110be98a7SChris Wilson continue; 205210be98a7SChris Wilson 205310be98a7SChris Wilson err = fn[bit](dst, src); 205410be98a7SChris Wilson if (err) 205510be98a7SChris Wilson return err; 205610be98a7SChris Wilson } 205710be98a7SChris Wilson 205810be98a7SChris Wilson return 0; 205910be98a7SChris Wilson } 206010be98a7SChris Wilson 206110be98a7SChris Wilson static const i915_user_extension_fn create_extensions[] = { 206210be98a7SChris Wilson [I915_CONTEXT_CREATE_EXT_SETPARAM] = create_setparam, 206310be98a7SChris Wilson [I915_CONTEXT_CREATE_EXT_CLONE] = create_clone, 206410be98a7SChris Wilson }; 206510be98a7SChris Wilson 206610be98a7SChris Wilson static bool client_is_banned(struct drm_i915_file_private *file_priv) 206710be98a7SChris Wilson { 206810be98a7SChris Wilson return atomic_read(&file_priv->ban_score) >= I915_CLIENT_SCORE_BANNED; 206910be98a7SChris Wilson } 207010be98a7SChris Wilson 207110be98a7SChris Wilson int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, 207210be98a7SChris Wilson struct drm_file *file) 207310be98a7SChris Wilson { 207410be98a7SChris Wilson struct drm_i915_private *i915 = to_i915(dev); 207510be98a7SChris Wilson struct drm_i915_gem_context_create_ext *args = data; 207610be98a7SChris Wilson struct create_ext ext_data; 207710be98a7SChris Wilson int ret; 207810be98a7SChris Wilson 207910be98a7SChris Wilson if (!DRIVER_CAPS(i915)->has_logical_contexts) 208010be98a7SChris Wilson return -ENODEV; 208110be98a7SChris Wilson 208210be98a7SChris Wilson if (args->flags & I915_CONTEXT_CREATE_FLAGS_UNKNOWN) 208310be98a7SChris Wilson return -EINVAL; 208410be98a7SChris Wilson 2085cb823ed9SChris Wilson ret = intel_gt_terminally_wedged(&i915->gt); 208610be98a7SChris Wilson if (ret) 208710be98a7SChris Wilson return ret; 208810be98a7SChris Wilson 208910be98a7SChris Wilson ext_data.fpriv = file->driver_priv; 209010be98a7SChris Wilson if (client_is_banned(ext_data.fpriv)) { 209110be98a7SChris Wilson DRM_DEBUG("client %s[%d] banned from creating ctx\n", 209210be98a7SChris Wilson current->comm, 209310be98a7SChris Wilson pid_nr(get_task_pid(current, PIDTYPE_PID))); 209410be98a7SChris Wilson return -EIO; 209510be98a7SChris Wilson } 209610be98a7SChris Wilson 209710be98a7SChris Wilson ret = i915_mutex_lock_interruptible(dev); 209810be98a7SChris Wilson if (ret) 209910be98a7SChris Wilson return ret; 210010be98a7SChris Wilson 210110be98a7SChris Wilson ext_data.ctx = i915_gem_create_context(i915, args->flags); 210210be98a7SChris Wilson mutex_unlock(&dev->struct_mutex); 210310be98a7SChris Wilson if (IS_ERR(ext_data.ctx)) 210410be98a7SChris Wilson return PTR_ERR(ext_data.ctx); 210510be98a7SChris Wilson 210610be98a7SChris Wilson if (args->flags & I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS) { 210710be98a7SChris Wilson ret = i915_user_extensions(u64_to_user_ptr(args->extensions), 210810be98a7SChris Wilson create_extensions, 210910be98a7SChris Wilson ARRAY_SIZE(create_extensions), 211010be98a7SChris Wilson &ext_data); 211110be98a7SChris Wilson if (ret) 211210be98a7SChris Wilson goto err_ctx; 211310be98a7SChris Wilson } 211410be98a7SChris Wilson 211510be98a7SChris Wilson ret = gem_context_register(ext_data.ctx, ext_data.fpriv); 211610be98a7SChris Wilson if (ret < 0) 211710be98a7SChris Wilson goto err_ctx; 211810be98a7SChris Wilson 211910be98a7SChris Wilson args->ctx_id = ret; 212010be98a7SChris Wilson DRM_DEBUG("HW context %d created\n", args->ctx_id); 212110be98a7SChris Wilson 212210be98a7SChris Wilson return 0; 212310be98a7SChris Wilson 212410be98a7SChris Wilson err_ctx: 212510be98a7SChris Wilson context_close(ext_data.ctx); 212610be98a7SChris Wilson return ret; 212710be98a7SChris Wilson } 212810be98a7SChris Wilson 212910be98a7SChris Wilson int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, 213010be98a7SChris Wilson struct drm_file *file) 213110be98a7SChris Wilson { 213210be98a7SChris Wilson struct drm_i915_gem_context_destroy *args = data; 213310be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 213410be98a7SChris Wilson struct i915_gem_context *ctx; 213510be98a7SChris Wilson 213610be98a7SChris Wilson if (args->pad != 0) 213710be98a7SChris Wilson return -EINVAL; 213810be98a7SChris Wilson 213910be98a7SChris Wilson if (!args->ctx_id) 214010be98a7SChris Wilson return -ENOENT; 214110be98a7SChris Wilson 214210be98a7SChris Wilson if (mutex_lock_interruptible(&file_priv->context_idr_lock)) 214310be98a7SChris Wilson return -EINTR; 214410be98a7SChris Wilson 214510be98a7SChris Wilson ctx = idr_remove(&file_priv->context_idr, args->ctx_id); 214610be98a7SChris Wilson mutex_unlock(&file_priv->context_idr_lock); 214710be98a7SChris Wilson if (!ctx) 214810be98a7SChris Wilson return -ENOENT; 214910be98a7SChris Wilson 215010be98a7SChris Wilson context_close(ctx); 215110be98a7SChris Wilson return 0; 215210be98a7SChris Wilson } 215310be98a7SChris Wilson 215410be98a7SChris Wilson static int get_sseu(struct i915_gem_context *ctx, 215510be98a7SChris Wilson struct drm_i915_gem_context_param *args) 215610be98a7SChris Wilson { 215710be98a7SChris Wilson struct drm_i915_gem_context_param_sseu user_sseu; 215810be98a7SChris Wilson struct intel_context *ce; 215910be98a7SChris Wilson unsigned long lookup; 216010be98a7SChris Wilson int err; 216110be98a7SChris Wilson 216210be98a7SChris Wilson if (args->size == 0) 216310be98a7SChris Wilson goto out; 216410be98a7SChris Wilson else if (args->size < sizeof(user_sseu)) 216510be98a7SChris Wilson return -EINVAL; 216610be98a7SChris Wilson 216710be98a7SChris Wilson if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), 216810be98a7SChris Wilson sizeof(user_sseu))) 216910be98a7SChris Wilson return -EFAULT; 217010be98a7SChris Wilson 217110be98a7SChris Wilson if (user_sseu.rsvd) 217210be98a7SChris Wilson return -EINVAL; 217310be98a7SChris Wilson 217410be98a7SChris Wilson if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) 217510be98a7SChris Wilson return -EINVAL; 217610be98a7SChris Wilson 217710be98a7SChris Wilson lookup = 0; 217810be98a7SChris Wilson if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) 217910be98a7SChris Wilson lookup |= LOOKUP_USER_INDEX; 218010be98a7SChris Wilson 218110be98a7SChris Wilson ce = lookup_user_engine(ctx, lookup, &user_sseu.engine); 218210be98a7SChris Wilson if (IS_ERR(ce)) 218310be98a7SChris Wilson return PTR_ERR(ce); 218410be98a7SChris Wilson 218510be98a7SChris Wilson err = intel_context_lock_pinned(ce); /* serialises with set_sseu */ 218610be98a7SChris Wilson if (err) { 218710be98a7SChris Wilson intel_context_put(ce); 218810be98a7SChris Wilson return err; 218910be98a7SChris Wilson } 219010be98a7SChris Wilson 219110be98a7SChris Wilson user_sseu.slice_mask = ce->sseu.slice_mask; 219210be98a7SChris Wilson user_sseu.subslice_mask = ce->sseu.subslice_mask; 219310be98a7SChris Wilson user_sseu.min_eus_per_subslice = ce->sseu.min_eus_per_subslice; 219410be98a7SChris Wilson user_sseu.max_eus_per_subslice = ce->sseu.max_eus_per_subslice; 219510be98a7SChris Wilson 219610be98a7SChris Wilson intel_context_unlock_pinned(ce); 219710be98a7SChris Wilson intel_context_put(ce); 219810be98a7SChris Wilson 219910be98a7SChris Wilson if (copy_to_user(u64_to_user_ptr(args->value), &user_sseu, 220010be98a7SChris Wilson sizeof(user_sseu))) 220110be98a7SChris Wilson return -EFAULT; 220210be98a7SChris Wilson 220310be98a7SChris Wilson out: 220410be98a7SChris Wilson args->size = sizeof(user_sseu); 220510be98a7SChris Wilson 220610be98a7SChris Wilson return 0; 220710be98a7SChris Wilson } 220810be98a7SChris Wilson 220910be98a7SChris Wilson int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, 221010be98a7SChris Wilson struct drm_file *file) 221110be98a7SChris Wilson { 221210be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 221310be98a7SChris Wilson struct drm_i915_gem_context_param *args = data; 221410be98a7SChris Wilson struct i915_gem_context *ctx; 221510be98a7SChris Wilson int ret = 0; 221610be98a7SChris Wilson 221710be98a7SChris Wilson ctx = i915_gem_context_lookup(file_priv, args->ctx_id); 221810be98a7SChris Wilson if (!ctx) 221910be98a7SChris Wilson return -ENOENT; 222010be98a7SChris Wilson 222110be98a7SChris Wilson switch (args->param) { 222210be98a7SChris Wilson case I915_CONTEXT_PARAM_NO_ZEROMAP: 222310be98a7SChris Wilson args->size = 0; 222410be98a7SChris Wilson args->value = test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); 222510be98a7SChris Wilson break; 222610be98a7SChris Wilson 222710be98a7SChris Wilson case I915_CONTEXT_PARAM_GTT_SIZE: 222810be98a7SChris Wilson args->size = 0; 2229e568ac38SChris Wilson if (ctx->vm) 2230e568ac38SChris Wilson args->value = ctx->vm->total; 2231c082afacSChris Wilson else if (to_i915(dev)->ggtt.alias) 2232c082afacSChris Wilson args->value = to_i915(dev)->ggtt.alias->vm.total; 223310be98a7SChris Wilson else 223410be98a7SChris Wilson args->value = to_i915(dev)->ggtt.vm.total; 223510be98a7SChris Wilson break; 223610be98a7SChris Wilson 223710be98a7SChris Wilson case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 223810be98a7SChris Wilson args->size = 0; 223910be98a7SChris Wilson args->value = i915_gem_context_no_error_capture(ctx); 224010be98a7SChris Wilson break; 224110be98a7SChris Wilson 224210be98a7SChris Wilson case I915_CONTEXT_PARAM_BANNABLE: 224310be98a7SChris Wilson args->size = 0; 224410be98a7SChris Wilson args->value = i915_gem_context_is_bannable(ctx); 224510be98a7SChris Wilson break; 224610be98a7SChris Wilson 224710be98a7SChris Wilson case I915_CONTEXT_PARAM_RECOVERABLE: 224810be98a7SChris Wilson args->size = 0; 224910be98a7SChris Wilson args->value = i915_gem_context_is_recoverable(ctx); 225010be98a7SChris Wilson break; 225110be98a7SChris Wilson 225210be98a7SChris Wilson case I915_CONTEXT_PARAM_PRIORITY: 225310be98a7SChris Wilson args->size = 0; 225410be98a7SChris Wilson args->value = ctx->sched.priority >> I915_USER_PRIORITY_SHIFT; 225510be98a7SChris Wilson break; 225610be98a7SChris Wilson 225710be98a7SChris Wilson case I915_CONTEXT_PARAM_SSEU: 225810be98a7SChris Wilson ret = get_sseu(ctx, args); 225910be98a7SChris Wilson break; 226010be98a7SChris Wilson 226110be98a7SChris Wilson case I915_CONTEXT_PARAM_VM: 226210be98a7SChris Wilson ret = get_ppgtt(file_priv, ctx, args); 226310be98a7SChris Wilson break; 226410be98a7SChris Wilson 226510be98a7SChris Wilson case I915_CONTEXT_PARAM_ENGINES: 226610be98a7SChris Wilson ret = get_engines(ctx, args); 226710be98a7SChris Wilson break; 226810be98a7SChris Wilson 226910be98a7SChris Wilson case I915_CONTEXT_PARAM_BAN_PERIOD: 227010be98a7SChris Wilson default: 227110be98a7SChris Wilson ret = -EINVAL; 227210be98a7SChris Wilson break; 227310be98a7SChris Wilson } 227410be98a7SChris Wilson 227510be98a7SChris Wilson i915_gem_context_put(ctx); 227610be98a7SChris Wilson return ret; 227710be98a7SChris Wilson } 227810be98a7SChris Wilson 227910be98a7SChris Wilson int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, 228010be98a7SChris Wilson struct drm_file *file) 228110be98a7SChris Wilson { 228210be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 228310be98a7SChris Wilson struct drm_i915_gem_context_param *args = data; 228410be98a7SChris Wilson struct i915_gem_context *ctx; 228510be98a7SChris Wilson int ret; 228610be98a7SChris Wilson 228710be98a7SChris Wilson ctx = i915_gem_context_lookup(file_priv, args->ctx_id); 228810be98a7SChris Wilson if (!ctx) 228910be98a7SChris Wilson return -ENOENT; 229010be98a7SChris Wilson 229110be98a7SChris Wilson ret = ctx_setparam(file_priv, ctx, args); 229210be98a7SChris Wilson 229310be98a7SChris Wilson i915_gem_context_put(ctx); 229410be98a7SChris Wilson return ret; 229510be98a7SChris Wilson } 229610be98a7SChris Wilson 229710be98a7SChris Wilson int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, 229810be98a7SChris Wilson void *data, struct drm_file *file) 229910be98a7SChris Wilson { 230010be98a7SChris Wilson struct drm_i915_private *dev_priv = to_i915(dev); 230110be98a7SChris Wilson struct drm_i915_reset_stats *args = data; 230210be98a7SChris Wilson struct i915_gem_context *ctx; 230310be98a7SChris Wilson int ret; 230410be98a7SChris Wilson 230510be98a7SChris Wilson if (args->flags || args->pad) 230610be98a7SChris Wilson return -EINVAL; 230710be98a7SChris Wilson 230810be98a7SChris Wilson ret = -ENOENT; 230910be98a7SChris Wilson rcu_read_lock(); 231010be98a7SChris Wilson ctx = __i915_gem_context_lookup_rcu(file->driver_priv, args->ctx_id); 231110be98a7SChris Wilson if (!ctx) 231210be98a7SChris Wilson goto out; 231310be98a7SChris Wilson 231410be98a7SChris Wilson /* 231510be98a7SChris Wilson * We opt for unserialised reads here. This may result in tearing 231610be98a7SChris Wilson * in the extremely unlikely event of a GPU hang on this context 231710be98a7SChris Wilson * as we are querying them. If we need that extra layer of protection, 231810be98a7SChris Wilson * we should wrap the hangstats with a seqlock. 231910be98a7SChris Wilson */ 232010be98a7SChris Wilson 232110be98a7SChris Wilson if (capable(CAP_SYS_ADMIN)) 232210be98a7SChris Wilson args->reset_count = i915_reset_count(&dev_priv->gpu_error); 232310be98a7SChris Wilson else 232410be98a7SChris Wilson args->reset_count = 0; 232510be98a7SChris Wilson 232610be98a7SChris Wilson args->batch_active = atomic_read(&ctx->guilty_count); 232710be98a7SChris Wilson args->batch_pending = atomic_read(&ctx->active_count); 232810be98a7SChris Wilson 232910be98a7SChris Wilson ret = 0; 233010be98a7SChris Wilson out: 233110be98a7SChris Wilson rcu_read_unlock(); 233210be98a7SChris Wilson return ret; 233310be98a7SChris Wilson } 233410be98a7SChris Wilson 233510be98a7SChris Wilson int __i915_gem_context_pin_hw_id(struct i915_gem_context *ctx) 233610be98a7SChris Wilson { 233710be98a7SChris Wilson struct drm_i915_private *i915 = ctx->i915; 233810be98a7SChris Wilson int err = 0; 233910be98a7SChris Wilson 234010be98a7SChris Wilson mutex_lock(&i915->contexts.mutex); 234110be98a7SChris Wilson 234210be98a7SChris Wilson GEM_BUG_ON(i915_gem_context_is_closed(ctx)); 234310be98a7SChris Wilson 234410be98a7SChris Wilson if (list_empty(&ctx->hw_id_link)) { 234510be98a7SChris Wilson GEM_BUG_ON(atomic_read(&ctx->hw_id_pin_count)); 234610be98a7SChris Wilson 234710be98a7SChris Wilson err = assign_hw_id(i915, &ctx->hw_id); 234810be98a7SChris Wilson if (err) 234910be98a7SChris Wilson goto out_unlock; 235010be98a7SChris Wilson 235110be98a7SChris Wilson list_add_tail(&ctx->hw_id_link, &i915->contexts.hw_id_list); 235210be98a7SChris Wilson } 235310be98a7SChris Wilson 235410be98a7SChris Wilson GEM_BUG_ON(atomic_read(&ctx->hw_id_pin_count) == ~0u); 235510be98a7SChris Wilson atomic_inc(&ctx->hw_id_pin_count); 235610be98a7SChris Wilson 235710be98a7SChris Wilson out_unlock: 235810be98a7SChris Wilson mutex_unlock(&i915->contexts.mutex); 235910be98a7SChris Wilson return err; 236010be98a7SChris Wilson } 236110be98a7SChris Wilson 236210be98a7SChris Wilson /* GEM context-engines iterator: for_each_gem_engine() */ 236310be98a7SChris Wilson struct intel_context * 236410be98a7SChris Wilson i915_gem_engines_iter_next(struct i915_gem_engines_iter *it) 236510be98a7SChris Wilson { 236610be98a7SChris Wilson const struct i915_gem_engines *e = it->engines; 236710be98a7SChris Wilson struct intel_context *ctx; 236810be98a7SChris Wilson 236910be98a7SChris Wilson do { 237010be98a7SChris Wilson if (it->idx >= e->num_engines) 237110be98a7SChris Wilson return NULL; 237210be98a7SChris Wilson 237310be98a7SChris Wilson ctx = e->engines[it->idx++]; 237410be98a7SChris Wilson } while (!ctx); 237510be98a7SChris Wilson 237610be98a7SChris Wilson return ctx; 237710be98a7SChris Wilson } 237810be98a7SChris Wilson 237910be98a7SChris Wilson #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 238010be98a7SChris Wilson #include "selftests/mock_context.c" 238110be98a7SChris Wilson #include "selftests/i915_gem_context.c" 238210be98a7SChris Wilson #endif 238310be98a7SChris Wilson 238410be98a7SChris Wilson static void i915_global_gem_context_shrink(void) 238510be98a7SChris Wilson { 238610be98a7SChris Wilson kmem_cache_shrink(global.slab_luts); 238710be98a7SChris Wilson } 238810be98a7SChris Wilson 238910be98a7SChris Wilson static void i915_global_gem_context_exit(void) 239010be98a7SChris Wilson { 239110be98a7SChris Wilson kmem_cache_destroy(global.slab_luts); 239210be98a7SChris Wilson } 239310be98a7SChris Wilson 239410be98a7SChris Wilson static struct i915_global_gem_context global = { { 239510be98a7SChris Wilson .shrink = i915_global_gem_context_shrink, 239610be98a7SChris Wilson .exit = i915_global_gem_context_exit, 239710be98a7SChris Wilson } }; 239810be98a7SChris Wilson 239910be98a7SChris Wilson int __init i915_global_gem_context_init(void) 240010be98a7SChris Wilson { 240110be98a7SChris Wilson global.slab_luts = KMEM_CACHE(i915_lut_handle, 0); 240210be98a7SChris Wilson if (!global.slab_luts) 240310be98a7SChris Wilson return -ENOMEM; 240410be98a7SChris Wilson 240510be98a7SChris Wilson i915_global_register(&global.base); 240610be98a7SChris Wilson return 0; 240710be98a7SChris Wilson } 2408