110be98a7SChris Wilson /* 210be98a7SChris Wilson * SPDX-License-Identifier: MIT 310be98a7SChris Wilson * 410be98a7SChris Wilson * Copyright © 2011-2012 Intel Corporation 510be98a7SChris Wilson */ 610be98a7SChris Wilson 710be98a7SChris Wilson /* 810be98a7SChris Wilson * This file implements HW context support. On gen5+ a HW context consists of an 910be98a7SChris Wilson * opaque GPU object which is referenced at times of context saves and restores. 1010be98a7SChris Wilson * With RC6 enabled, the context is also referenced as the GPU enters and exists 1110be98a7SChris Wilson * from RC6 (GPU has it's own internal power context, except on gen5). Though 1210be98a7SChris Wilson * something like a context does exist for the media ring, the code only 1310be98a7SChris Wilson * supports contexts for the render ring. 1410be98a7SChris Wilson * 1510be98a7SChris Wilson * In software, there is a distinction between contexts created by the user, 1610be98a7SChris Wilson * and the default HW context. The default HW context is used by GPU clients 1710be98a7SChris Wilson * that do not request setup of their own hardware context. The default 1810be98a7SChris Wilson * context's state is never restored to help prevent programming errors. This 1910be98a7SChris Wilson * would happen if a client ran and piggy-backed off another clients GPU state. 2010be98a7SChris Wilson * The default context only exists to give the GPU some offset to load as the 2110be98a7SChris Wilson * current to invoke a save of the context we actually care about. In fact, the 2210be98a7SChris Wilson * code could likely be constructed, albeit in a more complicated fashion, to 2310be98a7SChris Wilson * never use the default context, though that limits the driver's ability to 2410be98a7SChris Wilson * swap out, and/or destroy other contexts. 2510be98a7SChris Wilson * 2610be98a7SChris Wilson * All other contexts are created as a request by the GPU client. These contexts 2710be98a7SChris Wilson * store GPU state, and thus allow GPU clients to not re-emit state (and 2810be98a7SChris Wilson * potentially query certain state) at any time. The kernel driver makes 2910be98a7SChris Wilson * certain that the appropriate commands are inserted. 3010be98a7SChris Wilson * 3110be98a7SChris Wilson * The context life cycle is semi-complicated in that context BOs may live 3210be98a7SChris Wilson * longer than the context itself because of the way the hardware, and object 3310be98a7SChris Wilson * tracking works. Below is a very crude representation of the state machine 3410be98a7SChris Wilson * describing the context life. 3510be98a7SChris Wilson * refcount pincount active 3610be98a7SChris Wilson * S0: initial state 0 0 0 3710be98a7SChris Wilson * S1: context created 1 0 0 3810be98a7SChris Wilson * S2: context is currently running 2 1 X 3910be98a7SChris Wilson * S3: GPU referenced, but not current 2 0 1 4010be98a7SChris Wilson * S4: context is current, but destroyed 1 1 0 4110be98a7SChris Wilson * S5: like S3, but destroyed 1 0 1 4210be98a7SChris Wilson * 4310be98a7SChris Wilson * The most common (but not all) transitions: 4410be98a7SChris Wilson * S0->S1: client creates a context 4510be98a7SChris Wilson * S1->S2: client submits execbuf with context 4610be98a7SChris Wilson * S2->S3: other clients submits execbuf with context 4710be98a7SChris Wilson * S3->S1: context object was retired 4810be98a7SChris Wilson * S3->S2: clients submits another execbuf 4910be98a7SChris Wilson * S2->S4: context destroy called with current context 5010be98a7SChris Wilson * S3->S5->S0: destroy path 5110be98a7SChris Wilson * S4->S5->S0: destroy path on current context 5210be98a7SChris Wilson * 5310be98a7SChris Wilson * There are two confusing terms used above: 5410be98a7SChris Wilson * The "current context" means the context which is currently running on the 5510be98a7SChris Wilson * GPU. The GPU has loaded its state already and has stored away the gtt 5610be98a7SChris Wilson * offset of the BO. The GPU is not actively referencing the data at this 5710be98a7SChris Wilson * offset, but it will on the next context switch. The only way to avoid this 5810be98a7SChris Wilson * is to do a GPU reset. 5910be98a7SChris Wilson * 6010be98a7SChris Wilson * An "active context' is one which was previously the "current context" and is 6110be98a7SChris Wilson * on the active list waiting for the next context switch to occur. Until this 6210be98a7SChris Wilson * happens, the object must remain at the same gtt offset. It is therefore 6310be98a7SChris Wilson * possible to destroy a context, but it is still active. 6410be98a7SChris Wilson * 6510be98a7SChris Wilson */ 6610be98a7SChris Wilson 6710be98a7SChris Wilson #include <linux/log2.h> 6810be98a7SChris Wilson #include <linux/nospec.h> 6910be98a7SChris Wilson 7010be98a7SChris Wilson #include <drm/i915_drm.h> 7110be98a7SChris Wilson 7210be98a7SChris Wilson #include "gt/intel_lrc_reg.h" 73750e76b4SChris Wilson #include "gt/intel_engine_user.h" 7410be98a7SChris Wilson 7510be98a7SChris Wilson #include "i915_gem_context.h" 7610be98a7SChris Wilson #include "i915_globals.h" 7710be98a7SChris Wilson #include "i915_trace.h" 7810be98a7SChris Wilson #include "i915_user_extensions.h" 7910be98a7SChris Wilson 8010be98a7SChris Wilson #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1 8110be98a7SChris Wilson 8210be98a7SChris Wilson static struct i915_global_gem_context { 8310be98a7SChris Wilson struct i915_global base; 8410be98a7SChris Wilson struct kmem_cache *slab_luts; 8510be98a7SChris Wilson } global; 8610be98a7SChris Wilson 8710be98a7SChris Wilson struct i915_lut_handle *i915_lut_handle_alloc(void) 8810be98a7SChris Wilson { 8910be98a7SChris Wilson return kmem_cache_alloc(global.slab_luts, GFP_KERNEL); 9010be98a7SChris Wilson } 9110be98a7SChris Wilson 9210be98a7SChris Wilson void i915_lut_handle_free(struct i915_lut_handle *lut) 9310be98a7SChris Wilson { 9410be98a7SChris Wilson return kmem_cache_free(global.slab_luts, lut); 9510be98a7SChris Wilson } 9610be98a7SChris Wilson 9710be98a7SChris Wilson static void lut_close(struct i915_gem_context *ctx) 9810be98a7SChris Wilson { 9910be98a7SChris Wilson struct radix_tree_iter iter; 10010be98a7SChris Wilson void __rcu **slot; 10110be98a7SChris Wilson 102155ab883SChris Wilson lockdep_assert_held(&ctx->mutex); 10310be98a7SChris Wilson 10410be98a7SChris Wilson rcu_read_lock(); 10510be98a7SChris Wilson radix_tree_for_each_slot(slot, &ctx->handles_vma, &iter, 0) { 10610be98a7SChris Wilson struct i915_vma *vma = rcu_dereference_raw(*slot); 107155ab883SChris Wilson struct drm_i915_gem_object *obj = vma->obj; 108155ab883SChris Wilson struct i915_lut_handle *lut; 10910be98a7SChris Wilson 110155ab883SChris Wilson if (!kref_get_unless_zero(&obj->base.refcount)) 111155ab883SChris Wilson continue; 112155ab883SChris Wilson 113155ab883SChris Wilson rcu_read_unlock(); 114155ab883SChris Wilson i915_gem_object_lock(obj); 115155ab883SChris Wilson list_for_each_entry(lut, &obj->lut_list, obj_link) { 116155ab883SChris Wilson if (lut->ctx != ctx) 117155ab883SChris Wilson continue; 118155ab883SChris Wilson 119155ab883SChris Wilson if (lut->handle != iter.index) 120155ab883SChris Wilson continue; 121155ab883SChris Wilson 122155ab883SChris Wilson list_del(&lut->obj_link); 123155ab883SChris Wilson break; 124155ab883SChris Wilson } 125155ab883SChris Wilson i915_gem_object_unlock(obj); 126155ab883SChris Wilson rcu_read_lock(); 127155ab883SChris Wilson 128155ab883SChris Wilson if (&lut->obj_link != &obj->lut_list) { 129155ab883SChris Wilson i915_lut_handle_free(lut); 13010be98a7SChris Wilson radix_tree_iter_delete(&ctx->handles_vma, &iter, slot); 131155ab883SChris Wilson if (atomic_dec_and_test(&vma->open_count) && 132155ab883SChris Wilson !i915_vma_is_ggtt(vma)) 133155ab883SChris Wilson i915_vma_close(vma); 134155ab883SChris Wilson i915_gem_object_put(obj); 135155ab883SChris Wilson } 13610be98a7SChris Wilson 137155ab883SChris Wilson i915_gem_object_put(obj); 13810be98a7SChris Wilson } 13910be98a7SChris Wilson rcu_read_unlock(); 14010be98a7SChris Wilson } 14110be98a7SChris Wilson 14210be98a7SChris Wilson static struct intel_context * 14310be98a7SChris Wilson lookup_user_engine(struct i915_gem_context *ctx, 14410be98a7SChris Wilson unsigned long flags, 14510be98a7SChris Wilson const struct i915_engine_class_instance *ci) 14610be98a7SChris Wilson #define LOOKUP_USER_INDEX BIT(0) 14710be98a7SChris Wilson { 14810be98a7SChris Wilson int idx; 14910be98a7SChris Wilson 15010be98a7SChris Wilson if (!!(flags & LOOKUP_USER_INDEX) != i915_gem_context_user_engines(ctx)) 15110be98a7SChris Wilson return ERR_PTR(-EINVAL); 15210be98a7SChris Wilson 15310be98a7SChris Wilson if (!i915_gem_context_user_engines(ctx)) { 15410be98a7SChris Wilson struct intel_engine_cs *engine; 15510be98a7SChris Wilson 15610be98a7SChris Wilson engine = intel_engine_lookup_user(ctx->i915, 15710be98a7SChris Wilson ci->engine_class, 15810be98a7SChris Wilson ci->engine_instance); 15910be98a7SChris Wilson if (!engine) 16010be98a7SChris Wilson return ERR_PTR(-EINVAL); 16110be98a7SChris Wilson 162f1c4d157SChris Wilson idx = engine->legacy_idx; 16310be98a7SChris Wilson } else { 16410be98a7SChris Wilson idx = ci->engine_instance; 16510be98a7SChris Wilson } 16610be98a7SChris Wilson 16710be98a7SChris Wilson return i915_gem_context_get_engine(ctx, idx); 16810be98a7SChris Wilson } 16910be98a7SChris Wilson 17010be98a7SChris Wilson static inline int new_hw_id(struct drm_i915_private *i915, gfp_t gfp) 17110be98a7SChris Wilson { 17210be98a7SChris Wilson unsigned int max; 17310be98a7SChris Wilson 17410be98a7SChris Wilson lockdep_assert_held(&i915->contexts.mutex); 17510be98a7SChris Wilson 17610be98a7SChris Wilson if (INTEL_GEN(i915) >= 11) 17710be98a7SChris Wilson max = GEN11_MAX_CONTEXT_HW_ID; 17810be98a7SChris Wilson else if (USES_GUC_SUBMISSION(i915)) 17910be98a7SChris Wilson /* 18010be98a7SChris Wilson * When using GuC in proxy submission, GuC consumes the 18110be98a7SChris Wilson * highest bit in the context id to indicate proxy submission. 18210be98a7SChris Wilson */ 18310be98a7SChris Wilson max = MAX_GUC_CONTEXT_HW_ID; 18410be98a7SChris Wilson else 18510be98a7SChris Wilson max = MAX_CONTEXT_HW_ID; 18610be98a7SChris Wilson 18710be98a7SChris Wilson return ida_simple_get(&i915->contexts.hw_ida, 0, max, gfp); 18810be98a7SChris Wilson } 18910be98a7SChris Wilson 19010be98a7SChris Wilson static int steal_hw_id(struct drm_i915_private *i915) 19110be98a7SChris Wilson { 19210be98a7SChris Wilson struct i915_gem_context *ctx, *cn; 19310be98a7SChris Wilson LIST_HEAD(pinned); 19410be98a7SChris Wilson int id = -ENOSPC; 19510be98a7SChris Wilson 19610be98a7SChris Wilson lockdep_assert_held(&i915->contexts.mutex); 19710be98a7SChris Wilson 19810be98a7SChris Wilson list_for_each_entry_safe(ctx, cn, 19910be98a7SChris Wilson &i915->contexts.hw_id_list, hw_id_link) { 20010be98a7SChris Wilson if (atomic_read(&ctx->hw_id_pin_count)) { 20110be98a7SChris Wilson list_move_tail(&ctx->hw_id_link, &pinned); 20210be98a7SChris Wilson continue; 20310be98a7SChris Wilson } 20410be98a7SChris Wilson 20510be98a7SChris Wilson GEM_BUG_ON(!ctx->hw_id); /* perma-pinned kernel context */ 20610be98a7SChris Wilson list_del_init(&ctx->hw_id_link); 20710be98a7SChris Wilson id = ctx->hw_id; 20810be98a7SChris Wilson break; 20910be98a7SChris Wilson } 21010be98a7SChris Wilson 21110be98a7SChris Wilson /* 21210be98a7SChris Wilson * Remember how far we got up on the last repossesion scan, so the 21310be98a7SChris Wilson * list is kept in a "least recently scanned" order. 21410be98a7SChris Wilson */ 21510be98a7SChris Wilson list_splice_tail(&pinned, &i915->contexts.hw_id_list); 21610be98a7SChris Wilson return id; 21710be98a7SChris Wilson } 21810be98a7SChris Wilson 21910be98a7SChris Wilson static int assign_hw_id(struct drm_i915_private *i915, unsigned int *out) 22010be98a7SChris Wilson { 22110be98a7SChris Wilson int ret; 22210be98a7SChris Wilson 22310be98a7SChris Wilson lockdep_assert_held(&i915->contexts.mutex); 22410be98a7SChris Wilson 22510be98a7SChris Wilson /* 22610be98a7SChris Wilson * We prefer to steal/stall ourselves and our users over that of the 22710be98a7SChris Wilson * entire system. That may be a little unfair to our users, and 22810be98a7SChris Wilson * even hurt high priority clients. The choice is whether to oomkill 22910be98a7SChris Wilson * something else, or steal a context id. 23010be98a7SChris Wilson */ 23110be98a7SChris Wilson ret = new_hw_id(i915, GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); 23210be98a7SChris Wilson if (unlikely(ret < 0)) { 23310be98a7SChris Wilson ret = steal_hw_id(i915); 23410be98a7SChris Wilson if (ret < 0) /* once again for the correct errno code */ 23510be98a7SChris Wilson ret = new_hw_id(i915, GFP_KERNEL); 23610be98a7SChris Wilson if (ret < 0) 23710be98a7SChris Wilson return ret; 23810be98a7SChris Wilson } 23910be98a7SChris Wilson 24010be98a7SChris Wilson *out = ret; 24110be98a7SChris Wilson return 0; 24210be98a7SChris Wilson } 24310be98a7SChris Wilson 24410be98a7SChris Wilson static void release_hw_id(struct i915_gem_context *ctx) 24510be98a7SChris Wilson { 24610be98a7SChris Wilson struct drm_i915_private *i915 = ctx->i915; 24710be98a7SChris Wilson 24810be98a7SChris Wilson if (list_empty(&ctx->hw_id_link)) 24910be98a7SChris Wilson return; 25010be98a7SChris Wilson 25110be98a7SChris Wilson mutex_lock(&i915->contexts.mutex); 25210be98a7SChris Wilson if (!list_empty(&ctx->hw_id_link)) { 25310be98a7SChris Wilson ida_simple_remove(&i915->contexts.hw_ida, ctx->hw_id); 25410be98a7SChris Wilson list_del_init(&ctx->hw_id_link); 25510be98a7SChris Wilson } 25610be98a7SChris Wilson mutex_unlock(&i915->contexts.mutex); 25710be98a7SChris Wilson } 25810be98a7SChris Wilson 25910be98a7SChris Wilson static void __free_engines(struct i915_gem_engines *e, unsigned int count) 26010be98a7SChris Wilson { 26110be98a7SChris Wilson while (count--) { 26210be98a7SChris Wilson if (!e->engines[count]) 26310be98a7SChris Wilson continue; 26410be98a7SChris Wilson 26510be98a7SChris Wilson intel_context_put(e->engines[count]); 26610be98a7SChris Wilson } 26710be98a7SChris Wilson kfree(e); 26810be98a7SChris Wilson } 26910be98a7SChris Wilson 27010be98a7SChris Wilson static void free_engines(struct i915_gem_engines *e) 27110be98a7SChris Wilson { 27210be98a7SChris Wilson __free_engines(e, e->num_engines); 27310be98a7SChris Wilson } 27410be98a7SChris Wilson 275155ab883SChris Wilson static void free_engines_rcu(struct rcu_head *rcu) 27610be98a7SChris Wilson { 277155ab883SChris Wilson free_engines(container_of(rcu, struct i915_gem_engines, rcu)); 27810be98a7SChris Wilson } 27910be98a7SChris Wilson 28010be98a7SChris Wilson static struct i915_gem_engines *default_engines(struct i915_gem_context *ctx) 28110be98a7SChris Wilson { 282f1c4d157SChris Wilson const struct intel_gt *gt = &ctx->i915->gt; 28310be98a7SChris Wilson struct intel_engine_cs *engine; 28410be98a7SChris Wilson struct i915_gem_engines *e; 28510be98a7SChris Wilson enum intel_engine_id id; 28610be98a7SChris Wilson 28710be98a7SChris Wilson e = kzalloc(struct_size(e, engines, I915_NUM_ENGINES), GFP_KERNEL); 28810be98a7SChris Wilson if (!e) 28910be98a7SChris Wilson return ERR_PTR(-ENOMEM); 29010be98a7SChris Wilson 291155ab883SChris Wilson init_rcu_head(&e->rcu); 292f1c4d157SChris Wilson for_each_engine(engine, gt, id) { 29310be98a7SChris Wilson struct intel_context *ce; 29410be98a7SChris Wilson 29510be98a7SChris Wilson ce = intel_context_create(ctx, engine); 29610be98a7SChris Wilson if (IS_ERR(ce)) { 29710be98a7SChris Wilson __free_engines(e, id); 29810be98a7SChris Wilson return ERR_CAST(ce); 29910be98a7SChris Wilson } 30010be98a7SChris Wilson 30110be98a7SChris Wilson e->engines[id] = ce; 302f1c4d157SChris Wilson e->num_engines = id + 1; 30310be98a7SChris Wilson } 30410be98a7SChris Wilson 30510be98a7SChris Wilson return e; 30610be98a7SChris Wilson } 30710be98a7SChris Wilson 30810be98a7SChris Wilson static void i915_gem_context_free(struct i915_gem_context *ctx) 30910be98a7SChris Wilson { 31010be98a7SChris Wilson lockdep_assert_held(&ctx->i915->drm.struct_mutex); 31110be98a7SChris Wilson GEM_BUG_ON(!i915_gem_context_is_closed(ctx)); 31210be98a7SChris Wilson 31310be98a7SChris Wilson release_hw_id(ctx); 314e568ac38SChris Wilson if (ctx->vm) 315e568ac38SChris Wilson i915_vm_put(ctx->vm); 31610be98a7SChris Wilson 31710be98a7SChris Wilson free_engines(rcu_access_pointer(ctx->engines)); 31810be98a7SChris Wilson mutex_destroy(&ctx->engines_mutex); 31910be98a7SChris Wilson 32010be98a7SChris Wilson if (ctx->timeline) 321f0c02c1bSTvrtko Ursulin intel_timeline_put(ctx->timeline); 32210be98a7SChris Wilson 32310be98a7SChris Wilson kfree(ctx->name); 32410be98a7SChris Wilson put_pid(ctx->pid); 32510be98a7SChris Wilson 32610be98a7SChris Wilson list_del(&ctx->link); 32710be98a7SChris Wilson mutex_destroy(&ctx->mutex); 32810be98a7SChris Wilson 32910be98a7SChris Wilson kfree_rcu(ctx, rcu); 33010be98a7SChris Wilson } 33110be98a7SChris Wilson 33210be98a7SChris Wilson static void contexts_free(struct drm_i915_private *i915) 33310be98a7SChris Wilson { 33410be98a7SChris Wilson struct llist_node *freed = llist_del_all(&i915->contexts.free_list); 33510be98a7SChris Wilson struct i915_gem_context *ctx, *cn; 33610be98a7SChris Wilson 33710be98a7SChris Wilson lockdep_assert_held(&i915->drm.struct_mutex); 33810be98a7SChris Wilson 33910be98a7SChris Wilson llist_for_each_entry_safe(ctx, cn, freed, free_link) 34010be98a7SChris Wilson i915_gem_context_free(ctx); 34110be98a7SChris Wilson } 34210be98a7SChris Wilson 34310be98a7SChris Wilson static void contexts_free_first(struct drm_i915_private *i915) 34410be98a7SChris Wilson { 34510be98a7SChris Wilson struct i915_gem_context *ctx; 34610be98a7SChris Wilson struct llist_node *freed; 34710be98a7SChris Wilson 34810be98a7SChris Wilson lockdep_assert_held(&i915->drm.struct_mutex); 34910be98a7SChris Wilson 35010be98a7SChris Wilson freed = llist_del_first(&i915->contexts.free_list); 35110be98a7SChris Wilson if (!freed) 35210be98a7SChris Wilson return; 35310be98a7SChris Wilson 35410be98a7SChris Wilson ctx = container_of(freed, typeof(*ctx), free_link); 35510be98a7SChris Wilson i915_gem_context_free(ctx); 35610be98a7SChris Wilson } 35710be98a7SChris Wilson 35810be98a7SChris Wilson static void contexts_free_worker(struct work_struct *work) 35910be98a7SChris Wilson { 36010be98a7SChris Wilson struct drm_i915_private *i915 = 36110be98a7SChris Wilson container_of(work, typeof(*i915), contexts.free_work); 36210be98a7SChris Wilson 36310be98a7SChris Wilson mutex_lock(&i915->drm.struct_mutex); 36410be98a7SChris Wilson contexts_free(i915); 36510be98a7SChris Wilson mutex_unlock(&i915->drm.struct_mutex); 36610be98a7SChris Wilson } 36710be98a7SChris Wilson 36810be98a7SChris Wilson void i915_gem_context_release(struct kref *ref) 36910be98a7SChris Wilson { 37010be98a7SChris Wilson struct i915_gem_context *ctx = container_of(ref, typeof(*ctx), ref); 37110be98a7SChris Wilson struct drm_i915_private *i915 = ctx->i915; 37210be98a7SChris Wilson 37310be98a7SChris Wilson trace_i915_context_free(ctx); 37410be98a7SChris Wilson if (llist_add(&ctx->free_link, &i915->contexts.free_list)) 37510be98a7SChris Wilson queue_work(i915->wq, &i915->contexts.free_work); 37610be98a7SChris Wilson } 37710be98a7SChris Wilson 37810be98a7SChris Wilson static void context_close(struct i915_gem_context *ctx) 37910be98a7SChris Wilson { 380155ab883SChris Wilson mutex_lock(&ctx->mutex); 381155ab883SChris Wilson 38210be98a7SChris Wilson i915_gem_context_set_closed(ctx); 383155ab883SChris Wilson ctx->file_priv = ERR_PTR(-EBADF); 38410be98a7SChris Wilson 38510be98a7SChris Wilson /* 38610be98a7SChris Wilson * This context will never again be assinged to HW, so we can 38710be98a7SChris Wilson * reuse its ID for the next context. 38810be98a7SChris Wilson */ 38910be98a7SChris Wilson release_hw_id(ctx); 39010be98a7SChris Wilson 39110be98a7SChris Wilson /* 39210be98a7SChris Wilson * The LUT uses the VMA as a backpointer to unref the object, 39310be98a7SChris Wilson * so we need to clear the LUT before we close all the VMA (inside 39410be98a7SChris Wilson * the ppgtt). 39510be98a7SChris Wilson */ 39610be98a7SChris Wilson lut_close(ctx); 39710be98a7SChris Wilson 398155ab883SChris Wilson mutex_unlock(&ctx->mutex); 39910be98a7SChris Wilson i915_gem_context_put(ctx); 40010be98a7SChris Wilson } 40110be98a7SChris Wilson 40210be98a7SChris Wilson static struct i915_gem_context * 403e568ac38SChris Wilson __create_context(struct drm_i915_private *i915) 40410be98a7SChris Wilson { 40510be98a7SChris Wilson struct i915_gem_context *ctx; 40610be98a7SChris Wilson struct i915_gem_engines *e; 40710be98a7SChris Wilson int err; 40810be98a7SChris Wilson int i; 40910be98a7SChris Wilson 41010be98a7SChris Wilson ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 41110be98a7SChris Wilson if (!ctx) 41210be98a7SChris Wilson return ERR_PTR(-ENOMEM); 41310be98a7SChris Wilson 41410be98a7SChris Wilson kref_init(&ctx->ref); 415e568ac38SChris Wilson list_add_tail(&ctx->link, &i915->contexts.list); 416e568ac38SChris Wilson ctx->i915 = i915; 41710be98a7SChris Wilson ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_NORMAL); 41810be98a7SChris Wilson mutex_init(&ctx->mutex); 41910be98a7SChris Wilson 42010be98a7SChris Wilson mutex_init(&ctx->engines_mutex); 42110be98a7SChris Wilson e = default_engines(ctx); 42210be98a7SChris Wilson if (IS_ERR(e)) { 42310be98a7SChris Wilson err = PTR_ERR(e); 42410be98a7SChris Wilson goto err_free; 42510be98a7SChris Wilson } 42610be98a7SChris Wilson RCU_INIT_POINTER(ctx->engines, e); 42710be98a7SChris Wilson 42810be98a7SChris Wilson INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL); 42910be98a7SChris Wilson INIT_LIST_HEAD(&ctx->hw_id_link); 43010be98a7SChris Wilson 43110be98a7SChris Wilson /* NB: Mark all slices as needing a remap so that when the context first 43210be98a7SChris Wilson * loads it will restore whatever remap state already exists. If there 43310be98a7SChris Wilson * is no remap info, it will be a NOP. */ 434e568ac38SChris Wilson ctx->remap_slice = ALL_L3_SLICES(i915); 43510be98a7SChris Wilson 43610be98a7SChris Wilson i915_gem_context_set_bannable(ctx); 43710be98a7SChris Wilson i915_gem_context_set_recoverable(ctx); 43810be98a7SChris Wilson 43910be98a7SChris Wilson ctx->ring_size = 4 * PAGE_SIZE; 44010be98a7SChris Wilson 44110be98a7SChris Wilson for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++) 44210be98a7SChris Wilson ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES; 44310be98a7SChris Wilson 44410be98a7SChris Wilson return ctx; 44510be98a7SChris Wilson 44610be98a7SChris Wilson err_free: 44710be98a7SChris Wilson kfree(ctx); 44810be98a7SChris Wilson return ERR_PTR(err); 44910be98a7SChris Wilson } 45010be98a7SChris Wilson 451e568ac38SChris Wilson static struct i915_address_space * 452e568ac38SChris Wilson __set_ppgtt(struct i915_gem_context *ctx, struct i915_address_space *vm) 45310be98a7SChris Wilson { 454e568ac38SChris Wilson struct i915_address_space *old = ctx->vm; 455f5d974f9SChris Wilson struct i915_gem_engines_iter it; 456f5d974f9SChris Wilson struct intel_context *ce; 45710be98a7SChris Wilson 458a1c9ca22SChris Wilson GEM_BUG_ON(old && i915_vm_is_4lvl(vm) != i915_vm_is_4lvl(old)); 459a1c9ca22SChris Wilson 460e568ac38SChris Wilson ctx->vm = i915_vm_get(vm); 46110be98a7SChris Wilson 462f5d974f9SChris Wilson for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 463f5d974f9SChris Wilson i915_vm_put(ce->vm); 464f5d974f9SChris Wilson ce->vm = i915_vm_get(vm); 465f5d974f9SChris Wilson } 466f5d974f9SChris Wilson i915_gem_context_unlock_engines(ctx); 467f5d974f9SChris Wilson 46810be98a7SChris Wilson return old; 46910be98a7SChris Wilson } 47010be98a7SChris Wilson 47110be98a7SChris Wilson static void __assign_ppgtt(struct i915_gem_context *ctx, 472e568ac38SChris Wilson struct i915_address_space *vm) 47310be98a7SChris Wilson { 474e568ac38SChris Wilson if (vm == ctx->vm) 47510be98a7SChris Wilson return; 47610be98a7SChris Wilson 477e568ac38SChris Wilson vm = __set_ppgtt(ctx, vm); 478e568ac38SChris Wilson if (vm) 479e568ac38SChris Wilson i915_vm_put(vm); 48010be98a7SChris Wilson } 48110be98a7SChris Wilson 48210be98a7SChris Wilson static struct i915_gem_context * 48310be98a7SChris Wilson i915_gem_create_context(struct drm_i915_private *dev_priv, unsigned int flags) 48410be98a7SChris Wilson { 48510be98a7SChris Wilson struct i915_gem_context *ctx; 48610be98a7SChris Wilson 48710be98a7SChris Wilson lockdep_assert_held(&dev_priv->drm.struct_mutex); 48810be98a7SChris Wilson 48910be98a7SChris Wilson if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE && 49010be98a7SChris Wilson !HAS_EXECLISTS(dev_priv)) 49110be98a7SChris Wilson return ERR_PTR(-EINVAL); 49210be98a7SChris Wilson 49310be98a7SChris Wilson /* Reap the most stale context */ 49410be98a7SChris Wilson contexts_free_first(dev_priv); 49510be98a7SChris Wilson 49610be98a7SChris Wilson ctx = __create_context(dev_priv); 49710be98a7SChris Wilson if (IS_ERR(ctx)) 49810be98a7SChris Wilson return ctx; 49910be98a7SChris Wilson 50010be98a7SChris Wilson if (HAS_FULL_PPGTT(dev_priv)) { 501ab53497bSChris Wilson struct i915_ppgtt *ppgtt; 50210be98a7SChris Wilson 50310be98a7SChris Wilson ppgtt = i915_ppgtt_create(dev_priv); 50410be98a7SChris Wilson if (IS_ERR(ppgtt)) { 50510be98a7SChris Wilson DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n", 50610be98a7SChris Wilson PTR_ERR(ppgtt)); 50710be98a7SChris Wilson context_close(ctx); 50810be98a7SChris Wilson return ERR_CAST(ppgtt); 50910be98a7SChris Wilson } 51010be98a7SChris Wilson 511e568ac38SChris Wilson __assign_ppgtt(ctx, &ppgtt->vm); 512e568ac38SChris Wilson i915_vm_put(&ppgtt->vm); 51310be98a7SChris Wilson } 51410be98a7SChris Wilson 51510be98a7SChris Wilson if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE) { 516f0c02c1bSTvrtko Ursulin struct intel_timeline *timeline; 51710be98a7SChris Wilson 518f0c02c1bSTvrtko Ursulin timeline = intel_timeline_create(&dev_priv->gt, NULL); 51910be98a7SChris Wilson if (IS_ERR(timeline)) { 52010be98a7SChris Wilson context_close(ctx); 52110be98a7SChris Wilson return ERR_CAST(timeline); 52210be98a7SChris Wilson } 52310be98a7SChris Wilson 52410be98a7SChris Wilson ctx->timeline = timeline; 52510be98a7SChris Wilson } 52610be98a7SChris Wilson 52710be98a7SChris Wilson trace_i915_context_create(ctx); 52810be98a7SChris Wilson 52910be98a7SChris Wilson return ctx; 53010be98a7SChris Wilson } 53110be98a7SChris Wilson 53210be98a7SChris Wilson /** 53310be98a7SChris Wilson * i915_gem_context_create_gvt - create a GVT GEM context 53410be98a7SChris Wilson * @dev: drm device * 53510be98a7SChris Wilson * 53610be98a7SChris Wilson * This function is used to create a GVT specific GEM context. 53710be98a7SChris Wilson * 53810be98a7SChris Wilson * Returns: 53910be98a7SChris Wilson * pointer to i915_gem_context on success, error pointer if failed 54010be98a7SChris Wilson * 54110be98a7SChris Wilson */ 54210be98a7SChris Wilson struct i915_gem_context * 54310be98a7SChris Wilson i915_gem_context_create_gvt(struct drm_device *dev) 54410be98a7SChris Wilson { 54510be98a7SChris Wilson struct i915_gem_context *ctx; 54610be98a7SChris Wilson int ret; 54710be98a7SChris Wilson 54810be98a7SChris Wilson if (!IS_ENABLED(CONFIG_DRM_I915_GVT)) 54910be98a7SChris Wilson return ERR_PTR(-ENODEV); 55010be98a7SChris Wilson 55110be98a7SChris Wilson ret = i915_mutex_lock_interruptible(dev); 55210be98a7SChris Wilson if (ret) 55310be98a7SChris Wilson return ERR_PTR(ret); 55410be98a7SChris Wilson 55510be98a7SChris Wilson ctx = i915_gem_create_context(to_i915(dev), 0); 55610be98a7SChris Wilson if (IS_ERR(ctx)) 55710be98a7SChris Wilson goto out; 55810be98a7SChris Wilson 55910be98a7SChris Wilson ret = i915_gem_context_pin_hw_id(ctx); 56010be98a7SChris Wilson if (ret) { 56110be98a7SChris Wilson context_close(ctx); 56210be98a7SChris Wilson ctx = ERR_PTR(ret); 56310be98a7SChris Wilson goto out; 56410be98a7SChris Wilson } 56510be98a7SChris Wilson 56610be98a7SChris Wilson ctx->file_priv = ERR_PTR(-EBADF); 56710be98a7SChris Wilson i915_gem_context_set_closed(ctx); /* not user accessible */ 56810be98a7SChris Wilson i915_gem_context_clear_bannable(ctx); 56910be98a7SChris Wilson i915_gem_context_set_force_single_submission(ctx); 57010be98a7SChris Wilson if (!USES_GUC_SUBMISSION(to_i915(dev))) 57110be98a7SChris Wilson ctx->ring_size = 512 * PAGE_SIZE; /* Max ring buffer size */ 57210be98a7SChris Wilson 57310be98a7SChris Wilson GEM_BUG_ON(i915_gem_context_is_kernel(ctx)); 57410be98a7SChris Wilson out: 57510be98a7SChris Wilson mutex_unlock(&dev->struct_mutex); 57610be98a7SChris Wilson return ctx; 57710be98a7SChris Wilson } 57810be98a7SChris Wilson 57910be98a7SChris Wilson static void 58010be98a7SChris Wilson destroy_kernel_context(struct i915_gem_context **ctxp) 58110be98a7SChris Wilson { 58210be98a7SChris Wilson struct i915_gem_context *ctx; 58310be98a7SChris Wilson 58410be98a7SChris Wilson /* Keep the context ref so that we can free it immediately ourselves */ 58510be98a7SChris Wilson ctx = i915_gem_context_get(fetch_and_zero(ctxp)); 58610be98a7SChris Wilson GEM_BUG_ON(!i915_gem_context_is_kernel(ctx)); 58710be98a7SChris Wilson 58810be98a7SChris Wilson context_close(ctx); 58910be98a7SChris Wilson i915_gem_context_free(ctx); 59010be98a7SChris Wilson } 59110be98a7SChris Wilson 59210be98a7SChris Wilson struct i915_gem_context * 59310be98a7SChris Wilson i915_gem_context_create_kernel(struct drm_i915_private *i915, int prio) 59410be98a7SChris Wilson { 59510be98a7SChris Wilson struct i915_gem_context *ctx; 59610be98a7SChris Wilson int err; 59710be98a7SChris Wilson 59810be98a7SChris Wilson ctx = i915_gem_create_context(i915, 0); 59910be98a7SChris Wilson if (IS_ERR(ctx)) 60010be98a7SChris Wilson return ctx; 60110be98a7SChris Wilson 60210be98a7SChris Wilson err = i915_gem_context_pin_hw_id(ctx); 60310be98a7SChris Wilson if (err) { 60410be98a7SChris Wilson destroy_kernel_context(&ctx); 60510be98a7SChris Wilson return ERR_PTR(err); 60610be98a7SChris Wilson } 60710be98a7SChris Wilson 60810be98a7SChris Wilson i915_gem_context_clear_bannable(ctx); 60910be98a7SChris Wilson ctx->sched.priority = I915_USER_PRIORITY(prio); 61010be98a7SChris Wilson ctx->ring_size = PAGE_SIZE; 61110be98a7SChris Wilson 61210be98a7SChris Wilson GEM_BUG_ON(!i915_gem_context_is_kernel(ctx)); 61310be98a7SChris Wilson 61410be98a7SChris Wilson return ctx; 61510be98a7SChris Wilson } 61610be98a7SChris Wilson 61710be98a7SChris Wilson static void init_contexts(struct drm_i915_private *i915) 61810be98a7SChris Wilson { 61910be98a7SChris Wilson mutex_init(&i915->contexts.mutex); 62010be98a7SChris Wilson INIT_LIST_HEAD(&i915->contexts.list); 62110be98a7SChris Wilson 62210be98a7SChris Wilson /* Using the simple ida interface, the max is limited by sizeof(int) */ 62310be98a7SChris Wilson BUILD_BUG_ON(MAX_CONTEXT_HW_ID > INT_MAX); 62410be98a7SChris Wilson BUILD_BUG_ON(GEN11_MAX_CONTEXT_HW_ID > INT_MAX); 62510be98a7SChris Wilson ida_init(&i915->contexts.hw_ida); 62610be98a7SChris Wilson INIT_LIST_HEAD(&i915->contexts.hw_id_list); 62710be98a7SChris Wilson 62810be98a7SChris Wilson INIT_WORK(&i915->contexts.free_work, contexts_free_worker); 62910be98a7SChris Wilson init_llist_head(&i915->contexts.free_list); 63010be98a7SChris Wilson } 63110be98a7SChris Wilson 63210be98a7SChris Wilson int i915_gem_contexts_init(struct drm_i915_private *dev_priv) 63310be98a7SChris Wilson { 63410be98a7SChris Wilson struct i915_gem_context *ctx; 63510be98a7SChris Wilson 63610be98a7SChris Wilson /* Reassure ourselves we are only called once */ 63710be98a7SChris Wilson GEM_BUG_ON(dev_priv->kernel_context); 63810be98a7SChris Wilson 63910be98a7SChris Wilson init_contexts(dev_priv); 64010be98a7SChris Wilson 64110be98a7SChris Wilson /* lowest priority; idle task */ 64210be98a7SChris Wilson ctx = i915_gem_context_create_kernel(dev_priv, I915_PRIORITY_MIN); 64310be98a7SChris Wilson if (IS_ERR(ctx)) { 64410be98a7SChris Wilson DRM_ERROR("Failed to create default global context\n"); 64510be98a7SChris Wilson return PTR_ERR(ctx); 64610be98a7SChris Wilson } 64710be98a7SChris Wilson /* 64810be98a7SChris Wilson * For easy recognisablity, we want the kernel context to be 0 and then 64910be98a7SChris Wilson * all user contexts will have non-zero hw_id. Kernel contexts are 65010be98a7SChris Wilson * permanently pinned, so that we never suffer a stall and can 65110be98a7SChris Wilson * use them from any allocation context (e.g. for evicting other 65210be98a7SChris Wilson * contexts and from inside the shrinker). 65310be98a7SChris Wilson */ 65410be98a7SChris Wilson GEM_BUG_ON(ctx->hw_id); 65510be98a7SChris Wilson GEM_BUG_ON(!atomic_read(&ctx->hw_id_pin_count)); 65610be98a7SChris Wilson dev_priv->kernel_context = ctx; 65710be98a7SChris Wilson 65810be98a7SChris Wilson DRM_DEBUG_DRIVER("%s context support initialized\n", 65910be98a7SChris Wilson DRIVER_CAPS(dev_priv)->has_logical_contexts ? 66010be98a7SChris Wilson "logical" : "fake"); 66110be98a7SChris Wilson return 0; 66210be98a7SChris Wilson } 66310be98a7SChris Wilson 66410be98a7SChris Wilson void i915_gem_contexts_fini(struct drm_i915_private *i915) 66510be98a7SChris Wilson { 66610be98a7SChris Wilson lockdep_assert_held(&i915->drm.struct_mutex); 66710be98a7SChris Wilson 66810be98a7SChris Wilson destroy_kernel_context(&i915->kernel_context); 66910be98a7SChris Wilson 67010be98a7SChris Wilson /* Must free all deferred contexts (via flush_workqueue) first */ 67110be98a7SChris Wilson GEM_BUG_ON(!list_empty(&i915->contexts.hw_id_list)); 67210be98a7SChris Wilson ida_destroy(&i915->contexts.hw_ida); 67310be98a7SChris Wilson } 67410be98a7SChris Wilson 67510be98a7SChris Wilson static int context_idr_cleanup(int id, void *p, void *data) 67610be98a7SChris Wilson { 67710be98a7SChris Wilson context_close(p); 67810be98a7SChris Wilson return 0; 67910be98a7SChris Wilson } 68010be98a7SChris Wilson 68110be98a7SChris Wilson static int vm_idr_cleanup(int id, void *p, void *data) 68210be98a7SChris Wilson { 683e568ac38SChris Wilson i915_vm_put(p); 68410be98a7SChris Wilson return 0; 68510be98a7SChris Wilson } 68610be98a7SChris Wilson 68710be98a7SChris Wilson static int gem_context_register(struct i915_gem_context *ctx, 68810be98a7SChris Wilson struct drm_i915_file_private *fpriv) 68910be98a7SChris Wilson { 69010be98a7SChris Wilson int ret; 69110be98a7SChris Wilson 69210be98a7SChris Wilson ctx->file_priv = fpriv; 693e568ac38SChris Wilson if (ctx->vm) 694e568ac38SChris Wilson ctx->vm->file = fpriv; 69510be98a7SChris Wilson 69610be98a7SChris Wilson ctx->pid = get_task_pid(current, PIDTYPE_PID); 69710be98a7SChris Wilson ctx->name = kasprintf(GFP_KERNEL, "%s[%d]", 69810be98a7SChris Wilson current->comm, pid_nr(ctx->pid)); 69910be98a7SChris Wilson if (!ctx->name) { 70010be98a7SChris Wilson ret = -ENOMEM; 70110be98a7SChris Wilson goto err_pid; 70210be98a7SChris Wilson } 70310be98a7SChris Wilson 70410be98a7SChris Wilson /* And finally expose ourselves to userspace via the idr */ 70510be98a7SChris Wilson mutex_lock(&fpriv->context_idr_lock); 70610be98a7SChris Wilson ret = idr_alloc(&fpriv->context_idr, ctx, 0, 0, GFP_KERNEL); 70710be98a7SChris Wilson mutex_unlock(&fpriv->context_idr_lock); 70810be98a7SChris Wilson if (ret >= 0) 70910be98a7SChris Wilson goto out; 71010be98a7SChris Wilson 71110be98a7SChris Wilson kfree(fetch_and_zero(&ctx->name)); 71210be98a7SChris Wilson err_pid: 71310be98a7SChris Wilson put_pid(fetch_and_zero(&ctx->pid)); 71410be98a7SChris Wilson out: 71510be98a7SChris Wilson return ret; 71610be98a7SChris Wilson } 71710be98a7SChris Wilson 71810be98a7SChris Wilson int i915_gem_context_open(struct drm_i915_private *i915, 71910be98a7SChris Wilson struct drm_file *file) 72010be98a7SChris Wilson { 72110be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 72210be98a7SChris Wilson struct i915_gem_context *ctx; 72310be98a7SChris Wilson int err; 72410be98a7SChris Wilson 72510be98a7SChris Wilson mutex_init(&file_priv->context_idr_lock); 72610be98a7SChris Wilson mutex_init(&file_priv->vm_idr_lock); 72710be98a7SChris Wilson 72810be98a7SChris Wilson idr_init(&file_priv->context_idr); 72910be98a7SChris Wilson idr_init_base(&file_priv->vm_idr, 1); 73010be98a7SChris Wilson 73110be98a7SChris Wilson mutex_lock(&i915->drm.struct_mutex); 73210be98a7SChris Wilson ctx = i915_gem_create_context(i915, 0); 73310be98a7SChris Wilson mutex_unlock(&i915->drm.struct_mutex); 73410be98a7SChris Wilson if (IS_ERR(ctx)) { 73510be98a7SChris Wilson err = PTR_ERR(ctx); 73610be98a7SChris Wilson goto err; 73710be98a7SChris Wilson } 73810be98a7SChris Wilson 73910be98a7SChris Wilson err = gem_context_register(ctx, file_priv); 74010be98a7SChris Wilson if (err < 0) 74110be98a7SChris Wilson goto err_ctx; 74210be98a7SChris Wilson 74310be98a7SChris Wilson GEM_BUG_ON(i915_gem_context_is_kernel(ctx)); 74410be98a7SChris Wilson GEM_BUG_ON(err > 0); 74510be98a7SChris Wilson 74610be98a7SChris Wilson return 0; 74710be98a7SChris Wilson 74810be98a7SChris Wilson err_ctx: 74910be98a7SChris Wilson context_close(ctx); 75010be98a7SChris Wilson err: 75110be98a7SChris Wilson idr_destroy(&file_priv->vm_idr); 75210be98a7SChris Wilson idr_destroy(&file_priv->context_idr); 75310be98a7SChris Wilson mutex_destroy(&file_priv->vm_idr_lock); 75410be98a7SChris Wilson mutex_destroy(&file_priv->context_idr_lock); 75510be98a7SChris Wilson return err; 75610be98a7SChris Wilson } 75710be98a7SChris Wilson 75810be98a7SChris Wilson void i915_gem_context_close(struct drm_file *file) 75910be98a7SChris Wilson { 76010be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 76110be98a7SChris Wilson 76210be98a7SChris Wilson idr_for_each(&file_priv->context_idr, context_idr_cleanup, NULL); 76310be98a7SChris Wilson idr_destroy(&file_priv->context_idr); 76410be98a7SChris Wilson mutex_destroy(&file_priv->context_idr_lock); 76510be98a7SChris Wilson 76610be98a7SChris Wilson idr_for_each(&file_priv->vm_idr, vm_idr_cleanup, NULL); 76710be98a7SChris Wilson idr_destroy(&file_priv->vm_idr); 76810be98a7SChris Wilson mutex_destroy(&file_priv->vm_idr_lock); 76910be98a7SChris Wilson } 77010be98a7SChris Wilson 77110be98a7SChris Wilson int i915_gem_vm_create_ioctl(struct drm_device *dev, void *data, 77210be98a7SChris Wilson struct drm_file *file) 77310be98a7SChris Wilson { 77410be98a7SChris Wilson struct drm_i915_private *i915 = to_i915(dev); 77510be98a7SChris Wilson struct drm_i915_gem_vm_control *args = data; 77610be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 777ab53497bSChris Wilson struct i915_ppgtt *ppgtt; 77810be98a7SChris Wilson int err; 77910be98a7SChris Wilson 78010be98a7SChris Wilson if (!HAS_FULL_PPGTT(i915)) 78110be98a7SChris Wilson return -ENODEV; 78210be98a7SChris Wilson 78310be98a7SChris Wilson if (args->flags) 78410be98a7SChris Wilson return -EINVAL; 78510be98a7SChris Wilson 78610be98a7SChris Wilson ppgtt = i915_ppgtt_create(i915); 78710be98a7SChris Wilson if (IS_ERR(ppgtt)) 78810be98a7SChris Wilson return PTR_ERR(ppgtt); 78910be98a7SChris Wilson 79010be98a7SChris Wilson ppgtt->vm.file = file_priv; 79110be98a7SChris Wilson 79210be98a7SChris Wilson if (args->extensions) { 79310be98a7SChris Wilson err = i915_user_extensions(u64_to_user_ptr(args->extensions), 79410be98a7SChris Wilson NULL, 0, 79510be98a7SChris Wilson ppgtt); 79610be98a7SChris Wilson if (err) 79710be98a7SChris Wilson goto err_put; 79810be98a7SChris Wilson } 79910be98a7SChris Wilson 80010be98a7SChris Wilson err = mutex_lock_interruptible(&file_priv->vm_idr_lock); 80110be98a7SChris Wilson if (err) 80210be98a7SChris Wilson goto err_put; 80310be98a7SChris Wilson 804e568ac38SChris Wilson err = idr_alloc(&file_priv->vm_idr, &ppgtt->vm, 0, 0, GFP_KERNEL); 80510be98a7SChris Wilson if (err < 0) 80610be98a7SChris Wilson goto err_unlock; 80710be98a7SChris Wilson 80810be98a7SChris Wilson GEM_BUG_ON(err == 0); /* reserved for invalid/unassigned ppgtt */ 80910be98a7SChris Wilson 81010be98a7SChris Wilson mutex_unlock(&file_priv->vm_idr_lock); 81110be98a7SChris Wilson 81210be98a7SChris Wilson args->vm_id = err; 81310be98a7SChris Wilson return 0; 81410be98a7SChris Wilson 81510be98a7SChris Wilson err_unlock: 81610be98a7SChris Wilson mutex_unlock(&file_priv->vm_idr_lock); 81710be98a7SChris Wilson err_put: 818e568ac38SChris Wilson i915_vm_put(&ppgtt->vm); 81910be98a7SChris Wilson return err; 82010be98a7SChris Wilson } 82110be98a7SChris Wilson 82210be98a7SChris Wilson int i915_gem_vm_destroy_ioctl(struct drm_device *dev, void *data, 82310be98a7SChris Wilson struct drm_file *file) 82410be98a7SChris Wilson { 82510be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 82610be98a7SChris Wilson struct drm_i915_gem_vm_control *args = data; 827e568ac38SChris Wilson struct i915_address_space *vm; 82810be98a7SChris Wilson int err; 82910be98a7SChris Wilson u32 id; 83010be98a7SChris Wilson 83110be98a7SChris Wilson if (args->flags) 83210be98a7SChris Wilson return -EINVAL; 83310be98a7SChris Wilson 83410be98a7SChris Wilson if (args->extensions) 83510be98a7SChris Wilson return -EINVAL; 83610be98a7SChris Wilson 83710be98a7SChris Wilson id = args->vm_id; 83810be98a7SChris Wilson if (!id) 83910be98a7SChris Wilson return -ENOENT; 84010be98a7SChris Wilson 84110be98a7SChris Wilson err = mutex_lock_interruptible(&file_priv->vm_idr_lock); 84210be98a7SChris Wilson if (err) 84310be98a7SChris Wilson return err; 84410be98a7SChris Wilson 845e568ac38SChris Wilson vm = idr_remove(&file_priv->vm_idr, id); 84610be98a7SChris Wilson 84710be98a7SChris Wilson mutex_unlock(&file_priv->vm_idr_lock); 848e568ac38SChris Wilson if (!vm) 84910be98a7SChris Wilson return -ENOENT; 85010be98a7SChris Wilson 851e568ac38SChris Wilson i915_vm_put(vm); 85210be98a7SChris Wilson return 0; 85310be98a7SChris Wilson } 85410be98a7SChris Wilson 85510be98a7SChris Wilson struct context_barrier_task { 85610be98a7SChris Wilson struct i915_active base; 85710be98a7SChris Wilson void (*task)(void *data); 85810be98a7SChris Wilson void *data; 85910be98a7SChris Wilson }; 86010be98a7SChris Wilson 86110be98a7SChris Wilson static void cb_retire(struct i915_active *base) 86210be98a7SChris Wilson { 86310be98a7SChris Wilson struct context_barrier_task *cb = container_of(base, typeof(*cb), base); 86410be98a7SChris Wilson 86510be98a7SChris Wilson if (cb->task) 86610be98a7SChris Wilson cb->task(cb->data); 86710be98a7SChris Wilson 86810be98a7SChris Wilson i915_active_fini(&cb->base); 86910be98a7SChris Wilson kfree(cb); 87010be98a7SChris Wilson } 87110be98a7SChris Wilson 87210be98a7SChris Wilson I915_SELFTEST_DECLARE(static intel_engine_mask_t context_barrier_inject_fault); 87310be98a7SChris Wilson static int context_barrier_task(struct i915_gem_context *ctx, 87410be98a7SChris Wilson intel_engine_mask_t engines, 8751fe2d6f9SChris Wilson bool (*skip)(struct intel_context *ce, void *data), 87610be98a7SChris Wilson int (*emit)(struct i915_request *rq, void *data), 87710be98a7SChris Wilson void (*task)(void *data), 87810be98a7SChris Wilson void *data) 87910be98a7SChris Wilson { 88010be98a7SChris Wilson struct drm_i915_private *i915 = ctx->i915; 88110be98a7SChris Wilson struct context_barrier_task *cb; 88210be98a7SChris Wilson struct i915_gem_engines_iter it; 88310be98a7SChris Wilson struct intel_context *ce; 88410be98a7SChris Wilson int err = 0; 88510be98a7SChris Wilson 88610be98a7SChris Wilson lockdep_assert_held(&i915->drm.struct_mutex); 88710be98a7SChris Wilson GEM_BUG_ON(!task); 88810be98a7SChris Wilson 88910be98a7SChris Wilson cb = kmalloc(sizeof(*cb), GFP_KERNEL); 89010be98a7SChris Wilson if (!cb) 89110be98a7SChris Wilson return -ENOMEM; 89210be98a7SChris Wilson 89312c255b5SChris Wilson i915_active_init(i915, &cb->base, NULL, cb_retire); 89412c255b5SChris Wilson err = i915_active_acquire(&cb->base); 89512c255b5SChris Wilson if (err) { 89612c255b5SChris Wilson kfree(cb); 89712c255b5SChris Wilson return err; 89812c255b5SChris Wilson } 89910be98a7SChris Wilson 90010be98a7SChris Wilson for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 90110be98a7SChris Wilson struct i915_request *rq; 90210be98a7SChris Wilson 90310be98a7SChris Wilson if (I915_SELFTEST_ONLY(context_barrier_inject_fault & 90410be98a7SChris Wilson ce->engine->mask)) { 90510be98a7SChris Wilson err = -ENXIO; 90610be98a7SChris Wilson break; 90710be98a7SChris Wilson } 90810be98a7SChris Wilson 9091fe2d6f9SChris Wilson if (!(ce->engine->mask & engines)) 9101fe2d6f9SChris Wilson continue; 9111fe2d6f9SChris Wilson 9121fe2d6f9SChris Wilson if (skip && skip(ce, data)) 91310be98a7SChris Wilson continue; 91410be98a7SChris Wilson 91510be98a7SChris Wilson rq = intel_context_create_request(ce); 91610be98a7SChris Wilson if (IS_ERR(rq)) { 91710be98a7SChris Wilson err = PTR_ERR(rq); 91810be98a7SChris Wilson break; 91910be98a7SChris Wilson } 92010be98a7SChris Wilson 92110be98a7SChris Wilson err = 0; 92210be98a7SChris Wilson if (emit) 92310be98a7SChris Wilson err = emit(rq, data); 92410be98a7SChris Wilson if (err == 0) 92510be98a7SChris Wilson err = i915_active_ref(&cb->base, rq->fence.context, rq); 92610be98a7SChris Wilson 92710be98a7SChris Wilson i915_request_add(rq); 92810be98a7SChris Wilson if (err) 92910be98a7SChris Wilson break; 93010be98a7SChris Wilson } 93110be98a7SChris Wilson i915_gem_context_unlock_engines(ctx); 93210be98a7SChris Wilson 93310be98a7SChris Wilson cb->task = err ? NULL : task; /* caller needs to unwind instead */ 93410be98a7SChris Wilson cb->data = data; 93510be98a7SChris Wilson 93610be98a7SChris Wilson i915_active_release(&cb->base); 93710be98a7SChris Wilson 93810be98a7SChris Wilson return err; 93910be98a7SChris Wilson } 94010be98a7SChris Wilson 94110be98a7SChris Wilson static int get_ppgtt(struct drm_i915_file_private *file_priv, 94210be98a7SChris Wilson struct i915_gem_context *ctx, 94310be98a7SChris Wilson struct drm_i915_gem_context_param *args) 94410be98a7SChris Wilson { 945e568ac38SChris Wilson struct i915_address_space *vm; 94610be98a7SChris Wilson int ret; 94710be98a7SChris Wilson 948e568ac38SChris Wilson if (!ctx->vm) 94910be98a7SChris Wilson return -ENODEV; 95010be98a7SChris Wilson 95110be98a7SChris Wilson /* XXX rcu acquire? */ 95210be98a7SChris Wilson ret = mutex_lock_interruptible(&ctx->i915->drm.struct_mutex); 95310be98a7SChris Wilson if (ret) 95410be98a7SChris Wilson return ret; 95510be98a7SChris Wilson 956e568ac38SChris Wilson vm = i915_vm_get(ctx->vm); 95710be98a7SChris Wilson mutex_unlock(&ctx->i915->drm.struct_mutex); 95810be98a7SChris Wilson 95910be98a7SChris Wilson ret = mutex_lock_interruptible(&file_priv->vm_idr_lock); 96010be98a7SChris Wilson if (ret) 96110be98a7SChris Wilson goto err_put; 96210be98a7SChris Wilson 963e568ac38SChris Wilson ret = idr_alloc(&file_priv->vm_idr, vm, 0, 0, GFP_KERNEL); 96410be98a7SChris Wilson GEM_BUG_ON(!ret); 96510be98a7SChris Wilson if (ret < 0) 96610be98a7SChris Wilson goto err_unlock; 96710be98a7SChris Wilson 968e568ac38SChris Wilson i915_vm_get(vm); 96910be98a7SChris Wilson 97010be98a7SChris Wilson args->size = 0; 97110be98a7SChris Wilson args->value = ret; 97210be98a7SChris Wilson 97310be98a7SChris Wilson ret = 0; 97410be98a7SChris Wilson err_unlock: 97510be98a7SChris Wilson mutex_unlock(&file_priv->vm_idr_lock); 97610be98a7SChris Wilson err_put: 977e568ac38SChris Wilson i915_vm_put(vm); 97810be98a7SChris Wilson return ret; 97910be98a7SChris Wilson } 98010be98a7SChris Wilson 98110be98a7SChris Wilson static void set_ppgtt_barrier(void *data) 98210be98a7SChris Wilson { 983e568ac38SChris Wilson struct i915_address_space *old = data; 98410be98a7SChris Wilson 985e568ac38SChris Wilson if (INTEL_GEN(old->i915) < 8) 986e568ac38SChris Wilson gen6_ppgtt_unpin_all(i915_vm_to_ppgtt(old)); 98710be98a7SChris Wilson 988e568ac38SChris Wilson i915_vm_put(old); 98910be98a7SChris Wilson } 99010be98a7SChris Wilson 99110be98a7SChris Wilson static int emit_ppgtt_update(struct i915_request *rq, void *data) 99210be98a7SChris Wilson { 993f5d974f9SChris Wilson struct i915_address_space *vm = rq->hw_context->vm; 99410be98a7SChris Wilson struct intel_engine_cs *engine = rq->engine; 99510be98a7SChris Wilson u32 base = engine->mmio_base; 99610be98a7SChris Wilson u32 *cs; 99710be98a7SChris Wilson int i; 99810be98a7SChris Wilson 999e568ac38SChris Wilson if (i915_vm_is_4lvl(vm)) { 1000ab53497bSChris Wilson struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1001b5b7bef9SMika Kuoppala const dma_addr_t pd_daddr = px_dma(ppgtt->pd); 100210be98a7SChris Wilson 100310be98a7SChris Wilson cs = intel_ring_begin(rq, 6); 100410be98a7SChris Wilson if (IS_ERR(cs)) 100510be98a7SChris Wilson return PTR_ERR(cs); 100610be98a7SChris Wilson 100710be98a7SChris Wilson *cs++ = MI_LOAD_REGISTER_IMM(2); 100810be98a7SChris Wilson 100910be98a7SChris Wilson *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 0)); 101010be98a7SChris Wilson *cs++ = upper_32_bits(pd_daddr); 101110be98a7SChris Wilson *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 0)); 101210be98a7SChris Wilson *cs++ = lower_32_bits(pd_daddr); 101310be98a7SChris Wilson 101410be98a7SChris Wilson *cs++ = MI_NOOP; 101510be98a7SChris Wilson intel_ring_advance(rq, cs); 101610be98a7SChris Wilson } else if (HAS_LOGICAL_RING_CONTEXTS(engine->i915)) { 1017ab53497bSChris Wilson struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1018e568ac38SChris Wilson 101910be98a7SChris Wilson cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2); 102010be98a7SChris Wilson if (IS_ERR(cs)) 102110be98a7SChris Wilson return PTR_ERR(cs); 102210be98a7SChris Wilson 102310be98a7SChris Wilson *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES); 102410be98a7SChris Wilson for (i = GEN8_3LVL_PDPES; i--; ) { 102510be98a7SChris Wilson const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); 102610be98a7SChris Wilson 102710be98a7SChris Wilson *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i)); 102810be98a7SChris Wilson *cs++ = upper_32_bits(pd_daddr); 102910be98a7SChris Wilson *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i)); 103010be98a7SChris Wilson *cs++ = lower_32_bits(pd_daddr); 103110be98a7SChris Wilson } 103210be98a7SChris Wilson *cs++ = MI_NOOP; 103310be98a7SChris Wilson intel_ring_advance(rq, cs); 103410be98a7SChris Wilson } else { 103510be98a7SChris Wilson /* ppGTT is not part of the legacy context image */ 1036e568ac38SChris Wilson gen6_ppgtt_pin(i915_vm_to_ppgtt(vm)); 103710be98a7SChris Wilson } 103810be98a7SChris Wilson 103910be98a7SChris Wilson return 0; 104010be98a7SChris Wilson } 104110be98a7SChris Wilson 10421fe2d6f9SChris Wilson static bool skip_ppgtt_update(struct intel_context *ce, void *data) 10431fe2d6f9SChris Wilson { 10441fe2d6f9SChris Wilson if (HAS_LOGICAL_RING_CONTEXTS(ce->engine->i915)) 10451fe2d6f9SChris Wilson return !ce->state; 10461fe2d6f9SChris Wilson else 10471fe2d6f9SChris Wilson return !atomic_read(&ce->pin_count); 10481fe2d6f9SChris Wilson } 10491fe2d6f9SChris Wilson 105010be98a7SChris Wilson static int set_ppgtt(struct drm_i915_file_private *file_priv, 105110be98a7SChris Wilson struct i915_gem_context *ctx, 105210be98a7SChris Wilson struct drm_i915_gem_context_param *args) 105310be98a7SChris Wilson { 1054e568ac38SChris Wilson struct i915_address_space *vm, *old; 105510be98a7SChris Wilson int err; 105610be98a7SChris Wilson 105710be98a7SChris Wilson if (args->size) 105810be98a7SChris Wilson return -EINVAL; 105910be98a7SChris Wilson 1060e568ac38SChris Wilson if (!ctx->vm) 106110be98a7SChris Wilson return -ENODEV; 106210be98a7SChris Wilson 106310be98a7SChris Wilson if (upper_32_bits(args->value)) 106410be98a7SChris Wilson return -ENOENT; 106510be98a7SChris Wilson 106610be98a7SChris Wilson err = mutex_lock_interruptible(&file_priv->vm_idr_lock); 106710be98a7SChris Wilson if (err) 106810be98a7SChris Wilson return err; 106910be98a7SChris Wilson 1070e568ac38SChris Wilson vm = idr_find(&file_priv->vm_idr, args->value); 1071e568ac38SChris Wilson if (vm) 1072e568ac38SChris Wilson i915_vm_get(vm); 107310be98a7SChris Wilson mutex_unlock(&file_priv->vm_idr_lock); 1074e568ac38SChris Wilson if (!vm) 107510be98a7SChris Wilson return -ENOENT; 107610be98a7SChris Wilson 107710be98a7SChris Wilson err = mutex_lock_interruptible(&ctx->i915->drm.struct_mutex); 107810be98a7SChris Wilson if (err) 107910be98a7SChris Wilson goto out; 108010be98a7SChris Wilson 1081e568ac38SChris Wilson if (vm == ctx->vm) 108210be98a7SChris Wilson goto unlock; 108310be98a7SChris Wilson 108410be98a7SChris Wilson /* Teardown the existing obj:vma cache, it will have to be rebuilt. */ 1085155ab883SChris Wilson mutex_lock(&ctx->mutex); 108610be98a7SChris Wilson lut_close(ctx); 1087155ab883SChris Wilson mutex_unlock(&ctx->mutex); 108810be98a7SChris Wilson 1089e568ac38SChris Wilson old = __set_ppgtt(ctx, vm); 109010be98a7SChris Wilson 109110be98a7SChris Wilson /* 109210be98a7SChris Wilson * We need to flush any requests using the current ppgtt before 109310be98a7SChris Wilson * we release it as the requests do not hold a reference themselves, 109410be98a7SChris Wilson * only indirectly through the context. 109510be98a7SChris Wilson */ 109610be98a7SChris Wilson err = context_barrier_task(ctx, ALL_ENGINES, 10971fe2d6f9SChris Wilson skip_ppgtt_update, 109810be98a7SChris Wilson emit_ppgtt_update, 109910be98a7SChris Wilson set_ppgtt_barrier, 110010be98a7SChris Wilson old); 110110be98a7SChris Wilson if (err) { 1102f5d974f9SChris Wilson i915_vm_put(__set_ppgtt(ctx, old)); 1103f5d974f9SChris Wilson i915_vm_put(old); 110410be98a7SChris Wilson } 110510be98a7SChris Wilson 110610be98a7SChris Wilson unlock: 110710be98a7SChris Wilson mutex_unlock(&ctx->i915->drm.struct_mutex); 110810be98a7SChris Wilson 110910be98a7SChris Wilson out: 1110e568ac38SChris Wilson i915_vm_put(vm); 111110be98a7SChris Wilson return err; 111210be98a7SChris Wilson } 111310be98a7SChris Wilson 111410be98a7SChris Wilson static int gen8_emit_rpcs_config(struct i915_request *rq, 111510be98a7SChris Wilson struct intel_context *ce, 111610be98a7SChris Wilson struct intel_sseu sseu) 111710be98a7SChris Wilson { 111810be98a7SChris Wilson u64 offset; 111910be98a7SChris Wilson u32 *cs; 112010be98a7SChris Wilson 112110be98a7SChris Wilson cs = intel_ring_begin(rq, 4); 112210be98a7SChris Wilson if (IS_ERR(cs)) 112310be98a7SChris Wilson return PTR_ERR(cs); 112410be98a7SChris Wilson 112510be98a7SChris Wilson offset = i915_ggtt_offset(ce->state) + 112610be98a7SChris Wilson LRC_STATE_PN * PAGE_SIZE + 112710be98a7SChris Wilson (CTX_R_PWR_CLK_STATE + 1) * 4; 112810be98a7SChris Wilson 112910be98a7SChris Wilson *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 113010be98a7SChris Wilson *cs++ = lower_32_bits(offset); 113110be98a7SChris Wilson *cs++ = upper_32_bits(offset); 113210be98a7SChris Wilson *cs++ = intel_sseu_make_rpcs(rq->i915, &sseu); 113310be98a7SChris Wilson 113410be98a7SChris Wilson intel_ring_advance(rq, cs); 113510be98a7SChris Wilson 113610be98a7SChris Wilson return 0; 113710be98a7SChris Wilson } 113810be98a7SChris Wilson 113910be98a7SChris Wilson static int 114010be98a7SChris Wilson gen8_modify_rpcs(struct intel_context *ce, struct intel_sseu sseu) 114110be98a7SChris Wilson { 114210be98a7SChris Wilson struct i915_request *rq; 114310be98a7SChris Wilson int ret; 114410be98a7SChris Wilson 114510be98a7SChris Wilson lockdep_assert_held(&ce->pin_mutex); 114610be98a7SChris Wilson 114710be98a7SChris Wilson /* 114810be98a7SChris Wilson * If the context is not idle, we have to submit an ordered request to 114910be98a7SChris Wilson * modify its context image via the kernel context (writing to our own 115010be98a7SChris Wilson * image, or into the registers directory, does not stick). Pristine 115110be98a7SChris Wilson * and idle contexts will be configured on pinning. 115210be98a7SChris Wilson */ 115310be98a7SChris Wilson if (!intel_context_is_pinned(ce)) 115410be98a7SChris Wilson return 0; 115510be98a7SChris Wilson 115610be98a7SChris Wilson rq = i915_request_create(ce->engine->kernel_context); 115710be98a7SChris Wilson if (IS_ERR(rq)) 115810be98a7SChris Wilson return PTR_ERR(rq); 115910be98a7SChris Wilson 1160a9877da2SChris Wilson /* Serialise with the remote context */ 1161a9877da2SChris Wilson ret = intel_context_prepare_remote_request(ce, rq); 1162a9877da2SChris Wilson if (ret == 0) 1163ce476c80SChris Wilson ret = gen8_emit_rpcs_config(rq, ce, sseu); 116410be98a7SChris Wilson 116510be98a7SChris Wilson i915_request_add(rq); 116610be98a7SChris Wilson return ret; 116710be98a7SChris Wilson } 116810be98a7SChris Wilson 116910be98a7SChris Wilson static int 117010be98a7SChris Wilson __intel_context_reconfigure_sseu(struct intel_context *ce, 117110be98a7SChris Wilson struct intel_sseu sseu) 117210be98a7SChris Wilson { 117310be98a7SChris Wilson int ret; 117410be98a7SChris Wilson 1175cb0c43f3SChris Wilson GEM_BUG_ON(INTEL_GEN(ce->engine->i915) < 8); 117610be98a7SChris Wilson 117710be98a7SChris Wilson ret = intel_context_lock_pinned(ce); 117810be98a7SChris Wilson if (ret) 117910be98a7SChris Wilson return ret; 118010be98a7SChris Wilson 118110be98a7SChris Wilson /* Nothing to do if unmodified. */ 118210be98a7SChris Wilson if (!memcmp(&ce->sseu, &sseu, sizeof(sseu))) 118310be98a7SChris Wilson goto unlock; 118410be98a7SChris Wilson 118510be98a7SChris Wilson ret = gen8_modify_rpcs(ce, sseu); 118610be98a7SChris Wilson if (!ret) 118710be98a7SChris Wilson ce->sseu = sseu; 118810be98a7SChris Wilson 118910be98a7SChris Wilson unlock: 119010be98a7SChris Wilson intel_context_unlock_pinned(ce); 119110be98a7SChris Wilson return ret; 119210be98a7SChris Wilson } 119310be98a7SChris Wilson 119410be98a7SChris Wilson static int 119510be98a7SChris Wilson intel_context_reconfigure_sseu(struct intel_context *ce, struct intel_sseu sseu) 119610be98a7SChris Wilson { 1197cb0c43f3SChris Wilson struct drm_i915_private *i915 = ce->engine->i915; 119810be98a7SChris Wilson int ret; 119910be98a7SChris Wilson 120010be98a7SChris Wilson ret = mutex_lock_interruptible(&i915->drm.struct_mutex); 120110be98a7SChris Wilson if (ret) 120210be98a7SChris Wilson return ret; 120310be98a7SChris Wilson 120410be98a7SChris Wilson ret = __intel_context_reconfigure_sseu(ce, sseu); 120510be98a7SChris Wilson 120610be98a7SChris Wilson mutex_unlock(&i915->drm.struct_mutex); 120710be98a7SChris Wilson 120810be98a7SChris Wilson return ret; 120910be98a7SChris Wilson } 121010be98a7SChris Wilson 121110be98a7SChris Wilson static int 121210be98a7SChris Wilson user_to_context_sseu(struct drm_i915_private *i915, 121310be98a7SChris Wilson const struct drm_i915_gem_context_param_sseu *user, 121410be98a7SChris Wilson struct intel_sseu *context) 121510be98a7SChris Wilson { 121610be98a7SChris Wilson const struct sseu_dev_info *device = &RUNTIME_INFO(i915)->sseu; 121710be98a7SChris Wilson 121810be98a7SChris Wilson /* No zeros in any field. */ 121910be98a7SChris Wilson if (!user->slice_mask || !user->subslice_mask || 122010be98a7SChris Wilson !user->min_eus_per_subslice || !user->max_eus_per_subslice) 122110be98a7SChris Wilson return -EINVAL; 122210be98a7SChris Wilson 122310be98a7SChris Wilson /* Max > min. */ 122410be98a7SChris Wilson if (user->max_eus_per_subslice < user->min_eus_per_subslice) 122510be98a7SChris Wilson return -EINVAL; 122610be98a7SChris Wilson 122710be98a7SChris Wilson /* 122810be98a7SChris Wilson * Some future proofing on the types since the uAPI is wider than the 122910be98a7SChris Wilson * current internal implementation. 123010be98a7SChris Wilson */ 123110be98a7SChris Wilson if (overflows_type(user->slice_mask, context->slice_mask) || 123210be98a7SChris Wilson overflows_type(user->subslice_mask, context->subslice_mask) || 123310be98a7SChris Wilson overflows_type(user->min_eus_per_subslice, 123410be98a7SChris Wilson context->min_eus_per_subslice) || 123510be98a7SChris Wilson overflows_type(user->max_eus_per_subslice, 123610be98a7SChris Wilson context->max_eus_per_subslice)) 123710be98a7SChris Wilson return -EINVAL; 123810be98a7SChris Wilson 123910be98a7SChris Wilson /* Check validity against hardware. */ 124010be98a7SChris Wilson if (user->slice_mask & ~device->slice_mask) 124110be98a7SChris Wilson return -EINVAL; 124210be98a7SChris Wilson 124310be98a7SChris Wilson if (user->subslice_mask & ~device->subslice_mask[0]) 124410be98a7SChris Wilson return -EINVAL; 124510be98a7SChris Wilson 124610be98a7SChris Wilson if (user->max_eus_per_subslice > device->max_eus_per_subslice) 124710be98a7SChris Wilson return -EINVAL; 124810be98a7SChris Wilson 124910be98a7SChris Wilson context->slice_mask = user->slice_mask; 125010be98a7SChris Wilson context->subslice_mask = user->subslice_mask; 125110be98a7SChris Wilson context->min_eus_per_subslice = user->min_eus_per_subslice; 125210be98a7SChris Wilson context->max_eus_per_subslice = user->max_eus_per_subslice; 125310be98a7SChris Wilson 125410be98a7SChris Wilson /* Part specific restrictions. */ 125510be98a7SChris Wilson if (IS_GEN(i915, 11)) { 125610be98a7SChris Wilson unsigned int hw_s = hweight8(device->slice_mask); 125710be98a7SChris Wilson unsigned int hw_ss_per_s = hweight8(device->subslice_mask[0]); 125810be98a7SChris Wilson unsigned int req_s = hweight8(context->slice_mask); 125910be98a7SChris Wilson unsigned int req_ss = hweight8(context->subslice_mask); 126010be98a7SChris Wilson 126110be98a7SChris Wilson /* 126210be98a7SChris Wilson * Only full subslice enablement is possible if more than one 126310be98a7SChris Wilson * slice is turned on. 126410be98a7SChris Wilson */ 126510be98a7SChris Wilson if (req_s > 1 && req_ss != hw_ss_per_s) 126610be98a7SChris Wilson return -EINVAL; 126710be98a7SChris Wilson 126810be98a7SChris Wilson /* 126910be98a7SChris Wilson * If more than four (SScount bitfield limit) subslices are 127010be98a7SChris Wilson * requested then the number has to be even. 127110be98a7SChris Wilson */ 127210be98a7SChris Wilson if (req_ss > 4 && (req_ss & 1)) 127310be98a7SChris Wilson return -EINVAL; 127410be98a7SChris Wilson 127510be98a7SChris Wilson /* 127610be98a7SChris Wilson * If only one slice is enabled and subslice count is below the 127710be98a7SChris Wilson * device full enablement, it must be at most half of the all 127810be98a7SChris Wilson * available subslices. 127910be98a7SChris Wilson */ 128010be98a7SChris Wilson if (req_s == 1 && req_ss < hw_ss_per_s && 128110be98a7SChris Wilson req_ss > (hw_ss_per_s / 2)) 128210be98a7SChris Wilson return -EINVAL; 128310be98a7SChris Wilson 128410be98a7SChris Wilson /* ABI restriction - VME use case only. */ 128510be98a7SChris Wilson 128610be98a7SChris Wilson /* All slices or one slice only. */ 128710be98a7SChris Wilson if (req_s != 1 && req_s != hw_s) 128810be98a7SChris Wilson return -EINVAL; 128910be98a7SChris Wilson 129010be98a7SChris Wilson /* 129110be98a7SChris Wilson * Half subslices or full enablement only when one slice is 129210be98a7SChris Wilson * enabled. 129310be98a7SChris Wilson */ 129410be98a7SChris Wilson if (req_s == 1 && 129510be98a7SChris Wilson (req_ss != hw_ss_per_s && req_ss != (hw_ss_per_s / 2))) 129610be98a7SChris Wilson return -EINVAL; 129710be98a7SChris Wilson 129810be98a7SChris Wilson /* No EU configuration changes. */ 129910be98a7SChris Wilson if ((user->min_eus_per_subslice != 130010be98a7SChris Wilson device->max_eus_per_subslice) || 130110be98a7SChris Wilson (user->max_eus_per_subslice != 130210be98a7SChris Wilson device->max_eus_per_subslice)) 130310be98a7SChris Wilson return -EINVAL; 130410be98a7SChris Wilson } 130510be98a7SChris Wilson 130610be98a7SChris Wilson return 0; 130710be98a7SChris Wilson } 130810be98a7SChris Wilson 130910be98a7SChris Wilson static int set_sseu(struct i915_gem_context *ctx, 131010be98a7SChris Wilson struct drm_i915_gem_context_param *args) 131110be98a7SChris Wilson { 131210be98a7SChris Wilson struct drm_i915_private *i915 = ctx->i915; 131310be98a7SChris Wilson struct drm_i915_gem_context_param_sseu user_sseu; 131410be98a7SChris Wilson struct intel_context *ce; 131510be98a7SChris Wilson struct intel_sseu sseu; 131610be98a7SChris Wilson unsigned long lookup; 131710be98a7SChris Wilson int ret; 131810be98a7SChris Wilson 131910be98a7SChris Wilson if (args->size < sizeof(user_sseu)) 132010be98a7SChris Wilson return -EINVAL; 132110be98a7SChris Wilson 132210be98a7SChris Wilson if (!IS_GEN(i915, 11)) 132310be98a7SChris Wilson return -ENODEV; 132410be98a7SChris Wilson 132510be98a7SChris Wilson if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), 132610be98a7SChris Wilson sizeof(user_sseu))) 132710be98a7SChris Wilson return -EFAULT; 132810be98a7SChris Wilson 132910be98a7SChris Wilson if (user_sseu.rsvd) 133010be98a7SChris Wilson return -EINVAL; 133110be98a7SChris Wilson 133210be98a7SChris Wilson if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) 133310be98a7SChris Wilson return -EINVAL; 133410be98a7SChris Wilson 133510be98a7SChris Wilson lookup = 0; 133610be98a7SChris Wilson if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) 133710be98a7SChris Wilson lookup |= LOOKUP_USER_INDEX; 133810be98a7SChris Wilson 133910be98a7SChris Wilson ce = lookup_user_engine(ctx, lookup, &user_sseu.engine); 134010be98a7SChris Wilson if (IS_ERR(ce)) 134110be98a7SChris Wilson return PTR_ERR(ce); 134210be98a7SChris Wilson 134310be98a7SChris Wilson /* Only render engine supports RPCS configuration. */ 134410be98a7SChris Wilson if (ce->engine->class != RENDER_CLASS) { 134510be98a7SChris Wilson ret = -ENODEV; 134610be98a7SChris Wilson goto out_ce; 134710be98a7SChris Wilson } 134810be98a7SChris Wilson 134910be98a7SChris Wilson ret = user_to_context_sseu(i915, &user_sseu, &sseu); 135010be98a7SChris Wilson if (ret) 135110be98a7SChris Wilson goto out_ce; 135210be98a7SChris Wilson 135310be98a7SChris Wilson ret = intel_context_reconfigure_sseu(ce, sseu); 135410be98a7SChris Wilson if (ret) 135510be98a7SChris Wilson goto out_ce; 135610be98a7SChris Wilson 135710be98a7SChris Wilson args->size = sizeof(user_sseu); 135810be98a7SChris Wilson 135910be98a7SChris Wilson out_ce: 136010be98a7SChris Wilson intel_context_put(ce); 136110be98a7SChris Wilson return ret; 136210be98a7SChris Wilson } 136310be98a7SChris Wilson 136410be98a7SChris Wilson struct set_engines { 136510be98a7SChris Wilson struct i915_gem_context *ctx; 136610be98a7SChris Wilson struct i915_gem_engines *engines; 136710be98a7SChris Wilson }; 136810be98a7SChris Wilson 136910be98a7SChris Wilson static int 137010be98a7SChris Wilson set_engines__load_balance(struct i915_user_extension __user *base, void *data) 137110be98a7SChris Wilson { 137210be98a7SChris Wilson struct i915_context_engines_load_balance __user *ext = 137310be98a7SChris Wilson container_of_user(base, typeof(*ext), base); 137410be98a7SChris Wilson const struct set_engines *set = data; 137510be98a7SChris Wilson struct intel_engine_cs *stack[16]; 137610be98a7SChris Wilson struct intel_engine_cs **siblings; 137710be98a7SChris Wilson struct intel_context *ce; 137810be98a7SChris Wilson u16 num_siblings, idx; 137910be98a7SChris Wilson unsigned int n; 138010be98a7SChris Wilson int err; 138110be98a7SChris Wilson 138210be98a7SChris Wilson if (!HAS_EXECLISTS(set->ctx->i915)) 138310be98a7SChris Wilson return -ENODEV; 138410be98a7SChris Wilson 138510be98a7SChris Wilson if (USES_GUC_SUBMISSION(set->ctx->i915)) 138610be98a7SChris Wilson return -ENODEV; /* not implement yet */ 138710be98a7SChris Wilson 138810be98a7SChris Wilson if (get_user(idx, &ext->engine_index)) 138910be98a7SChris Wilson return -EFAULT; 139010be98a7SChris Wilson 139110be98a7SChris Wilson if (idx >= set->engines->num_engines) { 139210be98a7SChris Wilson DRM_DEBUG("Invalid placement value, %d >= %d\n", 139310be98a7SChris Wilson idx, set->engines->num_engines); 139410be98a7SChris Wilson return -EINVAL; 139510be98a7SChris Wilson } 139610be98a7SChris Wilson 139710be98a7SChris Wilson idx = array_index_nospec(idx, set->engines->num_engines); 139810be98a7SChris Wilson if (set->engines->engines[idx]) { 139910be98a7SChris Wilson DRM_DEBUG("Invalid placement[%d], already occupied\n", idx); 140010be98a7SChris Wilson return -EEXIST; 140110be98a7SChris Wilson } 140210be98a7SChris Wilson 140310be98a7SChris Wilson if (get_user(num_siblings, &ext->num_siblings)) 140410be98a7SChris Wilson return -EFAULT; 140510be98a7SChris Wilson 140610be98a7SChris Wilson err = check_user_mbz(&ext->flags); 140710be98a7SChris Wilson if (err) 140810be98a7SChris Wilson return err; 140910be98a7SChris Wilson 141010be98a7SChris Wilson err = check_user_mbz(&ext->mbz64); 141110be98a7SChris Wilson if (err) 141210be98a7SChris Wilson return err; 141310be98a7SChris Wilson 141410be98a7SChris Wilson siblings = stack; 141510be98a7SChris Wilson if (num_siblings > ARRAY_SIZE(stack)) { 141610be98a7SChris Wilson siblings = kmalloc_array(num_siblings, 141710be98a7SChris Wilson sizeof(*siblings), 141810be98a7SChris Wilson GFP_KERNEL); 141910be98a7SChris Wilson if (!siblings) 142010be98a7SChris Wilson return -ENOMEM; 142110be98a7SChris Wilson } 142210be98a7SChris Wilson 142310be98a7SChris Wilson for (n = 0; n < num_siblings; n++) { 142410be98a7SChris Wilson struct i915_engine_class_instance ci; 142510be98a7SChris Wilson 142610be98a7SChris Wilson if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) { 142710be98a7SChris Wilson err = -EFAULT; 142810be98a7SChris Wilson goto out_siblings; 142910be98a7SChris Wilson } 143010be98a7SChris Wilson 143110be98a7SChris Wilson siblings[n] = intel_engine_lookup_user(set->ctx->i915, 143210be98a7SChris Wilson ci.engine_class, 143310be98a7SChris Wilson ci.engine_instance); 143410be98a7SChris Wilson if (!siblings[n]) { 143510be98a7SChris Wilson DRM_DEBUG("Invalid sibling[%d]: { class:%d, inst:%d }\n", 143610be98a7SChris Wilson n, ci.engine_class, ci.engine_instance); 143710be98a7SChris Wilson err = -EINVAL; 143810be98a7SChris Wilson goto out_siblings; 143910be98a7SChris Wilson } 144010be98a7SChris Wilson } 144110be98a7SChris Wilson 144210be98a7SChris Wilson ce = intel_execlists_create_virtual(set->ctx, siblings, n); 144310be98a7SChris Wilson if (IS_ERR(ce)) { 144410be98a7SChris Wilson err = PTR_ERR(ce); 144510be98a7SChris Wilson goto out_siblings; 144610be98a7SChris Wilson } 144710be98a7SChris Wilson 144810be98a7SChris Wilson if (cmpxchg(&set->engines->engines[idx], NULL, ce)) { 144910be98a7SChris Wilson intel_context_put(ce); 145010be98a7SChris Wilson err = -EEXIST; 145110be98a7SChris Wilson goto out_siblings; 145210be98a7SChris Wilson } 145310be98a7SChris Wilson 145410be98a7SChris Wilson out_siblings: 145510be98a7SChris Wilson if (siblings != stack) 145610be98a7SChris Wilson kfree(siblings); 145710be98a7SChris Wilson 145810be98a7SChris Wilson return err; 145910be98a7SChris Wilson } 146010be98a7SChris Wilson 146110be98a7SChris Wilson static int 146210be98a7SChris Wilson set_engines__bond(struct i915_user_extension __user *base, void *data) 146310be98a7SChris Wilson { 146410be98a7SChris Wilson struct i915_context_engines_bond __user *ext = 146510be98a7SChris Wilson container_of_user(base, typeof(*ext), base); 146610be98a7SChris Wilson const struct set_engines *set = data; 146710be98a7SChris Wilson struct i915_engine_class_instance ci; 146810be98a7SChris Wilson struct intel_engine_cs *virtual; 146910be98a7SChris Wilson struct intel_engine_cs *master; 147010be98a7SChris Wilson u16 idx, num_bonds; 147110be98a7SChris Wilson int err, n; 147210be98a7SChris Wilson 147310be98a7SChris Wilson if (get_user(idx, &ext->virtual_index)) 147410be98a7SChris Wilson return -EFAULT; 147510be98a7SChris Wilson 147610be98a7SChris Wilson if (idx >= set->engines->num_engines) { 147710be98a7SChris Wilson DRM_DEBUG("Invalid index for virtual engine: %d >= %d\n", 147810be98a7SChris Wilson idx, set->engines->num_engines); 147910be98a7SChris Wilson return -EINVAL; 148010be98a7SChris Wilson } 148110be98a7SChris Wilson 148210be98a7SChris Wilson idx = array_index_nospec(idx, set->engines->num_engines); 148310be98a7SChris Wilson if (!set->engines->engines[idx]) { 148410be98a7SChris Wilson DRM_DEBUG("Invalid engine at %d\n", idx); 148510be98a7SChris Wilson return -EINVAL; 148610be98a7SChris Wilson } 148710be98a7SChris Wilson virtual = set->engines->engines[idx]->engine; 148810be98a7SChris Wilson 148910be98a7SChris Wilson err = check_user_mbz(&ext->flags); 149010be98a7SChris Wilson if (err) 149110be98a7SChris Wilson return err; 149210be98a7SChris Wilson 149310be98a7SChris Wilson for (n = 0; n < ARRAY_SIZE(ext->mbz64); n++) { 149410be98a7SChris Wilson err = check_user_mbz(&ext->mbz64[n]); 149510be98a7SChris Wilson if (err) 149610be98a7SChris Wilson return err; 149710be98a7SChris Wilson } 149810be98a7SChris Wilson 149910be98a7SChris Wilson if (copy_from_user(&ci, &ext->master, sizeof(ci))) 150010be98a7SChris Wilson return -EFAULT; 150110be98a7SChris Wilson 150210be98a7SChris Wilson master = intel_engine_lookup_user(set->ctx->i915, 150310be98a7SChris Wilson ci.engine_class, ci.engine_instance); 150410be98a7SChris Wilson if (!master) { 150510be98a7SChris Wilson DRM_DEBUG("Unrecognised master engine: { class:%u, instance:%u }\n", 150610be98a7SChris Wilson ci.engine_class, ci.engine_instance); 150710be98a7SChris Wilson return -EINVAL; 150810be98a7SChris Wilson } 150910be98a7SChris Wilson 151010be98a7SChris Wilson if (get_user(num_bonds, &ext->num_bonds)) 151110be98a7SChris Wilson return -EFAULT; 151210be98a7SChris Wilson 151310be98a7SChris Wilson for (n = 0; n < num_bonds; n++) { 151410be98a7SChris Wilson struct intel_engine_cs *bond; 151510be98a7SChris Wilson 151610be98a7SChris Wilson if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) 151710be98a7SChris Wilson return -EFAULT; 151810be98a7SChris Wilson 151910be98a7SChris Wilson bond = intel_engine_lookup_user(set->ctx->i915, 152010be98a7SChris Wilson ci.engine_class, 152110be98a7SChris Wilson ci.engine_instance); 152210be98a7SChris Wilson if (!bond) { 152310be98a7SChris Wilson DRM_DEBUG("Unrecognised engine[%d] for bonding: { class:%d, instance: %d }\n", 152410be98a7SChris Wilson n, ci.engine_class, ci.engine_instance); 152510be98a7SChris Wilson return -EINVAL; 152610be98a7SChris Wilson } 152710be98a7SChris Wilson 152810be98a7SChris Wilson /* 152910be98a7SChris Wilson * A non-virtual engine has no siblings to choose between; and 153010be98a7SChris Wilson * a submit fence will always be directed to the one engine. 153110be98a7SChris Wilson */ 153210be98a7SChris Wilson if (intel_engine_is_virtual(virtual)) { 153310be98a7SChris Wilson err = intel_virtual_engine_attach_bond(virtual, 153410be98a7SChris Wilson master, 153510be98a7SChris Wilson bond); 153610be98a7SChris Wilson if (err) 153710be98a7SChris Wilson return err; 153810be98a7SChris Wilson } 153910be98a7SChris Wilson } 154010be98a7SChris Wilson 154110be98a7SChris Wilson return 0; 154210be98a7SChris Wilson } 154310be98a7SChris Wilson 154410be98a7SChris Wilson static const i915_user_extension_fn set_engines__extensions[] = { 154510be98a7SChris Wilson [I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE] = set_engines__load_balance, 154610be98a7SChris Wilson [I915_CONTEXT_ENGINES_EXT_BOND] = set_engines__bond, 154710be98a7SChris Wilson }; 154810be98a7SChris Wilson 154910be98a7SChris Wilson static int 155010be98a7SChris Wilson set_engines(struct i915_gem_context *ctx, 155110be98a7SChris Wilson const struct drm_i915_gem_context_param *args) 155210be98a7SChris Wilson { 155310be98a7SChris Wilson struct i915_context_param_engines __user *user = 155410be98a7SChris Wilson u64_to_user_ptr(args->value); 155510be98a7SChris Wilson struct set_engines set = { .ctx = ctx }; 155610be98a7SChris Wilson unsigned int num_engines, n; 155710be98a7SChris Wilson u64 extensions; 155810be98a7SChris Wilson int err; 155910be98a7SChris Wilson 156010be98a7SChris Wilson if (!args->size) { /* switch back to legacy user_ring_map */ 156110be98a7SChris Wilson if (!i915_gem_context_user_engines(ctx)) 156210be98a7SChris Wilson return 0; 156310be98a7SChris Wilson 156410be98a7SChris Wilson set.engines = default_engines(ctx); 156510be98a7SChris Wilson if (IS_ERR(set.engines)) 156610be98a7SChris Wilson return PTR_ERR(set.engines); 156710be98a7SChris Wilson 156810be98a7SChris Wilson goto replace; 156910be98a7SChris Wilson } 157010be98a7SChris Wilson 157110be98a7SChris Wilson BUILD_BUG_ON(!IS_ALIGNED(sizeof(*user), sizeof(*user->engines))); 157210be98a7SChris Wilson if (args->size < sizeof(*user) || 157310be98a7SChris Wilson !IS_ALIGNED(args->size, sizeof(*user->engines))) { 157410be98a7SChris Wilson DRM_DEBUG("Invalid size for engine array: %d\n", 157510be98a7SChris Wilson args->size); 157610be98a7SChris Wilson return -EINVAL; 157710be98a7SChris Wilson } 157810be98a7SChris Wilson 157910be98a7SChris Wilson /* 158010be98a7SChris Wilson * Note that I915_EXEC_RING_MASK limits execbuf to only using the 158110be98a7SChris Wilson * first 64 engines defined here. 158210be98a7SChris Wilson */ 158310be98a7SChris Wilson num_engines = (args->size - sizeof(*user)) / sizeof(*user->engines); 158410be98a7SChris Wilson 158510be98a7SChris Wilson set.engines = kmalloc(struct_size(set.engines, engines, num_engines), 158610be98a7SChris Wilson GFP_KERNEL); 158710be98a7SChris Wilson if (!set.engines) 158810be98a7SChris Wilson return -ENOMEM; 158910be98a7SChris Wilson 1590155ab883SChris Wilson init_rcu_head(&set.engines->rcu); 159110be98a7SChris Wilson for (n = 0; n < num_engines; n++) { 159210be98a7SChris Wilson struct i915_engine_class_instance ci; 159310be98a7SChris Wilson struct intel_engine_cs *engine; 159410be98a7SChris Wilson 159510be98a7SChris Wilson if (copy_from_user(&ci, &user->engines[n], sizeof(ci))) { 159610be98a7SChris Wilson __free_engines(set.engines, n); 159710be98a7SChris Wilson return -EFAULT; 159810be98a7SChris Wilson } 159910be98a7SChris Wilson 160010be98a7SChris Wilson if (ci.engine_class == (u16)I915_ENGINE_CLASS_INVALID && 160110be98a7SChris Wilson ci.engine_instance == (u16)I915_ENGINE_CLASS_INVALID_NONE) { 160210be98a7SChris Wilson set.engines->engines[n] = NULL; 160310be98a7SChris Wilson continue; 160410be98a7SChris Wilson } 160510be98a7SChris Wilson 160610be98a7SChris Wilson engine = intel_engine_lookup_user(ctx->i915, 160710be98a7SChris Wilson ci.engine_class, 160810be98a7SChris Wilson ci.engine_instance); 160910be98a7SChris Wilson if (!engine) { 161010be98a7SChris Wilson DRM_DEBUG("Invalid engine[%d]: { class:%d, instance:%d }\n", 161110be98a7SChris Wilson n, ci.engine_class, ci.engine_instance); 161210be98a7SChris Wilson __free_engines(set.engines, n); 161310be98a7SChris Wilson return -ENOENT; 161410be98a7SChris Wilson } 161510be98a7SChris Wilson 161610be98a7SChris Wilson set.engines->engines[n] = intel_context_create(ctx, engine); 161710be98a7SChris Wilson if (!set.engines->engines[n]) { 161810be98a7SChris Wilson __free_engines(set.engines, n); 161910be98a7SChris Wilson return -ENOMEM; 162010be98a7SChris Wilson } 162110be98a7SChris Wilson } 162210be98a7SChris Wilson set.engines->num_engines = num_engines; 162310be98a7SChris Wilson 162410be98a7SChris Wilson err = -EFAULT; 162510be98a7SChris Wilson if (!get_user(extensions, &user->extensions)) 162610be98a7SChris Wilson err = i915_user_extensions(u64_to_user_ptr(extensions), 162710be98a7SChris Wilson set_engines__extensions, 162810be98a7SChris Wilson ARRAY_SIZE(set_engines__extensions), 162910be98a7SChris Wilson &set); 163010be98a7SChris Wilson if (err) { 163110be98a7SChris Wilson free_engines(set.engines); 163210be98a7SChris Wilson return err; 163310be98a7SChris Wilson } 163410be98a7SChris Wilson 163510be98a7SChris Wilson replace: 163610be98a7SChris Wilson mutex_lock(&ctx->engines_mutex); 163710be98a7SChris Wilson if (args->size) 163810be98a7SChris Wilson i915_gem_context_set_user_engines(ctx); 163910be98a7SChris Wilson else 164010be98a7SChris Wilson i915_gem_context_clear_user_engines(ctx); 164110be98a7SChris Wilson rcu_swap_protected(ctx->engines, set.engines, 1); 164210be98a7SChris Wilson mutex_unlock(&ctx->engines_mutex); 164310be98a7SChris Wilson 1644155ab883SChris Wilson call_rcu(&set.engines->rcu, free_engines_rcu); 164510be98a7SChris Wilson 164610be98a7SChris Wilson return 0; 164710be98a7SChris Wilson } 164810be98a7SChris Wilson 164910be98a7SChris Wilson static struct i915_gem_engines * 165010be98a7SChris Wilson __copy_engines(struct i915_gem_engines *e) 165110be98a7SChris Wilson { 165210be98a7SChris Wilson struct i915_gem_engines *copy; 165310be98a7SChris Wilson unsigned int n; 165410be98a7SChris Wilson 165510be98a7SChris Wilson copy = kmalloc(struct_size(e, engines, e->num_engines), GFP_KERNEL); 165610be98a7SChris Wilson if (!copy) 165710be98a7SChris Wilson return ERR_PTR(-ENOMEM); 165810be98a7SChris Wilson 1659155ab883SChris Wilson init_rcu_head(©->rcu); 166010be98a7SChris Wilson for (n = 0; n < e->num_engines; n++) { 166110be98a7SChris Wilson if (e->engines[n]) 166210be98a7SChris Wilson copy->engines[n] = intel_context_get(e->engines[n]); 166310be98a7SChris Wilson else 166410be98a7SChris Wilson copy->engines[n] = NULL; 166510be98a7SChris Wilson } 166610be98a7SChris Wilson copy->num_engines = n; 166710be98a7SChris Wilson 166810be98a7SChris Wilson return copy; 166910be98a7SChris Wilson } 167010be98a7SChris Wilson 167110be98a7SChris Wilson static int 167210be98a7SChris Wilson get_engines(struct i915_gem_context *ctx, 167310be98a7SChris Wilson struct drm_i915_gem_context_param *args) 167410be98a7SChris Wilson { 167510be98a7SChris Wilson struct i915_context_param_engines __user *user; 167610be98a7SChris Wilson struct i915_gem_engines *e; 167710be98a7SChris Wilson size_t n, count, size; 167810be98a7SChris Wilson int err = 0; 167910be98a7SChris Wilson 168010be98a7SChris Wilson err = mutex_lock_interruptible(&ctx->engines_mutex); 168110be98a7SChris Wilson if (err) 168210be98a7SChris Wilson return err; 168310be98a7SChris Wilson 168410be98a7SChris Wilson e = NULL; 168510be98a7SChris Wilson if (i915_gem_context_user_engines(ctx)) 168610be98a7SChris Wilson e = __copy_engines(i915_gem_context_engines(ctx)); 168710be98a7SChris Wilson mutex_unlock(&ctx->engines_mutex); 168810be98a7SChris Wilson if (IS_ERR_OR_NULL(e)) { 168910be98a7SChris Wilson args->size = 0; 169010be98a7SChris Wilson return PTR_ERR_OR_ZERO(e); 169110be98a7SChris Wilson } 169210be98a7SChris Wilson 169310be98a7SChris Wilson count = e->num_engines; 169410be98a7SChris Wilson 169510be98a7SChris Wilson /* Be paranoid in case we have an impedance mismatch */ 169610be98a7SChris Wilson if (!check_struct_size(user, engines, count, &size)) { 169710be98a7SChris Wilson err = -EINVAL; 169810be98a7SChris Wilson goto err_free; 169910be98a7SChris Wilson } 170010be98a7SChris Wilson if (overflows_type(size, args->size)) { 170110be98a7SChris Wilson err = -EINVAL; 170210be98a7SChris Wilson goto err_free; 170310be98a7SChris Wilson } 170410be98a7SChris Wilson 170510be98a7SChris Wilson if (!args->size) { 170610be98a7SChris Wilson args->size = size; 170710be98a7SChris Wilson goto err_free; 170810be98a7SChris Wilson } 170910be98a7SChris Wilson 171010be98a7SChris Wilson if (args->size < size) { 171110be98a7SChris Wilson err = -EINVAL; 171210be98a7SChris Wilson goto err_free; 171310be98a7SChris Wilson } 171410be98a7SChris Wilson 171510be98a7SChris Wilson user = u64_to_user_ptr(args->value); 171610be98a7SChris Wilson if (!access_ok(user, size)) { 171710be98a7SChris Wilson err = -EFAULT; 171810be98a7SChris Wilson goto err_free; 171910be98a7SChris Wilson } 172010be98a7SChris Wilson 172110be98a7SChris Wilson if (put_user(0, &user->extensions)) { 172210be98a7SChris Wilson err = -EFAULT; 172310be98a7SChris Wilson goto err_free; 172410be98a7SChris Wilson } 172510be98a7SChris Wilson 172610be98a7SChris Wilson for (n = 0; n < count; n++) { 172710be98a7SChris Wilson struct i915_engine_class_instance ci = { 172810be98a7SChris Wilson .engine_class = I915_ENGINE_CLASS_INVALID, 172910be98a7SChris Wilson .engine_instance = I915_ENGINE_CLASS_INVALID_NONE, 173010be98a7SChris Wilson }; 173110be98a7SChris Wilson 173210be98a7SChris Wilson if (e->engines[n]) { 173310be98a7SChris Wilson ci.engine_class = e->engines[n]->engine->uabi_class; 1734750e76b4SChris Wilson ci.engine_instance = e->engines[n]->engine->uabi_instance; 173510be98a7SChris Wilson } 173610be98a7SChris Wilson 173710be98a7SChris Wilson if (copy_to_user(&user->engines[n], &ci, sizeof(ci))) { 173810be98a7SChris Wilson err = -EFAULT; 173910be98a7SChris Wilson goto err_free; 174010be98a7SChris Wilson } 174110be98a7SChris Wilson } 174210be98a7SChris Wilson 174310be98a7SChris Wilson args->size = size; 174410be98a7SChris Wilson 174510be98a7SChris Wilson err_free: 1746155ab883SChris Wilson free_engines(e); 174710be98a7SChris Wilson return err; 174810be98a7SChris Wilson } 174910be98a7SChris Wilson 175010be98a7SChris Wilson static int ctx_setparam(struct drm_i915_file_private *fpriv, 175110be98a7SChris Wilson struct i915_gem_context *ctx, 175210be98a7SChris Wilson struct drm_i915_gem_context_param *args) 175310be98a7SChris Wilson { 175410be98a7SChris Wilson int ret = 0; 175510be98a7SChris Wilson 175610be98a7SChris Wilson switch (args->param) { 175710be98a7SChris Wilson case I915_CONTEXT_PARAM_NO_ZEROMAP: 175810be98a7SChris Wilson if (args->size) 175910be98a7SChris Wilson ret = -EINVAL; 176010be98a7SChris Wilson else if (args->value) 176110be98a7SChris Wilson set_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); 176210be98a7SChris Wilson else 176310be98a7SChris Wilson clear_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); 176410be98a7SChris Wilson break; 176510be98a7SChris Wilson 176610be98a7SChris Wilson case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 176710be98a7SChris Wilson if (args->size) 176810be98a7SChris Wilson ret = -EINVAL; 176910be98a7SChris Wilson else if (args->value) 177010be98a7SChris Wilson i915_gem_context_set_no_error_capture(ctx); 177110be98a7SChris Wilson else 177210be98a7SChris Wilson i915_gem_context_clear_no_error_capture(ctx); 177310be98a7SChris Wilson break; 177410be98a7SChris Wilson 177510be98a7SChris Wilson case I915_CONTEXT_PARAM_BANNABLE: 177610be98a7SChris Wilson if (args->size) 177710be98a7SChris Wilson ret = -EINVAL; 177810be98a7SChris Wilson else if (!capable(CAP_SYS_ADMIN) && !args->value) 177910be98a7SChris Wilson ret = -EPERM; 178010be98a7SChris Wilson else if (args->value) 178110be98a7SChris Wilson i915_gem_context_set_bannable(ctx); 178210be98a7SChris Wilson else 178310be98a7SChris Wilson i915_gem_context_clear_bannable(ctx); 178410be98a7SChris Wilson break; 178510be98a7SChris Wilson 178610be98a7SChris Wilson case I915_CONTEXT_PARAM_RECOVERABLE: 178710be98a7SChris Wilson if (args->size) 178810be98a7SChris Wilson ret = -EINVAL; 178910be98a7SChris Wilson else if (args->value) 179010be98a7SChris Wilson i915_gem_context_set_recoverable(ctx); 179110be98a7SChris Wilson else 179210be98a7SChris Wilson i915_gem_context_clear_recoverable(ctx); 179310be98a7SChris Wilson break; 179410be98a7SChris Wilson 179510be98a7SChris Wilson case I915_CONTEXT_PARAM_PRIORITY: 179610be98a7SChris Wilson { 179710be98a7SChris Wilson s64 priority = args->value; 179810be98a7SChris Wilson 179910be98a7SChris Wilson if (args->size) 180010be98a7SChris Wilson ret = -EINVAL; 180110be98a7SChris Wilson else if (!(ctx->i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY)) 180210be98a7SChris Wilson ret = -ENODEV; 180310be98a7SChris Wilson else if (priority > I915_CONTEXT_MAX_USER_PRIORITY || 180410be98a7SChris Wilson priority < I915_CONTEXT_MIN_USER_PRIORITY) 180510be98a7SChris Wilson ret = -EINVAL; 180610be98a7SChris Wilson else if (priority > I915_CONTEXT_DEFAULT_PRIORITY && 180710be98a7SChris Wilson !capable(CAP_SYS_NICE)) 180810be98a7SChris Wilson ret = -EPERM; 180910be98a7SChris Wilson else 181010be98a7SChris Wilson ctx->sched.priority = 181110be98a7SChris Wilson I915_USER_PRIORITY(priority); 181210be98a7SChris Wilson } 181310be98a7SChris Wilson break; 181410be98a7SChris Wilson 181510be98a7SChris Wilson case I915_CONTEXT_PARAM_SSEU: 181610be98a7SChris Wilson ret = set_sseu(ctx, args); 181710be98a7SChris Wilson break; 181810be98a7SChris Wilson 181910be98a7SChris Wilson case I915_CONTEXT_PARAM_VM: 182010be98a7SChris Wilson ret = set_ppgtt(fpriv, ctx, args); 182110be98a7SChris Wilson break; 182210be98a7SChris Wilson 182310be98a7SChris Wilson case I915_CONTEXT_PARAM_ENGINES: 182410be98a7SChris Wilson ret = set_engines(ctx, args); 182510be98a7SChris Wilson break; 182610be98a7SChris Wilson 182710be98a7SChris Wilson case I915_CONTEXT_PARAM_BAN_PERIOD: 182810be98a7SChris Wilson default: 182910be98a7SChris Wilson ret = -EINVAL; 183010be98a7SChris Wilson break; 183110be98a7SChris Wilson } 183210be98a7SChris Wilson 183310be98a7SChris Wilson return ret; 183410be98a7SChris Wilson } 183510be98a7SChris Wilson 183610be98a7SChris Wilson struct create_ext { 183710be98a7SChris Wilson struct i915_gem_context *ctx; 183810be98a7SChris Wilson struct drm_i915_file_private *fpriv; 183910be98a7SChris Wilson }; 184010be98a7SChris Wilson 184110be98a7SChris Wilson static int create_setparam(struct i915_user_extension __user *ext, void *data) 184210be98a7SChris Wilson { 184310be98a7SChris Wilson struct drm_i915_gem_context_create_ext_setparam local; 184410be98a7SChris Wilson const struct create_ext *arg = data; 184510be98a7SChris Wilson 184610be98a7SChris Wilson if (copy_from_user(&local, ext, sizeof(local))) 184710be98a7SChris Wilson return -EFAULT; 184810be98a7SChris Wilson 184910be98a7SChris Wilson if (local.param.ctx_id) 185010be98a7SChris Wilson return -EINVAL; 185110be98a7SChris Wilson 185210be98a7SChris Wilson return ctx_setparam(arg->fpriv, arg->ctx, &local.param); 185310be98a7SChris Wilson } 185410be98a7SChris Wilson 185510be98a7SChris Wilson static int clone_engines(struct i915_gem_context *dst, 185610be98a7SChris Wilson struct i915_gem_context *src) 185710be98a7SChris Wilson { 185810be98a7SChris Wilson struct i915_gem_engines *e = i915_gem_context_lock_engines(src); 185910be98a7SChris Wilson struct i915_gem_engines *clone; 186010be98a7SChris Wilson bool user_engines; 186110be98a7SChris Wilson unsigned long n; 186210be98a7SChris Wilson 186310be98a7SChris Wilson clone = kmalloc(struct_size(e, engines, e->num_engines), GFP_KERNEL); 186410be98a7SChris Wilson if (!clone) 186510be98a7SChris Wilson goto err_unlock; 186610be98a7SChris Wilson 1867155ab883SChris Wilson init_rcu_head(&clone->rcu); 186810be98a7SChris Wilson for (n = 0; n < e->num_engines; n++) { 186910be98a7SChris Wilson struct intel_engine_cs *engine; 187010be98a7SChris Wilson 187110be98a7SChris Wilson if (!e->engines[n]) { 187210be98a7SChris Wilson clone->engines[n] = NULL; 187310be98a7SChris Wilson continue; 187410be98a7SChris Wilson } 187510be98a7SChris Wilson engine = e->engines[n]->engine; 187610be98a7SChris Wilson 187710be98a7SChris Wilson /* 187810be98a7SChris Wilson * Virtual engines are singletons; they can only exist 187910be98a7SChris Wilson * inside a single context, because they embed their 188010be98a7SChris Wilson * HW context... As each virtual context implies a single 188110be98a7SChris Wilson * timeline (each engine can only dequeue a single request 188210be98a7SChris Wilson * at any time), it would be surprising for two contexts 188310be98a7SChris Wilson * to use the same engine. So let's create a copy of 188410be98a7SChris Wilson * the virtual engine instead. 188510be98a7SChris Wilson */ 188610be98a7SChris Wilson if (intel_engine_is_virtual(engine)) 188710be98a7SChris Wilson clone->engines[n] = 188810be98a7SChris Wilson intel_execlists_clone_virtual(dst, engine); 188910be98a7SChris Wilson else 189010be98a7SChris Wilson clone->engines[n] = intel_context_create(dst, engine); 189110be98a7SChris Wilson if (IS_ERR_OR_NULL(clone->engines[n])) { 189210be98a7SChris Wilson __free_engines(clone, n); 189310be98a7SChris Wilson goto err_unlock; 189410be98a7SChris Wilson } 189510be98a7SChris Wilson } 189610be98a7SChris Wilson clone->num_engines = n; 189710be98a7SChris Wilson 189810be98a7SChris Wilson user_engines = i915_gem_context_user_engines(src); 189910be98a7SChris Wilson i915_gem_context_unlock_engines(src); 190010be98a7SChris Wilson 190110be98a7SChris Wilson free_engines(dst->engines); 190210be98a7SChris Wilson RCU_INIT_POINTER(dst->engines, clone); 190310be98a7SChris Wilson if (user_engines) 190410be98a7SChris Wilson i915_gem_context_set_user_engines(dst); 190510be98a7SChris Wilson else 190610be98a7SChris Wilson i915_gem_context_clear_user_engines(dst); 190710be98a7SChris Wilson return 0; 190810be98a7SChris Wilson 190910be98a7SChris Wilson err_unlock: 191010be98a7SChris Wilson i915_gem_context_unlock_engines(src); 191110be98a7SChris Wilson return -ENOMEM; 191210be98a7SChris Wilson } 191310be98a7SChris Wilson 191410be98a7SChris Wilson static int clone_flags(struct i915_gem_context *dst, 191510be98a7SChris Wilson struct i915_gem_context *src) 191610be98a7SChris Wilson { 191710be98a7SChris Wilson dst->user_flags = src->user_flags; 191810be98a7SChris Wilson return 0; 191910be98a7SChris Wilson } 192010be98a7SChris Wilson 192110be98a7SChris Wilson static int clone_schedattr(struct i915_gem_context *dst, 192210be98a7SChris Wilson struct i915_gem_context *src) 192310be98a7SChris Wilson { 192410be98a7SChris Wilson dst->sched = src->sched; 192510be98a7SChris Wilson return 0; 192610be98a7SChris Wilson } 192710be98a7SChris Wilson 192810be98a7SChris Wilson static int clone_sseu(struct i915_gem_context *dst, 192910be98a7SChris Wilson struct i915_gem_context *src) 193010be98a7SChris Wilson { 193110be98a7SChris Wilson struct i915_gem_engines *e = i915_gem_context_lock_engines(src); 193210be98a7SChris Wilson struct i915_gem_engines *clone; 193310be98a7SChris Wilson unsigned long n; 193410be98a7SChris Wilson int err; 193510be98a7SChris Wilson 193610be98a7SChris Wilson clone = dst->engines; /* no locking required; sole access */ 193710be98a7SChris Wilson if (e->num_engines != clone->num_engines) { 193810be98a7SChris Wilson err = -EINVAL; 193910be98a7SChris Wilson goto unlock; 194010be98a7SChris Wilson } 194110be98a7SChris Wilson 194210be98a7SChris Wilson for (n = 0; n < e->num_engines; n++) { 194310be98a7SChris Wilson struct intel_context *ce = e->engines[n]; 194410be98a7SChris Wilson 194510be98a7SChris Wilson if (clone->engines[n]->engine->class != ce->engine->class) { 194610be98a7SChris Wilson /* Must have compatible engine maps! */ 194710be98a7SChris Wilson err = -EINVAL; 194810be98a7SChris Wilson goto unlock; 194910be98a7SChris Wilson } 195010be98a7SChris Wilson 195110be98a7SChris Wilson /* serialises with set_sseu */ 195210be98a7SChris Wilson err = intel_context_lock_pinned(ce); 195310be98a7SChris Wilson if (err) 195410be98a7SChris Wilson goto unlock; 195510be98a7SChris Wilson 195610be98a7SChris Wilson clone->engines[n]->sseu = ce->sseu; 195710be98a7SChris Wilson intel_context_unlock_pinned(ce); 195810be98a7SChris Wilson } 195910be98a7SChris Wilson 196010be98a7SChris Wilson err = 0; 196110be98a7SChris Wilson unlock: 196210be98a7SChris Wilson i915_gem_context_unlock_engines(src); 196310be98a7SChris Wilson return err; 196410be98a7SChris Wilson } 196510be98a7SChris Wilson 196610be98a7SChris Wilson static int clone_timeline(struct i915_gem_context *dst, 196710be98a7SChris Wilson struct i915_gem_context *src) 196810be98a7SChris Wilson { 196910be98a7SChris Wilson if (src->timeline) { 197010be98a7SChris Wilson GEM_BUG_ON(src->timeline == dst->timeline); 197110be98a7SChris Wilson 197210be98a7SChris Wilson if (dst->timeline) 1973f0c02c1bSTvrtko Ursulin intel_timeline_put(dst->timeline); 1974f0c02c1bSTvrtko Ursulin dst->timeline = intel_timeline_get(src->timeline); 197510be98a7SChris Wilson } 197610be98a7SChris Wilson 197710be98a7SChris Wilson return 0; 197810be98a7SChris Wilson } 197910be98a7SChris Wilson 198010be98a7SChris Wilson static int clone_vm(struct i915_gem_context *dst, 198110be98a7SChris Wilson struct i915_gem_context *src) 198210be98a7SChris Wilson { 1983e568ac38SChris Wilson struct i915_address_space *vm; 198410be98a7SChris Wilson 198510be98a7SChris Wilson rcu_read_lock(); 198610be98a7SChris Wilson do { 1987e568ac38SChris Wilson vm = READ_ONCE(src->vm); 1988e568ac38SChris Wilson if (!vm) 198910be98a7SChris Wilson break; 199010be98a7SChris Wilson 1991e568ac38SChris Wilson if (!kref_get_unless_zero(&vm->ref)) 199210be98a7SChris Wilson continue; 199310be98a7SChris Wilson 199410be98a7SChris Wilson /* 199510be98a7SChris Wilson * This ppgtt may have be reallocated between 199610be98a7SChris Wilson * the read and the kref, and reassigned to a third 199710be98a7SChris Wilson * context. In order to avoid inadvertent sharing 199810be98a7SChris Wilson * of this ppgtt with that third context (and not 199910be98a7SChris Wilson * src), we have to confirm that we have the same 200010be98a7SChris Wilson * ppgtt after passing through the strong memory 200110be98a7SChris Wilson * barrier implied by a successful 200210be98a7SChris Wilson * kref_get_unless_zero(). 200310be98a7SChris Wilson * 200410be98a7SChris Wilson * Once we have acquired the current ppgtt of src, 200510be98a7SChris Wilson * we no longer care if it is released from src, as 200610be98a7SChris Wilson * it cannot be reallocated elsewhere. 200710be98a7SChris Wilson */ 200810be98a7SChris Wilson 2009e568ac38SChris Wilson if (vm == READ_ONCE(src->vm)) 201010be98a7SChris Wilson break; 201110be98a7SChris Wilson 2012e568ac38SChris Wilson i915_vm_put(vm); 201310be98a7SChris Wilson } while (1); 201410be98a7SChris Wilson rcu_read_unlock(); 201510be98a7SChris Wilson 2016e568ac38SChris Wilson if (vm) { 2017e568ac38SChris Wilson __assign_ppgtt(dst, vm); 2018e568ac38SChris Wilson i915_vm_put(vm); 201910be98a7SChris Wilson } 202010be98a7SChris Wilson 202110be98a7SChris Wilson return 0; 202210be98a7SChris Wilson } 202310be98a7SChris Wilson 202410be98a7SChris Wilson static int create_clone(struct i915_user_extension __user *ext, void *data) 202510be98a7SChris Wilson { 202610be98a7SChris Wilson static int (* const fn[])(struct i915_gem_context *dst, 202710be98a7SChris Wilson struct i915_gem_context *src) = { 202810be98a7SChris Wilson #define MAP(x, y) [ilog2(I915_CONTEXT_CLONE_##x)] = y 202910be98a7SChris Wilson MAP(ENGINES, clone_engines), 203010be98a7SChris Wilson MAP(FLAGS, clone_flags), 203110be98a7SChris Wilson MAP(SCHEDATTR, clone_schedattr), 203210be98a7SChris Wilson MAP(SSEU, clone_sseu), 203310be98a7SChris Wilson MAP(TIMELINE, clone_timeline), 203410be98a7SChris Wilson MAP(VM, clone_vm), 203510be98a7SChris Wilson #undef MAP 203610be98a7SChris Wilson }; 203710be98a7SChris Wilson struct drm_i915_gem_context_create_ext_clone local; 203810be98a7SChris Wilson const struct create_ext *arg = data; 203910be98a7SChris Wilson struct i915_gem_context *dst = arg->ctx; 204010be98a7SChris Wilson struct i915_gem_context *src; 204110be98a7SChris Wilson int err, bit; 204210be98a7SChris Wilson 204310be98a7SChris Wilson if (copy_from_user(&local, ext, sizeof(local))) 204410be98a7SChris Wilson return -EFAULT; 204510be98a7SChris Wilson 204610be98a7SChris Wilson BUILD_BUG_ON(GENMASK(BITS_PER_TYPE(local.flags) - 1, ARRAY_SIZE(fn)) != 204710be98a7SChris Wilson I915_CONTEXT_CLONE_UNKNOWN); 204810be98a7SChris Wilson 204910be98a7SChris Wilson if (local.flags & I915_CONTEXT_CLONE_UNKNOWN) 205010be98a7SChris Wilson return -EINVAL; 205110be98a7SChris Wilson 205210be98a7SChris Wilson if (local.rsvd) 205310be98a7SChris Wilson return -EINVAL; 205410be98a7SChris Wilson 205510be98a7SChris Wilson rcu_read_lock(); 205610be98a7SChris Wilson src = __i915_gem_context_lookup_rcu(arg->fpriv, local.clone_id); 205710be98a7SChris Wilson rcu_read_unlock(); 205810be98a7SChris Wilson if (!src) 205910be98a7SChris Wilson return -ENOENT; 206010be98a7SChris Wilson 206110be98a7SChris Wilson GEM_BUG_ON(src == dst); 206210be98a7SChris Wilson 206310be98a7SChris Wilson for (bit = 0; bit < ARRAY_SIZE(fn); bit++) { 206410be98a7SChris Wilson if (!(local.flags & BIT(bit))) 206510be98a7SChris Wilson continue; 206610be98a7SChris Wilson 206710be98a7SChris Wilson err = fn[bit](dst, src); 206810be98a7SChris Wilson if (err) 206910be98a7SChris Wilson return err; 207010be98a7SChris Wilson } 207110be98a7SChris Wilson 207210be98a7SChris Wilson return 0; 207310be98a7SChris Wilson } 207410be98a7SChris Wilson 207510be98a7SChris Wilson static const i915_user_extension_fn create_extensions[] = { 207610be98a7SChris Wilson [I915_CONTEXT_CREATE_EXT_SETPARAM] = create_setparam, 207710be98a7SChris Wilson [I915_CONTEXT_CREATE_EXT_CLONE] = create_clone, 207810be98a7SChris Wilson }; 207910be98a7SChris Wilson 208010be98a7SChris Wilson static bool client_is_banned(struct drm_i915_file_private *file_priv) 208110be98a7SChris Wilson { 208210be98a7SChris Wilson return atomic_read(&file_priv->ban_score) >= I915_CLIENT_SCORE_BANNED; 208310be98a7SChris Wilson } 208410be98a7SChris Wilson 208510be98a7SChris Wilson int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, 208610be98a7SChris Wilson struct drm_file *file) 208710be98a7SChris Wilson { 208810be98a7SChris Wilson struct drm_i915_private *i915 = to_i915(dev); 208910be98a7SChris Wilson struct drm_i915_gem_context_create_ext *args = data; 209010be98a7SChris Wilson struct create_ext ext_data; 209110be98a7SChris Wilson int ret; 209210be98a7SChris Wilson 209310be98a7SChris Wilson if (!DRIVER_CAPS(i915)->has_logical_contexts) 209410be98a7SChris Wilson return -ENODEV; 209510be98a7SChris Wilson 209610be98a7SChris Wilson if (args->flags & I915_CONTEXT_CREATE_FLAGS_UNKNOWN) 209710be98a7SChris Wilson return -EINVAL; 209810be98a7SChris Wilson 2099cb823ed9SChris Wilson ret = intel_gt_terminally_wedged(&i915->gt); 210010be98a7SChris Wilson if (ret) 210110be98a7SChris Wilson return ret; 210210be98a7SChris Wilson 210310be98a7SChris Wilson ext_data.fpriv = file->driver_priv; 210410be98a7SChris Wilson if (client_is_banned(ext_data.fpriv)) { 210510be98a7SChris Wilson DRM_DEBUG("client %s[%d] banned from creating ctx\n", 210610be98a7SChris Wilson current->comm, 210710be98a7SChris Wilson pid_nr(get_task_pid(current, PIDTYPE_PID))); 210810be98a7SChris Wilson return -EIO; 210910be98a7SChris Wilson } 211010be98a7SChris Wilson 211110be98a7SChris Wilson ret = i915_mutex_lock_interruptible(dev); 211210be98a7SChris Wilson if (ret) 211310be98a7SChris Wilson return ret; 211410be98a7SChris Wilson 211510be98a7SChris Wilson ext_data.ctx = i915_gem_create_context(i915, args->flags); 211610be98a7SChris Wilson mutex_unlock(&dev->struct_mutex); 211710be98a7SChris Wilson if (IS_ERR(ext_data.ctx)) 211810be98a7SChris Wilson return PTR_ERR(ext_data.ctx); 211910be98a7SChris Wilson 212010be98a7SChris Wilson if (args->flags & I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS) { 212110be98a7SChris Wilson ret = i915_user_extensions(u64_to_user_ptr(args->extensions), 212210be98a7SChris Wilson create_extensions, 212310be98a7SChris Wilson ARRAY_SIZE(create_extensions), 212410be98a7SChris Wilson &ext_data); 212510be98a7SChris Wilson if (ret) 212610be98a7SChris Wilson goto err_ctx; 212710be98a7SChris Wilson } 212810be98a7SChris Wilson 212910be98a7SChris Wilson ret = gem_context_register(ext_data.ctx, ext_data.fpriv); 213010be98a7SChris Wilson if (ret < 0) 213110be98a7SChris Wilson goto err_ctx; 213210be98a7SChris Wilson 213310be98a7SChris Wilson args->ctx_id = ret; 213410be98a7SChris Wilson DRM_DEBUG("HW context %d created\n", args->ctx_id); 213510be98a7SChris Wilson 213610be98a7SChris Wilson return 0; 213710be98a7SChris Wilson 213810be98a7SChris Wilson err_ctx: 213910be98a7SChris Wilson context_close(ext_data.ctx); 214010be98a7SChris Wilson return ret; 214110be98a7SChris Wilson } 214210be98a7SChris Wilson 214310be98a7SChris Wilson int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, 214410be98a7SChris Wilson struct drm_file *file) 214510be98a7SChris Wilson { 214610be98a7SChris Wilson struct drm_i915_gem_context_destroy *args = data; 214710be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 214810be98a7SChris Wilson struct i915_gem_context *ctx; 214910be98a7SChris Wilson 215010be98a7SChris Wilson if (args->pad != 0) 215110be98a7SChris Wilson return -EINVAL; 215210be98a7SChris Wilson 215310be98a7SChris Wilson if (!args->ctx_id) 215410be98a7SChris Wilson return -ENOENT; 215510be98a7SChris Wilson 215610be98a7SChris Wilson if (mutex_lock_interruptible(&file_priv->context_idr_lock)) 215710be98a7SChris Wilson return -EINTR; 215810be98a7SChris Wilson 215910be98a7SChris Wilson ctx = idr_remove(&file_priv->context_idr, args->ctx_id); 216010be98a7SChris Wilson mutex_unlock(&file_priv->context_idr_lock); 216110be98a7SChris Wilson if (!ctx) 216210be98a7SChris Wilson return -ENOENT; 216310be98a7SChris Wilson 216410be98a7SChris Wilson context_close(ctx); 216510be98a7SChris Wilson return 0; 216610be98a7SChris Wilson } 216710be98a7SChris Wilson 216810be98a7SChris Wilson static int get_sseu(struct i915_gem_context *ctx, 216910be98a7SChris Wilson struct drm_i915_gem_context_param *args) 217010be98a7SChris Wilson { 217110be98a7SChris Wilson struct drm_i915_gem_context_param_sseu user_sseu; 217210be98a7SChris Wilson struct intel_context *ce; 217310be98a7SChris Wilson unsigned long lookup; 217410be98a7SChris Wilson int err; 217510be98a7SChris Wilson 217610be98a7SChris Wilson if (args->size == 0) 217710be98a7SChris Wilson goto out; 217810be98a7SChris Wilson else if (args->size < sizeof(user_sseu)) 217910be98a7SChris Wilson return -EINVAL; 218010be98a7SChris Wilson 218110be98a7SChris Wilson if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), 218210be98a7SChris Wilson sizeof(user_sseu))) 218310be98a7SChris Wilson return -EFAULT; 218410be98a7SChris Wilson 218510be98a7SChris Wilson if (user_sseu.rsvd) 218610be98a7SChris Wilson return -EINVAL; 218710be98a7SChris Wilson 218810be98a7SChris Wilson if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) 218910be98a7SChris Wilson return -EINVAL; 219010be98a7SChris Wilson 219110be98a7SChris Wilson lookup = 0; 219210be98a7SChris Wilson if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) 219310be98a7SChris Wilson lookup |= LOOKUP_USER_INDEX; 219410be98a7SChris Wilson 219510be98a7SChris Wilson ce = lookup_user_engine(ctx, lookup, &user_sseu.engine); 219610be98a7SChris Wilson if (IS_ERR(ce)) 219710be98a7SChris Wilson return PTR_ERR(ce); 219810be98a7SChris Wilson 219910be98a7SChris Wilson err = intel_context_lock_pinned(ce); /* serialises with set_sseu */ 220010be98a7SChris Wilson if (err) { 220110be98a7SChris Wilson intel_context_put(ce); 220210be98a7SChris Wilson return err; 220310be98a7SChris Wilson } 220410be98a7SChris Wilson 220510be98a7SChris Wilson user_sseu.slice_mask = ce->sseu.slice_mask; 220610be98a7SChris Wilson user_sseu.subslice_mask = ce->sseu.subslice_mask; 220710be98a7SChris Wilson user_sseu.min_eus_per_subslice = ce->sseu.min_eus_per_subslice; 220810be98a7SChris Wilson user_sseu.max_eus_per_subslice = ce->sseu.max_eus_per_subslice; 220910be98a7SChris Wilson 221010be98a7SChris Wilson intel_context_unlock_pinned(ce); 221110be98a7SChris Wilson intel_context_put(ce); 221210be98a7SChris Wilson 221310be98a7SChris Wilson if (copy_to_user(u64_to_user_ptr(args->value), &user_sseu, 221410be98a7SChris Wilson sizeof(user_sseu))) 221510be98a7SChris Wilson return -EFAULT; 221610be98a7SChris Wilson 221710be98a7SChris Wilson out: 221810be98a7SChris Wilson args->size = sizeof(user_sseu); 221910be98a7SChris Wilson 222010be98a7SChris Wilson return 0; 222110be98a7SChris Wilson } 222210be98a7SChris Wilson 222310be98a7SChris Wilson int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, 222410be98a7SChris Wilson struct drm_file *file) 222510be98a7SChris Wilson { 222610be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 222710be98a7SChris Wilson struct drm_i915_gem_context_param *args = data; 222810be98a7SChris Wilson struct i915_gem_context *ctx; 222910be98a7SChris Wilson int ret = 0; 223010be98a7SChris Wilson 223110be98a7SChris Wilson ctx = i915_gem_context_lookup(file_priv, args->ctx_id); 223210be98a7SChris Wilson if (!ctx) 223310be98a7SChris Wilson return -ENOENT; 223410be98a7SChris Wilson 223510be98a7SChris Wilson switch (args->param) { 223610be98a7SChris Wilson case I915_CONTEXT_PARAM_NO_ZEROMAP: 223710be98a7SChris Wilson args->size = 0; 223810be98a7SChris Wilson args->value = test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); 223910be98a7SChris Wilson break; 224010be98a7SChris Wilson 224110be98a7SChris Wilson case I915_CONTEXT_PARAM_GTT_SIZE: 224210be98a7SChris Wilson args->size = 0; 2243e568ac38SChris Wilson if (ctx->vm) 2244e568ac38SChris Wilson args->value = ctx->vm->total; 2245c082afacSChris Wilson else if (to_i915(dev)->ggtt.alias) 2246c082afacSChris Wilson args->value = to_i915(dev)->ggtt.alias->vm.total; 224710be98a7SChris Wilson else 224810be98a7SChris Wilson args->value = to_i915(dev)->ggtt.vm.total; 224910be98a7SChris Wilson break; 225010be98a7SChris Wilson 225110be98a7SChris Wilson case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 225210be98a7SChris Wilson args->size = 0; 225310be98a7SChris Wilson args->value = i915_gem_context_no_error_capture(ctx); 225410be98a7SChris Wilson break; 225510be98a7SChris Wilson 225610be98a7SChris Wilson case I915_CONTEXT_PARAM_BANNABLE: 225710be98a7SChris Wilson args->size = 0; 225810be98a7SChris Wilson args->value = i915_gem_context_is_bannable(ctx); 225910be98a7SChris Wilson break; 226010be98a7SChris Wilson 226110be98a7SChris Wilson case I915_CONTEXT_PARAM_RECOVERABLE: 226210be98a7SChris Wilson args->size = 0; 226310be98a7SChris Wilson args->value = i915_gem_context_is_recoverable(ctx); 226410be98a7SChris Wilson break; 226510be98a7SChris Wilson 226610be98a7SChris Wilson case I915_CONTEXT_PARAM_PRIORITY: 226710be98a7SChris Wilson args->size = 0; 226810be98a7SChris Wilson args->value = ctx->sched.priority >> I915_USER_PRIORITY_SHIFT; 226910be98a7SChris Wilson break; 227010be98a7SChris Wilson 227110be98a7SChris Wilson case I915_CONTEXT_PARAM_SSEU: 227210be98a7SChris Wilson ret = get_sseu(ctx, args); 227310be98a7SChris Wilson break; 227410be98a7SChris Wilson 227510be98a7SChris Wilson case I915_CONTEXT_PARAM_VM: 227610be98a7SChris Wilson ret = get_ppgtt(file_priv, ctx, args); 227710be98a7SChris Wilson break; 227810be98a7SChris Wilson 227910be98a7SChris Wilson case I915_CONTEXT_PARAM_ENGINES: 228010be98a7SChris Wilson ret = get_engines(ctx, args); 228110be98a7SChris Wilson break; 228210be98a7SChris Wilson 228310be98a7SChris Wilson case I915_CONTEXT_PARAM_BAN_PERIOD: 228410be98a7SChris Wilson default: 228510be98a7SChris Wilson ret = -EINVAL; 228610be98a7SChris Wilson break; 228710be98a7SChris Wilson } 228810be98a7SChris Wilson 228910be98a7SChris Wilson i915_gem_context_put(ctx); 229010be98a7SChris Wilson return ret; 229110be98a7SChris Wilson } 229210be98a7SChris Wilson 229310be98a7SChris Wilson int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, 229410be98a7SChris Wilson struct drm_file *file) 229510be98a7SChris Wilson { 229610be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 229710be98a7SChris Wilson struct drm_i915_gem_context_param *args = data; 229810be98a7SChris Wilson struct i915_gem_context *ctx; 229910be98a7SChris Wilson int ret; 230010be98a7SChris Wilson 230110be98a7SChris Wilson ctx = i915_gem_context_lookup(file_priv, args->ctx_id); 230210be98a7SChris Wilson if (!ctx) 230310be98a7SChris Wilson return -ENOENT; 230410be98a7SChris Wilson 230510be98a7SChris Wilson ret = ctx_setparam(file_priv, ctx, args); 230610be98a7SChris Wilson 230710be98a7SChris Wilson i915_gem_context_put(ctx); 230810be98a7SChris Wilson return ret; 230910be98a7SChris Wilson } 231010be98a7SChris Wilson 231110be98a7SChris Wilson int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, 231210be98a7SChris Wilson void *data, struct drm_file *file) 231310be98a7SChris Wilson { 231410be98a7SChris Wilson struct drm_i915_private *dev_priv = to_i915(dev); 231510be98a7SChris Wilson struct drm_i915_reset_stats *args = data; 231610be98a7SChris Wilson struct i915_gem_context *ctx; 231710be98a7SChris Wilson int ret; 231810be98a7SChris Wilson 231910be98a7SChris Wilson if (args->flags || args->pad) 232010be98a7SChris Wilson return -EINVAL; 232110be98a7SChris Wilson 232210be98a7SChris Wilson ret = -ENOENT; 232310be98a7SChris Wilson rcu_read_lock(); 232410be98a7SChris Wilson ctx = __i915_gem_context_lookup_rcu(file->driver_priv, args->ctx_id); 232510be98a7SChris Wilson if (!ctx) 232610be98a7SChris Wilson goto out; 232710be98a7SChris Wilson 232810be98a7SChris Wilson /* 232910be98a7SChris Wilson * We opt for unserialised reads here. This may result in tearing 233010be98a7SChris Wilson * in the extremely unlikely event of a GPU hang on this context 233110be98a7SChris Wilson * as we are querying them. If we need that extra layer of protection, 233210be98a7SChris Wilson * we should wrap the hangstats with a seqlock. 233310be98a7SChris Wilson */ 233410be98a7SChris Wilson 233510be98a7SChris Wilson if (capable(CAP_SYS_ADMIN)) 233610be98a7SChris Wilson args->reset_count = i915_reset_count(&dev_priv->gpu_error); 233710be98a7SChris Wilson else 233810be98a7SChris Wilson args->reset_count = 0; 233910be98a7SChris Wilson 234010be98a7SChris Wilson args->batch_active = atomic_read(&ctx->guilty_count); 234110be98a7SChris Wilson args->batch_pending = atomic_read(&ctx->active_count); 234210be98a7SChris Wilson 234310be98a7SChris Wilson ret = 0; 234410be98a7SChris Wilson out: 234510be98a7SChris Wilson rcu_read_unlock(); 234610be98a7SChris Wilson return ret; 234710be98a7SChris Wilson } 234810be98a7SChris Wilson 234910be98a7SChris Wilson int __i915_gem_context_pin_hw_id(struct i915_gem_context *ctx) 235010be98a7SChris Wilson { 235110be98a7SChris Wilson struct drm_i915_private *i915 = ctx->i915; 235210be98a7SChris Wilson int err = 0; 235310be98a7SChris Wilson 235410be98a7SChris Wilson mutex_lock(&i915->contexts.mutex); 235510be98a7SChris Wilson 235610be98a7SChris Wilson GEM_BUG_ON(i915_gem_context_is_closed(ctx)); 235710be98a7SChris Wilson 235810be98a7SChris Wilson if (list_empty(&ctx->hw_id_link)) { 235910be98a7SChris Wilson GEM_BUG_ON(atomic_read(&ctx->hw_id_pin_count)); 236010be98a7SChris Wilson 236110be98a7SChris Wilson err = assign_hw_id(i915, &ctx->hw_id); 236210be98a7SChris Wilson if (err) 236310be98a7SChris Wilson goto out_unlock; 236410be98a7SChris Wilson 236510be98a7SChris Wilson list_add_tail(&ctx->hw_id_link, &i915->contexts.hw_id_list); 236610be98a7SChris Wilson } 236710be98a7SChris Wilson 236810be98a7SChris Wilson GEM_BUG_ON(atomic_read(&ctx->hw_id_pin_count) == ~0u); 236910be98a7SChris Wilson atomic_inc(&ctx->hw_id_pin_count); 237010be98a7SChris Wilson 237110be98a7SChris Wilson out_unlock: 237210be98a7SChris Wilson mutex_unlock(&i915->contexts.mutex); 237310be98a7SChris Wilson return err; 237410be98a7SChris Wilson } 237510be98a7SChris Wilson 237610be98a7SChris Wilson /* GEM context-engines iterator: for_each_gem_engine() */ 237710be98a7SChris Wilson struct intel_context * 237810be98a7SChris Wilson i915_gem_engines_iter_next(struct i915_gem_engines_iter *it) 237910be98a7SChris Wilson { 238010be98a7SChris Wilson const struct i915_gem_engines *e = it->engines; 238110be98a7SChris Wilson struct intel_context *ctx; 238210be98a7SChris Wilson 238310be98a7SChris Wilson do { 238410be98a7SChris Wilson if (it->idx >= e->num_engines) 238510be98a7SChris Wilson return NULL; 238610be98a7SChris Wilson 238710be98a7SChris Wilson ctx = e->engines[it->idx++]; 238810be98a7SChris Wilson } while (!ctx); 238910be98a7SChris Wilson 239010be98a7SChris Wilson return ctx; 239110be98a7SChris Wilson } 239210be98a7SChris Wilson 239310be98a7SChris Wilson #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 239410be98a7SChris Wilson #include "selftests/mock_context.c" 239510be98a7SChris Wilson #include "selftests/i915_gem_context.c" 239610be98a7SChris Wilson #endif 239710be98a7SChris Wilson 239810be98a7SChris Wilson static void i915_global_gem_context_shrink(void) 239910be98a7SChris Wilson { 240010be98a7SChris Wilson kmem_cache_shrink(global.slab_luts); 240110be98a7SChris Wilson } 240210be98a7SChris Wilson 240310be98a7SChris Wilson static void i915_global_gem_context_exit(void) 240410be98a7SChris Wilson { 240510be98a7SChris Wilson kmem_cache_destroy(global.slab_luts); 240610be98a7SChris Wilson } 240710be98a7SChris Wilson 240810be98a7SChris Wilson static struct i915_global_gem_context global = { { 240910be98a7SChris Wilson .shrink = i915_global_gem_context_shrink, 241010be98a7SChris Wilson .exit = i915_global_gem_context_exit, 241110be98a7SChris Wilson } }; 241210be98a7SChris Wilson 241310be98a7SChris Wilson int __init i915_global_gem_context_init(void) 241410be98a7SChris Wilson { 241510be98a7SChris Wilson global.slab_luts = KMEM_CACHE(i915_lut_handle, 0); 241610be98a7SChris Wilson if (!global.slab_luts) 241710be98a7SChris Wilson return -ENOMEM; 241810be98a7SChris Wilson 241910be98a7SChris Wilson i915_global_register(&global.base); 242010be98a7SChris Wilson return 0; 242110be98a7SChris Wilson } 2422