110be98a7SChris Wilson /* 210be98a7SChris Wilson * SPDX-License-Identifier: MIT 310be98a7SChris Wilson * 410be98a7SChris Wilson * Copyright © 2011-2012 Intel Corporation 510be98a7SChris Wilson */ 610be98a7SChris Wilson 710be98a7SChris Wilson /* 810be98a7SChris Wilson * This file implements HW context support. On gen5+ a HW context consists of an 910be98a7SChris Wilson * opaque GPU object which is referenced at times of context saves and restores. 1010be98a7SChris Wilson * With RC6 enabled, the context is also referenced as the GPU enters and exists 1110be98a7SChris Wilson * from RC6 (GPU has it's own internal power context, except on gen5). Though 1210be98a7SChris Wilson * something like a context does exist for the media ring, the code only 1310be98a7SChris Wilson * supports contexts for the render ring. 1410be98a7SChris Wilson * 1510be98a7SChris Wilson * In software, there is a distinction between contexts created by the user, 1610be98a7SChris Wilson * and the default HW context. The default HW context is used by GPU clients 1710be98a7SChris Wilson * that do not request setup of their own hardware context. The default 1810be98a7SChris Wilson * context's state is never restored to help prevent programming errors. This 1910be98a7SChris Wilson * would happen if a client ran and piggy-backed off another clients GPU state. 2010be98a7SChris Wilson * The default context only exists to give the GPU some offset to load as the 2110be98a7SChris Wilson * current to invoke a save of the context we actually care about. In fact, the 2210be98a7SChris Wilson * code could likely be constructed, albeit in a more complicated fashion, to 2310be98a7SChris Wilson * never use the default context, though that limits the driver's ability to 2410be98a7SChris Wilson * swap out, and/or destroy other contexts. 2510be98a7SChris Wilson * 2610be98a7SChris Wilson * All other contexts are created as a request by the GPU client. These contexts 2710be98a7SChris Wilson * store GPU state, and thus allow GPU clients to not re-emit state (and 2810be98a7SChris Wilson * potentially query certain state) at any time. The kernel driver makes 2910be98a7SChris Wilson * certain that the appropriate commands are inserted. 3010be98a7SChris Wilson * 3110be98a7SChris Wilson * The context life cycle is semi-complicated in that context BOs may live 3210be98a7SChris Wilson * longer than the context itself because of the way the hardware, and object 3310be98a7SChris Wilson * tracking works. Below is a very crude representation of the state machine 3410be98a7SChris Wilson * describing the context life. 3510be98a7SChris Wilson * refcount pincount active 3610be98a7SChris Wilson * S0: initial state 0 0 0 3710be98a7SChris Wilson * S1: context created 1 0 0 3810be98a7SChris Wilson * S2: context is currently running 2 1 X 3910be98a7SChris Wilson * S3: GPU referenced, but not current 2 0 1 4010be98a7SChris Wilson * S4: context is current, but destroyed 1 1 0 4110be98a7SChris Wilson * S5: like S3, but destroyed 1 0 1 4210be98a7SChris Wilson * 4310be98a7SChris Wilson * The most common (but not all) transitions: 4410be98a7SChris Wilson * S0->S1: client creates a context 4510be98a7SChris Wilson * S1->S2: client submits execbuf with context 4610be98a7SChris Wilson * S2->S3: other clients submits execbuf with context 4710be98a7SChris Wilson * S3->S1: context object was retired 4810be98a7SChris Wilson * S3->S2: clients submits another execbuf 4910be98a7SChris Wilson * S2->S4: context destroy called with current context 5010be98a7SChris Wilson * S3->S5->S0: destroy path 5110be98a7SChris Wilson * S4->S5->S0: destroy path on current context 5210be98a7SChris Wilson * 5310be98a7SChris Wilson * There are two confusing terms used above: 5410be98a7SChris Wilson * The "current context" means the context which is currently running on the 5510be98a7SChris Wilson * GPU. The GPU has loaded its state already and has stored away the gtt 5610be98a7SChris Wilson * offset of the BO. The GPU is not actively referencing the data at this 5710be98a7SChris Wilson * offset, but it will on the next context switch. The only way to avoid this 5810be98a7SChris Wilson * is to do a GPU reset. 5910be98a7SChris Wilson * 6010be98a7SChris Wilson * An "active context' is one which was previously the "current context" and is 6110be98a7SChris Wilson * on the active list waiting for the next context switch to occur. Until this 6210be98a7SChris Wilson * happens, the object must remain at the same gtt offset. It is therefore 6310be98a7SChris Wilson * possible to destroy a context, but it is still active. 6410be98a7SChris Wilson * 6510be98a7SChris Wilson */ 6610be98a7SChris Wilson 6710be98a7SChris Wilson #include <linux/log2.h> 6810be98a7SChris Wilson #include <linux/nospec.h> 6910be98a7SChris Wilson 7010be98a7SChris Wilson #include <drm/i915_drm.h> 7110be98a7SChris Wilson 722e0986a5SChris Wilson #include "gt/intel_engine_heartbeat.h" 73750e76b4SChris Wilson #include "gt/intel_engine_user.h" 742871ea85SChris Wilson #include "gt/intel_lrc_reg.h" 752871ea85SChris Wilson #include "gt/intel_ring.h" 7610be98a7SChris Wilson 7710be98a7SChris Wilson #include "i915_gem_context.h" 7810be98a7SChris Wilson #include "i915_globals.h" 7910be98a7SChris Wilson #include "i915_trace.h" 8010be98a7SChris Wilson #include "i915_user_extensions.h" 8110be98a7SChris Wilson 8210be98a7SChris Wilson #define ALL_L3_SLICES(dev) (1 << NUM_L3_SLICES(dev)) - 1 8310be98a7SChris Wilson 8410be98a7SChris Wilson static struct i915_global_gem_context { 8510be98a7SChris Wilson struct i915_global base; 8610be98a7SChris Wilson struct kmem_cache *slab_luts; 8710be98a7SChris Wilson } global; 8810be98a7SChris Wilson 8910be98a7SChris Wilson struct i915_lut_handle *i915_lut_handle_alloc(void) 9010be98a7SChris Wilson { 9110be98a7SChris Wilson return kmem_cache_alloc(global.slab_luts, GFP_KERNEL); 9210be98a7SChris Wilson } 9310be98a7SChris Wilson 9410be98a7SChris Wilson void i915_lut_handle_free(struct i915_lut_handle *lut) 9510be98a7SChris Wilson { 9610be98a7SChris Wilson return kmem_cache_free(global.slab_luts, lut); 9710be98a7SChris Wilson } 9810be98a7SChris Wilson 9910be98a7SChris Wilson static void lut_close(struct i915_gem_context *ctx) 10010be98a7SChris Wilson { 10110be98a7SChris Wilson struct radix_tree_iter iter; 10210be98a7SChris Wilson void __rcu **slot; 10310be98a7SChris Wilson 104155ab883SChris Wilson lockdep_assert_held(&ctx->mutex); 10510be98a7SChris Wilson 10610be98a7SChris Wilson rcu_read_lock(); 10710be98a7SChris Wilson radix_tree_for_each_slot(slot, &ctx->handles_vma, &iter, 0) { 10810be98a7SChris Wilson struct i915_vma *vma = rcu_dereference_raw(*slot); 109155ab883SChris Wilson struct drm_i915_gem_object *obj = vma->obj; 110155ab883SChris Wilson struct i915_lut_handle *lut; 11110be98a7SChris Wilson 112155ab883SChris Wilson if (!kref_get_unless_zero(&obj->base.refcount)) 113155ab883SChris Wilson continue; 114155ab883SChris Wilson 115155ab883SChris Wilson rcu_read_unlock(); 116155ab883SChris Wilson i915_gem_object_lock(obj); 117155ab883SChris Wilson list_for_each_entry(lut, &obj->lut_list, obj_link) { 118155ab883SChris Wilson if (lut->ctx != ctx) 119155ab883SChris Wilson continue; 120155ab883SChris Wilson 121155ab883SChris Wilson if (lut->handle != iter.index) 122155ab883SChris Wilson continue; 123155ab883SChris Wilson 124155ab883SChris Wilson list_del(&lut->obj_link); 125155ab883SChris Wilson break; 126155ab883SChris Wilson } 127155ab883SChris Wilson i915_gem_object_unlock(obj); 128155ab883SChris Wilson rcu_read_lock(); 129155ab883SChris Wilson 130155ab883SChris Wilson if (&lut->obj_link != &obj->lut_list) { 131155ab883SChris Wilson i915_lut_handle_free(lut); 13210be98a7SChris Wilson radix_tree_iter_delete(&ctx->handles_vma, &iter, slot); 133155ab883SChris Wilson if (atomic_dec_and_test(&vma->open_count) && 134155ab883SChris Wilson !i915_vma_is_ggtt(vma)) 135155ab883SChris Wilson i915_vma_close(vma); 136155ab883SChris Wilson i915_gem_object_put(obj); 137155ab883SChris Wilson } 13810be98a7SChris Wilson 139155ab883SChris Wilson i915_gem_object_put(obj); 14010be98a7SChris Wilson } 14110be98a7SChris Wilson rcu_read_unlock(); 14210be98a7SChris Wilson } 14310be98a7SChris Wilson 14410be98a7SChris Wilson static struct intel_context * 14510be98a7SChris Wilson lookup_user_engine(struct i915_gem_context *ctx, 14610be98a7SChris Wilson unsigned long flags, 14710be98a7SChris Wilson const struct i915_engine_class_instance *ci) 14810be98a7SChris Wilson #define LOOKUP_USER_INDEX BIT(0) 14910be98a7SChris Wilson { 15010be98a7SChris Wilson int idx; 15110be98a7SChris Wilson 15210be98a7SChris Wilson if (!!(flags & LOOKUP_USER_INDEX) != i915_gem_context_user_engines(ctx)) 15310be98a7SChris Wilson return ERR_PTR(-EINVAL); 15410be98a7SChris Wilson 15510be98a7SChris Wilson if (!i915_gem_context_user_engines(ctx)) { 15610be98a7SChris Wilson struct intel_engine_cs *engine; 15710be98a7SChris Wilson 15810be98a7SChris Wilson engine = intel_engine_lookup_user(ctx->i915, 15910be98a7SChris Wilson ci->engine_class, 16010be98a7SChris Wilson ci->engine_instance); 16110be98a7SChris Wilson if (!engine) 16210be98a7SChris Wilson return ERR_PTR(-EINVAL); 16310be98a7SChris Wilson 164f1c4d157SChris Wilson idx = engine->legacy_idx; 16510be98a7SChris Wilson } else { 16610be98a7SChris Wilson idx = ci->engine_instance; 16710be98a7SChris Wilson } 16810be98a7SChris Wilson 16910be98a7SChris Wilson return i915_gem_context_get_engine(ctx, idx); 17010be98a7SChris Wilson } 17110be98a7SChris Wilson 17210be98a7SChris Wilson static void __free_engines(struct i915_gem_engines *e, unsigned int count) 17310be98a7SChris Wilson { 17410be98a7SChris Wilson while (count--) { 17510be98a7SChris Wilson if (!e->engines[count]) 17610be98a7SChris Wilson continue; 17710be98a7SChris Wilson 17810be98a7SChris Wilson intel_context_put(e->engines[count]); 17910be98a7SChris Wilson } 18010be98a7SChris Wilson kfree(e); 18110be98a7SChris Wilson } 18210be98a7SChris Wilson 18310be98a7SChris Wilson static void free_engines(struct i915_gem_engines *e) 18410be98a7SChris Wilson { 18510be98a7SChris Wilson __free_engines(e, e->num_engines); 18610be98a7SChris Wilson } 18710be98a7SChris Wilson 188155ab883SChris Wilson static void free_engines_rcu(struct rcu_head *rcu) 18910be98a7SChris Wilson { 190155ab883SChris Wilson free_engines(container_of(rcu, struct i915_gem_engines, rcu)); 19110be98a7SChris Wilson } 19210be98a7SChris Wilson 19310be98a7SChris Wilson static struct i915_gem_engines *default_engines(struct i915_gem_context *ctx) 19410be98a7SChris Wilson { 195f1c4d157SChris Wilson const struct intel_gt *gt = &ctx->i915->gt; 19610be98a7SChris Wilson struct intel_engine_cs *engine; 19710be98a7SChris Wilson struct i915_gem_engines *e; 19810be98a7SChris Wilson enum intel_engine_id id; 19910be98a7SChris Wilson 20010be98a7SChris Wilson e = kzalloc(struct_size(e, engines, I915_NUM_ENGINES), GFP_KERNEL); 20110be98a7SChris Wilson if (!e) 20210be98a7SChris Wilson return ERR_PTR(-ENOMEM); 20310be98a7SChris Wilson 204155ab883SChris Wilson init_rcu_head(&e->rcu); 205f1c4d157SChris Wilson for_each_engine(engine, gt, id) { 20610be98a7SChris Wilson struct intel_context *ce; 20710be98a7SChris Wilson 208a50134b1STvrtko Ursulin if (engine->legacy_idx == INVALID_ENGINE) 209a50134b1STvrtko Ursulin continue; 210a50134b1STvrtko Ursulin 211a50134b1STvrtko Ursulin GEM_BUG_ON(engine->legacy_idx >= I915_NUM_ENGINES); 212a50134b1STvrtko Ursulin GEM_BUG_ON(e->engines[engine->legacy_idx]); 213a50134b1STvrtko Ursulin 21410be98a7SChris Wilson ce = intel_context_create(ctx, engine); 21510be98a7SChris Wilson if (IS_ERR(ce)) { 216a50134b1STvrtko Ursulin __free_engines(e, e->num_engines + 1); 21710be98a7SChris Wilson return ERR_CAST(ce); 21810be98a7SChris Wilson } 21910be98a7SChris Wilson 220a50134b1STvrtko Ursulin e->engines[engine->legacy_idx] = ce; 221a50134b1STvrtko Ursulin e->num_engines = max(e->num_engines, engine->legacy_idx); 22210be98a7SChris Wilson } 223a50134b1STvrtko Ursulin e->num_engines++; 22410be98a7SChris Wilson 22510be98a7SChris Wilson return e; 22610be98a7SChris Wilson } 22710be98a7SChris Wilson 22810be98a7SChris Wilson static void i915_gem_context_free(struct i915_gem_context *ctx) 22910be98a7SChris Wilson { 23010be98a7SChris Wilson GEM_BUG_ON(!i915_gem_context_is_closed(ctx)); 23110be98a7SChris Wilson 232a4e7ccdaSChris Wilson spin_lock(&ctx->i915->gem.contexts.lock); 233a4e7ccdaSChris Wilson list_del(&ctx->link); 234a4e7ccdaSChris Wilson spin_unlock(&ctx->i915->gem.contexts.lock); 235a4e7ccdaSChris Wilson 23610be98a7SChris Wilson free_engines(rcu_access_pointer(ctx->engines)); 23710be98a7SChris Wilson mutex_destroy(&ctx->engines_mutex); 23810be98a7SChris Wilson 23910be98a7SChris Wilson if (ctx->timeline) 240f0c02c1bSTvrtko Ursulin intel_timeline_put(ctx->timeline); 24110be98a7SChris Wilson 24210be98a7SChris Wilson kfree(ctx->name); 24310be98a7SChris Wilson put_pid(ctx->pid); 24410be98a7SChris Wilson 24510be98a7SChris Wilson mutex_destroy(&ctx->mutex); 24610be98a7SChris Wilson 24710be98a7SChris Wilson kfree_rcu(ctx, rcu); 24810be98a7SChris Wilson } 24910be98a7SChris Wilson 250a4e7ccdaSChris Wilson static void contexts_free_all(struct llist_node *list) 25110be98a7SChris Wilson { 25210be98a7SChris Wilson struct i915_gem_context *ctx, *cn; 25310be98a7SChris Wilson 254a4e7ccdaSChris Wilson llist_for_each_entry_safe(ctx, cn, list, free_link) 25510be98a7SChris Wilson i915_gem_context_free(ctx); 25610be98a7SChris Wilson } 25710be98a7SChris Wilson 258a4e7ccdaSChris Wilson static void contexts_flush_free(struct i915_gem_contexts *gc) 25910be98a7SChris Wilson { 260a4e7ccdaSChris Wilson contexts_free_all(llist_del_all(&gc->free_list)); 26110be98a7SChris Wilson } 26210be98a7SChris Wilson 26310be98a7SChris Wilson static void contexts_free_worker(struct work_struct *work) 26410be98a7SChris Wilson { 265a4e7ccdaSChris Wilson struct i915_gem_contexts *gc = 266a4e7ccdaSChris Wilson container_of(work, typeof(*gc), free_work); 26710be98a7SChris Wilson 268a4e7ccdaSChris Wilson contexts_flush_free(gc); 26910be98a7SChris Wilson } 27010be98a7SChris Wilson 27110be98a7SChris Wilson void i915_gem_context_release(struct kref *ref) 27210be98a7SChris Wilson { 27310be98a7SChris Wilson struct i915_gem_context *ctx = container_of(ref, typeof(*ctx), ref); 274a4e7ccdaSChris Wilson struct i915_gem_contexts *gc = &ctx->i915->gem.contexts; 27510be98a7SChris Wilson 27610be98a7SChris Wilson trace_i915_context_free(ctx); 277a4e7ccdaSChris Wilson if (llist_add(&ctx->free_link, &gc->free_list)) 278a4e7ccdaSChris Wilson schedule_work(&gc->free_work); 27910be98a7SChris Wilson } 28010be98a7SChris Wilson 2812e0986a5SChris Wilson static inline struct i915_gem_engines * 2822e0986a5SChris Wilson __context_engines_static(const struct i915_gem_context *ctx) 2832e0986a5SChris Wilson { 2842e0986a5SChris Wilson return rcu_dereference_protected(ctx->engines, true); 2852e0986a5SChris Wilson } 2862e0986a5SChris Wilson 2872e0986a5SChris Wilson static bool __reset_engine(struct intel_engine_cs *engine) 2882e0986a5SChris Wilson { 2892e0986a5SChris Wilson struct intel_gt *gt = engine->gt; 2902e0986a5SChris Wilson bool success = false; 2912e0986a5SChris Wilson 2922e0986a5SChris Wilson if (!intel_has_reset_engine(gt)) 2932e0986a5SChris Wilson return false; 2942e0986a5SChris Wilson 2952e0986a5SChris Wilson if (!test_and_set_bit(I915_RESET_ENGINE + engine->id, 2962e0986a5SChris Wilson >->reset.flags)) { 2972e0986a5SChris Wilson success = intel_engine_reset(engine, NULL) == 0; 2982e0986a5SChris Wilson clear_and_wake_up_bit(I915_RESET_ENGINE + engine->id, 2992e0986a5SChris Wilson >->reset.flags); 3002e0986a5SChris Wilson } 3012e0986a5SChris Wilson 3022e0986a5SChris Wilson return success; 3032e0986a5SChris Wilson } 3042e0986a5SChris Wilson 3052e0986a5SChris Wilson static void __reset_context(struct i915_gem_context *ctx, 3062e0986a5SChris Wilson struct intel_engine_cs *engine) 3072e0986a5SChris Wilson { 3082e0986a5SChris Wilson intel_gt_handle_error(engine->gt, engine->mask, 0, 3092e0986a5SChris Wilson "context closure in %s", ctx->name); 3102e0986a5SChris Wilson } 3112e0986a5SChris Wilson 3122e0986a5SChris Wilson static bool __cancel_engine(struct intel_engine_cs *engine) 3132e0986a5SChris Wilson { 3142e0986a5SChris Wilson /* 3152e0986a5SChris Wilson * Send a "high priority pulse" down the engine to cause the 3162e0986a5SChris Wilson * current request to be momentarily preempted. (If it fails to 3172e0986a5SChris Wilson * be preempted, it will be reset). As we have marked our context 3182e0986a5SChris Wilson * as banned, any incomplete request, including any running, will 3192e0986a5SChris Wilson * be skipped following the preemption. 3202e0986a5SChris Wilson * 3212e0986a5SChris Wilson * If there is no hangchecking (one of the reasons why we try to 3222e0986a5SChris Wilson * cancel the context) and no forced preemption, there may be no 3232e0986a5SChris Wilson * means by which we reset the GPU and evict the persistent hog. 3242e0986a5SChris Wilson * Ergo if we are unable to inject a preemptive pulse that can 3252e0986a5SChris Wilson * kill the banned context, we fallback to doing a local reset 3262e0986a5SChris Wilson * instead. 3272e0986a5SChris Wilson */ 328babaab2fSChris Wilson if (IS_ACTIVE(CONFIG_DRM_I915_PREEMPT_TIMEOUT) && 329babaab2fSChris Wilson !intel_engine_pulse(engine)) 3302e0986a5SChris Wilson return true; 3312e0986a5SChris Wilson 3322e0986a5SChris Wilson /* If we are unable to send a pulse, try resetting this engine. */ 3332e0986a5SChris Wilson return __reset_engine(engine); 3342e0986a5SChris Wilson } 3352e0986a5SChris Wilson 3362e0986a5SChris Wilson static struct intel_engine_cs * 3372e0986a5SChris Wilson active_engine(struct dma_fence *fence, struct intel_context *ce) 3382e0986a5SChris Wilson { 3392e0986a5SChris Wilson struct i915_request *rq = to_request(fence); 3402e0986a5SChris Wilson struct intel_engine_cs *engine, *locked; 3412e0986a5SChris Wilson 3422e0986a5SChris Wilson /* 3432e0986a5SChris Wilson * Serialise with __i915_request_submit() so that it sees 3442e0986a5SChris Wilson * is-banned?, or we know the request is already inflight. 3452e0986a5SChris Wilson */ 3462e0986a5SChris Wilson locked = READ_ONCE(rq->engine); 3472e0986a5SChris Wilson spin_lock_irq(&locked->active.lock); 3482e0986a5SChris Wilson while (unlikely(locked != (engine = READ_ONCE(rq->engine)))) { 3492e0986a5SChris Wilson spin_unlock(&locked->active.lock); 3502e0986a5SChris Wilson spin_lock(&engine->active.lock); 3512e0986a5SChris Wilson locked = engine; 3522e0986a5SChris Wilson } 3532e0986a5SChris Wilson 3542e0986a5SChris Wilson engine = NULL; 3552e0986a5SChris Wilson if (i915_request_is_active(rq) && !rq->fence.error) 3562e0986a5SChris Wilson engine = rq->engine; 3572e0986a5SChris Wilson 3582e0986a5SChris Wilson spin_unlock_irq(&locked->active.lock); 3592e0986a5SChris Wilson 3602e0986a5SChris Wilson return engine; 3612e0986a5SChris Wilson } 3622e0986a5SChris Wilson 3632e0986a5SChris Wilson static void kill_context(struct i915_gem_context *ctx) 3642e0986a5SChris Wilson { 3652e0986a5SChris Wilson struct i915_gem_engines_iter it; 3662e0986a5SChris Wilson struct intel_context *ce; 3672e0986a5SChris Wilson 3682e0986a5SChris Wilson /* 3692e0986a5SChris Wilson * If we are already banned, it was due to a guilty request causing 3702e0986a5SChris Wilson * a reset and the entire context being evicted from the GPU. 3712e0986a5SChris Wilson */ 3722e0986a5SChris Wilson if (i915_gem_context_is_banned(ctx)) 3732e0986a5SChris Wilson return; 3742e0986a5SChris Wilson 3752e0986a5SChris Wilson i915_gem_context_set_banned(ctx); 3762e0986a5SChris Wilson 3772e0986a5SChris Wilson /* 3782e0986a5SChris Wilson * Map the user's engine back to the actual engines; one virtual 3792e0986a5SChris Wilson * engine will be mapped to multiple engines, and using ctx->engine[] 3802e0986a5SChris Wilson * the same engine may be have multiple instances in the user's map. 3812e0986a5SChris Wilson * However, we only care about pending requests, so only include 3822e0986a5SChris Wilson * engines on which there are incomplete requests. 3832e0986a5SChris Wilson */ 3842e0986a5SChris Wilson for_each_gem_engine(ce, __context_engines_static(ctx), it) { 3852e0986a5SChris Wilson struct intel_engine_cs *engine; 3862e0986a5SChris Wilson struct dma_fence *fence; 3872e0986a5SChris Wilson 3882e0986a5SChris Wilson if (!ce->timeline) 3892e0986a5SChris Wilson continue; 3902e0986a5SChris Wilson 3912e0986a5SChris Wilson fence = i915_active_fence_get(&ce->timeline->last_request); 3922e0986a5SChris Wilson if (!fence) 3932e0986a5SChris Wilson continue; 3942e0986a5SChris Wilson 3952e0986a5SChris Wilson /* Check with the backend if the request is still inflight */ 3962e0986a5SChris Wilson engine = active_engine(fence, ce); 3972e0986a5SChris Wilson 3982e0986a5SChris Wilson /* First attempt to gracefully cancel the context */ 3992e0986a5SChris Wilson if (engine && !__cancel_engine(engine)) 4002e0986a5SChris Wilson /* 4012e0986a5SChris Wilson * If we are unable to send a preemptive pulse to bump 4022e0986a5SChris Wilson * the context from the GPU, we have to resort to a full 4032e0986a5SChris Wilson * reset. We hope the collateral damage is worth it. 4042e0986a5SChris Wilson */ 4052e0986a5SChris Wilson __reset_context(ctx, engine); 4062e0986a5SChris Wilson 4072e0986a5SChris Wilson dma_fence_put(fence); 4082e0986a5SChris Wilson } 4092e0986a5SChris Wilson } 4102e0986a5SChris Wilson 41110be98a7SChris Wilson static void context_close(struct i915_gem_context *ctx) 41210be98a7SChris Wilson { 413a4e7ccdaSChris Wilson struct i915_address_space *vm; 414a4e7ccdaSChris Wilson 4152850748eSChris Wilson i915_gem_context_set_closed(ctx); 4162850748eSChris Wilson 417155ab883SChris Wilson mutex_lock(&ctx->mutex); 418155ab883SChris Wilson 419a4e7ccdaSChris Wilson vm = i915_gem_context_vm(ctx); 420a4e7ccdaSChris Wilson if (vm) 421a4e7ccdaSChris Wilson i915_vm_close(vm); 422a4e7ccdaSChris Wilson 423155ab883SChris Wilson ctx->file_priv = ERR_PTR(-EBADF); 42410be98a7SChris Wilson 42510be98a7SChris Wilson /* 42610be98a7SChris Wilson * The LUT uses the VMA as a backpointer to unref the object, 42710be98a7SChris Wilson * so we need to clear the LUT before we close all the VMA (inside 42810be98a7SChris Wilson * the ppgtt). 42910be98a7SChris Wilson */ 43010be98a7SChris Wilson lut_close(ctx); 43110be98a7SChris Wilson 432155ab883SChris Wilson mutex_unlock(&ctx->mutex); 4332e0986a5SChris Wilson 4342e0986a5SChris Wilson /* 4352e0986a5SChris Wilson * If the user has disabled hangchecking, we can not be sure that 4362e0986a5SChris Wilson * the batches will ever complete after the context is closed, 4372e0986a5SChris Wilson * keeping the context and all resources pinned forever. So in this 4382e0986a5SChris Wilson * case we opt to forcibly kill off all remaining requests on 4392e0986a5SChris Wilson * context close. 4402e0986a5SChris Wilson */ 4412e0986a5SChris Wilson if (!i915_modparams.enable_hangcheck) 4422e0986a5SChris Wilson kill_context(ctx); 4432e0986a5SChris Wilson 44410be98a7SChris Wilson i915_gem_context_put(ctx); 44510be98a7SChris Wilson } 44610be98a7SChris Wilson 44710be98a7SChris Wilson static struct i915_gem_context * 448e568ac38SChris Wilson __create_context(struct drm_i915_private *i915) 44910be98a7SChris Wilson { 45010be98a7SChris Wilson struct i915_gem_context *ctx; 45110be98a7SChris Wilson struct i915_gem_engines *e; 45210be98a7SChris Wilson int err; 45310be98a7SChris Wilson int i; 45410be98a7SChris Wilson 45510be98a7SChris Wilson ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); 45610be98a7SChris Wilson if (!ctx) 45710be98a7SChris Wilson return ERR_PTR(-ENOMEM); 45810be98a7SChris Wilson 45910be98a7SChris Wilson kref_init(&ctx->ref); 460e568ac38SChris Wilson ctx->i915 = i915; 46110be98a7SChris Wilson ctx->sched.priority = I915_USER_PRIORITY(I915_PRIORITY_NORMAL); 46210be98a7SChris Wilson mutex_init(&ctx->mutex); 46310be98a7SChris Wilson 46410be98a7SChris Wilson mutex_init(&ctx->engines_mutex); 46510be98a7SChris Wilson e = default_engines(ctx); 46610be98a7SChris Wilson if (IS_ERR(e)) { 46710be98a7SChris Wilson err = PTR_ERR(e); 46810be98a7SChris Wilson goto err_free; 46910be98a7SChris Wilson } 47010be98a7SChris Wilson RCU_INIT_POINTER(ctx->engines, e); 47110be98a7SChris Wilson 47210be98a7SChris Wilson INIT_RADIX_TREE(&ctx->handles_vma, GFP_KERNEL); 47310be98a7SChris Wilson 47410be98a7SChris Wilson /* NB: Mark all slices as needing a remap so that when the context first 47510be98a7SChris Wilson * loads it will restore whatever remap state already exists. If there 47610be98a7SChris Wilson * is no remap info, it will be a NOP. */ 477e568ac38SChris Wilson ctx->remap_slice = ALL_L3_SLICES(i915); 47810be98a7SChris Wilson 47910be98a7SChris Wilson i915_gem_context_set_bannable(ctx); 48010be98a7SChris Wilson i915_gem_context_set_recoverable(ctx); 48110be98a7SChris Wilson 48210be98a7SChris Wilson for (i = 0; i < ARRAY_SIZE(ctx->hang_timestamp); i++) 48310be98a7SChris Wilson ctx->hang_timestamp[i] = jiffies - CONTEXT_FAST_HANG_JIFFIES; 48410be98a7SChris Wilson 485a4e7ccdaSChris Wilson spin_lock(&i915->gem.contexts.lock); 486a4e7ccdaSChris Wilson list_add_tail(&ctx->link, &i915->gem.contexts.list); 487a4e7ccdaSChris Wilson spin_unlock(&i915->gem.contexts.lock); 488a4e7ccdaSChris Wilson 48910be98a7SChris Wilson return ctx; 49010be98a7SChris Wilson 49110be98a7SChris Wilson err_free: 49210be98a7SChris Wilson kfree(ctx); 49310be98a7SChris Wilson return ERR_PTR(err); 49410be98a7SChris Wilson } 49510be98a7SChris Wilson 49648ae397bSChris Wilson static void 49748ae397bSChris Wilson context_apply_all(struct i915_gem_context *ctx, 49848ae397bSChris Wilson void (*fn)(struct intel_context *ce, void *data), 49948ae397bSChris Wilson void *data) 50048ae397bSChris Wilson { 50148ae397bSChris Wilson struct i915_gem_engines_iter it; 50248ae397bSChris Wilson struct intel_context *ce; 50348ae397bSChris Wilson 50448ae397bSChris Wilson for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) 50548ae397bSChris Wilson fn(ce, data); 50648ae397bSChris Wilson i915_gem_context_unlock_engines(ctx); 50748ae397bSChris Wilson } 50848ae397bSChris Wilson 50948ae397bSChris Wilson static void __apply_ppgtt(struct intel_context *ce, void *vm) 51048ae397bSChris Wilson { 51148ae397bSChris Wilson i915_vm_put(ce->vm); 51248ae397bSChris Wilson ce->vm = i915_vm_get(vm); 51348ae397bSChris Wilson } 51448ae397bSChris Wilson 515e568ac38SChris Wilson static struct i915_address_space * 516e568ac38SChris Wilson __set_ppgtt(struct i915_gem_context *ctx, struct i915_address_space *vm) 51710be98a7SChris Wilson { 518a4e7ccdaSChris Wilson struct i915_address_space *old = i915_gem_context_vm(ctx); 51910be98a7SChris Wilson 520a1c9ca22SChris Wilson GEM_BUG_ON(old && i915_vm_is_4lvl(vm) != i915_vm_is_4lvl(old)); 521a1c9ca22SChris Wilson 522a4e7ccdaSChris Wilson rcu_assign_pointer(ctx->vm, i915_vm_open(vm)); 52348ae397bSChris Wilson context_apply_all(ctx, __apply_ppgtt, vm); 524f5d974f9SChris Wilson 52510be98a7SChris Wilson return old; 52610be98a7SChris Wilson } 52710be98a7SChris Wilson 52810be98a7SChris Wilson static void __assign_ppgtt(struct i915_gem_context *ctx, 529e568ac38SChris Wilson struct i915_address_space *vm) 53010be98a7SChris Wilson { 531a4e7ccdaSChris Wilson if (vm == rcu_access_pointer(ctx->vm)) 53210be98a7SChris Wilson return; 53310be98a7SChris Wilson 534e568ac38SChris Wilson vm = __set_ppgtt(ctx, vm); 535e568ac38SChris Wilson if (vm) 5362850748eSChris Wilson i915_vm_close(vm); 53710be98a7SChris Wilson } 53810be98a7SChris Wilson 53975d0a7f3SChris Wilson static void __set_timeline(struct intel_timeline **dst, 54075d0a7f3SChris Wilson struct intel_timeline *src) 54175d0a7f3SChris Wilson { 54275d0a7f3SChris Wilson struct intel_timeline *old = *dst; 54375d0a7f3SChris Wilson 54475d0a7f3SChris Wilson *dst = src ? intel_timeline_get(src) : NULL; 54575d0a7f3SChris Wilson 54675d0a7f3SChris Wilson if (old) 54775d0a7f3SChris Wilson intel_timeline_put(old); 54875d0a7f3SChris Wilson } 54975d0a7f3SChris Wilson 55075d0a7f3SChris Wilson static void __apply_timeline(struct intel_context *ce, void *timeline) 55175d0a7f3SChris Wilson { 55275d0a7f3SChris Wilson __set_timeline(&ce->timeline, timeline); 55375d0a7f3SChris Wilson } 55475d0a7f3SChris Wilson 55575d0a7f3SChris Wilson static void __assign_timeline(struct i915_gem_context *ctx, 55675d0a7f3SChris Wilson struct intel_timeline *timeline) 55775d0a7f3SChris Wilson { 55875d0a7f3SChris Wilson __set_timeline(&ctx->timeline, timeline); 55975d0a7f3SChris Wilson context_apply_all(ctx, __apply_timeline, timeline); 56075d0a7f3SChris Wilson } 56175d0a7f3SChris Wilson 56210be98a7SChris Wilson static struct i915_gem_context * 563a4e7ccdaSChris Wilson i915_gem_create_context(struct drm_i915_private *i915, unsigned int flags) 56410be98a7SChris Wilson { 56510be98a7SChris Wilson struct i915_gem_context *ctx; 56610be98a7SChris Wilson 56710be98a7SChris Wilson if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE && 568a4e7ccdaSChris Wilson !HAS_EXECLISTS(i915)) 56910be98a7SChris Wilson return ERR_PTR(-EINVAL); 57010be98a7SChris Wilson 571a4e7ccdaSChris Wilson /* Reap the stale contexts */ 572a4e7ccdaSChris Wilson contexts_flush_free(&i915->gem.contexts); 57310be98a7SChris Wilson 574a4e7ccdaSChris Wilson ctx = __create_context(i915); 57510be98a7SChris Wilson if (IS_ERR(ctx)) 57610be98a7SChris Wilson return ctx; 57710be98a7SChris Wilson 578a4e7ccdaSChris Wilson if (HAS_FULL_PPGTT(i915)) { 579ab53497bSChris Wilson struct i915_ppgtt *ppgtt; 58010be98a7SChris Wilson 581a4e7ccdaSChris Wilson ppgtt = i915_ppgtt_create(i915); 58210be98a7SChris Wilson if (IS_ERR(ppgtt)) { 58310be98a7SChris Wilson DRM_DEBUG_DRIVER("PPGTT setup failed (%ld)\n", 58410be98a7SChris Wilson PTR_ERR(ppgtt)); 58510be98a7SChris Wilson context_close(ctx); 58610be98a7SChris Wilson return ERR_CAST(ppgtt); 58710be98a7SChris Wilson } 58810be98a7SChris Wilson 589a4e7ccdaSChris Wilson mutex_lock(&ctx->mutex); 590e568ac38SChris Wilson __assign_ppgtt(ctx, &ppgtt->vm); 591a4e7ccdaSChris Wilson mutex_unlock(&ctx->mutex); 592a4e7ccdaSChris Wilson 593e568ac38SChris Wilson i915_vm_put(&ppgtt->vm); 59410be98a7SChris Wilson } 59510be98a7SChris Wilson 59610be98a7SChris Wilson if (flags & I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE) { 597f0c02c1bSTvrtko Ursulin struct intel_timeline *timeline; 59810be98a7SChris Wilson 599a4e7ccdaSChris Wilson timeline = intel_timeline_create(&i915->gt, NULL); 60010be98a7SChris Wilson if (IS_ERR(timeline)) { 60110be98a7SChris Wilson context_close(ctx); 60210be98a7SChris Wilson return ERR_CAST(timeline); 60310be98a7SChris Wilson } 60410be98a7SChris Wilson 60575d0a7f3SChris Wilson __assign_timeline(ctx, timeline); 60675d0a7f3SChris Wilson intel_timeline_put(timeline); 60710be98a7SChris Wilson } 60810be98a7SChris Wilson 60910be98a7SChris Wilson trace_i915_context_create(ctx); 61010be98a7SChris Wilson 61110be98a7SChris Wilson return ctx; 61210be98a7SChris Wilson } 61310be98a7SChris Wilson 61410be98a7SChris Wilson static void 61510be98a7SChris Wilson destroy_kernel_context(struct i915_gem_context **ctxp) 61610be98a7SChris Wilson { 61710be98a7SChris Wilson struct i915_gem_context *ctx; 61810be98a7SChris Wilson 61910be98a7SChris Wilson /* Keep the context ref so that we can free it immediately ourselves */ 62010be98a7SChris Wilson ctx = i915_gem_context_get(fetch_and_zero(ctxp)); 62110be98a7SChris Wilson GEM_BUG_ON(!i915_gem_context_is_kernel(ctx)); 62210be98a7SChris Wilson 62310be98a7SChris Wilson context_close(ctx); 62410be98a7SChris Wilson i915_gem_context_free(ctx); 62510be98a7SChris Wilson } 62610be98a7SChris Wilson 62710be98a7SChris Wilson struct i915_gem_context * 62810be98a7SChris Wilson i915_gem_context_create_kernel(struct drm_i915_private *i915, int prio) 62910be98a7SChris Wilson { 63010be98a7SChris Wilson struct i915_gem_context *ctx; 63110be98a7SChris Wilson 63210be98a7SChris Wilson ctx = i915_gem_create_context(i915, 0); 63310be98a7SChris Wilson if (IS_ERR(ctx)) 63410be98a7SChris Wilson return ctx; 63510be98a7SChris Wilson 63610be98a7SChris Wilson i915_gem_context_clear_bannable(ctx); 63710be98a7SChris Wilson ctx->sched.priority = I915_USER_PRIORITY(prio); 63810be98a7SChris Wilson 63910be98a7SChris Wilson GEM_BUG_ON(!i915_gem_context_is_kernel(ctx)); 64010be98a7SChris Wilson 64110be98a7SChris Wilson return ctx; 64210be98a7SChris Wilson } 64310be98a7SChris Wilson 644a4e7ccdaSChris Wilson static void init_contexts(struct i915_gem_contexts *gc) 64510be98a7SChris Wilson { 646a4e7ccdaSChris Wilson spin_lock_init(&gc->lock); 647a4e7ccdaSChris Wilson INIT_LIST_HEAD(&gc->list); 64810be98a7SChris Wilson 649a4e7ccdaSChris Wilson INIT_WORK(&gc->free_work, contexts_free_worker); 650a4e7ccdaSChris Wilson init_llist_head(&gc->free_list); 65110be98a7SChris Wilson } 65210be98a7SChris Wilson 653a4e7ccdaSChris Wilson int i915_gem_init_contexts(struct drm_i915_private *i915) 65410be98a7SChris Wilson { 65510be98a7SChris Wilson struct i915_gem_context *ctx; 65610be98a7SChris Wilson 65710be98a7SChris Wilson /* Reassure ourselves we are only called once */ 658a4e7ccdaSChris Wilson GEM_BUG_ON(i915->kernel_context); 65910be98a7SChris Wilson 660a4e7ccdaSChris Wilson init_contexts(&i915->gem.contexts); 66110be98a7SChris Wilson 66210be98a7SChris Wilson /* lowest priority; idle task */ 663a4e7ccdaSChris Wilson ctx = i915_gem_context_create_kernel(i915, I915_PRIORITY_MIN); 66410be98a7SChris Wilson if (IS_ERR(ctx)) { 66510be98a7SChris Wilson DRM_ERROR("Failed to create default global context\n"); 66610be98a7SChris Wilson return PTR_ERR(ctx); 66710be98a7SChris Wilson } 668a4e7ccdaSChris Wilson i915->kernel_context = ctx; 66910be98a7SChris Wilson 67010be98a7SChris Wilson DRM_DEBUG_DRIVER("%s context support initialized\n", 671a4e7ccdaSChris Wilson DRIVER_CAPS(i915)->has_logical_contexts ? 67210be98a7SChris Wilson "logical" : "fake"); 67310be98a7SChris Wilson return 0; 67410be98a7SChris Wilson } 67510be98a7SChris Wilson 676a4e7ccdaSChris Wilson void i915_gem_driver_release__contexts(struct drm_i915_private *i915) 67710be98a7SChris Wilson { 67810be98a7SChris Wilson destroy_kernel_context(&i915->kernel_context); 67910be98a7SChris Wilson } 68010be98a7SChris Wilson 68110be98a7SChris Wilson static int context_idr_cleanup(int id, void *p, void *data) 68210be98a7SChris Wilson { 68310be98a7SChris Wilson context_close(p); 68410be98a7SChris Wilson return 0; 68510be98a7SChris Wilson } 68610be98a7SChris Wilson 68710be98a7SChris Wilson static int vm_idr_cleanup(int id, void *p, void *data) 68810be98a7SChris Wilson { 689e568ac38SChris Wilson i915_vm_put(p); 69010be98a7SChris Wilson return 0; 69110be98a7SChris Wilson } 69210be98a7SChris Wilson 69310be98a7SChris Wilson static int gem_context_register(struct i915_gem_context *ctx, 69410be98a7SChris Wilson struct drm_i915_file_private *fpriv) 69510be98a7SChris Wilson { 696a4e7ccdaSChris Wilson struct i915_address_space *vm; 69710be98a7SChris Wilson int ret; 69810be98a7SChris Wilson 69910be98a7SChris Wilson ctx->file_priv = fpriv; 700a4e7ccdaSChris Wilson 701a4e7ccdaSChris Wilson mutex_lock(&ctx->mutex); 702a4e7ccdaSChris Wilson vm = i915_gem_context_vm(ctx); 703a4e7ccdaSChris Wilson if (vm) 704a4e7ccdaSChris Wilson WRITE_ONCE(vm->file, fpriv); /* XXX */ 705a4e7ccdaSChris Wilson mutex_unlock(&ctx->mutex); 70610be98a7SChris Wilson 70710be98a7SChris Wilson ctx->pid = get_task_pid(current, PIDTYPE_PID); 70810be98a7SChris Wilson ctx->name = kasprintf(GFP_KERNEL, "%s[%d]", 70910be98a7SChris Wilson current->comm, pid_nr(ctx->pid)); 71010be98a7SChris Wilson if (!ctx->name) { 71110be98a7SChris Wilson ret = -ENOMEM; 71210be98a7SChris Wilson goto err_pid; 71310be98a7SChris Wilson } 71410be98a7SChris Wilson 71510be98a7SChris Wilson /* And finally expose ourselves to userspace via the idr */ 71610be98a7SChris Wilson mutex_lock(&fpriv->context_idr_lock); 71710be98a7SChris Wilson ret = idr_alloc(&fpriv->context_idr, ctx, 0, 0, GFP_KERNEL); 71810be98a7SChris Wilson mutex_unlock(&fpriv->context_idr_lock); 71910be98a7SChris Wilson if (ret >= 0) 72010be98a7SChris Wilson goto out; 72110be98a7SChris Wilson 72210be98a7SChris Wilson kfree(fetch_and_zero(&ctx->name)); 72310be98a7SChris Wilson err_pid: 72410be98a7SChris Wilson put_pid(fetch_and_zero(&ctx->pid)); 72510be98a7SChris Wilson out: 72610be98a7SChris Wilson return ret; 72710be98a7SChris Wilson } 72810be98a7SChris Wilson 72910be98a7SChris Wilson int i915_gem_context_open(struct drm_i915_private *i915, 73010be98a7SChris Wilson struct drm_file *file) 73110be98a7SChris Wilson { 73210be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 73310be98a7SChris Wilson struct i915_gem_context *ctx; 73410be98a7SChris Wilson int err; 73510be98a7SChris Wilson 73610be98a7SChris Wilson mutex_init(&file_priv->context_idr_lock); 73710be98a7SChris Wilson mutex_init(&file_priv->vm_idr_lock); 73810be98a7SChris Wilson 73910be98a7SChris Wilson idr_init(&file_priv->context_idr); 74010be98a7SChris Wilson idr_init_base(&file_priv->vm_idr, 1); 74110be98a7SChris Wilson 74210be98a7SChris Wilson ctx = i915_gem_create_context(i915, 0); 74310be98a7SChris Wilson if (IS_ERR(ctx)) { 74410be98a7SChris Wilson err = PTR_ERR(ctx); 74510be98a7SChris Wilson goto err; 74610be98a7SChris Wilson } 74710be98a7SChris Wilson 74810be98a7SChris Wilson err = gem_context_register(ctx, file_priv); 74910be98a7SChris Wilson if (err < 0) 75010be98a7SChris Wilson goto err_ctx; 75110be98a7SChris Wilson 75210be98a7SChris Wilson GEM_BUG_ON(i915_gem_context_is_kernel(ctx)); 75310be98a7SChris Wilson GEM_BUG_ON(err > 0); 75410be98a7SChris Wilson 75510be98a7SChris Wilson return 0; 75610be98a7SChris Wilson 75710be98a7SChris Wilson err_ctx: 75810be98a7SChris Wilson context_close(ctx); 75910be98a7SChris Wilson err: 76010be98a7SChris Wilson idr_destroy(&file_priv->vm_idr); 76110be98a7SChris Wilson idr_destroy(&file_priv->context_idr); 76210be98a7SChris Wilson mutex_destroy(&file_priv->vm_idr_lock); 76310be98a7SChris Wilson mutex_destroy(&file_priv->context_idr_lock); 76410be98a7SChris Wilson return err; 76510be98a7SChris Wilson } 76610be98a7SChris Wilson 76710be98a7SChris Wilson void i915_gem_context_close(struct drm_file *file) 76810be98a7SChris Wilson { 76910be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 770a4e7ccdaSChris Wilson struct drm_i915_private *i915 = file_priv->dev_priv; 77110be98a7SChris Wilson 77210be98a7SChris Wilson idr_for_each(&file_priv->context_idr, context_idr_cleanup, NULL); 77310be98a7SChris Wilson idr_destroy(&file_priv->context_idr); 77410be98a7SChris Wilson mutex_destroy(&file_priv->context_idr_lock); 77510be98a7SChris Wilson 77610be98a7SChris Wilson idr_for_each(&file_priv->vm_idr, vm_idr_cleanup, NULL); 77710be98a7SChris Wilson idr_destroy(&file_priv->vm_idr); 77810be98a7SChris Wilson mutex_destroy(&file_priv->vm_idr_lock); 779a4e7ccdaSChris Wilson 780a4e7ccdaSChris Wilson contexts_flush_free(&i915->gem.contexts); 78110be98a7SChris Wilson } 78210be98a7SChris Wilson 78310be98a7SChris Wilson int i915_gem_vm_create_ioctl(struct drm_device *dev, void *data, 78410be98a7SChris Wilson struct drm_file *file) 78510be98a7SChris Wilson { 78610be98a7SChris Wilson struct drm_i915_private *i915 = to_i915(dev); 78710be98a7SChris Wilson struct drm_i915_gem_vm_control *args = data; 78810be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 789ab53497bSChris Wilson struct i915_ppgtt *ppgtt; 79010be98a7SChris Wilson int err; 79110be98a7SChris Wilson 79210be98a7SChris Wilson if (!HAS_FULL_PPGTT(i915)) 79310be98a7SChris Wilson return -ENODEV; 79410be98a7SChris Wilson 79510be98a7SChris Wilson if (args->flags) 79610be98a7SChris Wilson return -EINVAL; 79710be98a7SChris Wilson 79810be98a7SChris Wilson ppgtt = i915_ppgtt_create(i915); 79910be98a7SChris Wilson if (IS_ERR(ppgtt)) 80010be98a7SChris Wilson return PTR_ERR(ppgtt); 80110be98a7SChris Wilson 80210be98a7SChris Wilson ppgtt->vm.file = file_priv; 80310be98a7SChris Wilson 80410be98a7SChris Wilson if (args->extensions) { 80510be98a7SChris Wilson err = i915_user_extensions(u64_to_user_ptr(args->extensions), 80610be98a7SChris Wilson NULL, 0, 80710be98a7SChris Wilson ppgtt); 80810be98a7SChris Wilson if (err) 80910be98a7SChris Wilson goto err_put; 81010be98a7SChris Wilson } 81110be98a7SChris Wilson 81210be98a7SChris Wilson err = mutex_lock_interruptible(&file_priv->vm_idr_lock); 81310be98a7SChris Wilson if (err) 81410be98a7SChris Wilson goto err_put; 81510be98a7SChris Wilson 816e568ac38SChris Wilson err = idr_alloc(&file_priv->vm_idr, &ppgtt->vm, 0, 0, GFP_KERNEL); 81710be98a7SChris Wilson if (err < 0) 81810be98a7SChris Wilson goto err_unlock; 81910be98a7SChris Wilson 82010be98a7SChris Wilson GEM_BUG_ON(err == 0); /* reserved for invalid/unassigned ppgtt */ 82110be98a7SChris Wilson 82210be98a7SChris Wilson mutex_unlock(&file_priv->vm_idr_lock); 82310be98a7SChris Wilson 82410be98a7SChris Wilson args->vm_id = err; 82510be98a7SChris Wilson return 0; 82610be98a7SChris Wilson 82710be98a7SChris Wilson err_unlock: 82810be98a7SChris Wilson mutex_unlock(&file_priv->vm_idr_lock); 82910be98a7SChris Wilson err_put: 830e568ac38SChris Wilson i915_vm_put(&ppgtt->vm); 83110be98a7SChris Wilson return err; 83210be98a7SChris Wilson } 83310be98a7SChris Wilson 83410be98a7SChris Wilson int i915_gem_vm_destroy_ioctl(struct drm_device *dev, void *data, 83510be98a7SChris Wilson struct drm_file *file) 83610be98a7SChris Wilson { 83710be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 83810be98a7SChris Wilson struct drm_i915_gem_vm_control *args = data; 839e568ac38SChris Wilson struct i915_address_space *vm; 84010be98a7SChris Wilson int err; 84110be98a7SChris Wilson u32 id; 84210be98a7SChris Wilson 84310be98a7SChris Wilson if (args->flags) 84410be98a7SChris Wilson return -EINVAL; 84510be98a7SChris Wilson 84610be98a7SChris Wilson if (args->extensions) 84710be98a7SChris Wilson return -EINVAL; 84810be98a7SChris Wilson 84910be98a7SChris Wilson id = args->vm_id; 85010be98a7SChris Wilson if (!id) 85110be98a7SChris Wilson return -ENOENT; 85210be98a7SChris Wilson 85310be98a7SChris Wilson err = mutex_lock_interruptible(&file_priv->vm_idr_lock); 85410be98a7SChris Wilson if (err) 85510be98a7SChris Wilson return err; 85610be98a7SChris Wilson 857e568ac38SChris Wilson vm = idr_remove(&file_priv->vm_idr, id); 85810be98a7SChris Wilson 85910be98a7SChris Wilson mutex_unlock(&file_priv->vm_idr_lock); 860e568ac38SChris Wilson if (!vm) 86110be98a7SChris Wilson return -ENOENT; 86210be98a7SChris Wilson 863e568ac38SChris Wilson i915_vm_put(vm); 86410be98a7SChris Wilson return 0; 86510be98a7SChris Wilson } 86610be98a7SChris Wilson 86710be98a7SChris Wilson struct context_barrier_task { 86810be98a7SChris Wilson struct i915_active base; 86910be98a7SChris Wilson void (*task)(void *data); 87010be98a7SChris Wilson void *data; 87110be98a7SChris Wilson }; 87210be98a7SChris Wilson 873274cbf20SChris Wilson __i915_active_call 87410be98a7SChris Wilson static void cb_retire(struct i915_active *base) 87510be98a7SChris Wilson { 87610be98a7SChris Wilson struct context_barrier_task *cb = container_of(base, typeof(*cb), base); 87710be98a7SChris Wilson 87810be98a7SChris Wilson if (cb->task) 87910be98a7SChris Wilson cb->task(cb->data); 88010be98a7SChris Wilson 88110be98a7SChris Wilson i915_active_fini(&cb->base); 88210be98a7SChris Wilson kfree(cb); 88310be98a7SChris Wilson } 88410be98a7SChris Wilson 88510be98a7SChris Wilson I915_SELFTEST_DECLARE(static intel_engine_mask_t context_barrier_inject_fault); 88610be98a7SChris Wilson static int context_barrier_task(struct i915_gem_context *ctx, 88710be98a7SChris Wilson intel_engine_mask_t engines, 8881fe2d6f9SChris Wilson bool (*skip)(struct intel_context *ce, void *data), 88910be98a7SChris Wilson int (*emit)(struct i915_request *rq, void *data), 89010be98a7SChris Wilson void (*task)(void *data), 89110be98a7SChris Wilson void *data) 89210be98a7SChris Wilson { 89310be98a7SChris Wilson struct context_barrier_task *cb; 89410be98a7SChris Wilson struct i915_gem_engines_iter it; 89510be98a7SChris Wilson struct intel_context *ce; 89610be98a7SChris Wilson int err = 0; 89710be98a7SChris Wilson 89810be98a7SChris Wilson GEM_BUG_ON(!task); 89910be98a7SChris Wilson 90010be98a7SChris Wilson cb = kmalloc(sizeof(*cb), GFP_KERNEL); 90110be98a7SChris Wilson if (!cb) 90210be98a7SChris Wilson return -ENOMEM; 90310be98a7SChris Wilson 904b1e3177bSChris Wilson i915_active_init(&cb->base, NULL, cb_retire); 90512c255b5SChris Wilson err = i915_active_acquire(&cb->base); 90612c255b5SChris Wilson if (err) { 90712c255b5SChris Wilson kfree(cb); 90812c255b5SChris Wilson return err; 90912c255b5SChris Wilson } 91010be98a7SChris Wilson 91110be98a7SChris Wilson for_each_gem_engine(ce, i915_gem_context_lock_engines(ctx), it) { 91210be98a7SChris Wilson struct i915_request *rq; 91310be98a7SChris Wilson 91410be98a7SChris Wilson if (I915_SELFTEST_ONLY(context_barrier_inject_fault & 91510be98a7SChris Wilson ce->engine->mask)) { 91610be98a7SChris Wilson err = -ENXIO; 91710be98a7SChris Wilson break; 91810be98a7SChris Wilson } 91910be98a7SChris Wilson 9201fe2d6f9SChris Wilson if (!(ce->engine->mask & engines)) 9211fe2d6f9SChris Wilson continue; 9221fe2d6f9SChris Wilson 9231fe2d6f9SChris Wilson if (skip && skip(ce, data)) 92410be98a7SChris Wilson continue; 92510be98a7SChris Wilson 92610be98a7SChris Wilson rq = intel_context_create_request(ce); 92710be98a7SChris Wilson if (IS_ERR(rq)) { 92810be98a7SChris Wilson err = PTR_ERR(rq); 92910be98a7SChris Wilson break; 93010be98a7SChris Wilson } 93110be98a7SChris Wilson 93210be98a7SChris Wilson err = 0; 93310be98a7SChris Wilson if (emit) 93410be98a7SChris Wilson err = emit(rq, data); 93510be98a7SChris Wilson if (err == 0) 936d19d71fcSChris Wilson err = i915_active_add_request(&cb->base, rq); 93710be98a7SChris Wilson 93810be98a7SChris Wilson i915_request_add(rq); 93910be98a7SChris Wilson if (err) 94010be98a7SChris Wilson break; 94110be98a7SChris Wilson } 94210be98a7SChris Wilson i915_gem_context_unlock_engines(ctx); 94310be98a7SChris Wilson 94410be98a7SChris Wilson cb->task = err ? NULL : task; /* caller needs to unwind instead */ 94510be98a7SChris Wilson cb->data = data; 94610be98a7SChris Wilson 94710be98a7SChris Wilson i915_active_release(&cb->base); 94810be98a7SChris Wilson 94910be98a7SChris Wilson return err; 95010be98a7SChris Wilson } 95110be98a7SChris Wilson 95210be98a7SChris Wilson static int get_ppgtt(struct drm_i915_file_private *file_priv, 95310be98a7SChris Wilson struct i915_gem_context *ctx, 95410be98a7SChris Wilson struct drm_i915_gem_context_param *args) 95510be98a7SChris Wilson { 956e568ac38SChris Wilson struct i915_address_space *vm; 95710be98a7SChris Wilson int ret; 95810be98a7SChris Wilson 959a4e7ccdaSChris Wilson if (!rcu_access_pointer(ctx->vm)) 96010be98a7SChris Wilson return -ENODEV; 96110be98a7SChris Wilson 962a4e7ccdaSChris Wilson rcu_read_lock(); 963e568ac38SChris Wilson vm = i915_vm_get(ctx->vm); 964a4e7ccdaSChris Wilson rcu_read_unlock(); 96510be98a7SChris Wilson 96610be98a7SChris Wilson ret = mutex_lock_interruptible(&file_priv->vm_idr_lock); 96710be98a7SChris Wilson if (ret) 96810be98a7SChris Wilson goto err_put; 96910be98a7SChris Wilson 970e568ac38SChris Wilson ret = idr_alloc(&file_priv->vm_idr, vm, 0, 0, GFP_KERNEL); 97110be98a7SChris Wilson GEM_BUG_ON(!ret); 97210be98a7SChris Wilson if (ret < 0) 97310be98a7SChris Wilson goto err_unlock; 97410be98a7SChris Wilson 9752850748eSChris Wilson i915_vm_open(vm); 97610be98a7SChris Wilson 97710be98a7SChris Wilson args->size = 0; 97810be98a7SChris Wilson args->value = ret; 97910be98a7SChris Wilson 98010be98a7SChris Wilson ret = 0; 98110be98a7SChris Wilson err_unlock: 98210be98a7SChris Wilson mutex_unlock(&file_priv->vm_idr_lock); 98310be98a7SChris Wilson err_put: 984e568ac38SChris Wilson i915_vm_put(vm); 98510be98a7SChris Wilson return ret; 98610be98a7SChris Wilson } 98710be98a7SChris Wilson 98810be98a7SChris Wilson static void set_ppgtt_barrier(void *data) 98910be98a7SChris Wilson { 990e568ac38SChris Wilson struct i915_address_space *old = data; 99110be98a7SChris Wilson 992e568ac38SChris Wilson if (INTEL_GEN(old->i915) < 8) 993e568ac38SChris Wilson gen6_ppgtt_unpin_all(i915_vm_to_ppgtt(old)); 99410be98a7SChris Wilson 9952850748eSChris Wilson i915_vm_close(old); 99610be98a7SChris Wilson } 99710be98a7SChris Wilson 99810be98a7SChris Wilson static int emit_ppgtt_update(struct i915_request *rq, void *data) 99910be98a7SChris Wilson { 1000f5d974f9SChris Wilson struct i915_address_space *vm = rq->hw_context->vm; 100110be98a7SChris Wilson struct intel_engine_cs *engine = rq->engine; 100210be98a7SChris Wilson u32 base = engine->mmio_base; 100310be98a7SChris Wilson u32 *cs; 100410be98a7SChris Wilson int i; 100510be98a7SChris Wilson 1006e568ac38SChris Wilson if (i915_vm_is_4lvl(vm)) { 1007ab53497bSChris Wilson struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1008b5b7bef9SMika Kuoppala const dma_addr_t pd_daddr = px_dma(ppgtt->pd); 100910be98a7SChris Wilson 101010be98a7SChris Wilson cs = intel_ring_begin(rq, 6); 101110be98a7SChris Wilson if (IS_ERR(cs)) 101210be98a7SChris Wilson return PTR_ERR(cs); 101310be98a7SChris Wilson 101410be98a7SChris Wilson *cs++ = MI_LOAD_REGISTER_IMM(2); 101510be98a7SChris Wilson 101610be98a7SChris Wilson *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, 0)); 101710be98a7SChris Wilson *cs++ = upper_32_bits(pd_daddr); 101810be98a7SChris Wilson *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, 0)); 101910be98a7SChris Wilson *cs++ = lower_32_bits(pd_daddr); 102010be98a7SChris Wilson 102110be98a7SChris Wilson *cs++ = MI_NOOP; 102210be98a7SChris Wilson intel_ring_advance(rq, cs); 102310be98a7SChris Wilson } else if (HAS_LOGICAL_RING_CONTEXTS(engine->i915)) { 1024ab53497bSChris Wilson struct i915_ppgtt *ppgtt = i915_vm_to_ppgtt(vm); 1025191797a8SChris Wilson int err; 1026191797a8SChris Wilson 1027191797a8SChris Wilson /* Magic required to prevent forcewake errors! */ 1028191797a8SChris Wilson err = engine->emit_flush(rq, EMIT_INVALIDATE); 1029191797a8SChris Wilson if (err) 1030191797a8SChris Wilson return err; 1031e568ac38SChris Wilson 103210be98a7SChris Wilson cs = intel_ring_begin(rq, 4 * GEN8_3LVL_PDPES + 2); 103310be98a7SChris Wilson if (IS_ERR(cs)) 103410be98a7SChris Wilson return PTR_ERR(cs); 103510be98a7SChris Wilson 1036191797a8SChris Wilson *cs++ = MI_LOAD_REGISTER_IMM(2 * GEN8_3LVL_PDPES) | MI_LRI_FORCE_POSTED; 103710be98a7SChris Wilson for (i = GEN8_3LVL_PDPES; i--; ) { 103810be98a7SChris Wilson const dma_addr_t pd_daddr = i915_page_dir_dma_addr(ppgtt, i); 103910be98a7SChris Wilson 104010be98a7SChris Wilson *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_UDW(base, i)); 104110be98a7SChris Wilson *cs++ = upper_32_bits(pd_daddr); 104210be98a7SChris Wilson *cs++ = i915_mmio_reg_offset(GEN8_RING_PDP_LDW(base, i)); 104310be98a7SChris Wilson *cs++ = lower_32_bits(pd_daddr); 104410be98a7SChris Wilson } 104510be98a7SChris Wilson *cs++ = MI_NOOP; 104610be98a7SChris Wilson intel_ring_advance(rq, cs); 104710be98a7SChris Wilson } else { 104810be98a7SChris Wilson /* ppGTT is not part of the legacy context image */ 1049e568ac38SChris Wilson gen6_ppgtt_pin(i915_vm_to_ppgtt(vm)); 105010be98a7SChris Wilson } 105110be98a7SChris Wilson 105210be98a7SChris Wilson return 0; 105310be98a7SChris Wilson } 105410be98a7SChris Wilson 10551fe2d6f9SChris Wilson static bool skip_ppgtt_update(struct intel_context *ce, void *data) 10561fe2d6f9SChris Wilson { 10571fe2d6f9SChris Wilson if (HAS_LOGICAL_RING_CONTEXTS(ce->engine->i915)) 10581fe2d6f9SChris Wilson return !ce->state; 10591fe2d6f9SChris Wilson else 10601fe2d6f9SChris Wilson return !atomic_read(&ce->pin_count); 10611fe2d6f9SChris Wilson } 10621fe2d6f9SChris Wilson 106310be98a7SChris Wilson static int set_ppgtt(struct drm_i915_file_private *file_priv, 106410be98a7SChris Wilson struct i915_gem_context *ctx, 106510be98a7SChris Wilson struct drm_i915_gem_context_param *args) 106610be98a7SChris Wilson { 1067e568ac38SChris Wilson struct i915_address_space *vm, *old; 106810be98a7SChris Wilson int err; 106910be98a7SChris Wilson 107010be98a7SChris Wilson if (args->size) 107110be98a7SChris Wilson return -EINVAL; 107210be98a7SChris Wilson 1073a4e7ccdaSChris Wilson if (!rcu_access_pointer(ctx->vm)) 107410be98a7SChris Wilson return -ENODEV; 107510be98a7SChris Wilson 107610be98a7SChris Wilson if (upper_32_bits(args->value)) 107710be98a7SChris Wilson return -ENOENT; 107810be98a7SChris Wilson 1079aabbe344SChris Wilson rcu_read_lock(); 1080e568ac38SChris Wilson vm = idr_find(&file_priv->vm_idr, args->value); 1081aabbe344SChris Wilson if (vm && !kref_get_unless_zero(&vm->ref)) 1082aabbe344SChris Wilson vm = NULL; 1083aabbe344SChris Wilson rcu_read_unlock(); 1084e568ac38SChris Wilson if (!vm) 108510be98a7SChris Wilson return -ENOENT; 108610be98a7SChris Wilson 1087a4e7ccdaSChris Wilson err = mutex_lock_interruptible(&ctx->mutex); 108810be98a7SChris Wilson if (err) 108910be98a7SChris Wilson goto out; 109010be98a7SChris Wilson 1091a4e7ccdaSChris Wilson if (i915_gem_context_is_closed(ctx)) { 1092a4e7ccdaSChris Wilson err = -ENOENT; 1093a4e7ccdaSChris Wilson goto out; 1094a4e7ccdaSChris Wilson } 1095a4e7ccdaSChris Wilson 1096a4e7ccdaSChris Wilson if (vm == rcu_access_pointer(ctx->vm)) 109710be98a7SChris Wilson goto unlock; 109810be98a7SChris Wilson 109910be98a7SChris Wilson /* Teardown the existing obj:vma cache, it will have to be rebuilt. */ 110010be98a7SChris Wilson lut_close(ctx); 110110be98a7SChris Wilson 1102e568ac38SChris Wilson old = __set_ppgtt(ctx, vm); 110310be98a7SChris Wilson 110410be98a7SChris Wilson /* 110510be98a7SChris Wilson * We need to flush any requests using the current ppgtt before 110610be98a7SChris Wilson * we release it as the requests do not hold a reference themselves, 110710be98a7SChris Wilson * only indirectly through the context. 110810be98a7SChris Wilson */ 110910be98a7SChris Wilson err = context_barrier_task(ctx, ALL_ENGINES, 11101fe2d6f9SChris Wilson skip_ppgtt_update, 111110be98a7SChris Wilson emit_ppgtt_update, 111210be98a7SChris Wilson set_ppgtt_barrier, 111310be98a7SChris Wilson old); 111410be98a7SChris Wilson if (err) { 11152850748eSChris Wilson i915_vm_close(__set_ppgtt(ctx, old)); 11162850748eSChris Wilson i915_vm_close(old); 111710be98a7SChris Wilson } 111810be98a7SChris Wilson 111910be98a7SChris Wilson unlock: 1120a4e7ccdaSChris Wilson mutex_unlock(&ctx->mutex); 112110be98a7SChris Wilson out: 1122e568ac38SChris Wilson i915_vm_put(vm); 112310be98a7SChris Wilson return err; 112410be98a7SChris Wilson } 112510be98a7SChris Wilson 112610be98a7SChris Wilson static int gen8_emit_rpcs_config(struct i915_request *rq, 112710be98a7SChris Wilson struct intel_context *ce, 112810be98a7SChris Wilson struct intel_sseu sseu) 112910be98a7SChris Wilson { 113010be98a7SChris Wilson u64 offset; 113110be98a7SChris Wilson u32 *cs; 113210be98a7SChris Wilson 113310be98a7SChris Wilson cs = intel_ring_begin(rq, 4); 113410be98a7SChris Wilson if (IS_ERR(cs)) 113510be98a7SChris Wilson return PTR_ERR(cs); 113610be98a7SChris Wilson 113710be98a7SChris Wilson offset = i915_ggtt_offset(ce->state) + 113810be98a7SChris Wilson LRC_STATE_PN * PAGE_SIZE + 11397dc56af5SChris Wilson CTX_R_PWR_CLK_STATE * 4; 114010be98a7SChris Wilson 114110be98a7SChris Wilson *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT; 114210be98a7SChris Wilson *cs++ = lower_32_bits(offset); 114310be98a7SChris Wilson *cs++ = upper_32_bits(offset); 114410be98a7SChris Wilson *cs++ = intel_sseu_make_rpcs(rq->i915, &sseu); 114510be98a7SChris Wilson 114610be98a7SChris Wilson intel_ring_advance(rq, cs); 114710be98a7SChris Wilson 114810be98a7SChris Wilson return 0; 114910be98a7SChris Wilson } 115010be98a7SChris Wilson 115110be98a7SChris Wilson static int 115210be98a7SChris Wilson gen8_modify_rpcs(struct intel_context *ce, struct intel_sseu sseu) 115310be98a7SChris Wilson { 115410be98a7SChris Wilson struct i915_request *rq; 115510be98a7SChris Wilson int ret; 115610be98a7SChris Wilson 115710be98a7SChris Wilson lockdep_assert_held(&ce->pin_mutex); 115810be98a7SChris Wilson 115910be98a7SChris Wilson /* 116010be98a7SChris Wilson * If the context is not idle, we have to submit an ordered request to 116110be98a7SChris Wilson * modify its context image via the kernel context (writing to our own 116210be98a7SChris Wilson * image, or into the registers directory, does not stick). Pristine 116310be98a7SChris Wilson * and idle contexts will be configured on pinning. 116410be98a7SChris Wilson */ 116510be98a7SChris Wilson if (!intel_context_is_pinned(ce)) 116610be98a7SChris Wilson return 0; 116710be98a7SChris Wilson 116810be98a7SChris Wilson rq = i915_request_create(ce->engine->kernel_context); 116910be98a7SChris Wilson if (IS_ERR(rq)) 117010be98a7SChris Wilson return PTR_ERR(rq); 117110be98a7SChris Wilson 1172a9877da2SChris Wilson /* Serialise with the remote context */ 1173a9877da2SChris Wilson ret = intel_context_prepare_remote_request(ce, rq); 1174a9877da2SChris Wilson if (ret == 0) 1175ce476c80SChris Wilson ret = gen8_emit_rpcs_config(rq, ce, sseu); 117610be98a7SChris Wilson 117710be98a7SChris Wilson i915_request_add(rq); 117810be98a7SChris Wilson return ret; 117910be98a7SChris Wilson } 118010be98a7SChris Wilson 118110be98a7SChris Wilson static int 11827e805762SChris Wilson intel_context_reconfigure_sseu(struct intel_context *ce, struct intel_sseu sseu) 118310be98a7SChris Wilson { 118410be98a7SChris Wilson int ret; 118510be98a7SChris Wilson 1186cb0c43f3SChris Wilson GEM_BUG_ON(INTEL_GEN(ce->engine->i915) < 8); 118710be98a7SChris Wilson 118810be98a7SChris Wilson ret = intel_context_lock_pinned(ce); 118910be98a7SChris Wilson if (ret) 119010be98a7SChris Wilson return ret; 119110be98a7SChris Wilson 119210be98a7SChris Wilson /* Nothing to do if unmodified. */ 119310be98a7SChris Wilson if (!memcmp(&ce->sseu, &sseu, sizeof(sseu))) 119410be98a7SChris Wilson goto unlock; 119510be98a7SChris Wilson 119610be98a7SChris Wilson ret = gen8_modify_rpcs(ce, sseu); 119710be98a7SChris Wilson if (!ret) 119810be98a7SChris Wilson ce->sseu = sseu; 119910be98a7SChris Wilson 120010be98a7SChris Wilson unlock: 120110be98a7SChris Wilson intel_context_unlock_pinned(ce); 120210be98a7SChris Wilson return ret; 120310be98a7SChris Wilson } 120410be98a7SChris Wilson 120510be98a7SChris Wilson static int 120610be98a7SChris Wilson user_to_context_sseu(struct drm_i915_private *i915, 120710be98a7SChris Wilson const struct drm_i915_gem_context_param_sseu *user, 120810be98a7SChris Wilson struct intel_sseu *context) 120910be98a7SChris Wilson { 121010be98a7SChris Wilson const struct sseu_dev_info *device = &RUNTIME_INFO(i915)->sseu; 121110be98a7SChris Wilson 121210be98a7SChris Wilson /* No zeros in any field. */ 121310be98a7SChris Wilson if (!user->slice_mask || !user->subslice_mask || 121410be98a7SChris Wilson !user->min_eus_per_subslice || !user->max_eus_per_subslice) 121510be98a7SChris Wilson return -EINVAL; 121610be98a7SChris Wilson 121710be98a7SChris Wilson /* Max > min. */ 121810be98a7SChris Wilson if (user->max_eus_per_subslice < user->min_eus_per_subslice) 121910be98a7SChris Wilson return -EINVAL; 122010be98a7SChris Wilson 122110be98a7SChris Wilson /* 122210be98a7SChris Wilson * Some future proofing on the types since the uAPI is wider than the 122310be98a7SChris Wilson * current internal implementation. 122410be98a7SChris Wilson */ 122510be98a7SChris Wilson if (overflows_type(user->slice_mask, context->slice_mask) || 122610be98a7SChris Wilson overflows_type(user->subslice_mask, context->subslice_mask) || 122710be98a7SChris Wilson overflows_type(user->min_eus_per_subslice, 122810be98a7SChris Wilson context->min_eus_per_subslice) || 122910be98a7SChris Wilson overflows_type(user->max_eus_per_subslice, 123010be98a7SChris Wilson context->max_eus_per_subslice)) 123110be98a7SChris Wilson return -EINVAL; 123210be98a7SChris Wilson 123310be98a7SChris Wilson /* Check validity against hardware. */ 123410be98a7SChris Wilson if (user->slice_mask & ~device->slice_mask) 123510be98a7SChris Wilson return -EINVAL; 123610be98a7SChris Wilson 123710be98a7SChris Wilson if (user->subslice_mask & ~device->subslice_mask[0]) 123810be98a7SChris Wilson return -EINVAL; 123910be98a7SChris Wilson 124010be98a7SChris Wilson if (user->max_eus_per_subslice > device->max_eus_per_subslice) 124110be98a7SChris Wilson return -EINVAL; 124210be98a7SChris Wilson 124310be98a7SChris Wilson context->slice_mask = user->slice_mask; 124410be98a7SChris Wilson context->subslice_mask = user->subslice_mask; 124510be98a7SChris Wilson context->min_eus_per_subslice = user->min_eus_per_subslice; 124610be98a7SChris Wilson context->max_eus_per_subslice = user->max_eus_per_subslice; 124710be98a7SChris Wilson 124810be98a7SChris Wilson /* Part specific restrictions. */ 124910be98a7SChris Wilson if (IS_GEN(i915, 11)) { 125010be98a7SChris Wilson unsigned int hw_s = hweight8(device->slice_mask); 125110be98a7SChris Wilson unsigned int hw_ss_per_s = hweight8(device->subslice_mask[0]); 125210be98a7SChris Wilson unsigned int req_s = hweight8(context->slice_mask); 125310be98a7SChris Wilson unsigned int req_ss = hweight8(context->subslice_mask); 125410be98a7SChris Wilson 125510be98a7SChris Wilson /* 125610be98a7SChris Wilson * Only full subslice enablement is possible if more than one 125710be98a7SChris Wilson * slice is turned on. 125810be98a7SChris Wilson */ 125910be98a7SChris Wilson if (req_s > 1 && req_ss != hw_ss_per_s) 126010be98a7SChris Wilson return -EINVAL; 126110be98a7SChris Wilson 126210be98a7SChris Wilson /* 126310be98a7SChris Wilson * If more than four (SScount bitfield limit) subslices are 126410be98a7SChris Wilson * requested then the number has to be even. 126510be98a7SChris Wilson */ 126610be98a7SChris Wilson if (req_ss > 4 && (req_ss & 1)) 126710be98a7SChris Wilson return -EINVAL; 126810be98a7SChris Wilson 126910be98a7SChris Wilson /* 127010be98a7SChris Wilson * If only one slice is enabled and subslice count is below the 127110be98a7SChris Wilson * device full enablement, it must be at most half of the all 127210be98a7SChris Wilson * available subslices. 127310be98a7SChris Wilson */ 127410be98a7SChris Wilson if (req_s == 1 && req_ss < hw_ss_per_s && 127510be98a7SChris Wilson req_ss > (hw_ss_per_s / 2)) 127610be98a7SChris Wilson return -EINVAL; 127710be98a7SChris Wilson 127810be98a7SChris Wilson /* ABI restriction - VME use case only. */ 127910be98a7SChris Wilson 128010be98a7SChris Wilson /* All slices or one slice only. */ 128110be98a7SChris Wilson if (req_s != 1 && req_s != hw_s) 128210be98a7SChris Wilson return -EINVAL; 128310be98a7SChris Wilson 128410be98a7SChris Wilson /* 128510be98a7SChris Wilson * Half subslices or full enablement only when one slice is 128610be98a7SChris Wilson * enabled. 128710be98a7SChris Wilson */ 128810be98a7SChris Wilson if (req_s == 1 && 128910be98a7SChris Wilson (req_ss != hw_ss_per_s && req_ss != (hw_ss_per_s / 2))) 129010be98a7SChris Wilson return -EINVAL; 129110be98a7SChris Wilson 129210be98a7SChris Wilson /* No EU configuration changes. */ 129310be98a7SChris Wilson if ((user->min_eus_per_subslice != 129410be98a7SChris Wilson device->max_eus_per_subslice) || 129510be98a7SChris Wilson (user->max_eus_per_subslice != 129610be98a7SChris Wilson device->max_eus_per_subslice)) 129710be98a7SChris Wilson return -EINVAL; 129810be98a7SChris Wilson } 129910be98a7SChris Wilson 130010be98a7SChris Wilson return 0; 130110be98a7SChris Wilson } 130210be98a7SChris Wilson 130310be98a7SChris Wilson static int set_sseu(struct i915_gem_context *ctx, 130410be98a7SChris Wilson struct drm_i915_gem_context_param *args) 130510be98a7SChris Wilson { 130610be98a7SChris Wilson struct drm_i915_private *i915 = ctx->i915; 130710be98a7SChris Wilson struct drm_i915_gem_context_param_sseu user_sseu; 130810be98a7SChris Wilson struct intel_context *ce; 130910be98a7SChris Wilson struct intel_sseu sseu; 131010be98a7SChris Wilson unsigned long lookup; 131110be98a7SChris Wilson int ret; 131210be98a7SChris Wilson 131310be98a7SChris Wilson if (args->size < sizeof(user_sseu)) 131410be98a7SChris Wilson return -EINVAL; 131510be98a7SChris Wilson 131610be98a7SChris Wilson if (!IS_GEN(i915, 11)) 131710be98a7SChris Wilson return -ENODEV; 131810be98a7SChris Wilson 131910be98a7SChris Wilson if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), 132010be98a7SChris Wilson sizeof(user_sseu))) 132110be98a7SChris Wilson return -EFAULT; 132210be98a7SChris Wilson 132310be98a7SChris Wilson if (user_sseu.rsvd) 132410be98a7SChris Wilson return -EINVAL; 132510be98a7SChris Wilson 132610be98a7SChris Wilson if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) 132710be98a7SChris Wilson return -EINVAL; 132810be98a7SChris Wilson 132910be98a7SChris Wilson lookup = 0; 133010be98a7SChris Wilson if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) 133110be98a7SChris Wilson lookup |= LOOKUP_USER_INDEX; 133210be98a7SChris Wilson 133310be98a7SChris Wilson ce = lookup_user_engine(ctx, lookup, &user_sseu.engine); 133410be98a7SChris Wilson if (IS_ERR(ce)) 133510be98a7SChris Wilson return PTR_ERR(ce); 133610be98a7SChris Wilson 133710be98a7SChris Wilson /* Only render engine supports RPCS configuration. */ 133810be98a7SChris Wilson if (ce->engine->class != RENDER_CLASS) { 133910be98a7SChris Wilson ret = -ENODEV; 134010be98a7SChris Wilson goto out_ce; 134110be98a7SChris Wilson } 134210be98a7SChris Wilson 134310be98a7SChris Wilson ret = user_to_context_sseu(i915, &user_sseu, &sseu); 134410be98a7SChris Wilson if (ret) 134510be98a7SChris Wilson goto out_ce; 134610be98a7SChris Wilson 134710be98a7SChris Wilson ret = intel_context_reconfigure_sseu(ce, sseu); 134810be98a7SChris Wilson if (ret) 134910be98a7SChris Wilson goto out_ce; 135010be98a7SChris Wilson 135110be98a7SChris Wilson args->size = sizeof(user_sseu); 135210be98a7SChris Wilson 135310be98a7SChris Wilson out_ce: 135410be98a7SChris Wilson intel_context_put(ce); 135510be98a7SChris Wilson return ret; 135610be98a7SChris Wilson } 135710be98a7SChris Wilson 135810be98a7SChris Wilson struct set_engines { 135910be98a7SChris Wilson struct i915_gem_context *ctx; 136010be98a7SChris Wilson struct i915_gem_engines *engines; 136110be98a7SChris Wilson }; 136210be98a7SChris Wilson 136310be98a7SChris Wilson static int 136410be98a7SChris Wilson set_engines__load_balance(struct i915_user_extension __user *base, void *data) 136510be98a7SChris Wilson { 136610be98a7SChris Wilson struct i915_context_engines_load_balance __user *ext = 136710be98a7SChris Wilson container_of_user(base, typeof(*ext), base); 136810be98a7SChris Wilson const struct set_engines *set = data; 136910be98a7SChris Wilson struct intel_engine_cs *stack[16]; 137010be98a7SChris Wilson struct intel_engine_cs **siblings; 137110be98a7SChris Wilson struct intel_context *ce; 137210be98a7SChris Wilson u16 num_siblings, idx; 137310be98a7SChris Wilson unsigned int n; 137410be98a7SChris Wilson int err; 137510be98a7SChris Wilson 137610be98a7SChris Wilson if (!HAS_EXECLISTS(set->ctx->i915)) 137710be98a7SChris Wilson return -ENODEV; 137810be98a7SChris Wilson 137910be98a7SChris Wilson if (USES_GUC_SUBMISSION(set->ctx->i915)) 138010be98a7SChris Wilson return -ENODEV; /* not implement yet */ 138110be98a7SChris Wilson 138210be98a7SChris Wilson if (get_user(idx, &ext->engine_index)) 138310be98a7SChris Wilson return -EFAULT; 138410be98a7SChris Wilson 138510be98a7SChris Wilson if (idx >= set->engines->num_engines) { 138610be98a7SChris Wilson DRM_DEBUG("Invalid placement value, %d >= %d\n", 138710be98a7SChris Wilson idx, set->engines->num_engines); 138810be98a7SChris Wilson return -EINVAL; 138910be98a7SChris Wilson } 139010be98a7SChris Wilson 139110be98a7SChris Wilson idx = array_index_nospec(idx, set->engines->num_engines); 139210be98a7SChris Wilson if (set->engines->engines[idx]) { 139310be98a7SChris Wilson DRM_DEBUG("Invalid placement[%d], already occupied\n", idx); 139410be98a7SChris Wilson return -EEXIST; 139510be98a7SChris Wilson } 139610be98a7SChris Wilson 139710be98a7SChris Wilson if (get_user(num_siblings, &ext->num_siblings)) 139810be98a7SChris Wilson return -EFAULT; 139910be98a7SChris Wilson 140010be98a7SChris Wilson err = check_user_mbz(&ext->flags); 140110be98a7SChris Wilson if (err) 140210be98a7SChris Wilson return err; 140310be98a7SChris Wilson 140410be98a7SChris Wilson err = check_user_mbz(&ext->mbz64); 140510be98a7SChris Wilson if (err) 140610be98a7SChris Wilson return err; 140710be98a7SChris Wilson 140810be98a7SChris Wilson siblings = stack; 140910be98a7SChris Wilson if (num_siblings > ARRAY_SIZE(stack)) { 141010be98a7SChris Wilson siblings = kmalloc_array(num_siblings, 141110be98a7SChris Wilson sizeof(*siblings), 141210be98a7SChris Wilson GFP_KERNEL); 141310be98a7SChris Wilson if (!siblings) 141410be98a7SChris Wilson return -ENOMEM; 141510be98a7SChris Wilson } 141610be98a7SChris Wilson 141710be98a7SChris Wilson for (n = 0; n < num_siblings; n++) { 141810be98a7SChris Wilson struct i915_engine_class_instance ci; 141910be98a7SChris Wilson 142010be98a7SChris Wilson if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) { 142110be98a7SChris Wilson err = -EFAULT; 142210be98a7SChris Wilson goto out_siblings; 142310be98a7SChris Wilson } 142410be98a7SChris Wilson 142510be98a7SChris Wilson siblings[n] = intel_engine_lookup_user(set->ctx->i915, 142610be98a7SChris Wilson ci.engine_class, 142710be98a7SChris Wilson ci.engine_instance); 142810be98a7SChris Wilson if (!siblings[n]) { 142910be98a7SChris Wilson DRM_DEBUG("Invalid sibling[%d]: { class:%d, inst:%d }\n", 143010be98a7SChris Wilson n, ci.engine_class, ci.engine_instance); 143110be98a7SChris Wilson err = -EINVAL; 143210be98a7SChris Wilson goto out_siblings; 143310be98a7SChris Wilson } 143410be98a7SChris Wilson } 143510be98a7SChris Wilson 143610be98a7SChris Wilson ce = intel_execlists_create_virtual(set->ctx, siblings, n); 143710be98a7SChris Wilson if (IS_ERR(ce)) { 143810be98a7SChris Wilson err = PTR_ERR(ce); 143910be98a7SChris Wilson goto out_siblings; 144010be98a7SChris Wilson } 144110be98a7SChris Wilson 144210be98a7SChris Wilson if (cmpxchg(&set->engines->engines[idx], NULL, ce)) { 144310be98a7SChris Wilson intel_context_put(ce); 144410be98a7SChris Wilson err = -EEXIST; 144510be98a7SChris Wilson goto out_siblings; 144610be98a7SChris Wilson } 144710be98a7SChris Wilson 144810be98a7SChris Wilson out_siblings: 144910be98a7SChris Wilson if (siblings != stack) 145010be98a7SChris Wilson kfree(siblings); 145110be98a7SChris Wilson 145210be98a7SChris Wilson return err; 145310be98a7SChris Wilson } 145410be98a7SChris Wilson 145510be98a7SChris Wilson static int 145610be98a7SChris Wilson set_engines__bond(struct i915_user_extension __user *base, void *data) 145710be98a7SChris Wilson { 145810be98a7SChris Wilson struct i915_context_engines_bond __user *ext = 145910be98a7SChris Wilson container_of_user(base, typeof(*ext), base); 146010be98a7SChris Wilson const struct set_engines *set = data; 146110be98a7SChris Wilson struct i915_engine_class_instance ci; 146210be98a7SChris Wilson struct intel_engine_cs *virtual; 146310be98a7SChris Wilson struct intel_engine_cs *master; 146410be98a7SChris Wilson u16 idx, num_bonds; 146510be98a7SChris Wilson int err, n; 146610be98a7SChris Wilson 146710be98a7SChris Wilson if (get_user(idx, &ext->virtual_index)) 146810be98a7SChris Wilson return -EFAULT; 146910be98a7SChris Wilson 147010be98a7SChris Wilson if (idx >= set->engines->num_engines) { 147110be98a7SChris Wilson DRM_DEBUG("Invalid index for virtual engine: %d >= %d\n", 147210be98a7SChris Wilson idx, set->engines->num_engines); 147310be98a7SChris Wilson return -EINVAL; 147410be98a7SChris Wilson } 147510be98a7SChris Wilson 147610be98a7SChris Wilson idx = array_index_nospec(idx, set->engines->num_engines); 147710be98a7SChris Wilson if (!set->engines->engines[idx]) { 147810be98a7SChris Wilson DRM_DEBUG("Invalid engine at %d\n", idx); 147910be98a7SChris Wilson return -EINVAL; 148010be98a7SChris Wilson } 148110be98a7SChris Wilson virtual = set->engines->engines[idx]->engine; 148210be98a7SChris Wilson 148310be98a7SChris Wilson err = check_user_mbz(&ext->flags); 148410be98a7SChris Wilson if (err) 148510be98a7SChris Wilson return err; 148610be98a7SChris Wilson 148710be98a7SChris Wilson for (n = 0; n < ARRAY_SIZE(ext->mbz64); n++) { 148810be98a7SChris Wilson err = check_user_mbz(&ext->mbz64[n]); 148910be98a7SChris Wilson if (err) 149010be98a7SChris Wilson return err; 149110be98a7SChris Wilson } 149210be98a7SChris Wilson 149310be98a7SChris Wilson if (copy_from_user(&ci, &ext->master, sizeof(ci))) 149410be98a7SChris Wilson return -EFAULT; 149510be98a7SChris Wilson 149610be98a7SChris Wilson master = intel_engine_lookup_user(set->ctx->i915, 149710be98a7SChris Wilson ci.engine_class, ci.engine_instance); 149810be98a7SChris Wilson if (!master) { 149910be98a7SChris Wilson DRM_DEBUG("Unrecognised master engine: { class:%u, instance:%u }\n", 150010be98a7SChris Wilson ci.engine_class, ci.engine_instance); 150110be98a7SChris Wilson return -EINVAL; 150210be98a7SChris Wilson } 150310be98a7SChris Wilson 150410be98a7SChris Wilson if (get_user(num_bonds, &ext->num_bonds)) 150510be98a7SChris Wilson return -EFAULT; 150610be98a7SChris Wilson 150710be98a7SChris Wilson for (n = 0; n < num_bonds; n++) { 150810be98a7SChris Wilson struct intel_engine_cs *bond; 150910be98a7SChris Wilson 151010be98a7SChris Wilson if (copy_from_user(&ci, &ext->engines[n], sizeof(ci))) 151110be98a7SChris Wilson return -EFAULT; 151210be98a7SChris Wilson 151310be98a7SChris Wilson bond = intel_engine_lookup_user(set->ctx->i915, 151410be98a7SChris Wilson ci.engine_class, 151510be98a7SChris Wilson ci.engine_instance); 151610be98a7SChris Wilson if (!bond) { 151710be98a7SChris Wilson DRM_DEBUG("Unrecognised engine[%d] for bonding: { class:%d, instance: %d }\n", 151810be98a7SChris Wilson n, ci.engine_class, ci.engine_instance); 151910be98a7SChris Wilson return -EINVAL; 152010be98a7SChris Wilson } 152110be98a7SChris Wilson 152210be98a7SChris Wilson /* 152310be98a7SChris Wilson * A non-virtual engine has no siblings to choose between; and 152410be98a7SChris Wilson * a submit fence will always be directed to the one engine. 152510be98a7SChris Wilson */ 152610be98a7SChris Wilson if (intel_engine_is_virtual(virtual)) { 152710be98a7SChris Wilson err = intel_virtual_engine_attach_bond(virtual, 152810be98a7SChris Wilson master, 152910be98a7SChris Wilson bond); 153010be98a7SChris Wilson if (err) 153110be98a7SChris Wilson return err; 153210be98a7SChris Wilson } 153310be98a7SChris Wilson } 153410be98a7SChris Wilson 153510be98a7SChris Wilson return 0; 153610be98a7SChris Wilson } 153710be98a7SChris Wilson 153810be98a7SChris Wilson static const i915_user_extension_fn set_engines__extensions[] = { 153910be98a7SChris Wilson [I915_CONTEXT_ENGINES_EXT_LOAD_BALANCE] = set_engines__load_balance, 154010be98a7SChris Wilson [I915_CONTEXT_ENGINES_EXT_BOND] = set_engines__bond, 154110be98a7SChris Wilson }; 154210be98a7SChris Wilson 154310be98a7SChris Wilson static int 154410be98a7SChris Wilson set_engines(struct i915_gem_context *ctx, 154510be98a7SChris Wilson const struct drm_i915_gem_context_param *args) 154610be98a7SChris Wilson { 154710be98a7SChris Wilson struct i915_context_param_engines __user *user = 154810be98a7SChris Wilson u64_to_user_ptr(args->value); 154910be98a7SChris Wilson struct set_engines set = { .ctx = ctx }; 155010be98a7SChris Wilson unsigned int num_engines, n; 155110be98a7SChris Wilson u64 extensions; 155210be98a7SChris Wilson int err; 155310be98a7SChris Wilson 155410be98a7SChris Wilson if (!args->size) { /* switch back to legacy user_ring_map */ 155510be98a7SChris Wilson if (!i915_gem_context_user_engines(ctx)) 155610be98a7SChris Wilson return 0; 155710be98a7SChris Wilson 155810be98a7SChris Wilson set.engines = default_engines(ctx); 155910be98a7SChris Wilson if (IS_ERR(set.engines)) 156010be98a7SChris Wilson return PTR_ERR(set.engines); 156110be98a7SChris Wilson 156210be98a7SChris Wilson goto replace; 156310be98a7SChris Wilson } 156410be98a7SChris Wilson 156510be98a7SChris Wilson BUILD_BUG_ON(!IS_ALIGNED(sizeof(*user), sizeof(*user->engines))); 156610be98a7SChris Wilson if (args->size < sizeof(*user) || 156710be98a7SChris Wilson !IS_ALIGNED(args->size, sizeof(*user->engines))) { 156810be98a7SChris Wilson DRM_DEBUG("Invalid size for engine array: %d\n", 156910be98a7SChris Wilson args->size); 157010be98a7SChris Wilson return -EINVAL; 157110be98a7SChris Wilson } 157210be98a7SChris Wilson 157310be98a7SChris Wilson /* 157410be98a7SChris Wilson * Note that I915_EXEC_RING_MASK limits execbuf to only using the 157510be98a7SChris Wilson * first 64 engines defined here. 157610be98a7SChris Wilson */ 157710be98a7SChris Wilson num_engines = (args->size - sizeof(*user)) / sizeof(*user->engines); 157810be98a7SChris Wilson 157910be98a7SChris Wilson set.engines = kmalloc(struct_size(set.engines, engines, num_engines), 158010be98a7SChris Wilson GFP_KERNEL); 158110be98a7SChris Wilson if (!set.engines) 158210be98a7SChris Wilson return -ENOMEM; 158310be98a7SChris Wilson 1584155ab883SChris Wilson init_rcu_head(&set.engines->rcu); 158510be98a7SChris Wilson for (n = 0; n < num_engines; n++) { 158610be98a7SChris Wilson struct i915_engine_class_instance ci; 158710be98a7SChris Wilson struct intel_engine_cs *engine; 158848ae397bSChris Wilson struct intel_context *ce; 158910be98a7SChris Wilson 159010be98a7SChris Wilson if (copy_from_user(&ci, &user->engines[n], sizeof(ci))) { 159110be98a7SChris Wilson __free_engines(set.engines, n); 159210be98a7SChris Wilson return -EFAULT; 159310be98a7SChris Wilson } 159410be98a7SChris Wilson 159510be98a7SChris Wilson if (ci.engine_class == (u16)I915_ENGINE_CLASS_INVALID && 159610be98a7SChris Wilson ci.engine_instance == (u16)I915_ENGINE_CLASS_INVALID_NONE) { 159710be98a7SChris Wilson set.engines->engines[n] = NULL; 159810be98a7SChris Wilson continue; 159910be98a7SChris Wilson } 160010be98a7SChris Wilson 160110be98a7SChris Wilson engine = intel_engine_lookup_user(ctx->i915, 160210be98a7SChris Wilson ci.engine_class, 160310be98a7SChris Wilson ci.engine_instance); 160410be98a7SChris Wilson if (!engine) { 160510be98a7SChris Wilson DRM_DEBUG("Invalid engine[%d]: { class:%d, instance:%d }\n", 160610be98a7SChris Wilson n, ci.engine_class, ci.engine_instance); 160710be98a7SChris Wilson __free_engines(set.engines, n); 160810be98a7SChris Wilson return -ENOENT; 160910be98a7SChris Wilson } 161010be98a7SChris Wilson 161148ae397bSChris Wilson ce = intel_context_create(ctx, engine); 161248ae397bSChris Wilson if (IS_ERR(ce)) { 161310be98a7SChris Wilson __free_engines(set.engines, n); 161448ae397bSChris Wilson return PTR_ERR(ce); 161510be98a7SChris Wilson } 161648ae397bSChris Wilson 161748ae397bSChris Wilson set.engines->engines[n] = ce; 161810be98a7SChris Wilson } 161910be98a7SChris Wilson set.engines->num_engines = num_engines; 162010be98a7SChris Wilson 162110be98a7SChris Wilson err = -EFAULT; 162210be98a7SChris Wilson if (!get_user(extensions, &user->extensions)) 162310be98a7SChris Wilson err = i915_user_extensions(u64_to_user_ptr(extensions), 162410be98a7SChris Wilson set_engines__extensions, 162510be98a7SChris Wilson ARRAY_SIZE(set_engines__extensions), 162610be98a7SChris Wilson &set); 162710be98a7SChris Wilson if (err) { 162810be98a7SChris Wilson free_engines(set.engines); 162910be98a7SChris Wilson return err; 163010be98a7SChris Wilson } 163110be98a7SChris Wilson 163210be98a7SChris Wilson replace: 163310be98a7SChris Wilson mutex_lock(&ctx->engines_mutex); 163410be98a7SChris Wilson if (args->size) 163510be98a7SChris Wilson i915_gem_context_set_user_engines(ctx); 163610be98a7SChris Wilson else 163710be98a7SChris Wilson i915_gem_context_clear_user_engines(ctx); 163810be98a7SChris Wilson rcu_swap_protected(ctx->engines, set.engines, 1); 163910be98a7SChris Wilson mutex_unlock(&ctx->engines_mutex); 164010be98a7SChris Wilson 1641155ab883SChris Wilson call_rcu(&set.engines->rcu, free_engines_rcu); 164210be98a7SChris Wilson 164310be98a7SChris Wilson return 0; 164410be98a7SChris Wilson } 164510be98a7SChris Wilson 164610be98a7SChris Wilson static struct i915_gem_engines * 164710be98a7SChris Wilson __copy_engines(struct i915_gem_engines *e) 164810be98a7SChris Wilson { 164910be98a7SChris Wilson struct i915_gem_engines *copy; 165010be98a7SChris Wilson unsigned int n; 165110be98a7SChris Wilson 165210be98a7SChris Wilson copy = kmalloc(struct_size(e, engines, e->num_engines), GFP_KERNEL); 165310be98a7SChris Wilson if (!copy) 165410be98a7SChris Wilson return ERR_PTR(-ENOMEM); 165510be98a7SChris Wilson 1656155ab883SChris Wilson init_rcu_head(©->rcu); 165710be98a7SChris Wilson for (n = 0; n < e->num_engines; n++) { 165810be98a7SChris Wilson if (e->engines[n]) 165910be98a7SChris Wilson copy->engines[n] = intel_context_get(e->engines[n]); 166010be98a7SChris Wilson else 166110be98a7SChris Wilson copy->engines[n] = NULL; 166210be98a7SChris Wilson } 166310be98a7SChris Wilson copy->num_engines = n; 166410be98a7SChris Wilson 166510be98a7SChris Wilson return copy; 166610be98a7SChris Wilson } 166710be98a7SChris Wilson 166810be98a7SChris Wilson static int 166910be98a7SChris Wilson get_engines(struct i915_gem_context *ctx, 167010be98a7SChris Wilson struct drm_i915_gem_context_param *args) 167110be98a7SChris Wilson { 167210be98a7SChris Wilson struct i915_context_param_engines __user *user; 167310be98a7SChris Wilson struct i915_gem_engines *e; 167410be98a7SChris Wilson size_t n, count, size; 167510be98a7SChris Wilson int err = 0; 167610be98a7SChris Wilson 167710be98a7SChris Wilson err = mutex_lock_interruptible(&ctx->engines_mutex); 167810be98a7SChris Wilson if (err) 167910be98a7SChris Wilson return err; 168010be98a7SChris Wilson 168110be98a7SChris Wilson e = NULL; 168210be98a7SChris Wilson if (i915_gem_context_user_engines(ctx)) 168310be98a7SChris Wilson e = __copy_engines(i915_gem_context_engines(ctx)); 168410be98a7SChris Wilson mutex_unlock(&ctx->engines_mutex); 168510be98a7SChris Wilson if (IS_ERR_OR_NULL(e)) { 168610be98a7SChris Wilson args->size = 0; 168710be98a7SChris Wilson return PTR_ERR_OR_ZERO(e); 168810be98a7SChris Wilson } 168910be98a7SChris Wilson 169010be98a7SChris Wilson count = e->num_engines; 169110be98a7SChris Wilson 169210be98a7SChris Wilson /* Be paranoid in case we have an impedance mismatch */ 169310be98a7SChris Wilson if (!check_struct_size(user, engines, count, &size)) { 169410be98a7SChris Wilson err = -EINVAL; 169510be98a7SChris Wilson goto err_free; 169610be98a7SChris Wilson } 169710be98a7SChris Wilson if (overflows_type(size, args->size)) { 169810be98a7SChris Wilson err = -EINVAL; 169910be98a7SChris Wilson goto err_free; 170010be98a7SChris Wilson } 170110be98a7SChris Wilson 170210be98a7SChris Wilson if (!args->size) { 170310be98a7SChris Wilson args->size = size; 170410be98a7SChris Wilson goto err_free; 170510be98a7SChris Wilson } 170610be98a7SChris Wilson 170710be98a7SChris Wilson if (args->size < size) { 170810be98a7SChris Wilson err = -EINVAL; 170910be98a7SChris Wilson goto err_free; 171010be98a7SChris Wilson } 171110be98a7SChris Wilson 171210be98a7SChris Wilson user = u64_to_user_ptr(args->value); 171310be98a7SChris Wilson if (!access_ok(user, size)) { 171410be98a7SChris Wilson err = -EFAULT; 171510be98a7SChris Wilson goto err_free; 171610be98a7SChris Wilson } 171710be98a7SChris Wilson 171810be98a7SChris Wilson if (put_user(0, &user->extensions)) { 171910be98a7SChris Wilson err = -EFAULT; 172010be98a7SChris Wilson goto err_free; 172110be98a7SChris Wilson } 172210be98a7SChris Wilson 172310be98a7SChris Wilson for (n = 0; n < count; n++) { 172410be98a7SChris Wilson struct i915_engine_class_instance ci = { 172510be98a7SChris Wilson .engine_class = I915_ENGINE_CLASS_INVALID, 172610be98a7SChris Wilson .engine_instance = I915_ENGINE_CLASS_INVALID_NONE, 172710be98a7SChris Wilson }; 172810be98a7SChris Wilson 172910be98a7SChris Wilson if (e->engines[n]) { 173010be98a7SChris Wilson ci.engine_class = e->engines[n]->engine->uabi_class; 1731750e76b4SChris Wilson ci.engine_instance = e->engines[n]->engine->uabi_instance; 173210be98a7SChris Wilson } 173310be98a7SChris Wilson 173410be98a7SChris Wilson if (copy_to_user(&user->engines[n], &ci, sizeof(ci))) { 173510be98a7SChris Wilson err = -EFAULT; 173610be98a7SChris Wilson goto err_free; 173710be98a7SChris Wilson } 173810be98a7SChris Wilson } 173910be98a7SChris Wilson 174010be98a7SChris Wilson args->size = size; 174110be98a7SChris Wilson 174210be98a7SChris Wilson err_free: 1743155ab883SChris Wilson free_engines(e); 174410be98a7SChris Wilson return err; 174510be98a7SChris Wilson } 174610be98a7SChris Wilson 174710be98a7SChris Wilson static int ctx_setparam(struct drm_i915_file_private *fpriv, 174810be98a7SChris Wilson struct i915_gem_context *ctx, 174910be98a7SChris Wilson struct drm_i915_gem_context_param *args) 175010be98a7SChris Wilson { 175110be98a7SChris Wilson int ret = 0; 175210be98a7SChris Wilson 175310be98a7SChris Wilson switch (args->param) { 175410be98a7SChris Wilson case I915_CONTEXT_PARAM_NO_ZEROMAP: 175510be98a7SChris Wilson if (args->size) 175610be98a7SChris Wilson ret = -EINVAL; 175710be98a7SChris Wilson else if (args->value) 175810be98a7SChris Wilson set_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); 175910be98a7SChris Wilson else 176010be98a7SChris Wilson clear_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); 176110be98a7SChris Wilson break; 176210be98a7SChris Wilson 176310be98a7SChris Wilson case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 176410be98a7SChris Wilson if (args->size) 176510be98a7SChris Wilson ret = -EINVAL; 176610be98a7SChris Wilson else if (args->value) 176710be98a7SChris Wilson i915_gem_context_set_no_error_capture(ctx); 176810be98a7SChris Wilson else 176910be98a7SChris Wilson i915_gem_context_clear_no_error_capture(ctx); 177010be98a7SChris Wilson break; 177110be98a7SChris Wilson 177210be98a7SChris Wilson case I915_CONTEXT_PARAM_BANNABLE: 177310be98a7SChris Wilson if (args->size) 177410be98a7SChris Wilson ret = -EINVAL; 177510be98a7SChris Wilson else if (!capable(CAP_SYS_ADMIN) && !args->value) 177610be98a7SChris Wilson ret = -EPERM; 177710be98a7SChris Wilson else if (args->value) 177810be98a7SChris Wilson i915_gem_context_set_bannable(ctx); 177910be98a7SChris Wilson else 178010be98a7SChris Wilson i915_gem_context_clear_bannable(ctx); 178110be98a7SChris Wilson break; 178210be98a7SChris Wilson 178310be98a7SChris Wilson case I915_CONTEXT_PARAM_RECOVERABLE: 178410be98a7SChris Wilson if (args->size) 178510be98a7SChris Wilson ret = -EINVAL; 178610be98a7SChris Wilson else if (args->value) 178710be98a7SChris Wilson i915_gem_context_set_recoverable(ctx); 178810be98a7SChris Wilson else 178910be98a7SChris Wilson i915_gem_context_clear_recoverable(ctx); 179010be98a7SChris Wilson break; 179110be98a7SChris Wilson 179210be98a7SChris Wilson case I915_CONTEXT_PARAM_PRIORITY: 179310be98a7SChris Wilson { 179410be98a7SChris Wilson s64 priority = args->value; 179510be98a7SChris Wilson 179610be98a7SChris Wilson if (args->size) 179710be98a7SChris Wilson ret = -EINVAL; 179810be98a7SChris Wilson else if (!(ctx->i915->caps.scheduler & I915_SCHEDULER_CAP_PRIORITY)) 179910be98a7SChris Wilson ret = -ENODEV; 180010be98a7SChris Wilson else if (priority > I915_CONTEXT_MAX_USER_PRIORITY || 180110be98a7SChris Wilson priority < I915_CONTEXT_MIN_USER_PRIORITY) 180210be98a7SChris Wilson ret = -EINVAL; 180310be98a7SChris Wilson else if (priority > I915_CONTEXT_DEFAULT_PRIORITY && 180410be98a7SChris Wilson !capable(CAP_SYS_NICE)) 180510be98a7SChris Wilson ret = -EPERM; 180610be98a7SChris Wilson else 180710be98a7SChris Wilson ctx->sched.priority = 180810be98a7SChris Wilson I915_USER_PRIORITY(priority); 180910be98a7SChris Wilson } 181010be98a7SChris Wilson break; 181110be98a7SChris Wilson 181210be98a7SChris Wilson case I915_CONTEXT_PARAM_SSEU: 181310be98a7SChris Wilson ret = set_sseu(ctx, args); 181410be98a7SChris Wilson break; 181510be98a7SChris Wilson 181610be98a7SChris Wilson case I915_CONTEXT_PARAM_VM: 181710be98a7SChris Wilson ret = set_ppgtt(fpriv, ctx, args); 181810be98a7SChris Wilson break; 181910be98a7SChris Wilson 182010be98a7SChris Wilson case I915_CONTEXT_PARAM_ENGINES: 182110be98a7SChris Wilson ret = set_engines(ctx, args); 182210be98a7SChris Wilson break; 182310be98a7SChris Wilson 182410be98a7SChris Wilson case I915_CONTEXT_PARAM_BAN_PERIOD: 182510be98a7SChris Wilson default: 182610be98a7SChris Wilson ret = -EINVAL; 182710be98a7SChris Wilson break; 182810be98a7SChris Wilson } 182910be98a7SChris Wilson 183010be98a7SChris Wilson return ret; 183110be98a7SChris Wilson } 183210be98a7SChris Wilson 183310be98a7SChris Wilson struct create_ext { 183410be98a7SChris Wilson struct i915_gem_context *ctx; 183510be98a7SChris Wilson struct drm_i915_file_private *fpriv; 183610be98a7SChris Wilson }; 183710be98a7SChris Wilson 183810be98a7SChris Wilson static int create_setparam(struct i915_user_extension __user *ext, void *data) 183910be98a7SChris Wilson { 184010be98a7SChris Wilson struct drm_i915_gem_context_create_ext_setparam local; 184110be98a7SChris Wilson const struct create_ext *arg = data; 184210be98a7SChris Wilson 184310be98a7SChris Wilson if (copy_from_user(&local, ext, sizeof(local))) 184410be98a7SChris Wilson return -EFAULT; 184510be98a7SChris Wilson 184610be98a7SChris Wilson if (local.param.ctx_id) 184710be98a7SChris Wilson return -EINVAL; 184810be98a7SChris Wilson 184910be98a7SChris Wilson return ctx_setparam(arg->fpriv, arg->ctx, &local.param); 185010be98a7SChris Wilson } 185110be98a7SChris Wilson 185210be98a7SChris Wilson static int clone_engines(struct i915_gem_context *dst, 185310be98a7SChris Wilson struct i915_gem_context *src) 185410be98a7SChris Wilson { 185510be98a7SChris Wilson struct i915_gem_engines *e = i915_gem_context_lock_engines(src); 185610be98a7SChris Wilson struct i915_gem_engines *clone; 185710be98a7SChris Wilson bool user_engines; 185810be98a7SChris Wilson unsigned long n; 185910be98a7SChris Wilson 186010be98a7SChris Wilson clone = kmalloc(struct_size(e, engines, e->num_engines), GFP_KERNEL); 186110be98a7SChris Wilson if (!clone) 186210be98a7SChris Wilson goto err_unlock; 186310be98a7SChris Wilson 1864155ab883SChris Wilson init_rcu_head(&clone->rcu); 186510be98a7SChris Wilson for (n = 0; n < e->num_engines; n++) { 186610be98a7SChris Wilson struct intel_engine_cs *engine; 186710be98a7SChris Wilson 186810be98a7SChris Wilson if (!e->engines[n]) { 186910be98a7SChris Wilson clone->engines[n] = NULL; 187010be98a7SChris Wilson continue; 187110be98a7SChris Wilson } 187210be98a7SChris Wilson engine = e->engines[n]->engine; 187310be98a7SChris Wilson 187410be98a7SChris Wilson /* 187510be98a7SChris Wilson * Virtual engines are singletons; they can only exist 187610be98a7SChris Wilson * inside a single context, because they embed their 187710be98a7SChris Wilson * HW context... As each virtual context implies a single 187810be98a7SChris Wilson * timeline (each engine can only dequeue a single request 187910be98a7SChris Wilson * at any time), it would be surprising for two contexts 188010be98a7SChris Wilson * to use the same engine. So let's create a copy of 188110be98a7SChris Wilson * the virtual engine instead. 188210be98a7SChris Wilson */ 188310be98a7SChris Wilson if (intel_engine_is_virtual(engine)) 188410be98a7SChris Wilson clone->engines[n] = 188510be98a7SChris Wilson intel_execlists_clone_virtual(dst, engine); 188610be98a7SChris Wilson else 188710be98a7SChris Wilson clone->engines[n] = intel_context_create(dst, engine); 188810be98a7SChris Wilson if (IS_ERR_OR_NULL(clone->engines[n])) { 188910be98a7SChris Wilson __free_engines(clone, n); 189010be98a7SChris Wilson goto err_unlock; 189110be98a7SChris Wilson } 189210be98a7SChris Wilson } 189310be98a7SChris Wilson clone->num_engines = n; 189410be98a7SChris Wilson 189510be98a7SChris Wilson user_engines = i915_gem_context_user_engines(src); 189610be98a7SChris Wilson i915_gem_context_unlock_engines(src); 189710be98a7SChris Wilson 189810be98a7SChris Wilson free_engines(dst->engines); 189910be98a7SChris Wilson RCU_INIT_POINTER(dst->engines, clone); 190010be98a7SChris Wilson if (user_engines) 190110be98a7SChris Wilson i915_gem_context_set_user_engines(dst); 190210be98a7SChris Wilson else 190310be98a7SChris Wilson i915_gem_context_clear_user_engines(dst); 190410be98a7SChris Wilson return 0; 190510be98a7SChris Wilson 190610be98a7SChris Wilson err_unlock: 190710be98a7SChris Wilson i915_gem_context_unlock_engines(src); 190810be98a7SChris Wilson return -ENOMEM; 190910be98a7SChris Wilson } 191010be98a7SChris Wilson 191110be98a7SChris Wilson static int clone_flags(struct i915_gem_context *dst, 191210be98a7SChris Wilson struct i915_gem_context *src) 191310be98a7SChris Wilson { 191410be98a7SChris Wilson dst->user_flags = src->user_flags; 191510be98a7SChris Wilson return 0; 191610be98a7SChris Wilson } 191710be98a7SChris Wilson 191810be98a7SChris Wilson static int clone_schedattr(struct i915_gem_context *dst, 191910be98a7SChris Wilson struct i915_gem_context *src) 192010be98a7SChris Wilson { 192110be98a7SChris Wilson dst->sched = src->sched; 192210be98a7SChris Wilson return 0; 192310be98a7SChris Wilson } 192410be98a7SChris Wilson 192510be98a7SChris Wilson static int clone_sseu(struct i915_gem_context *dst, 192610be98a7SChris Wilson struct i915_gem_context *src) 192710be98a7SChris Wilson { 192810be98a7SChris Wilson struct i915_gem_engines *e = i915_gem_context_lock_engines(src); 192910be98a7SChris Wilson struct i915_gem_engines *clone; 193010be98a7SChris Wilson unsigned long n; 193110be98a7SChris Wilson int err; 193210be98a7SChris Wilson 193310be98a7SChris Wilson clone = dst->engines; /* no locking required; sole access */ 193410be98a7SChris Wilson if (e->num_engines != clone->num_engines) { 193510be98a7SChris Wilson err = -EINVAL; 193610be98a7SChris Wilson goto unlock; 193710be98a7SChris Wilson } 193810be98a7SChris Wilson 193910be98a7SChris Wilson for (n = 0; n < e->num_engines; n++) { 194010be98a7SChris Wilson struct intel_context *ce = e->engines[n]; 194110be98a7SChris Wilson 194210be98a7SChris Wilson if (clone->engines[n]->engine->class != ce->engine->class) { 194310be98a7SChris Wilson /* Must have compatible engine maps! */ 194410be98a7SChris Wilson err = -EINVAL; 194510be98a7SChris Wilson goto unlock; 194610be98a7SChris Wilson } 194710be98a7SChris Wilson 194810be98a7SChris Wilson /* serialises with set_sseu */ 194910be98a7SChris Wilson err = intel_context_lock_pinned(ce); 195010be98a7SChris Wilson if (err) 195110be98a7SChris Wilson goto unlock; 195210be98a7SChris Wilson 195310be98a7SChris Wilson clone->engines[n]->sseu = ce->sseu; 195410be98a7SChris Wilson intel_context_unlock_pinned(ce); 195510be98a7SChris Wilson } 195610be98a7SChris Wilson 195710be98a7SChris Wilson err = 0; 195810be98a7SChris Wilson unlock: 195910be98a7SChris Wilson i915_gem_context_unlock_engines(src); 196010be98a7SChris Wilson return err; 196110be98a7SChris Wilson } 196210be98a7SChris Wilson 196310be98a7SChris Wilson static int clone_timeline(struct i915_gem_context *dst, 196410be98a7SChris Wilson struct i915_gem_context *src) 196510be98a7SChris Wilson { 196675d0a7f3SChris Wilson if (src->timeline) 196775d0a7f3SChris Wilson __assign_timeline(dst, src->timeline); 196810be98a7SChris Wilson 196910be98a7SChris Wilson return 0; 197010be98a7SChris Wilson } 197110be98a7SChris Wilson 197210be98a7SChris Wilson static int clone_vm(struct i915_gem_context *dst, 197310be98a7SChris Wilson struct i915_gem_context *src) 197410be98a7SChris Wilson { 1975e568ac38SChris Wilson struct i915_address_space *vm; 1976a4e7ccdaSChris Wilson int err = 0; 197710be98a7SChris Wilson 197810be98a7SChris Wilson rcu_read_lock(); 197910be98a7SChris Wilson do { 1980a4e7ccdaSChris Wilson vm = rcu_dereference(src->vm); 1981e568ac38SChris Wilson if (!vm) 198210be98a7SChris Wilson break; 198310be98a7SChris Wilson 1984e568ac38SChris Wilson if (!kref_get_unless_zero(&vm->ref)) 198510be98a7SChris Wilson continue; 198610be98a7SChris Wilson 198710be98a7SChris Wilson /* 198810be98a7SChris Wilson * This ppgtt may have be reallocated between 198910be98a7SChris Wilson * the read and the kref, and reassigned to a third 199010be98a7SChris Wilson * context. In order to avoid inadvertent sharing 199110be98a7SChris Wilson * of this ppgtt with that third context (and not 199210be98a7SChris Wilson * src), we have to confirm that we have the same 199310be98a7SChris Wilson * ppgtt after passing through the strong memory 199410be98a7SChris Wilson * barrier implied by a successful 199510be98a7SChris Wilson * kref_get_unless_zero(). 199610be98a7SChris Wilson * 199710be98a7SChris Wilson * Once we have acquired the current ppgtt of src, 199810be98a7SChris Wilson * we no longer care if it is released from src, as 199910be98a7SChris Wilson * it cannot be reallocated elsewhere. 200010be98a7SChris Wilson */ 200110be98a7SChris Wilson 2002a4e7ccdaSChris Wilson if (vm == rcu_access_pointer(src->vm)) 200310be98a7SChris Wilson break; 200410be98a7SChris Wilson 2005e568ac38SChris Wilson i915_vm_put(vm); 200610be98a7SChris Wilson } while (1); 200710be98a7SChris Wilson rcu_read_unlock(); 200810be98a7SChris Wilson 2009e568ac38SChris Wilson if (vm) { 2010a4e7ccdaSChris Wilson if (!mutex_lock_interruptible(&dst->mutex)) { 2011e568ac38SChris Wilson __assign_ppgtt(dst, vm); 2012a4e7ccdaSChris Wilson mutex_unlock(&dst->mutex); 2013a4e7ccdaSChris Wilson } else { 2014a4e7ccdaSChris Wilson err = -EINTR; 2015a4e7ccdaSChris Wilson } 2016e568ac38SChris Wilson i915_vm_put(vm); 201710be98a7SChris Wilson } 201810be98a7SChris Wilson 2019a4e7ccdaSChris Wilson return err; 202010be98a7SChris Wilson } 202110be98a7SChris Wilson 202210be98a7SChris Wilson static int create_clone(struct i915_user_extension __user *ext, void *data) 202310be98a7SChris Wilson { 202410be98a7SChris Wilson static int (* const fn[])(struct i915_gem_context *dst, 202510be98a7SChris Wilson struct i915_gem_context *src) = { 202610be98a7SChris Wilson #define MAP(x, y) [ilog2(I915_CONTEXT_CLONE_##x)] = y 202710be98a7SChris Wilson MAP(ENGINES, clone_engines), 202810be98a7SChris Wilson MAP(FLAGS, clone_flags), 202910be98a7SChris Wilson MAP(SCHEDATTR, clone_schedattr), 203010be98a7SChris Wilson MAP(SSEU, clone_sseu), 203110be98a7SChris Wilson MAP(TIMELINE, clone_timeline), 203210be98a7SChris Wilson MAP(VM, clone_vm), 203310be98a7SChris Wilson #undef MAP 203410be98a7SChris Wilson }; 203510be98a7SChris Wilson struct drm_i915_gem_context_create_ext_clone local; 203610be98a7SChris Wilson const struct create_ext *arg = data; 203710be98a7SChris Wilson struct i915_gem_context *dst = arg->ctx; 203810be98a7SChris Wilson struct i915_gem_context *src; 203910be98a7SChris Wilson int err, bit; 204010be98a7SChris Wilson 204110be98a7SChris Wilson if (copy_from_user(&local, ext, sizeof(local))) 204210be98a7SChris Wilson return -EFAULT; 204310be98a7SChris Wilson 204410be98a7SChris Wilson BUILD_BUG_ON(GENMASK(BITS_PER_TYPE(local.flags) - 1, ARRAY_SIZE(fn)) != 204510be98a7SChris Wilson I915_CONTEXT_CLONE_UNKNOWN); 204610be98a7SChris Wilson 204710be98a7SChris Wilson if (local.flags & I915_CONTEXT_CLONE_UNKNOWN) 204810be98a7SChris Wilson return -EINVAL; 204910be98a7SChris Wilson 205010be98a7SChris Wilson if (local.rsvd) 205110be98a7SChris Wilson return -EINVAL; 205210be98a7SChris Wilson 205310be98a7SChris Wilson rcu_read_lock(); 205410be98a7SChris Wilson src = __i915_gem_context_lookup_rcu(arg->fpriv, local.clone_id); 205510be98a7SChris Wilson rcu_read_unlock(); 205610be98a7SChris Wilson if (!src) 205710be98a7SChris Wilson return -ENOENT; 205810be98a7SChris Wilson 205910be98a7SChris Wilson GEM_BUG_ON(src == dst); 206010be98a7SChris Wilson 206110be98a7SChris Wilson for (bit = 0; bit < ARRAY_SIZE(fn); bit++) { 206210be98a7SChris Wilson if (!(local.flags & BIT(bit))) 206310be98a7SChris Wilson continue; 206410be98a7SChris Wilson 206510be98a7SChris Wilson err = fn[bit](dst, src); 206610be98a7SChris Wilson if (err) 206710be98a7SChris Wilson return err; 206810be98a7SChris Wilson } 206910be98a7SChris Wilson 207010be98a7SChris Wilson return 0; 207110be98a7SChris Wilson } 207210be98a7SChris Wilson 207310be98a7SChris Wilson static const i915_user_extension_fn create_extensions[] = { 207410be98a7SChris Wilson [I915_CONTEXT_CREATE_EXT_SETPARAM] = create_setparam, 207510be98a7SChris Wilson [I915_CONTEXT_CREATE_EXT_CLONE] = create_clone, 207610be98a7SChris Wilson }; 207710be98a7SChris Wilson 207810be98a7SChris Wilson static bool client_is_banned(struct drm_i915_file_private *file_priv) 207910be98a7SChris Wilson { 208010be98a7SChris Wilson return atomic_read(&file_priv->ban_score) >= I915_CLIENT_SCORE_BANNED; 208110be98a7SChris Wilson } 208210be98a7SChris Wilson 208310be98a7SChris Wilson int i915_gem_context_create_ioctl(struct drm_device *dev, void *data, 208410be98a7SChris Wilson struct drm_file *file) 208510be98a7SChris Wilson { 208610be98a7SChris Wilson struct drm_i915_private *i915 = to_i915(dev); 208710be98a7SChris Wilson struct drm_i915_gem_context_create_ext *args = data; 208810be98a7SChris Wilson struct create_ext ext_data; 208910be98a7SChris Wilson int ret; 209010be98a7SChris Wilson 209110be98a7SChris Wilson if (!DRIVER_CAPS(i915)->has_logical_contexts) 209210be98a7SChris Wilson return -ENODEV; 209310be98a7SChris Wilson 209410be98a7SChris Wilson if (args->flags & I915_CONTEXT_CREATE_FLAGS_UNKNOWN) 209510be98a7SChris Wilson return -EINVAL; 209610be98a7SChris Wilson 2097cb823ed9SChris Wilson ret = intel_gt_terminally_wedged(&i915->gt); 209810be98a7SChris Wilson if (ret) 209910be98a7SChris Wilson return ret; 210010be98a7SChris Wilson 210110be98a7SChris Wilson ext_data.fpriv = file->driver_priv; 210210be98a7SChris Wilson if (client_is_banned(ext_data.fpriv)) { 210310be98a7SChris Wilson DRM_DEBUG("client %s[%d] banned from creating ctx\n", 210410be98a7SChris Wilson current->comm, 210510be98a7SChris Wilson pid_nr(get_task_pid(current, PIDTYPE_PID))); 210610be98a7SChris Wilson return -EIO; 210710be98a7SChris Wilson } 210810be98a7SChris Wilson 210910be98a7SChris Wilson ext_data.ctx = i915_gem_create_context(i915, args->flags); 211010be98a7SChris Wilson if (IS_ERR(ext_data.ctx)) 211110be98a7SChris Wilson return PTR_ERR(ext_data.ctx); 211210be98a7SChris Wilson 211310be98a7SChris Wilson if (args->flags & I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS) { 211410be98a7SChris Wilson ret = i915_user_extensions(u64_to_user_ptr(args->extensions), 211510be98a7SChris Wilson create_extensions, 211610be98a7SChris Wilson ARRAY_SIZE(create_extensions), 211710be98a7SChris Wilson &ext_data); 211810be98a7SChris Wilson if (ret) 211910be98a7SChris Wilson goto err_ctx; 212010be98a7SChris Wilson } 212110be98a7SChris Wilson 212210be98a7SChris Wilson ret = gem_context_register(ext_data.ctx, ext_data.fpriv); 212310be98a7SChris Wilson if (ret < 0) 212410be98a7SChris Wilson goto err_ctx; 212510be98a7SChris Wilson 212610be98a7SChris Wilson args->ctx_id = ret; 212710be98a7SChris Wilson DRM_DEBUG("HW context %d created\n", args->ctx_id); 212810be98a7SChris Wilson 212910be98a7SChris Wilson return 0; 213010be98a7SChris Wilson 213110be98a7SChris Wilson err_ctx: 213210be98a7SChris Wilson context_close(ext_data.ctx); 213310be98a7SChris Wilson return ret; 213410be98a7SChris Wilson } 213510be98a7SChris Wilson 213610be98a7SChris Wilson int i915_gem_context_destroy_ioctl(struct drm_device *dev, void *data, 213710be98a7SChris Wilson struct drm_file *file) 213810be98a7SChris Wilson { 213910be98a7SChris Wilson struct drm_i915_gem_context_destroy *args = data; 214010be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 214110be98a7SChris Wilson struct i915_gem_context *ctx; 214210be98a7SChris Wilson 214310be98a7SChris Wilson if (args->pad != 0) 214410be98a7SChris Wilson return -EINVAL; 214510be98a7SChris Wilson 214610be98a7SChris Wilson if (!args->ctx_id) 214710be98a7SChris Wilson return -ENOENT; 214810be98a7SChris Wilson 214910be98a7SChris Wilson if (mutex_lock_interruptible(&file_priv->context_idr_lock)) 215010be98a7SChris Wilson return -EINTR; 215110be98a7SChris Wilson 215210be98a7SChris Wilson ctx = idr_remove(&file_priv->context_idr, args->ctx_id); 215310be98a7SChris Wilson mutex_unlock(&file_priv->context_idr_lock); 215410be98a7SChris Wilson if (!ctx) 215510be98a7SChris Wilson return -ENOENT; 215610be98a7SChris Wilson 215710be98a7SChris Wilson context_close(ctx); 215810be98a7SChris Wilson return 0; 215910be98a7SChris Wilson } 216010be98a7SChris Wilson 216110be98a7SChris Wilson static int get_sseu(struct i915_gem_context *ctx, 216210be98a7SChris Wilson struct drm_i915_gem_context_param *args) 216310be98a7SChris Wilson { 216410be98a7SChris Wilson struct drm_i915_gem_context_param_sseu user_sseu; 216510be98a7SChris Wilson struct intel_context *ce; 216610be98a7SChris Wilson unsigned long lookup; 216710be98a7SChris Wilson int err; 216810be98a7SChris Wilson 216910be98a7SChris Wilson if (args->size == 0) 217010be98a7SChris Wilson goto out; 217110be98a7SChris Wilson else if (args->size < sizeof(user_sseu)) 217210be98a7SChris Wilson return -EINVAL; 217310be98a7SChris Wilson 217410be98a7SChris Wilson if (copy_from_user(&user_sseu, u64_to_user_ptr(args->value), 217510be98a7SChris Wilson sizeof(user_sseu))) 217610be98a7SChris Wilson return -EFAULT; 217710be98a7SChris Wilson 217810be98a7SChris Wilson if (user_sseu.rsvd) 217910be98a7SChris Wilson return -EINVAL; 218010be98a7SChris Wilson 218110be98a7SChris Wilson if (user_sseu.flags & ~(I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX)) 218210be98a7SChris Wilson return -EINVAL; 218310be98a7SChris Wilson 218410be98a7SChris Wilson lookup = 0; 218510be98a7SChris Wilson if (user_sseu.flags & I915_CONTEXT_SSEU_FLAG_ENGINE_INDEX) 218610be98a7SChris Wilson lookup |= LOOKUP_USER_INDEX; 218710be98a7SChris Wilson 218810be98a7SChris Wilson ce = lookup_user_engine(ctx, lookup, &user_sseu.engine); 218910be98a7SChris Wilson if (IS_ERR(ce)) 219010be98a7SChris Wilson return PTR_ERR(ce); 219110be98a7SChris Wilson 219210be98a7SChris Wilson err = intel_context_lock_pinned(ce); /* serialises with set_sseu */ 219310be98a7SChris Wilson if (err) { 219410be98a7SChris Wilson intel_context_put(ce); 219510be98a7SChris Wilson return err; 219610be98a7SChris Wilson } 219710be98a7SChris Wilson 219810be98a7SChris Wilson user_sseu.slice_mask = ce->sseu.slice_mask; 219910be98a7SChris Wilson user_sseu.subslice_mask = ce->sseu.subslice_mask; 220010be98a7SChris Wilson user_sseu.min_eus_per_subslice = ce->sseu.min_eus_per_subslice; 220110be98a7SChris Wilson user_sseu.max_eus_per_subslice = ce->sseu.max_eus_per_subslice; 220210be98a7SChris Wilson 220310be98a7SChris Wilson intel_context_unlock_pinned(ce); 220410be98a7SChris Wilson intel_context_put(ce); 220510be98a7SChris Wilson 220610be98a7SChris Wilson if (copy_to_user(u64_to_user_ptr(args->value), &user_sseu, 220710be98a7SChris Wilson sizeof(user_sseu))) 220810be98a7SChris Wilson return -EFAULT; 220910be98a7SChris Wilson 221010be98a7SChris Wilson out: 221110be98a7SChris Wilson args->size = sizeof(user_sseu); 221210be98a7SChris Wilson 221310be98a7SChris Wilson return 0; 221410be98a7SChris Wilson } 221510be98a7SChris Wilson 221610be98a7SChris Wilson int i915_gem_context_getparam_ioctl(struct drm_device *dev, void *data, 221710be98a7SChris Wilson struct drm_file *file) 221810be98a7SChris Wilson { 221910be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 222010be98a7SChris Wilson struct drm_i915_gem_context_param *args = data; 222110be98a7SChris Wilson struct i915_gem_context *ctx; 222210be98a7SChris Wilson int ret = 0; 222310be98a7SChris Wilson 222410be98a7SChris Wilson ctx = i915_gem_context_lookup(file_priv, args->ctx_id); 222510be98a7SChris Wilson if (!ctx) 222610be98a7SChris Wilson return -ENOENT; 222710be98a7SChris Wilson 222810be98a7SChris Wilson switch (args->param) { 222910be98a7SChris Wilson case I915_CONTEXT_PARAM_NO_ZEROMAP: 223010be98a7SChris Wilson args->size = 0; 223110be98a7SChris Wilson args->value = test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags); 223210be98a7SChris Wilson break; 223310be98a7SChris Wilson 223410be98a7SChris Wilson case I915_CONTEXT_PARAM_GTT_SIZE: 223510be98a7SChris Wilson args->size = 0; 2236a4e7ccdaSChris Wilson rcu_read_lock(); 2237a4e7ccdaSChris Wilson if (rcu_access_pointer(ctx->vm)) 2238a4e7ccdaSChris Wilson args->value = rcu_dereference(ctx->vm)->total; 223910be98a7SChris Wilson else 224010be98a7SChris Wilson args->value = to_i915(dev)->ggtt.vm.total; 2241a4e7ccdaSChris Wilson rcu_read_unlock(); 224210be98a7SChris Wilson break; 224310be98a7SChris Wilson 224410be98a7SChris Wilson case I915_CONTEXT_PARAM_NO_ERROR_CAPTURE: 224510be98a7SChris Wilson args->size = 0; 224610be98a7SChris Wilson args->value = i915_gem_context_no_error_capture(ctx); 224710be98a7SChris Wilson break; 224810be98a7SChris Wilson 224910be98a7SChris Wilson case I915_CONTEXT_PARAM_BANNABLE: 225010be98a7SChris Wilson args->size = 0; 225110be98a7SChris Wilson args->value = i915_gem_context_is_bannable(ctx); 225210be98a7SChris Wilson break; 225310be98a7SChris Wilson 225410be98a7SChris Wilson case I915_CONTEXT_PARAM_RECOVERABLE: 225510be98a7SChris Wilson args->size = 0; 225610be98a7SChris Wilson args->value = i915_gem_context_is_recoverable(ctx); 225710be98a7SChris Wilson break; 225810be98a7SChris Wilson 225910be98a7SChris Wilson case I915_CONTEXT_PARAM_PRIORITY: 226010be98a7SChris Wilson args->size = 0; 226110be98a7SChris Wilson args->value = ctx->sched.priority >> I915_USER_PRIORITY_SHIFT; 226210be98a7SChris Wilson break; 226310be98a7SChris Wilson 226410be98a7SChris Wilson case I915_CONTEXT_PARAM_SSEU: 226510be98a7SChris Wilson ret = get_sseu(ctx, args); 226610be98a7SChris Wilson break; 226710be98a7SChris Wilson 226810be98a7SChris Wilson case I915_CONTEXT_PARAM_VM: 226910be98a7SChris Wilson ret = get_ppgtt(file_priv, ctx, args); 227010be98a7SChris Wilson break; 227110be98a7SChris Wilson 227210be98a7SChris Wilson case I915_CONTEXT_PARAM_ENGINES: 227310be98a7SChris Wilson ret = get_engines(ctx, args); 227410be98a7SChris Wilson break; 227510be98a7SChris Wilson 227610be98a7SChris Wilson case I915_CONTEXT_PARAM_BAN_PERIOD: 227710be98a7SChris Wilson default: 227810be98a7SChris Wilson ret = -EINVAL; 227910be98a7SChris Wilson break; 228010be98a7SChris Wilson } 228110be98a7SChris Wilson 228210be98a7SChris Wilson i915_gem_context_put(ctx); 228310be98a7SChris Wilson return ret; 228410be98a7SChris Wilson } 228510be98a7SChris Wilson 228610be98a7SChris Wilson int i915_gem_context_setparam_ioctl(struct drm_device *dev, void *data, 228710be98a7SChris Wilson struct drm_file *file) 228810be98a7SChris Wilson { 228910be98a7SChris Wilson struct drm_i915_file_private *file_priv = file->driver_priv; 229010be98a7SChris Wilson struct drm_i915_gem_context_param *args = data; 229110be98a7SChris Wilson struct i915_gem_context *ctx; 229210be98a7SChris Wilson int ret; 229310be98a7SChris Wilson 229410be98a7SChris Wilson ctx = i915_gem_context_lookup(file_priv, args->ctx_id); 229510be98a7SChris Wilson if (!ctx) 229610be98a7SChris Wilson return -ENOENT; 229710be98a7SChris Wilson 229810be98a7SChris Wilson ret = ctx_setparam(file_priv, ctx, args); 229910be98a7SChris Wilson 230010be98a7SChris Wilson i915_gem_context_put(ctx); 230110be98a7SChris Wilson return ret; 230210be98a7SChris Wilson } 230310be98a7SChris Wilson 230410be98a7SChris Wilson int i915_gem_context_reset_stats_ioctl(struct drm_device *dev, 230510be98a7SChris Wilson void *data, struct drm_file *file) 230610be98a7SChris Wilson { 2307a4e7ccdaSChris Wilson struct drm_i915_private *i915 = to_i915(dev); 230810be98a7SChris Wilson struct drm_i915_reset_stats *args = data; 230910be98a7SChris Wilson struct i915_gem_context *ctx; 231010be98a7SChris Wilson int ret; 231110be98a7SChris Wilson 231210be98a7SChris Wilson if (args->flags || args->pad) 231310be98a7SChris Wilson return -EINVAL; 231410be98a7SChris Wilson 231510be98a7SChris Wilson ret = -ENOENT; 231610be98a7SChris Wilson rcu_read_lock(); 231710be98a7SChris Wilson ctx = __i915_gem_context_lookup_rcu(file->driver_priv, args->ctx_id); 231810be98a7SChris Wilson if (!ctx) 231910be98a7SChris Wilson goto out; 232010be98a7SChris Wilson 232110be98a7SChris Wilson /* 232210be98a7SChris Wilson * We opt for unserialised reads here. This may result in tearing 232310be98a7SChris Wilson * in the extremely unlikely event of a GPU hang on this context 232410be98a7SChris Wilson * as we are querying them. If we need that extra layer of protection, 232510be98a7SChris Wilson * we should wrap the hangstats with a seqlock. 232610be98a7SChris Wilson */ 232710be98a7SChris Wilson 232810be98a7SChris Wilson if (capable(CAP_SYS_ADMIN)) 2329a4e7ccdaSChris Wilson args->reset_count = i915_reset_count(&i915->gpu_error); 233010be98a7SChris Wilson else 233110be98a7SChris Wilson args->reset_count = 0; 233210be98a7SChris Wilson 233310be98a7SChris Wilson args->batch_active = atomic_read(&ctx->guilty_count); 233410be98a7SChris Wilson args->batch_pending = atomic_read(&ctx->active_count); 233510be98a7SChris Wilson 233610be98a7SChris Wilson ret = 0; 233710be98a7SChris Wilson out: 233810be98a7SChris Wilson rcu_read_unlock(); 233910be98a7SChris Wilson return ret; 234010be98a7SChris Wilson } 234110be98a7SChris Wilson 234210be98a7SChris Wilson /* GEM context-engines iterator: for_each_gem_engine() */ 234310be98a7SChris Wilson struct intel_context * 234410be98a7SChris Wilson i915_gem_engines_iter_next(struct i915_gem_engines_iter *it) 234510be98a7SChris Wilson { 234610be98a7SChris Wilson const struct i915_gem_engines *e = it->engines; 234710be98a7SChris Wilson struct intel_context *ctx; 234810be98a7SChris Wilson 234910be98a7SChris Wilson do { 235010be98a7SChris Wilson if (it->idx >= e->num_engines) 235110be98a7SChris Wilson return NULL; 235210be98a7SChris Wilson 235310be98a7SChris Wilson ctx = e->engines[it->idx++]; 235410be98a7SChris Wilson } while (!ctx); 235510be98a7SChris Wilson 235610be98a7SChris Wilson return ctx; 235710be98a7SChris Wilson } 235810be98a7SChris Wilson 235910be98a7SChris Wilson #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 236010be98a7SChris Wilson #include "selftests/mock_context.c" 236110be98a7SChris Wilson #include "selftests/i915_gem_context.c" 236210be98a7SChris Wilson #endif 236310be98a7SChris Wilson 236410be98a7SChris Wilson static void i915_global_gem_context_shrink(void) 236510be98a7SChris Wilson { 236610be98a7SChris Wilson kmem_cache_shrink(global.slab_luts); 236710be98a7SChris Wilson } 236810be98a7SChris Wilson 236910be98a7SChris Wilson static void i915_global_gem_context_exit(void) 237010be98a7SChris Wilson { 237110be98a7SChris Wilson kmem_cache_destroy(global.slab_luts); 237210be98a7SChris Wilson } 237310be98a7SChris Wilson 237410be98a7SChris Wilson static struct i915_global_gem_context global = { { 237510be98a7SChris Wilson .shrink = i915_global_gem_context_shrink, 237610be98a7SChris Wilson .exit = i915_global_gem_context_exit, 237710be98a7SChris Wilson } }; 237810be98a7SChris Wilson 237910be98a7SChris Wilson int __init i915_global_gem_context_init(void) 238010be98a7SChris Wilson { 238110be98a7SChris Wilson global.slab_luts = KMEM_CACHE(i915_lut_handle, 0); 238210be98a7SChris Wilson if (!global.slab_luts) 238310be98a7SChris Wilson return -ENOMEM; 238410be98a7SChris Wilson 238510be98a7SChris Wilson i915_global_register(&global.base); 238610be98a7SChris Wilson return 0; 238710be98a7SChris Wilson } 2388