124f90d66SChris Wilson /* SPDX-License-Identifier: MIT */ 2112ed2d3SChris Wilson /* 3112ed2d3SChris Wilson * Copyright © 2019 Intel Corporation 4112ed2d3SChris Wilson */ 5112ed2d3SChris Wilson 6112ed2d3SChris Wilson #ifndef __INTEL_CONTEXT_H__ 7112ed2d3SChris Wilson #define __INTEL_CONTEXT_H__ 8112ed2d3SChris Wilson 99f3ccd40SChris Wilson #include <linux/bitops.h> 10112ed2d3SChris Wilson #include <linux/lockdep.h> 119f3ccd40SChris Wilson #include <linux/types.h> 12112ed2d3SChris Wilson 1312c255b5SChris Wilson #include "i915_active.h" 141883a0a4STvrtko Ursulin #include "i915_drv.h" 15112ed2d3SChris Wilson #include "intel_context_types.h" 16112ed2d3SChris Wilson #include "intel_engine_types.h" 172871ea85SChris Wilson #include "intel_ring_types.h" 18e5dadff4SChris Wilson #include "intel_timeline_types.h" 19ae8ac10dSMatthew Brost #include "i915_trace.h" 20112ed2d3SChris Wilson 21639f2f24SVenkata Sandeep Dhanalakota #define CE_TRACE(ce, fmt, ...) do { \ 22639f2f24SVenkata Sandeep Dhanalakota const struct intel_context *ce__ = (ce); \ 23639f2f24SVenkata Sandeep Dhanalakota ENGINE_TRACE(ce__->engine, "context:%llx " fmt, \ 24639f2f24SVenkata Sandeep Dhanalakota ce__->timeline->fence_context, \ 25639f2f24SVenkata Sandeep Dhanalakota ##__VA_ARGS__); \ 26639f2f24SVenkata Sandeep Dhanalakota } while (0) 27639f2f24SVenkata Sandeep Dhanalakota 2847b08693SMaarten Lankhorst struct i915_gem_ww_ctx; 2947b08693SMaarten Lankhorst 30112ed2d3SChris Wilson void intel_context_init(struct intel_context *ce, 31112ed2d3SChris Wilson struct intel_engine_cs *engine); 32df8cf31eSChris Wilson void intel_context_fini(struct intel_context *ce); 33112ed2d3SChris Wilson 342dcec7d3SDaniel Vetter void i915_context_module_exit(void); 352dcec7d3SDaniel Vetter int i915_context_module_init(void); 362dcec7d3SDaniel Vetter 37112ed2d3SChris Wilson struct intel_context * 38e6ba7648SChris Wilson intel_context_create(struct intel_engine_cs *engine); 39112ed2d3SChris Wilson 4089f98d63SChris Wilson int intel_context_alloc_state(struct intel_context *ce); 4189f98d63SChris Wilson 425e2a0419SChris Wilson void intel_context_free(struct intel_context *ce); 435e2a0419SChris Wilson 44b4d3acaaSChris Wilson int intel_context_reconfigure_sseu(struct intel_context *ce, 45b4d3acaaSChris Wilson const struct intel_sseu sseu); 46b4d3acaaSChris Wilson 47c2aa552fSMatthew Brost #define PARENT_SCRATCH_SIZE PAGE_SIZE 48c2aa552fSMatthew Brost 493897df4cSMatthew Brost static inline bool intel_context_is_child(struct intel_context *ce) 503897df4cSMatthew Brost { 513897df4cSMatthew Brost return !!ce->parallel.parent; 523897df4cSMatthew Brost } 533897df4cSMatthew Brost 543897df4cSMatthew Brost static inline bool intel_context_is_parent(struct intel_context *ce) 553897df4cSMatthew Brost { 563897df4cSMatthew Brost return !!ce->parallel.number_children; 573897df4cSMatthew Brost } 583897df4cSMatthew Brost 593897df4cSMatthew Brost static inline bool intel_context_is_pinned(struct intel_context *ce); 603897df4cSMatthew Brost 613897df4cSMatthew Brost static inline struct intel_context * 623897df4cSMatthew Brost intel_context_to_parent(struct intel_context *ce) 633897df4cSMatthew Brost { 643897df4cSMatthew Brost if (intel_context_is_child(ce)) { 653897df4cSMatthew Brost /* 663897df4cSMatthew Brost * The parent holds ref count to the child so it is always safe 673897df4cSMatthew Brost * for the parent to access the child, but the child has a 683897df4cSMatthew Brost * pointer to the parent without a ref. To ensure this is safe 693897df4cSMatthew Brost * the child should only access the parent pointer while the 703897df4cSMatthew Brost * parent is pinned. 713897df4cSMatthew Brost */ 723897df4cSMatthew Brost GEM_BUG_ON(!intel_context_is_pinned(ce->parallel.parent)); 733897df4cSMatthew Brost 743897df4cSMatthew Brost return ce->parallel.parent; 753897df4cSMatthew Brost } else { 763897df4cSMatthew Brost return ce; 773897df4cSMatthew Brost } 783897df4cSMatthew Brost } 793897df4cSMatthew Brost 80*bc955204SMatthew Brost static inline bool intel_context_is_parallel(struct intel_context *ce) 81*bc955204SMatthew Brost { 82*bc955204SMatthew Brost return intel_context_is_child(ce) || intel_context_is_parent(ce); 83*bc955204SMatthew Brost } 84*bc955204SMatthew Brost 853897df4cSMatthew Brost void intel_context_bind_parent_child(struct intel_context *parent, 863897df4cSMatthew Brost struct intel_context *child); 873897df4cSMatthew Brost 883897df4cSMatthew Brost #define for_each_child(parent, ce)\ 893897df4cSMatthew Brost list_for_each_entry(ce, &(parent)->parallel.child_list,\ 903897df4cSMatthew Brost parallel.child_link) 913897df4cSMatthew Brost #define for_each_child_safe(parent, ce, cn)\ 923897df4cSMatthew Brost list_for_each_entry_safe(ce, cn, &(parent)->parallel.child_list,\ 933897df4cSMatthew Brost parallel.child_link) 943897df4cSMatthew Brost 95112ed2d3SChris Wilson /** 966b736de5SChris Wilson * intel_context_lock_pinned - Stablises the 'pinned' status of the HW context 976b736de5SChris Wilson * @ce - the context 98112ed2d3SChris Wilson * 99112ed2d3SChris Wilson * Acquire a lock on the pinned status of the HW context, such that the context 100112ed2d3SChris Wilson * can neither be bound to the GPU or unbound whilst the lock is held, i.e. 101112ed2d3SChris Wilson * intel_context_is_pinned() remains stable. 102112ed2d3SChris Wilson */ 1036b736de5SChris Wilson static inline int intel_context_lock_pinned(struct intel_context *ce) 1046b736de5SChris Wilson __acquires(ce->pin_mutex) 1056b736de5SChris Wilson { 1066b736de5SChris Wilson return mutex_lock_interruptible(&ce->pin_mutex); 1076b736de5SChris Wilson } 108112ed2d3SChris Wilson 1096b736de5SChris Wilson /** 1106b736de5SChris Wilson * intel_context_is_pinned - Reports the 'pinned' status 1116b736de5SChris Wilson * @ce - the context 1126b736de5SChris Wilson * 1136b736de5SChris Wilson * While in use by the GPU, the context, along with its ring and page 1146b736de5SChris Wilson * tables is pinned into memory and the GTT. 1156b736de5SChris Wilson * 1166b736de5SChris Wilson * Returns: true if the context is currently pinned for use by the GPU. 1176b736de5SChris Wilson */ 118112ed2d3SChris Wilson static inline bool 119112ed2d3SChris Wilson intel_context_is_pinned(struct intel_context *ce) 120112ed2d3SChris Wilson { 121112ed2d3SChris Wilson return atomic_read(&ce->pin_count); 122112ed2d3SChris Wilson } 123112ed2d3SChris Wilson 12462eaf0aeSMatthew Brost static inline void intel_context_cancel_request(struct intel_context *ce, 12562eaf0aeSMatthew Brost struct i915_request *rq) 12662eaf0aeSMatthew Brost { 12762eaf0aeSMatthew Brost GEM_BUG_ON(!ce->ops->cancel_request); 12862eaf0aeSMatthew Brost return ce->ops->cancel_request(ce, rq); 12962eaf0aeSMatthew Brost } 13062eaf0aeSMatthew Brost 1316b736de5SChris Wilson /** 1326b736de5SChris Wilson * intel_context_unlock_pinned - Releases the earlier locking of 'pinned' status 1336b736de5SChris Wilson * @ce - the context 1346b736de5SChris Wilson * 1356b736de5SChris Wilson * Releases the lock earlier acquired by intel_context_unlock_pinned(). 1366b736de5SChris Wilson */ 1376b736de5SChris Wilson static inline void intel_context_unlock_pinned(struct intel_context *ce) 1386b736de5SChris Wilson __releases(ce->pin_mutex) 1396b736de5SChris Wilson { 1406b736de5SChris Wilson mutex_unlock(&ce->pin_mutex); 1416b736de5SChris Wilson } 142112ed2d3SChris Wilson 143fa9f6681SChris Wilson int __intel_context_do_pin(struct intel_context *ce); 14447b08693SMaarten Lankhorst int __intel_context_do_pin_ww(struct intel_context *ce, 14547b08693SMaarten Lankhorst struct i915_gem_ww_ctx *ww); 146fa9f6681SChris Wilson 147feed5c7bSChris Wilson static inline bool intel_context_pin_if_active(struct intel_context *ce) 148feed5c7bSChris Wilson { 149feed5c7bSChris Wilson return atomic_inc_not_zero(&ce->pin_count); 150feed5c7bSChris Wilson } 151feed5c7bSChris Wilson 152fa9f6681SChris Wilson static inline int intel_context_pin(struct intel_context *ce) 153fa9f6681SChris Wilson { 154feed5c7bSChris Wilson if (likely(intel_context_pin_if_active(ce))) 155fa9f6681SChris Wilson return 0; 156fa9f6681SChris Wilson 157fa9f6681SChris Wilson return __intel_context_do_pin(ce); 158fa9f6681SChris Wilson } 159112ed2d3SChris Wilson 16047b08693SMaarten Lankhorst static inline int intel_context_pin_ww(struct intel_context *ce, 16147b08693SMaarten Lankhorst struct i915_gem_ww_ctx *ww) 16247b08693SMaarten Lankhorst { 16347b08693SMaarten Lankhorst if (likely(intel_context_pin_if_active(ce))) 16447b08693SMaarten Lankhorst return 0; 16547b08693SMaarten Lankhorst 16647b08693SMaarten Lankhorst return __intel_context_do_pin_ww(ce, ww); 16747b08693SMaarten Lankhorst } 16847b08693SMaarten Lankhorst 169112ed2d3SChris Wilson static inline void __intel_context_pin(struct intel_context *ce) 170112ed2d3SChris Wilson { 171112ed2d3SChris Wilson GEM_BUG_ON(!intel_context_is_pinned(ce)); 172112ed2d3SChris Wilson atomic_inc(&ce->pin_count); 173112ed2d3SChris Wilson } 174112ed2d3SChris Wilson 175e0717063SMatthew Brost void __intel_context_do_unpin(struct intel_context *ce, int sub); 176e0717063SMatthew Brost 177e0717063SMatthew Brost static inline void intel_context_sched_disable_unpin(struct intel_context *ce) 178e0717063SMatthew Brost { 179e0717063SMatthew Brost __intel_context_do_unpin(ce, 2); 180e0717063SMatthew Brost } 181e0717063SMatthew Brost 182e0717063SMatthew Brost static inline void intel_context_unpin(struct intel_context *ce) 183e0717063SMatthew Brost { 184e0717063SMatthew Brost if (!ce->ops->sched_disable) { 185e0717063SMatthew Brost __intel_context_do_unpin(ce, 1); 186e0717063SMatthew Brost } else { 187e0717063SMatthew Brost /* 188e0717063SMatthew Brost * Move ownership of this pin to the scheduling disable which is 189e0717063SMatthew Brost * an async operation. When that operation completes the above 190e0717063SMatthew Brost * intel_context_sched_disable_unpin is called potentially 191e0717063SMatthew Brost * unpinning the context. 192e0717063SMatthew Brost */ 193e0717063SMatthew Brost while (!atomic_add_unless(&ce->pin_count, -1, 1)) { 194e0717063SMatthew Brost if (atomic_cmpxchg(&ce->pin_count, 1, 2) == 1) { 195e0717063SMatthew Brost ce->ops->sched_disable(ce); 196e0717063SMatthew Brost break; 197e0717063SMatthew Brost } 198e0717063SMatthew Brost } 199e0717063SMatthew Brost } 200e0717063SMatthew Brost } 201112ed2d3SChris Wilson 2026eee33e8SChris Wilson void intel_context_enter_engine(struct intel_context *ce); 2036eee33e8SChris Wilson void intel_context_exit_engine(struct intel_context *ce); 2046eee33e8SChris Wilson 2056eee33e8SChris Wilson static inline void intel_context_enter(struct intel_context *ce) 2066eee33e8SChris Wilson { 2076c69a454SChris Wilson lockdep_assert_held(&ce->timeline->mutex); 2086eee33e8SChris Wilson if (!ce->active_count++) 2096eee33e8SChris Wilson ce->ops->enter(ce); 2106eee33e8SChris Wilson } 2116eee33e8SChris Wilson 2126eee33e8SChris Wilson static inline void intel_context_mark_active(struct intel_context *ce) 2136eee33e8SChris Wilson { 2146c69a454SChris Wilson lockdep_assert_held(&ce->timeline->mutex); 2156eee33e8SChris Wilson ++ce->active_count; 2166eee33e8SChris Wilson } 2176eee33e8SChris Wilson 2186eee33e8SChris Wilson static inline void intel_context_exit(struct intel_context *ce) 2196eee33e8SChris Wilson { 2206c69a454SChris Wilson lockdep_assert_held(&ce->timeline->mutex); 2216eee33e8SChris Wilson GEM_BUG_ON(!ce->active_count); 2226eee33e8SChris Wilson if (!--ce->active_count) 2236eee33e8SChris Wilson ce->ops->exit(ce); 2246eee33e8SChris Wilson } 2256eee33e8SChris Wilson 226112ed2d3SChris Wilson static inline struct intel_context *intel_context_get(struct intel_context *ce) 227112ed2d3SChris Wilson { 228112ed2d3SChris Wilson kref_get(&ce->ref); 229112ed2d3SChris Wilson return ce; 230112ed2d3SChris Wilson } 231112ed2d3SChris Wilson 232112ed2d3SChris Wilson static inline void intel_context_put(struct intel_context *ce) 233112ed2d3SChris Wilson { 234112ed2d3SChris Wilson kref_put(&ce->ref, ce->ops->destroy); 235112ed2d3SChris Wilson } 236112ed2d3SChris Wilson 237e5dadff4SChris Wilson static inline struct intel_timeline *__must_check 238f4d57d83SChris Wilson intel_context_timeline_lock(struct intel_context *ce) 23975d0a7f3SChris Wilson __acquires(&ce->timeline->mutex) 2402ccdf6a1SChris Wilson { 241e5dadff4SChris Wilson struct intel_timeline *tl = ce->timeline; 242e5dadff4SChris Wilson int err; 243e5dadff4SChris Wilson 244e5dadff4SChris Wilson err = mutex_lock_interruptible(&tl->mutex); 245e5dadff4SChris Wilson if (err) 246e5dadff4SChris Wilson return ERR_PTR(err); 247e5dadff4SChris Wilson 248e5dadff4SChris Wilson return tl; 2492ccdf6a1SChris Wilson } 2502ccdf6a1SChris Wilson 251e5dadff4SChris Wilson static inline void intel_context_timeline_unlock(struct intel_timeline *tl) 252e5dadff4SChris Wilson __releases(&tl->mutex) 2532ccdf6a1SChris Wilson { 254e5dadff4SChris Wilson mutex_unlock(&tl->mutex); 2552ccdf6a1SChris Wilson } 2562ccdf6a1SChris Wilson 257a9877da2SChris Wilson int intel_context_prepare_remote_request(struct intel_context *ce, 258a9877da2SChris Wilson struct i915_request *rq); 259a9877da2SChris Wilson 2605e2a0419SChris Wilson struct i915_request *intel_context_create_request(struct intel_context *ce); 2615e2a0419SChris Wilson 262573ba126SMatthew Brost struct i915_request * 263573ba126SMatthew Brost intel_context_find_active_request(struct intel_context *ce); 264573ba126SMatthew Brost 265e6ba7648SChris Wilson static inline bool intel_context_is_barrier(const struct intel_context *ce) 266e6ba7648SChris Wilson { 267e6ba7648SChris Wilson return test_bit(CONTEXT_BARRIER_BIT, &ce->flags); 268e6ba7648SChris Wilson } 269e6ba7648SChris Wilson 2702e46a2a0SChris Wilson static inline bool intel_context_is_closed(const struct intel_context *ce) 2712e46a2a0SChris Wilson { 2722e46a2a0SChris Wilson return test_bit(CONTEXT_CLOSED_BIT, &ce->flags); 2732e46a2a0SChris Wilson } 2742e46a2a0SChris Wilson 275cc1557caSChris Wilson static inline bool intel_context_has_inflight(const struct intel_context *ce) 276cc1557caSChris Wilson { 277cc1557caSChris Wilson return test_bit(COPS_HAS_INFLIGHT_BIT, &ce->ops->flags); 278cc1557caSChris Wilson } 279cc1557caSChris Wilson 2800f100b70SChris Wilson static inline bool intel_context_use_semaphores(const struct intel_context *ce) 2810f100b70SChris Wilson { 2820f100b70SChris Wilson return test_bit(CONTEXT_USE_SEMAPHORES, &ce->flags); 2830f100b70SChris Wilson } 2840f100b70SChris Wilson 2850f100b70SChris Wilson static inline void intel_context_set_use_semaphores(struct intel_context *ce) 2860f100b70SChris Wilson { 2870f100b70SChris Wilson set_bit(CONTEXT_USE_SEMAPHORES, &ce->flags); 2880f100b70SChris Wilson } 2890f100b70SChris Wilson 2900f100b70SChris Wilson static inline void intel_context_clear_use_semaphores(struct intel_context *ce) 2910f100b70SChris Wilson { 2920f100b70SChris Wilson clear_bit(CONTEXT_USE_SEMAPHORES, &ce->flags); 2930f100b70SChris Wilson } 2940f100b70SChris Wilson 2959f3ccd40SChris Wilson static inline bool intel_context_is_banned(const struct intel_context *ce) 2969f3ccd40SChris Wilson { 2979f3ccd40SChris Wilson return test_bit(CONTEXT_BANNED, &ce->flags); 2989f3ccd40SChris Wilson } 2999f3ccd40SChris Wilson 3009f3ccd40SChris Wilson static inline bool intel_context_set_banned(struct intel_context *ce) 3019f3ccd40SChris Wilson { 3029f3ccd40SChris Wilson return test_and_set_bit(CONTEXT_BANNED, &ce->flags); 3039f3ccd40SChris Wilson } 3049f3ccd40SChris Wilson 305ae8ac10dSMatthew Brost static inline bool intel_context_ban(struct intel_context *ce, 306ae8ac10dSMatthew Brost struct i915_request *rq) 307ae8ac10dSMatthew Brost { 308ae8ac10dSMatthew Brost bool ret = intel_context_set_banned(ce); 309ae8ac10dSMatthew Brost 310ae8ac10dSMatthew Brost trace_intel_context_ban(ce); 311ae8ac10dSMatthew Brost if (ce->ops->ban) 312ae8ac10dSMatthew Brost ce->ops->ban(ce, rq); 313ae8ac10dSMatthew Brost 314ae8ac10dSMatthew Brost return ret; 315ae8ac10dSMatthew Brost } 316ae8ac10dSMatthew Brost 3179f3ccd40SChris Wilson static inline bool 3189f3ccd40SChris Wilson intel_context_force_single_submission(const struct intel_context *ce) 3199f3ccd40SChris Wilson { 3209f3ccd40SChris Wilson return test_bit(CONTEXT_FORCE_SINGLE_SUBMISSION, &ce->flags); 3219f3ccd40SChris Wilson } 3229f3ccd40SChris Wilson 3239f3ccd40SChris Wilson static inline void 3249f3ccd40SChris Wilson intel_context_set_single_submission(struct intel_context *ce) 3259f3ccd40SChris Wilson { 3269f3ccd40SChris Wilson __set_bit(CONTEXT_FORCE_SINGLE_SUBMISSION, &ce->flags); 3279f3ccd40SChris Wilson } 3289f3ccd40SChris Wilson 3299f3ccd40SChris Wilson static inline bool 3309f3ccd40SChris Wilson intel_context_nopreempt(const struct intel_context *ce) 3319f3ccd40SChris Wilson { 3329f3ccd40SChris Wilson return test_bit(CONTEXT_NOPREEMPT, &ce->flags); 3339f3ccd40SChris Wilson } 3349f3ccd40SChris Wilson 3359f3ccd40SChris Wilson static inline void 3369f3ccd40SChris Wilson intel_context_set_nopreempt(struct intel_context *ce) 3379f3ccd40SChris Wilson { 3389f3ccd40SChris Wilson set_bit(CONTEXT_NOPREEMPT, &ce->flags); 3399f3ccd40SChris Wilson } 3409f3ccd40SChris Wilson 3419f3ccd40SChris Wilson static inline void 3429f3ccd40SChris Wilson intel_context_clear_nopreempt(struct intel_context *ce) 3439f3ccd40SChris Wilson { 3449f3ccd40SChris Wilson clear_bit(CONTEXT_NOPREEMPT, &ce->flags); 3459f3ccd40SChris Wilson } 3469f3ccd40SChris Wilson 3471883a0a4STvrtko Ursulin static inline u64 intel_context_get_total_runtime_ns(struct intel_context *ce) 3481883a0a4STvrtko Ursulin { 349f170523aSChris Wilson const u32 period = ce->engine->gt->clock_period_ns; 3501883a0a4STvrtko Ursulin 3511883a0a4STvrtko Ursulin return READ_ONCE(ce->runtime.total) * period; 3521883a0a4STvrtko Ursulin } 3531883a0a4STvrtko Ursulin 3541883a0a4STvrtko Ursulin static inline u64 intel_context_get_avg_runtime_ns(struct intel_context *ce) 3551883a0a4STvrtko Ursulin { 356f170523aSChris Wilson const u32 period = ce->engine->gt->clock_period_ns; 3571883a0a4STvrtko Ursulin 3581883a0a4STvrtko Ursulin return mul_u32_u32(ewma_runtime_read(&ce->runtime.avg), period); 3591883a0a4STvrtko Ursulin } 3601883a0a4STvrtko Ursulin 361112ed2d3SChris Wilson #endif /* __INTEL_CONTEXT_H__ */ 362