124f90d66SChris Wilson /* SPDX-License-Identifier: MIT */ 2112ed2d3SChris Wilson /* 3112ed2d3SChris Wilson * Copyright © 2019 Intel Corporation 4112ed2d3SChris Wilson */ 5112ed2d3SChris Wilson 6112ed2d3SChris Wilson #ifndef __INTEL_CONTEXT_H__ 7112ed2d3SChris Wilson #define __INTEL_CONTEXT_H__ 8112ed2d3SChris Wilson 99f3ccd40SChris Wilson #include <linux/bitops.h> 10112ed2d3SChris Wilson #include <linux/lockdep.h> 119f3ccd40SChris Wilson #include <linux/types.h> 12112ed2d3SChris Wilson 1312c255b5SChris Wilson #include "i915_active.h" 141883a0a4STvrtko Ursulin #include "i915_drv.h" 15112ed2d3SChris Wilson #include "intel_context_types.h" 16112ed2d3SChris Wilson #include "intel_engine_types.h" 172871ea85SChris Wilson #include "intel_ring_types.h" 18e5dadff4SChris Wilson #include "intel_timeline_types.h" 19ae8ac10dSMatthew Brost #include "i915_trace.h" 20112ed2d3SChris Wilson 21639f2f24SVenkata Sandeep Dhanalakota #define CE_TRACE(ce, fmt, ...) do { \ 22639f2f24SVenkata Sandeep Dhanalakota const struct intel_context *ce__ = (ce); \ 23639f2f24SVenkata Sandeep Dhanalakota ENGINE_TRACE(ce__->engine, "context:%llx " fmt, \ 24639f2f24SVenkata Sandeep Dhanalakota ce__->timeline->fence_context, \ 25639f2f24SVenkata Sandeep Dhanalakota ##__VA_ARGS__); \ 26639f2f24SVenkata Sandeep Dhanalakota } while (0) 27639f2f24SVenkata Sandeep Dhanalakota 2847b08693SMaarten Lankhorst struct i915_gem_ww_ctx; 2947b08693SMaarten Lankhorst 30112ed2d3SChris Wilson void intel_context_init(struct intel_context *ce, 31112ed2d3SChris Wilson struct intel_engine_cs *engine); 32df8cf31eSChris Wilson void intel_context_fini(struct intel_context *ce); 33112ed2d3SChris Wilson 342dcec7d3SDaniel Vetter void i915_context_module_exit(void); 352dcec7d3SDaniel Vetter int i915_context_module_init(void); 362dcec7d3SDaniel Vetter 37112ed2d3SChris Wilson struct intel_context * 38e6ba7648SChris Wilson intel_context_create(struct intel_engine_cs *engine); 39112ed2d3SChris Wilson 4089f98d63SChris Wilson int intel_context_alloc_state(struct intel_context *ce); 4189f98d63SChris Wilson 425e2a0419SChris Wilson void intel_context_free(struct intel_context *ce); 435e2a0419SChris Wilson 44b4d3acaaSChris Wilson int intel_context_reconfigure_sseu(struct intel_context *ce, 45b4d3acaaSChris Wilson const struct intel_sseu sseu); 46b4d3acaaSChris Wilson 47c2aa552fSMatthew Brost #define PARENT_SCRATCH_SIZE PAGE_SIZE 48c2aa552fSMatthew Brost 493897df4cSMatthew Brost static inline bool intel_context_is_child(struct intel_context *ce) 503897df4cSMatthew Brost { 513897df4cSMatthew Brost return !!ce->parallel.parent; 523897df4cSMatthew Brost } 533897df4cSMatthew Brost 543897df4cSMatthew Brost static inline bool intel_context_is_parent(struct intel_context *ce) 553897df4cSMatthew Brost { 563897df4cSMatthew Brost return !!ce->parallel.number_children; 573897df4cSMatthew Brost } 583897df4cSMatthew Brost 593897df4cSMatthew Brost static inline bool intel_context_is_pinned(struct intel_context *ce); 603897df4cSMatthew Brost 613897df4cSMatthew Brost static inline struct intel_context * 623897df4cSMatthew Brost intel_context_to_parent(struct intel_context *ce) 633897df4cSMatthew Brost { 643897df4cSMatthew Brost if (intel_context_is_child(ce)) { 653897df4cSMatthew Brost /* 663897df4cSMatthew Brost * The parent holds ref count to the child so it is always safe 673897df4cSMatthew Brost * for the parent to access the child, but the child has a 683897df4cSMatthew Brost * pointer to the parent without a ref. To ensure this is safe 693897df4cSMatthew Brost * the child should only access the parent pointer while the 703897df4cSMatthew Brost * parent is pinned. 713897df4cSMatthew Brost */ 723897df4cSMatthew Brost GEM_BUG_ON(!intel_context_is_pinned(ce->parallel.parent)); 733897df4cSMatthew Brost 743897df4cSMatthew Brost return ce->parallel.parent; 753897df4cSMatthew Brost } else { 763897df4cSMatthew Brost return ce; 773897df4cSMatthew Brost } 783897df4cSMatthew Brost } 793897df4cSMatthew Brost 80bc955204SMatthew Brost static inline bool intel_context_is_parallel(struct intel_context *ce) 81bc955204SMatthew Brost { 82bc955204SMatthew Brost return intel_context_is_child(ce) || intel_context_is_parent(ce); 83bc955204SMatthew Brost } 84bc955204SMatthew Brost 853897df4cSMatthew Brost void intel_context_bind_parent_child(struct intel_context *parent, 863897df4cSMatthew Brost struct intel_context *child); 873897df4cSMatthew Brost 883897df4cSMatthew Brost #define for_each_child(parent, ce)\ 893897df4cSMatthew Brost list_for_each_entry(ce, &(parent)->parallel.child_list,\ 903897df4cSMatthew Brost parallel.child_link) 913897df4cSMatthew Brost #define for_each_child_safe(parent, ce, cn)\ 923897df4cSMatthew Brost list_for_each_entry_safe(ce, cn, &(parent)->parallel.child_list,\ 933897df4cSMatthew Brost parallel.child_link) 943897df4cSMatthew Brost 95112ed2d3SChris Wilson /** 966b736de5SChris Wilson * intel_context_lock_pinned - Stablises the 'pinned' status of the HW context 976b736de5SChris Wilson * @ce - the context 98112ed2d3SChris Wilson * 99112ed2d3SChris Wilson * Acquire a lock on the pinned status of the HW context, such that the context 100112ed2d3SChris Wilson * can neither be bound to the GPU or unbound whilst the lock is held, i.e. 101112ed2d3SChris Wilson * intel_context_is_pinned() remains stable. 102112ed2d3SChris Wilson */ 1036b736de5SChris Wilson static inline int intel_context_lock_pinned(struct intel_context *ce) 1046b736de5SChris Wilson __acquires(ce->pin_mutex) 1056b736de5SChris Wilson { 1066b736de5SChris Wilson return mutex_lock_interruptible(&ce->pin_mutex); 1076b736de5SChris Wilson } 108112ed2d3SChris Wilson 1096b736de5SChris Wilson /** 1106b736de5SChris Wilson * intel_context_is_pinned - Reports the 'pinned' status 1116b736de5SChris Wilson * @ce - the context 1126b736de5SChris Wilson * 1136b736de5SChris Wilson * While in use by the GPU, the context, along with its ring and page 1146b736de5SChris Wilson * tables is pinned into memory and the GTT. 1156b736de5SChris Wilson * 1166b736de5SChris Wilson * Returns: true if the context is currently pinned for use by the GPU. 1176b736de5SChris Wilson */ 118112ed2d3SChris Wilson static inline bool 119112ed2d3SChris Wilson intel_context_is_pinned(struct intel_context *ce) 120112ed2d3SChris Wilson { 121112ed2d3SChris Wilson return atomic_read(&ce->pin_count); 122112ed2d3SChris Wilson } 123112ed2d3SChris Wilson 12462eaf0aeSMatthew Brost static inline void intel_context_cancel_request(struct intel_context *ce, 12562eaf0aeSMatthew Brost struct i915_request *rq) 12662eaf0aeSMatthew Brost { 12762eaf0aeSMatthew Brost GEM_BUG_ON(!ce->ops->cancel_request); 12862eaf0aeSMatthew Brost return ce->ops->cancel_request(ce, rq); 12962eaf0aeSMatthew Brost } 13062eaf0aeSMatthew Brost 1316b736de5SChris Wilson /** 1326b736de5SChris Wilson * intel_context_unlock_pinned - Releases the earlier locking of 'pinned' status 1336b736de5SChris Wilson * @ce - the context 1346b736de5SChris Wilson * 1356b736de5SChris Wilson * Releases the lock earlier acquired by intel_context_unlock_pinned(). 1366b736de5SChris Wilson */ 1376b736de5SChris Wilson static inline void intel_context_unlock_pinned(struct intel_context *ce) 1386b736de5SChris Wilson __releases(ce->pin_mutex) 1396b736de5SChris Wilson { 1406b736de5SChris Wilson mutex_unlock(&ce->pin_mutex); 1416b736de5SChris Wilson } 142112ed2d3SChris Wilson 143fa9f6681SChris Wilson int __intel_context_do_pin(struct intel_context *ce); 14447b08693SMaarten Lankhorst int __intel_context_do_pin_ww(struct intel_context *ce, 14547b08693SMaarten Lankhorst struct i915_gem_ww_ctx *ww); 146fa9f6681SChris Wilson 147feed5c7bSChris Wilson static inline bool intel_context_pin_if_active(struct intel_context *ce) 148feed5c7bSChris Wilson { 149feed5c7bSChris Wilson return atomic_inc_not_zero(&ce->pin_count); 150feed5c7bSChris Wilson } 151feed5c7bSChris Wilson 152fa9f6681SChris Wilson static inline int intel_context_pin(struct intel_context *ce) 153fa9f6681SChris Wilson { 154feed5c7bSChris Wilson if (likely(intel_context_pin_if_active(ce))) 155fa9f6681SChris Wilson return 0; 156fa9f6681SChris Wilson 157fa9f6681SChris Wilson return __intel_context_do_pin(ce); 158fa9f6681SChris Wilson } 159112ed2d3SChris Wilson 16047b08693SMaarten Lankhorst static inline int intel_context_pin_ww(struct intel_context *ce, 16147b08693SMaarten Lankhorst struct i915_gem_ww_ctx *ww) 16247b08693SMaarten Lankhorst { 16347b08693SMaarten Lankhorst if (likely(intel_context_pin_if_active(ce))) 16447b08693SMaarten Lankhorst return 0; 16547b08693SMaarten Lankhorst 16647b08693SMaarten Lankhorst return __intel_context_do_pin_ww(ce, ww); 16747b08693SMaarten Lankhorst } 16847b08693SMaarten Lankhorst 169112ed2d3SChris Wilson static inline void __intel_context_pin(struct intel_context *ce) 170112ed2d3SChris Wilson { 171112ed2d3SChris Wilson GEM_BUG_ON(!intel_context_is_pinned(ce)); 172112ed2d3SChris Wilson atomic_inc(&ce->pin_count); 173112ed2d3SChris Wilson } 174112ed2d3SChris Wilson 175e0717063SMatthew Brost void __intel_context_do_unpin(struct intel_context *ce, int sub); 176e0717063SMatthew Brost 177e0717063SMatthew Brost static inline void intel_context_sched_disable_unpin(struct intel_context *ce) 178e0717063SMatthew Brost { 179e0717063SMatthew Brost __intel_context_do_unpin(ce, 2); 180e0717063SMatthew Brost } 181e0717063SMatthew Brost 182e0717063SMatthew Brost static inline void intel_context_unpin(struct intel_context *ce) 183e0717063SMatthew Brost { 184e0717063SMatthew Brost if (!ce->ops->sched_disable) { 185e0717063SMatthew Brost __intel_context_do_unpin(ce, 1); 186e0717063SMatthew Brost } else { 187e0717063SMatthew Brost /* 188e0717063SMatthew Brost * Move ownership of this pin to the scheduling disable which is 189e0717063SMatthew Brost * an async operation. When that operation completes the above 190e0717063SMatthew Brost * intel_context_sched_disable_unpin is called potentially 191e0717063SMatthew Brost * unpinning the context. 192e0717063SMatthew Brost */ 193e0717063SMatthew Brost while (!atomic_add_unless(&ce->pin_count, -1, 1)) { 194e0717063SMatthew Brost if (atomic_cmpxchg(&ce->pin_count, 1, 2) == 1) { 195e0717063SMatthew Brost ce->ops->sched_disable(ce); 196e0717063SMatthew Brost break; 197e0717063SMatthew Brost } 198e0717063SMatthew Brost } 199e0717063SMatthew Brost } 200e0717063SMatthew Brost } 201112ed2d3SChris Wilson 2026eee33e8SChris Wilson void intel_context_enter_engine(struct intel_context *ce); 2036eee33e8SChris Wilson void intel_context_exit_engine(struct intel_context *ce); 2046eee33e8SChris Wilson 2056eee33e8SChris Wilson static inline void intel_context_enter(struct intel_context *ce) 2066eee33e8SChris Wilson { 2076c69a454SChris Wilson lockdep_assert_held(&ce->timeline->mutex); 2086eee33e8SChris Wilson if (!ce->active_count++) 2096eee33e8SChris Wilson ce->ops->enter(ce); 2106eee33e8SChris Wilson } 2116eee33e8SChris Wilson 2126eee33e8SChris Wilson static inline void intel_context_mark_active(struct intel_context *ce) 2136eee33e8SChris Wilson { 214*bce45c26SSebastian Andrzej Siewior lockdep_assert(lockdep_is_held(&ce->timeline->mutex) || 215*bce45c26SSebastian Andrzej Siewior test_bit(CONTEXT_IS_PARKING, &ce->flags)); 2166eee33e8SChris Wilson ++ce->active_count; 2176eee33e8SChris Wilson } 2186eee33e8SChris Wilson 2196eee33e8SChris Wilson static inline void intel_context_exit(struct intel_context *ce) 2206eee33e8SChris Wilson { 2216c69a454SChris Wilson lockdep_assert_held(&ce->timeline->mutex); 2226eee33e8SChris Wilson GEM_BUG_ON(!ce->active_count); 2236eee33e8SChris Wilson if (!--ce->active_count) 2246eee33e8SChris Wilson ce->ops->exit(ce); 2256eee33e8SChris Wilson } 2266eee33e8SChris Wilson 227112ed2d3SChris Wilson static inline struct intel_context *intel_context_get(struct intel_context *ce) 228112ed2d3SChris Wilson { 229112ed2d3SChris Wilson kref_get(&ce->ref); 230112ed2d3SChris Wilson return ce; 231112ed2d3SChris Wilson } 232112ed2d3SChris Wilson 233112ed2d3SChris Wilson static inline void intel_context_put(struct intel_context *ce) 234112ed2d3SChris Wilson { 235112ed2d3SChris Wilson kref_put(&ce->ref, ce->ops->destroy); 236112ed2d3SChris Wilson } 237112ed2d3SChris Wilson 238e5dadff4SChris Wilson static inline struct intel_timeline *__must_check 239f4d57d83SChris Wilson intel_context_timeline_lock(struct intel_context *ce) 24075d0a7f3SChris Wilson __acquires(&ce->timeline->mutex) 2412ccdf6a1SChris Wilson { 242e5dadff4SChris Wilson struct intel_timeline *tl = ce->timeline; 243e5dadff4SChris Wilson int err; 244e5dadff4SChris Wilson 245544460c3SMatthew Brost if (intel_context_is_parent(ce)) 246544460c3SMatthew Brost err = mutex_lock_interruptible_nested(&tl->mutex, 0); 247544460c3SMatthew Brost else if (intel_context_is_child(ce)) 248544460c3SMatthew Brost err = mutex_lock_interruptible_nested(&tl->mutex, 249544460c3SMatthew Brost ce->parallel.child_index + 1); 250544460c3SMatthew Brost else 251e5dadff4SChris Wilson err = mutex_lock_interruptible(&tl->mutex); 252e5dadff4SChris Wilson if (err) 253e5dadff4SChris Wilson return ERR_PTR(err); 254e5dadff4SChris Wilson 255e5dadff4SChris Wilson return tl; 2562ccdf6a1SChris Wilson } 2572ccdf6a1SChris Wilson 258e5dadff4SChris Wilson static inline void intel_context_timeline_unlock(struct intel_timeline *tl) 259e5dadff4SChris Wilson __releases(&tl->mutex) 2602ccdf6a1SChris Wilson { 261e5dadff4SChris Wilson mutex_unlock(&tl->mutex); 2622ccdf6a1SChris Wilson } 2632ccdf6a1SChris Wilson 264a9877da2SChris Wilson int intel_context_prepare_remote_request(struct intel_context *ce, 265a9877da2SChris Wilson struct i915_request *rq); 266a9877da2SChris Wilson 2675e2a0419SChris Wilson struct i915_request *intel_context_create_request(struct intel_context *ce); 2685e2a0419SChris Wilson 269573ba126SMatthew Brost struct i915_request * 270573ba126SMatthew Brost intel_context_find_active_request(struct intel_context *ce); 271573ba126SMatthew Brost 272e6ba7648SChris Wilson static inline bool intel_context_is_barrier(const struct intel_context *ce) 273e6ba7648SChris Wilson { 274e6ba7648SChris Wilson return test_bit(CONTEXT_BARRIER_BIT, &ce->flags); 275e6ba7648SChris Wilson } 276e6ba7648SChris Wilson 2772e46a2a0SChris Wilson static inline bool intel_context_is_closed(const struct intel_context *ce) 2782e46a2a0SChris Wilson { 2792e46a2a0SChris Wilson return test_bit(CONTEXT_CLOSED_BIT, &ce->flags); 2802e46a2a0SChris Wilson } 2812e46a2a0SChris Wilson 282cc1557caSChris Wilson static inline bool intel_context_has_inflight(const struct intel_context *ce) 283cc1557caSChris Wilson { 284cc1557caSChris Wilson return test_bit(COPS_HAS_INFLIGHT_BIT, &ce->ops->flags); 285cc1557caSChris Wilson } 286cc1557caSChris Wilson 2870f100b70SChris Wilson static inline bool intel_context_use_semaphores(const struct intel_context *ce) 2880f100b70SChris Wilson { 2890f100b70SChris Wilson return test_bit(CONTEXT_USE_SEMAPHORES, &ce->flags); 2900f100b70SChris Wilson } 2910f100b70SChris Wilson 2920f100b70SChris Wilson static inline void intel_context_set_use_semaphores(struct intel_context *ce) 2930f100b70SChris Wilson { 2940f100b70SChris Wilson set_bit(CONTEXT_USE_SEMAPHORES, &ce->flags); 2950f100b70SChris Wilson } 2960f100b70SChris Wilson 2970f100b70SChris Wilson static inline void intel_context_clear_use_semaphores(struct intel_context *ce) 2980f100b70SChris Wilson { 2990f100b70SChris Wilson clear_bit(CONTEXT_USE_SEMAPHORES, &ce->flags); 3000f100b70SChris Wilson } 3010f100b70SChris Wilson 3029f3ccd40SChris Wilson static inline bool intel_context_is_banned(const struct intel_context *ce) 3039f3ccd40SChris Wilson { 3049f3ccd40SChris Wilson return test_bit(CONTEXT_BANNED, &ce->flags); 3059f3ccd40SChris Wilson } 3069f3ccd40SChris Wilson 3079f3ccd40SChris Wilson static inline bool intel_context_set_banned(struct intel_context *ce) 3089f3ccd40SChris Wilson { 3099f3ccd40SChris Wilson return test_and_set_bit(CONTEXT_BANNED, &ce->flags); 3109f3ccd40SChris Wilson } 3119f3ccd40SChris Wilson 312ae8ac10dSMatthew Brost static inline bool intel_context_ban(struct intel_context *ce, 313ae8ac10dSMatthew Brost struct i915_request *rq) 314ae8ac10dSMatthew Brost { 315ae8ac10dSMatthew Brost bool ret = intel_context_set_banned(ce); 316ae8ac10dSMatthew Brost 317ae8ac10dSMatthew Brost trace_intel_context_ban(ce); 318ae8ac10dSMatthew Brost if (ce->ops->ban) 319ae8ac10dSMatthew Brost ce->ops->ban(ce, rq); 320ae8ac10dSMatthew Brost 321ae8ac10dSMatthew Brost return ret; 322ae8ac10dSMatthew Brost } 323ae8ac10dSMatthew Brost 3249f3ccd40SChris Wilson static inline bool 3259f3ccd40SChris Wilson intel_context_force_single_submission(const struct intel_context *ce) 3269f3ccd40SChris Wilson { 3279f3ccd40SChris Wilson return test_bit(CONTEXT_FORCE_SINGLE_SUBMISSION, &ce->flags); 3289f3ccd40SChris Wilson } 3299f3ccd40SChris Wilson 3309f3ccd40SChris Wilson static inline void 3319f3ccd40SChris Wilson intel_context_set_single_submission(struct intel_context *ce) 3329f3ccd40SChris Wilson { 3339f3ccd40SChris Wilson __set_bit(CONTEXT_FORCE_SINGLE_SUBMISSION, &ce->flags); 3349f3ccd40SChris Wilson } 3359f3ccd40SChris Wilson 3369f3ccd40SChris Wilson static inline bool 3379f3ccd40SChris Wilson intel_context_nopreempt(const struct intel_context *ce) 3389f3ccd40SChris Wilson { 3399f3ccd40SChris Wilson return test_bit(CONTEXT_NOPREEMPT, &ce->flags); 3409f3ccd40SChris Wilson } 3419f3ccd40SChris Wilson 3429f3ccd40SChris Wilson static inline void 3439f3ccd40SChris Wilson intel_context_set_nopreempt(struct intel_context *ce) 3449f3ccd40SChris Wilson { 3459f3ccd40SChris Wilson set_bit(CONTEXT_NOPREEMPT, &ce->flags); 3469f3ccd40SChris Wilson } 3479f3ccd40SChris Wilson 3489f3ccd40SChris Wilson static inline void 3499f3ccd40SChris Wilson intel_context_clear_nopreempt(struct intel_context *ce) 3509f3ccd40SChris Wilson { 3519f3ccd40SChris Wilson clear_bit(CONTEXT_NOPREEMPT, &ce->flags); 3529f3ccd40SChris Wilson } 3539f3ccd40SChris Wilson 3541883a0a4STvrtko Ursulin static inline u64 intel_context_get_total_runtime_ns(struct intel_context *ce) 3551883a0a4STvrtko Ursulin { 356f170523aSChris Wilson const u32 period = ce->engine->gt->clock_period_ns; 3571883a0a4STvrtko Ursulin 3581883a0a4STvrtko Ursulin return READ_ONCE(ce->runtime.total) * period; 3591883a0a4STvrtko Ursulin } 3601883a0a4STvrtko Ursulin 3611883a0a4STvrtko Ursulin static inline u64 intel_context_get_avg_runtime_ns(struct intel_context *ce) 3621883a0a4STvrtko Ursulin { 363f170523aSChris Wilson const u32 period = ce->engine->gt->clock_period_ns; 3641883a0a4STvrtko Ursulin 3651883a0a4STvrtko Ursulin return mul_u32_u32(ewma_runtime_read(&ce->runtime.avg), period); 3661883a0a4STvrtko Ursulin } 3671883a0a4STvrtko Ursulin 368112ed2d3SChris Wilson #endif /* __INTEL_CONTEXT_H__ */ 369