1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include "i915_drv.h" 8 #include "i915_active.h" 9 10 #define BKL(ref) (&(ref)->i915->drm.struct_mutex) 11 12 /* 13 * Active refs memory management 14 * 15 * To be more economical with memory, we reap all the i915_active trees as 16 * they idle (when we know the active requests are inactive) and allocate the 17 * nodes from a local slab cache to hopefully reduce the fragmentation. 18 */ 19 static struct i915_global_active { 20 struct kmem_cache *slab_cache; 21 } global; 22 23 struct active_node { 24 struct i915_active_request base; 25 struct i915_active *ref; 26 struct rb_node node; 27 u64 timeline; 28 }; 29 30 static void 31 __active_park(struct i915_active *ref) 32 { 33 struct active_node *it, *n; 34 35 rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { 36 GEM_BUG_ON(i915_active_request_isset(&it->base)); 37 kmem_cache_free(global.slab_cache, it); 38 } 39 ref->tree = RB_ROOT; 40 } 41 42 static void 43 __active_retire(struct i915_active *ref) 44 { 45 GEM_BUG_ON(!ref->count); 46 if (--ref->count) 47 return; 48 49 /* return the unused nodes to our slabcache */ 50 __active_park(ref); 51 52 ref->retire(ref); 53 } 54 55 static void 56 node_retire(struct i915_active_request *base, struct i915_request *rq) 57 { 58 __active_retire(container_of(base, struct active_node, base)->ref); 59 } 60 61 static void 62 last_retire(struct i915_active_request *base, struct i915_request *rq) 63 { 64 __active_retire(container_of(base, struct i915_active, last)); 65 } 66 67 static struct i915_active_request * 68 active_instance(struct i915_active *ref, u64 idx) 69 { 70 struct active_node *node; 71 struct rb_node **p, *parent; 72 struct i915_request *old; 73 74 /* 75 * We track the most recently used timeline to skip a rbtree search 76 * for the common case, under typical loads we never need the rbtree 77 * at all. We can reuse the last slot if it is empty, that is 78 * after the previous activity has been retired, or if it matches the 79 * current timeline. 80 * 81 * Note that we allow the timeline to be active simultaneously in 82 * the rbtree and the last cache. We do this to avoid having 83 * to search and replace the rbtree element for a new timeline, with 84 * the cost being that we must be aware that the ref may be retired 85 * twice for the same timeline (as the older rbtree element will be 86 * retired before the new request added to last). 87 */ 88 old = i915_active_request_raw(&ref->last, BKL(ref)); 89 if (!old || old->fence.context == idx) 90 goto out; 91 92 /* Move the currently active fence into the rbtree */ 93 idx = old->fence.context; 94 95 parent = NULL; 96 p = &ref->tree.rb_node; 97 while (*p) { 98 parent = *p; 99 100 node = rb_entry(parent, struct active_node, node); 101 if (node->timeline == idx) 102 goto replace; 103 104 if (node->timeline < idx) 105 p = &parent->rb_right; 106 else 107 p = &parent->rb_left; 108 } 109 110 node = kmem_cache_alloc(global.slab_cache, GFP_KERNEL); 111 112 /* kmalloc may retire the ref->last (thanks shrinker)! */ 113 if (unlikely(!i915_active_request_raw(&ref->last, BKL(ref)))) { 114 kmem_cache_free(global.slab_cache, node); 115 goto out; 116 } 117 118 if (unlikely(!node)) 119 return ERR_PTR(-ENOMEM); 120 121 i915_active_request_init(&node->base, NULL, node_retire); 122 node->ref = ref; 123 node->timeline = idx; 124 125 rb_link_node(&node->node, parent, p); 126 rb_insert_color(&node->node, &ref->tree); 127 128 replace: 129 /* 130 * Overwrite the previous active slot in the rbtree with last, 131 * leaving last zeroed. If the previous slot is still active, 132 * we must be careful as we now only expect to receive one retire 133 * callback not two, and so much undo the active counting for the 134 * overwritten slot. 135 */ 136 if (i915_active_request_isset(&node->base)) { 137 /* Retire ourselves from the old rq->active_list */ 138 __list_del_entry(&node->base.link); 139 ref->count--; 140 GEM_BUG_ON(!ref->count); 141 } 142 GEM_BUG_ON(list_empty(&ref->last.link)); 143 list_replace_init(&ref->last.link, &node->base.link); 144 node->base.request = fetch_and_zero(&ref->last.request); 145 146 out: 147 return &ref->last; 148 } 149 150 void i915_active_init(struct drm_i915_private *i915, 151 struct i915_active *ref, 152 void (*retire)(struct i915_active *ref)) 153 { 154 ref->i915 = i915; 155 ref->retire = retire; 156 ref->tree = RB_ROOT; 157 i915_active_request_init(&ref->last, NULL, last_retire); 158 ref->count = 0; 159 } 160 161 int i915_active_ref(struct i915_active *ref, 162 u64 timeline, 163 struct i915_request *rq) 164 { 165 struct i915_active_request *active; 166 int err = 0; 167 168 /* Prevent reaping in case we malloc/wait while building the tree */ 169 i915_active_acquire(ref); 170 171 active = active_instance(ref, timeline); 172 if (IS_ERR(active)) { 173 err = PTR_ERR(active); 174 goto out; 175 } 176 177 if (!i915_active_request_isset(active)) 178 ref->count++; 179 __i915_active_request_set(active, rq); 180 181 GEM_BUG_ON(!ref->count); 182 out: 183 i915_active_release(ref); 184 return err; 185 } 186 187 bool i915_active_acquire(struct i915_active *ref) 188 { 189 lockdep_assert_held(BKL(ref)); 190 return !ref->count++; 191 } 192 193 void i915_active_release(struct i915_active *ref) 194 { 195 lockdep_assert_held(BKL(ref)); 196 __active_retire(ref); 197 } 198 199 int i915_active_wait(struct i915_active *ref) 200 { 201 struct active_node *it, *n; 202 int ret = 0; 203 204 if (i915_active_acquire(ref)) 205 goto out_release; 206 207 ret = i915_active_request_retire(&ref->last, BKL(ref)); 208 if (ret) 209 goto out_release; 210 211 rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { 212 ret = i915_active_request_retire(&it->base, BKL(ref)); 213 if (ret) 214 break; 215 } 216 217 out_release: 218 i915_active_release(ref); 219 return ret; 220 } 221 222 int i915_request_await_active_request(struct i915_request *rq, 223 struct i915_active_request *active) 224 { 225 struct i915_request *barrier = 226 i915_active_request_raw(active, &rq->i915->drm.struct_mutex); 227 228 return barrier ? i915_request_await_dma_fence(rq, &barrier->fence) : 0; 229 } 230 231 int i915_request_await_active(struct i915_request *rq, struct i915_active *ref) 232 { 233 struct active_node *it, *n; 234 int err = 0; 235 236 /* await allocates and so we need to avoid hitting the shrinker */ 237 if (i915_active_acquire(ref)) 238 goto out; /* was idle */ 239 240 err = i915_request_await_active_request(rq, &ref->last); 241 if (err) 242 goto out; 243 244 rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { 245 err = i915_request_await_active_request(rq, &it->base); 246 if (err) 247 goto out; 248 } 249 250 out: 251 i915_active_release(ref); 252 return err; 253 } 254 255 #if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM) 256 void i915_active_fini(struct i915_active *ref) 257 { 258 GEM_BUG_ON(i915_active_request_isset(&ref->last)); 259 GEM_BUG_ON(!RB_EMPTY_ROOT(&ref->tree)); 260 GEM_BUG_ON(ref->count); 261 } 262 #endif 263 264 int i915_active_request_set(struct i915_active_request *active, 265 struct i915_request *rq) 266 { 267 int err; 268 269 /* Must maintain ordering wrt previous active requests */ 270 err = i915_request_await_active_request(rq, active); 271 if (err) 272 return err; 273 274 __i915_active_request_set(active, rq); 275 return 0; 276 } 277 278 void i915_active_retire_noop(struct i915_active_request *active, 279 struct i915_request *request) 280 { 281 /* Space left intentionally blank */ 282 } 283 284 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 285 #include "selftests/i915_active.c" 286 #endif 287 288 int __init i915_global_active_init(void) 289 { 290 global.slab_cache = KMEM_CACHE(active_node, SLAB_HWCACHE_ALIGN); 291 if (!global.slab_cache) 292 return -ENOMEM; 293 294 return 0; 295 } 296 297 void __exit i915_global_active_exit(void) 298 { 299 kmem_cache_destroy(global.slab_cache); 300 } 301