1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2018 Intel Corporation 5 */ 6 7 #include <linux/kref.h> 8 9 #include "gem/i915_gem_pm.h" 10 #include "gt/intel_gt.h" 11 12 #include "i915_selftest.h" 13 14 #include "igt_flush_test.h" 15 #include "lib_sw_fence.h" 16 17 struct live_active { 18 struct i915_active base; 19 struct kref ref; 20 bool retired; 21 }; 22 23 static void __live_get(struct live_active *active) 24 { 25 kref_get(&active->ref); 26 } 27 28 static void __live_free(struct live_active *active) 29 { 30 i915_active_fini(&active->base); 31 kfree(active); 32 } 33 34 static void __live_release(struct kref *ref) 35 { 36 struct live_active *active = container_of(ref, typeof(*active), ref); 37 38 __live_free(active); 39 } 40 41 static void __live_put(struct live_active *active) 42 { 43 kref_put(&active->ref, __live_release); 44 } 45 46 static int __live_active(struct i915_active *base) 47 { 48 struct live_active *active = container_of(base, typeof(*active), base); 49 50 __live_get(active); 51 return 0; 52 } 53 54 static void __live_retire(struct i915_active *base) 55 { 56 struct live_active *active = container_of(base, typeof(*active), base); 57 58 active->retired = true; 59 __live_put(active); 60 } 61 62 static struct live_active *__live_alloc(struct drm_i915_private *i915) 63 { 64 struct live_active *active; 65 66 active = kzalloc(sizeof(*active), GFP_KERNEL); 67 if (!active) 68 return NULL; 69 70 kref_init(&active->ref); 71 i915_active_init(&active->base, __live_active, __live_retire); 72 73 return active; 74 } 75 76 static struct live_active * 77 __live_active_setup(struct drm_i915_private *i915) 78 { 79 struct intel_engine_cs *engine; 80 struct i915_sw_fence *submit; 81 struct live_active *active; 82 unsigned int count = 0; 83 int err = 0; 84 85 active = __live_alloc(i915); 86 if (!active) 87 return ERR_PTR(-ENOMEM); 88 89 submit = heap_fence_create(GFP_KERNEL); 90 if (!submit) { 91 kfree(active); 92 return ERR_PTR(-ENOMEM); 93 } 94 95 err = i915_active_acquire(&active->base); 96 if (err) 97 goto out; 98 99 for_each_uabi_engine(engine, i915) { 100 struct i915_request *rq; 101 102 rq = intel_engine_create_kernel_request(engine); 103 if (IS_ERR(rq)) { 104 err = PTR_ERR(rq); 105 break; 106 } 107 108 err = i915_sw_fence_await_sw_fence_gfp(&rq->submit, 109 submit, 110 GFP_KERNEL); 111 if (err >= 0) 112 err = i915_active_add_request(&active->base, rq); 113 i915_request_add(rq); 114 if (err) { 115 pr_err("Failed to track active ref!\n"); 116 break; 117 } 118 119 count++; 120 } 121 122 i915_active_release(&active->base); 123 if (READ_ONCE(active->retired) && count) { 124 pr_err("i915_active retired before submission!\n"); 125 err = -EINVAL; 126 } 127 if (atomic_read(&active->base.count) != count) { 128 pr_err("i915_active not tracking all requests, found %d, expected %d\n", 129 atomic_read(&active->base.count), count); 130 err = -EINVAL; 131 } 132 133 out: 134 i915_sw_fence_commit(submit); 135 heap_fence_put(submit); 136 if (err) { 137 __live_put(active); 138 active = ERR_PTR(err); 139 } 140 141 return active; 142 } 143 144 static int live_active_wait(void *arg) 145 { 146 struct drm_i915_private *i915 = arg; 147 struct live_active *active; 148 int err = 0; 149 150 /* Check that we get a callback when requests retire upon waiting */ 151 152 active = __live_active_setup(i915); 153 if (IS_ERR(active)) 154 return PTR_ERR(active); 155 156 __i915_active_wait(&active->base, TASK_UNINTERRUPTIBLE); 157 if (!READ_ONCE(active->retired)) { 158 struct drm_printer p = drm_err_printer(__func__); 159 160 pr_err("i915_active not retired after waiting!\n"); 161 i915_active_print(&active->base, &p); 162 163 err = -EINVAL; 164 } 165 166 __live_put(active); 167 168 if (igt_flush_test(i915)) 169 err = -EIO; 170 171 return err; 172 } 173 174 static int live_active_retire(void *arg) 175 { 176 struct drm_i915_private *i915 = arg; 177 struct live_active *active; 178 int err = 0; 179 180 /* Check that we get a callback when requests are indirectly retired */ 181 182 active = __live_active_setup(i915); 183 if (IS_ERR(active)) 184 return PTR_ERR(active); 185 186 /* waits for & retires all requests */ 187 if (igt_flush_test(i915)) 188 err = -EIO; 189 190 if (!READ_ONCE(active->retired)) { 191 struct drm_printer p = drm_err_printer(__func__); 192 193 pr_err("i915_active not retired after flushing!\n"); 194 i915_active_print(&active->base, &p); 195 196 err = -EINVAL; 197 } 198 199 __live_put(active); 200 201 return err; 202 } 203 204 static int live_active_barrier(void *arg) 205 { 206 struct drm_i915_private *i915 = arg; 207 struct intel_engine_cs *engine; 208 struct live_active *active; 209 int err = 0; 210 211 /* Check that we get a callback when requests retire upon waiting */ 212 213 active = __live_alloc(i915); 214 if (!active) 215 return -ENOMEM; 216 217 err = i915_active_acquire(&active->base); 218 if (err) 219 goto out; 220 221 for_each_uabi_engine(engine, i915) { 222 err = i915_active_acquire_preallocate_barrier(&active->base, 223 engine); 224 if (err) 225 break; 226 227 i915_active_acquire_barrier(&active->base); 228 } 229 230 i915_active_release(&active->base); 231 if (err) 232 goto out; 233 234 __i915_active_wait(&active->base, TASK_UNINTERRUPTIBLE); 235 if (!READ_ONCE(active->retired)) { 236 pr_err("i915_active not retired after flushing barriers!\n"); 237 err = -EINVAL; 238 } 239 240 out: 241 __live_put(active); 242 243 if (igt_flush_test(i915)) 244 err = -EIO; 245 246 return err; 247 } 248 249 int i915_active_live_selftests(struct drm_i915_private *i915) 250 { 251 static const struct i915_subtest tests[] = { 252 SUBTEST(live_active_wait), 253 SUBTEST(live_active_retire), 254 SUBTEST(live_active_barrier), 255 }; 256 257 if (intel_gt_is_wedged(&i915->gt)) 258 return 0; 259 260 return i915_subtests(tests, i915); 261 } 262 263 static struct intel_engine_cs *node_to_barrier(struct active_node *it) 264 { 265 struct intel_engine_cs *engine; 266 267 if (!is_barrier(&it->base)) 268 return NULL; 269 270 engine = __barrier_to_engine(it); 271 smp_rmb(); /* serialise with add_active_barriers */ 272 if (!is_barrier(&it->base)) 273 return NULL; 274 275 return engine; 276 } 277 278 void i915_active_print(struct i915_active *ref, struct drm_printer *m) 279 { 280 drm_printf(m, "active %ps:%ps\n", ref->active, ref->retire); 281 drm_printf(m, "\tcount: %d\n", atomic_read(&ref->count)); 282 drm_printf(m, "\tpreallocated barriers? %s\n", 283 yesno(!llist_empty(&ref->preallocated_barriers))); 284 285 if (i915_active_acquire_if_busy(ref)) { 286 struct active_node *it, *n; 287 288 rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) { 289 struct intel_engine_cs *engine; 290 291 engine = node_to_barrier(it); 292 if (engine) { 293 drm_printf(m, "\tbarrier: %s\n", engine->name); 294 continue; 295 } 296 297 if (i915_active_fence_isset(&it->base)) { 298 drm_printf(m, 299 "\ttimeline: %llx\n", it->timeline); 300 continue; 301 } 302 } 303 304 i915_active_release(ref); 305 } 306 } 307 308 static void spin_unlock_wait(spinlock_t *lock) 309 { 310 spin_lock_irq(lock); 311 spin_unlock_irq(lock); 312 } 313 314 static void active_flush(struct i915_active *ref, 315 struct i915_active_fence *active) 316 { 317 struct dma_fence *fence; 318 319 fence = xchg(__active_fence_slot(active), NULL); 320 if (!fence) 321 return; 322 323 spin_lock_irq(fence->lock); 324 __list_del_entry(&active->cb.node); 325 spin_unlock_irq(fence->lock); /* serialise with fence->cb_list */ 326 atomic_dec(&ref->count); 327 328 GEM_BUG_ON(!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags)); 329 } 330 331 void i915_active_unlock_wait(struct i915_active *ref) 332 { 333 if (i915_active_acquire_if_busy(ref)) { 334 struct active_node *it, *n; 335 336 /* Wait for all active callbacks */ 337 rcu_read_lock(); 338 active_flush(ref, &ref->excl); 339 rbtree_postorder_for_each_entry_safe(it, n, &ref->tree, node) 340 active_flush(ref, &it->base); 341 rcu_read_unlock(); 342 343 i915_active_release(ref); 344 } 345 346 /* And wait for the retire callback */ 347 spin_unlock_wait(&ref->tree_lock); 348 349 /* ... which may have been on a thread instead */ 350 flush_work(&ref->work); 351 } 352