1dc2f7e67SChris Wilson // SPDX-License-Identifier: MIT 2dc2f7e67SChris Wilson 3dc2f7e67SChris Wilson /* 4dc2f7e67SChris Wilson * Copyright © 2019 Intel Corporation 5dc2f7e67SChris Wilson */ 6dc2f7e67SChris Wilson 7dc2f7e67SChris Wilson #include <linux/delay.h> 8dc2f7e67SChris Wilson #include <linux/dma-fence.h> 9dc2f7e67SChris Wilson #include <linux/dma-fence-chain.h> 10dc2f7e67SChris Wilson #include <linux/kernel.h> 11dc2f7e67SChris Wilson #include <linux/kthread.h> 12dc2f7e67SChris Wilson #include <linux/mm.h> 13dc2f7e67SChris Wilson #include <linux/sched/signal.h> 14dc2f7e67SChris Wilson #include <linux/slab.h> 15dc2f7e67SChris Wilson #include <linux/spinlock.h> 16dc2f7e67SChris Wilson #include <linux/random.h> 17dc2f7e67SChris Wilson 18dc2f7e67SChris Wilson #include "selftest.h" 19dc2f7e67SChris Wilson 20dc2f7e67SChris Wilson #define CHAIN_SZ (4 << 10) 21dc2f7e67SChris Wilson 22dc2f7e67SChris Wilson static struct kmem_cache *slab_fences; 23dc2f7e67SChris Wilson 24dc2f7e67SChris Wilson static inline struct mock_fence { 25dc2f7e67SChris Wilson struct dma_fence base; 26dc2f7e67SChris Wilson spinlock_t lock; 27dc2f7e67SChris Wilson } *to_mock_fence(struct dma_fence *f) { 28dc2f7e67SChris Wilson return container_of(f, struct mock_fence, base); 29dc2f7e67SChris Wilson } 30dc2f7e67SChris Wilson 31dc2f7e67SChris Wilson static const char *mock_name(struct dma_fence *f) 32dc2f7e67SChris Wilson { 33dc2f7e67SChris Wilson return "mock"; 34dc2f7e67SChris Wilson } 35dc2f7e67SChris Wilson 36dc2f7e67SChris Wilson static void mock_fence_release(struct dma_fence *f) 37dc2f7e67SChris Wilson { 38dc2f7e67SChris Wilson kmem_cache_free(slab_fences, to_mock_fence(f)); 39dc2f7e67SChris Wilson } 40dc2f7e67SChris Wilson 41dc2f7e67SChris Wilson static const struct dma_fence_ops mock_ops = { 42dc2f7e67SChris Wilson .get_driver_name = mock_name, 43dc2f7e67SChris Wilson .get_timeline_name = mock_name, 44dc2f7e67SChris Wilson .release = mock_fence_release, 45dc2f7e67SChris Wilson }; 46dc2f7e67SChris Wilson 47dc2f7e67SChris Wilson static struct dma_fence *mock_fence(void) 48dc2f7e67SChris Wilson { 49dc2f7e67SChris Wilson struct mock_fence *f; 50dc2f7e67SChris Wilson 51dc2f7e67SChris Wilson f = kmem_cache_alloc(slab_fences, GFP_KERNEL); 52dc2f7e67SChris Wilson if (!f) 53dc2f7e67SChris Wilson return NULL; 54dc2f7e67SChris Wilson 55dc2f7e67SChris Wilson spin_lock_init(&f->lock); 56dc2f7e67SChris Wilson dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0); 57dc2f7e67SChris Wilson 58dc2f7e67SChris Wilson return &f->base; 59dc2f7e67SChris Wilson } 60dc2f7e67SChris Wilson 61dc2f7e67SChris Wilson static inline struct mock_chain { 62dc2f7e67SChris Wilson struct dma_fence_chain base; 63dc2f7e67SChris Wilson } *to_mock_chain(struct dma_fence *f) { 64dc2f7e67SChris Wilson return container_of(f, struct mock_chain, base.base); 65dc2f7e67SChris Wilson } 66dc2f7e67SChris Wilson 67dc2f7e67SChris Wilson static struct dma_fence *mock_chain(struct dma_fence *prev, 68dc2f7e67SChris Wilson struct dma_fence *fence, 69dc2f7e67SChris Wilson u64 seqno) 70dc2f7e67SChris Wilson { 71dc2f7e67SChris Wilson struct mock_chain *f; 72dc2f7e67SChris Wilson 73dc2f7e67SChris Wilson f = kmalloc(sizeof(*f), GFP_KERNEL); 74dc2f7e67SChris Wilson if (!f) 75dc2f7e67SChris Wilson return NULL; 76dc2f7e67SChris Wilson 77dc2f7e67SChris Wilson dma_fence_chain_init(&f->base, 78dc2f7e67SChris Wilson dma_fence_get(prev), 79dc2f7e67SChris Wilson dma_fence_get(fence), 80dc2f7e67SChris Wilson seqno); 81dc2f7e67SChris Wilson 82dc2f7e67SChris Wilson return &f->base.base; 83dc2f7e67SChris Wilson } 84dc2f7e67SChris Wilson 85dc2f7e67SChris Wilson static int sanitycheck(void *arg) 86dc2f7e67SChris Wilson { 87dc2f7e67SChris Wilson struct dma_fence *f, *chain; 88dc2f7e67SChris Wilson int err = 0; 89dc2f7e67SChris Wilson 90dc2f7e67SChris Wilson f = mock_fence(); 91dc2f7e67SChris Wilson if (!f) 92dc2f7e67SChris Wilson return -ENOMEM; 93dc2f7e67SChris Wilson 94dc2f7e67SChris Wilson chain = mock_chain(NULL, f, 1); 95dc2f7e67SChris Wilson if (!chain) 96dc2f7e67SChris Wilson err = -ENOMEM; 97dc2f7e67SChris Wilson 98dc2f7e67SChris Wilson dma_fence_signal(f); 99dc2f7e67SChris Wilson dma_fence_put(f); 100dc2f7e67SChris Wilson 101dc2f7e67SChris Wilson dma_fence_put(chain); 102dc2f7e67SChris Wilson 103dc2f7e67SChris Wilson return err; 104dc2f7e67SChris Wilson } 105dc2f7e67SChris Wilson 106dc2f7e67SChris Wilson struct fence_chains { 107dc2f7e67SChris Wilson unsigned int chain_length; 108dc2f7e67SChris Wilson struct dma_fence **fences; 109dc2f7e67SChris Wilson struct dma_fence **chains; 110dc2f7e67SChris Wilson 111dc2f7e67SChris Wilson struct dma_fence *tail; 112dc2f7e67SChris Wilson }; 113dc2f7e67SChris Wilson 114dc2f7e67SChris Wilson static uint64_t seqno_inc(unsigned int i) 115dc2f7e67SChris Wilson { 116dc2f7e67SChris Wilson return i + 1; 117dc2f7e67SChris Wilson } 118dc2f7e67SChris Wilson 119dc2f7e67SChris Wilson static int fence_chains_init(struct fence_chains *fc, unsigned int count, 120dc2f7e67SChris Wilson uint64_t (*seqno_fn)(unsigned int)) 121dc2f7e67SChris Wilson { 122dc2f7e67SChris Wilson unsigned int i; 123dc2f7e67SChris Wilson int err = 0; 124dc2f7e67SChris Wilson 125dc2f7e67SChris Wilson fc->chains = kvmalloc_array(count, sizeof(*fc->chains), 126dc2f7e67SChris Wilson GFP_KERNEL | __GFP_ZERO); 127dc2f7e67SChris Wilson if (!fc->chains) 128dc2f7e67SChris Wilson return -ENOMEM; 129dc2f7e67SChris Wilson 130dc2f7e67SChris Wilson fc->fences = kvmalloc_array(count, sizeof(*fc->fences), 131dc2f7e67SChris Wilson GFP_KERNEL | __GFP_ZERO); 132dc2f7e67SChris Wilson if (!fc->fences) { 133dc2f7e67SChris Wilson err = -ENOMEM; 134dc2f7e67SChris Wilson goto err_chains; 135dc2f7e67SChris Wilson } 136dc2f7e67SChris Wilson 137dc2f7e67SChris Wilson fc->tail = NULL; 138dc2f7e67SChris Wilson for (i = 0; i < count; i++) { 139dc2f7e67SChris Wilson fc->fences[i] = mock_fence(); 140dc2f7e67SChris Wilson if (!fc->fences[i]) { 141dc2f7e67SChris Wilson err = -ENOMEM; 142dc2f7e67SChris Wilson goto unwind; 143dc2f7e67SChris Wilson } 144dc2f7e67SChris Wilson 145dc2f7e67SChris Wilson fc->chains[i] = mock_chain(fc->tail, 146dc2f7e67SChris Wilson fc->fences[i], 147dc2f7e67SChris Wilson seqno_fn(i)); 148dc2f7e67SChris Wilson if (!fc->chains[i]) { 149dc2f7e67SChris Wilson err = -ENOMEM; 150dc2f7e67SChris Wilson goto unwind; 151dc2f7e67SChris Wilson } 152dc2f7e67SChris Wilson 153dc2f7e67SChris Wilson fc->tail = fc->chains[i]; 154dc2f7e67SChris Wilson } 155dc2f7e67SChris Wilson 156dc2f7e67SChris Wilson fc->chain_length = i; 157dc2f7e67SChris Wilson return 0; 158dc2f7e67SChris Wilson 159dc2f7e67SChris Wilson unwind: 160dc2f7e67SChris Wilson for (i = 0; i < count; i++) { 161dc2f7e67SChris Wilson dma_fence_put(fc->fences[i]); 162dc2f7e67SChris Wilson dma_fence_put(fc->chains[i]); 163dc2f7e67SChris Wilson } 164dc2f7e67SChris Wilson kvfree(fc->fences); 165dc2f7e67SChris Wilson err_chains: 166dc2f7e67SChris Wilson kvfree(fc->chains); 167dc2f7e67SChris Wilson return err; 168dc2f7e67SChris Wilson } 169dc2f7e67SChris Wilson 170dc2f7e67SChris Wilson static void fence_chains_fini(struct fence_chains *fc) 171dc2f7e67SChris Wilson { 172dc2f7e67SChris Wilson unsigned int i; 173dc2f7e67SChris Wilson 174dc2f7e67SChris Wilson for (i = 0; i < fc->chain_length; i++) { 175dc2f7e67SChris Wilson dma_fence_signal(fc->fences[i]); 176dc2f7e67SChris Wilson dma_fence_put(fc->fences[i]); 177dc2f7e67SChris Wilson } 178dc2f7e67SChris Wilson kvfree(fc->fences); 179dc2f7e67SChris Wilson 180dc2f7e67SChris Wilson for (i = 0; i < fc->chain_length; i++) 181dc2f7e67SChris Wilson dma_fence_put(fc->chains[i]); 182dc2f7e67SChris Wilson kvfree(fc->chains); 183dc2f7e67SChris Wilson } 184dc2f7e67SChris Wilson 185dc2f7e67SChris Wilson static int find_seqno(void *arg) 186dc2f7e67SChris Wilson { 187dc2f7e67SChris Wilson struct fence_chains fc; 188dc2f7e67SChris Wilson struct dma_fence *fence; 189dc2f7e67SChris Wilson int err; 190dc2f7e67SChris Wilson int i; 191dc2f7e67SChris Wilson 192dc2f7e67SChris Wilson err = fence_chains_init(&fc, 64, seqno_inc); 193dc2f7e67SChris Wilson if (err) 194dc2f7e67SChris Wilson return err; 195dc2f7e67SChris Wilson 196dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 197dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 0); 198dc2f7e67SChris Wilson dma_fence_put(fence); 199dc2f7e67SChris Wilson if (err) { 200dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno(0)!\n", err); 201dc2f7e67SChris Wilson goto err; 202dc2f7e67SChris Wilson } 203dc2f7e67SChris Wilson 204dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) { 205dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 206dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, i + 1); 207dc2f7e67SChris Wilson dma_fence_put(fence); 208dc2f7e67SChris Wilson if (err) { 209dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno(%d:%d)!\n", 210dc2f7e67SChris Wilson err, fc.chain_length + 1, i + 1); 211dc2f7e67SChris Wilson goto err; 212dc2f7e67SChris Wilson } 213dc2f7e67SChris Wilson if (fence != fc.chains[i]) { 214dc2f7e67SChris Wilson pr_err("Incorrect fence reported by find_seqno(%d:%d)\n", 215dc2f7e67SChris Wilson fc.chain_length + 1, i + 1); 216dc2f7e67SChris Wilson err = -EINVAL; 217dc2f7e67SChris Wilson goto err; 218dc2f7e67SChris Wilson } 219dc2f7e67SChris Wilson 220dc2f7e67SChris Wilson dma_fence_get(fence); 221dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, i + 1); 222dc2f7e67SChris Wilson dma_fence_put(fence); 223dc2f7e67SChris Wilson if (err) { 224dc2f7e67SChris Wilson pr_err("Error reported for finding self\n"); 225dc2f7e67SChris Wilson goto err; 226dc2f7e67SChris Wilson } 227dc2f7e67SChris Wilson if (fence != fc.chains[i]) { 228dc2f7e67SChris Wilson pr_err("Incorrect fence reported by find self\n"); 229dc2f7e67SChris Wilson err = -EINVAL; 230dc2f7e67SChris Wilson goto err; 231dc2f7e67SChris Wilson } 232dc2f7e67SChris Wilson 233dc2f7e67SChris Wilson dma_fence_get(fence); 234dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, i + 2); 235dc2f7e67SChris Wilson dma_fence_put(fence); 236dc2f7e67SChris Wilson if (!err) { 237dc2f7e67SChris Wilson pr_err("Error not reported for future fence: find_seqno(%d:%d)!\n", 238dc2f7e67SChris Wilson i + 1, i + 2); 239dc2f7e67SChris Wilson err = -EINVAL; 240dc2f7e67SChris Wilson goto err; 241dc2f7e67SChris Wilson } 242dc2f7e67SChris Wilson 243dc2f7e67SChris Wilson dma_fence_get(fence); 244dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, i); 245dc2f7e67SChris Wilson dma_fence_put(fence); 246dc2f7e67SChris Wilson if (err) { 247dc2f7e67SChris Wilson pr_err("Error reported for previous fence!\n"); 248dc2f7e67SChris Wilson goto err; 249dc2f7e67SChris Wilson } 250dc2f7e67SChris Wilson if (i > 0 && fence != fc.chains[i - 1]) { 251dc2f7e67SChris Wilson pr_err("Incorrect fence reported by find_seqno(%d:%d)\n", 252dc2f7e67SChris Wilson i + 1, i); 253dc2f7e67SChris Wilson err = -EINVAL; 254dc2f7e67SChris Wilson goto err; 255dc2f7e67SChris Wilson } 256dc2f7e67SChris Wilson } 257dc2f7e67SChris Wilson 258dc2f7e67SChris Wilson err: 259dc2f7e67SChris Wilson fence_chains_fini(&fc); 260dc2f7e67SChris Wilson return err; 261dc2f7e67SChris Wilson } 262dc2f7e67SChris Wilson 263dc2f7e67SChris Wilson static int find_signaled(void *arg) 264dc2f7e67SChris Wilson { 265dc2f7e67SChris Wilson struct fence_chains fc; 266dc2f7e67SChris Wilson struct dma_fence *fence; 267dc2f7e67SChris Wilson int err; 268dc2f7e67SChris Wilson 269dc2f7e67SChris Wilson err = fence_chains_init(&fc, 2, seqno_inc); 270dc2f7e67SChris Wilson if (err) 271dc2f7e67SChris Wilson return err; 272dc2f7e67SChris Wilson 273dc2f7e67SChris Wilson dma_fence_signal(fc.fences[0]); 274dc2f7e67SChris Wilson 275dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 276dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 1); 277dc2f7e67SChris Wilson dma_fence_put(fence); 278dc2f7e67SChris Wilson if (err) { 279dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno()!\n", err); 280dc2f7e67SChris Wilson goto err; 281dc2f7e67SChris Wilson } 282dc2f7e67SChris Wilson 283dc2f7e67SChris Wilson if (fence && fence != fc.chains[0]) { 284dc2f7e67SChris Wilson pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:1\n", 285dc2f7e67SChris Wilson fence->seqno); 286dc2f7e67SChris Wilson 287dc2f7e67SChris Wilson dma_fence_get(fence); 288dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 1); 289dc2f7e67SChris Wilson dma_fence_put(fence); 290dc2f7e67SChris Wilson if (err) 291dc2f7e67SChris Wilson pr_err("Reported %d for finding self!\n", err); 292dc2f7e67SChris Wilson 293dc2f7e67SChris Wilson err = -EINVAL; 294dc2f7e67SChris Wilson } 295dc2f7e67SChris Wilson 296dc2f7e67SChris Wilson err: 297dc2f7e67SChris Wilson fence_chains_fini(&fc); 298dc2f7e67SChris Wilson return err; 299dc2f7e67SChris Wilson } 300dc2f7e67SChris Wilson 301dc2f7e67SChris Wilson static int find_out_of_order(void *arg) 302dc2f7e67SChris Wilson { 303dc2f7e67SChris Wilson struct fence_chains fc; 304dc2f7e67SChris Wilson struct dma_fence *fence; 305dc2f7e67SChris Wilson int err; 306dc2f7e67SChris Wilson 307dc2f7e67SChris Wilson err = fence_chains_init(&fc, 3, seqno_inc); 308dc2f7e67SChris Wilson if (err) 309dc2f7e67SChris Wilson return err; 310dc2f7e67SChris Wilson 311dc2f7e67SChris Wilson dma_fence_signal(fc.fences[1]); 312dc2f7e67SChris Wilson 313dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 314dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 2); 315dc2f7e67SChris Wilson dma_fence_put(fence); 316dc2f7e67SChris Wilson if (err) { 317dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno()!\n", err); 318dc2f7e67SChris Wilson goto err; 319dc2f7e67SChris Wilson } 320dc2f7e67SChris Wilson 321*4cca2e64SLionel Landwerlin /* 322*4cca2e64SLionel Landwerlin * We signaled the middle fence (2) of the 1-2-3 chain. The behavior 323*4cca2e64SLionel Landwerlin * of the dma-fence-chain is to make us wait for all the fences up to 324*4cca2e64SLionel Landwerlin * the point we want. Since fence 1 is still not signaled, this what 325*4cca2e64SLionel Landwerlin * we should get as fence to wait upon (fence 2 being garbage 326*4cca2e64SLionel Landwerlin * collected during the traversal of the chain). 327*4cca2e64SLionel Landwerlin */ 328*4cca2e64SLionel Landwerlin if (fence != fc.chains[0]) { 329dc2f7e67SChris Wilson pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:2\n", 330*4cca2e64SLionel Landwerlin fence ? fence->seqno : 0); 331dc2f7e67SChris Wilson 332dc2f7e67SChris Wilson err = -EINVAL; 333dc2f7e67SChris Wilson } 334dc2f7e67SChris Wilson 335dc2f7e67SChris Wilson err: 336dc2f7e67SChris Wilson fence_chains_fini(&fc); 337dc2f7e67SChris Wilson return err; 338dc2f7e67SChris Wilson } 339dc2f7e67SChris Wilson 340dc2f7e67SChris Wilson static uint64_t seqno_inc2(unsigned int i) 341dc2f7e67SChris Wilson { 342dc2f7e67SChris Wilson return 2 * i + 2; 343dc2f7e67SChris Wilson } 344dc2f7e67SChris Wilson 345dc2f7e67SChris Wilson static int find_gap(void *arg) 346dc2f7e67SChris Wilson { 347dc2f7e67SChris Wilson struct fence_chains fc; 348dc2f7e67SChris Wilson struct dma_fence *fence; 349dc2f7e67SChris Wilson int err; 350dc2f7e67SChris Wilson int i; 351dc2f7e67SChris Wilson 352dc2f7e67SChris Wilson err = fence_chains_init(&fc, 64, seqno_inc2); 353dc2f7e67SChris Wilson if (err) 354dc2f7e67SChris Wilson return err; 355dc2f7e67SChris Wilson 356dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) { 357dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 358dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 2 * i + 1); 359dc2f7e67SChris Wilson dma_fence_put(fence); 360dc2f7e67SChris Wilson if (err) { 361dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno(%d:%d)!\n", 362dc2f7e67SChris Wilson err, fc.chain_length + 1, 2 * i + 1); 363dc2f7e67SChris Wilson goto err; 364dc2f7e67SChris Wilson } 365dc2f7e67SChris Wilson if (fence != fc.chains[i]) { 366dc2f7e67SChris Wilson pr_err("Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)\n", 367dc2f7e67SChris Wilson fence->seqno, 368dc2f7e67SChris Wilson fc.chain_length + 1, 369dc2f7e67SChris Wilson 2 * i + 1); 370dc2f7e67SChris Wilson err = -EINVAL; 371dc2f7e67SChris Wilson goto err; 372dc2f7e67SChris Wilson } 373dc2f7e67SChris Wilson 374dc2f7e67SChris Wilson dma_fence_get(fence); 375dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 2 * i + 2); 376dc2f7e67SChris Wilson dma_fence_put(fence); 377dc2f7e67SChris Wilson if (err) { 378dc2f7e67SChris Wilson pr_err("Error reported for finding self\n"); 379dc2f7e67SChris Wilson goto err; 380dc2f7e67SChris Wilson } 381dc2f7e67SChris Wilson if (fence != fc.chains[i]) { 382dc2f7e67SChris Wilson pr_err("Incorrect fence reported by find self\n"); 383dc2f7e67SChris Wilson err = -EINVAL; 384dc2f7e67SChris Wilson goto err; 385dc2f7e67SChris Wilson } 386dc2f7e67SChris Wilson } 387dc2f7e67SChris Wilson 388dc2f7e67SChris Wilson err: 389dc2f7e67SChris Wilson fence_chains_fini(&fc); 390dc2f7e67SChris Wilson return err; 391dc2f7e67SChris Wilson } 392dc2f7e67SChris Wilson 393dc2f7e67SChris Wilson struct find_race { 394dc2f7e67SChris Wilson struct fence_chains fc; 395dc2f7e67SChris Wilson atomic_t children; 396dc2f7e67SChris Wilson }; 397dc2f7e67SChris Wilson 398dc2f7e67SChris Wilson static int __find_race(void *arg) 399dc2f7e67SChris Wilson { 400dc2f7e67SChris Wilson struct find_race *data = arg; 401dc2f7e67SChris Wilson int err = 0; 402dc2f7e67SChris Wilson 403dc2f7e67SChris Wilson while (!kthread_should_stop()) { 404dc2f7e67SChris Wilson struct dma_fence *fence = dma_fence_get(data->fc.tail); 405dc2f7e67SChris Wilson int seqno; 406dc2f7e67SChris Wilson 407dc2f7e67SChris Wilson seqno = prandom_u32_max(data->fc.chain_length) + 1; 408dc2f7e67SChris Wilson 409dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, seqno); 410dc2f7e67SChris Wilson if (err) { 411dc2f7e67SChris Wilson pr_err("Failed to find fence seqno:%d\n", 412dc2f7e67SChris Wilson seqno); 413dc2f7e67SChris Wilson dma_fence_put(fence); 414dc2f7e67SChris Wilson break; 415dc2f7e67SChris Wilson } 416dc2f7e67SChris Wilson if (!fence) 417dc2f7e67SChris Wilson goto signal; 418dc2f7e67SChris Wilson 419*4cca2e64SLionel Landwerlin /* 420*4cca2e64SLionel Landwerlin * We can only find ourselves if we are on fence we were 421*4cca2e64SLionel Landwerlin * looking for. 422*4cca2e64SLionel Landwerlin */ 423*4cca2e64SLionel Landwerlin if (fence->seqno == seqno) { 424dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, seqno); 425dc2f7e67SChris Wilson if (err) { 426dc2f7e67SChris Wilson pr_err("Reported an invalid fence for find-self:%d\n", 427dc2f7e67SChris Wilson seqno); 428dc2f7e67SChris Wilson dma_fence_put(fence); 429dc2f7e67SChris Wilson break; 430dc2f7e67SChris Wilson } 431dc2f7e67SChris Wilson } 432dc2f7e67SChris Wilson 433dc2f7e67SChris Wilson dma_fence_put(fence); 434dc2f7e67SChris Wilson 435dc2f7e67SChris Wilson signal: 436dc2f7e67SChris Wilson seqno = prandom_u32_max(data->fc.chain_length - 1); 437dc2f7e67SChris Wilson dma_fence_signal(data->fc.fences[seqno]); 438dc2f7e67SChris Wilson cond_resched(); 439dc2f7e67SChris Wilson } 440dc2f7e67SChris Wilson 441dc2f7e67SChris Wilson if (atomic_dec_and_test(&data->children)) 442dc2f7e67SChris Wilson wake_up_var(&data->children); 443dc2f7e67SChris Wilson return err; 444dc2f7e67SChris Wilson } 445dc2f7e67SChris Wilson 446dc2f7e67SChris Wilson static int find_race(void *arg) 447dc2f7e67SChris Wilson { 448dc2f7e67SChris Wilson struct find_race data; 449dc2f7e67SChris Wilson int ncpus = num_online_cpus(); 450dc2f7e67SChris Wilson struct task_struct **threads; 451dc2f7e67SChris Wilson unsigned long count; 452dc2f7e67SChris Wilson int err; 453dc2f7e67SChris Wilson int i; 454dc2f7e67SChris Wilson 455dc2f7e67SChris Wilson err = fence_chains_init(&data.fc, CHAIN_SZ, seqno_inc); 456dc2f7e67SChris Wilson if (err) 457dc2f7e67SChris Wilson return err; 458dc2f7e67SChris Wilson 459dc2f7e67SChris Wilson threads = kmalloc_array(ncpus, sizeof(*threads), GFP_KERNEL); 460dc2f7e67SChris Wilson if (!threads) { 461dc2f7e67SChris Wilson err = -ENOMEM; 462dc2f7e67SChris Wilson goto err; 463dc2f7e67SChris Wilson } 464dc2f7e67SChris Wilson 465dc2f7e67SChris Wilson atomic_set(&data.children, 0); 466dc2f7e67SChris Wilson for (i = 0; i < ncpus; i++) { 467dc2f7e67SChris Wilson threads[i] = kthread_run(__find_race, &data, "dmabuf/%d", i); 468dc2f7e67SChris Wilson if (IS_ERR(threads[i])) { 469dc2f7e67SChris Wilson ncpus = i; 470dc2f7e67SChris Wilson break; 471dc2f7e67SChris Wilson } 472dc2f7e67SChris Wilson atomic_inc(&data.children); 473dc2f7e67SChris Wilson get_task_struct(threads[i]); 474dc2f7e67SChris Wilson } 475dc2f7e67SChris Wilson 476dc2f7e67SChris Wilson wait_var_event_timeout(&data.children, 477dc2f7e67SChris Wilson !atomic_read(&data.children), 478dc2f7e67SChris Wilson 5 * HZ); 479dc2f7e67SChris Wilson 480dc2f7e67SChris Wilson for (i = 0; i < ncpus; i++) { 481dc2f7e67SChris Wilson int ret; 482dc2f7e67SChris Wilson 483dc2f7e67SChris Wilson ret = kthread_stop(threads[i]); 484dc2f7e67SChris Wilson if (ret && !err) 485dc2f7e67SChris Wilson err = ret; 486dc2f7e67SChris Wilson put_task_struct(threads[i]); 487dc2f7e67SChris Wilson } 488dc2f7e67SChris Wilson kfree(threads); 489dc2f7e67SChris Wilson 490dc2f7e67SChris Wilson count = 0; 491dc2f7e67SChris Wilson for (i = 0; i < data.fc.chain_length; i++) 492dc2f7e67SChris Wilson if (dma_fence_is_signaled(data.fc.fences[i])) 493dc2f7e67SChris Wilson count++; 494dc2f7e67SChris Wilson pr_info("Completed %lu cycles\n", count); 495dc2f7e67SChris Wilson 496dc2f7e67SChris Wilson err: 497dc2f7e67SChris Wilson fence_chains_fini(&data.fc); 498dc2f7e67SChris Wilson return err; 499dc2f7e67SChris Wilson } 500dc2f7e67SChris Wilson 501dc2f7e67SChris Wilson static int signal_forward(void *arg) 502dc2f7e67SChris Wilson { 503dc2f7e67SChris Wilson struct fence_chains fc; 504dc2f7e67SChris Wilson int err; 505dc2f7e67SChris Wilson int i; 506dc2f7e67SChris Wilson 507dc2f7e67SChris Wilson err = fence_chains_init(&fc, 64, seqno_inc); 508dc2f7e67SChris Wilson if (err) 509dc2f7e67SChris Wilson return err; 510dc2f7e67SChris Wilson 511dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) { 512dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 513dc2f7e67SChris Wilson 514dc2f7e67SChris Wilson if (!dma_fence_is_signaled(fc.chains[i])) { 515dc2f7e67SChris Wilson pr_err("chain[%d] not signaled!\n", i); 516dc2f7e67SChris Wilson err = -EINVAL; 517dc2f7e67SChris Wilson goto err; 518dc2f7e67SChris Wilson } 519dc2f7e67SChris Wilson 520dc2f7e67SChris Wilson if (i + 1 < fc.chain_length && 521dc2f7e67SChris Wilson dma_fence_is_signaled(fc.chains[i + 1])) { 522dc2f7e67SChris Wilson pr_err("chain[%d] is signaled!\n", i); 523dc2f7e67SChris Wilson err = -EINVAL; 524dc2f7e67SChris Wilson goto err; 525dc2f7e67SChris Wilson } 526dc2f7e67SChris Wilson } 527dc2f7e67SChris Wilson 528dc2f7e67SChris Wilson err: 529dc2f7e67SChris Wilson fence_chains_fini(&fc); 530dc2f7e67SChris Wilson return err; 531dc2f7e67SChris Wilson } 532dc2f7e67SChris Wilson 533dc2f7e67SChris Wilson static int signal_backward(void *arg) 534dc2f7e67SChris Wilson { 535dc2f7e67SChris Wilson struct fence_chains fc; 536dc2f7e67SChris Wilson int err; 537dc2f7e67SChris Wilson int i; 538dc2f7e67SChris Wilson 539dc2f7e67SChris Wilson err = fence_chains_init(&fc, 64, seqno_inc); 540dc2f7e67SChris Wilson if (err) 541dc2f7e67SChris Wilson return err; 542dc2f7e67SChris Wilson 543dc2f7e67SChris Wilson for (i = fc.chain_length; i--; ) { 544dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 545dc2f7e67SChris Wilson 546dc2f7e67SChris Wilson if (i > 0 && dma_fence_is_signaled(fc.chains[i])) { 547dc2f7e67SChris Wilson pr_err("chain[%d] is signaled!\n", i); 548dc2f7e67SChris Wilson err = -EINVAL; 549dc2f7e67SChris Wilson goto err; 550dc2f7e67SChris Wilson } 551dc2f7e67SChris Wilson } 552dc2f7e67SChris Wilson 553dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) { 554dc2f7e67SChris Wilson if (!dma_fence_is_signaled(fc.chains[i])) { 555dc2f7e67SChris Wilson pr_err("chain[%d] was not signaled!\n", i); 556dc2f7e67SChris Wilson err = -EINVAL; 557dc2f7e67SChris Wilson goto err; 558dc2f7e67SChris Wilson } 559dc2f7e67SChris Wilson } 560dc2f7e67SChris Wilson 561dc2f7e67SChris Wilson err: 562dc2f7e67SChris Wilson fence_chains_fini(&fc); 563dc2f7e67SChris Wilson return err; 564dc2f7e67SChris Wilson } 565dc2f7e67SChris Wilson 566dc2f7e67SChris Wilson static int __wait_fence_chains(void *arg) 567dc2f7e67SChris Wilson { 568dc2f7e67SChris Wilson struct fence_chains *fc = arg; 569dc2f7e67SChris Wilson 570dc2f7e67SChris Wilson if (dma_fence_wait(fc->tail, false)) 571dc2f7e67SChris Wilson return -EIO; 572dc2f7e67SChris Wilson 573dc2f7e67SChris Wilson return 0; 574dc2f7e67SChris Wilson } 575dc2f7e67SChris Wilson 576dc2f7e67SChris Wilson static int wait_forward(void *arg) 577dc2f7e67SChris Wilson { 578dc2f7e67SChris Wilson struct fence_chains fc; 579dc2f7e67SChris Wilson struct task_struct *tsk; 580dc2f7e67SChris Wilson int err; 581dc2f7e67SChris Wilson int i; 582dc2f7e67SChris Wilson 583dc2f7e67SChris Wilson err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 584dc2f7e67SChris Wilson if (err) 585dc2f7e67SChris Wilson return err; 586dc2f7e67SChris Wilson 587dc2f7e67SChris Wilson tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 588dc2f7e67SChris Wilson if (IS_ERR(tsk)) { 589dc2f7e67SChris Wilson err = PTR_ERR(tsk); 590dc2f7e67SChris Wilson goto err; 591dc2f7e67SChris Wilson } 592dc2f7e67SChris Wilson get_task_struct(tsk); 593dc2f7e67SChris Wilson yield_to(tsk, true); 594dc2f7e67SChris Wilson 595dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) 596dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 597dc2f7e67SChris Wilson 598dc2f7e67SChris Wilson err = kthread_stop(tsk); 599dc2f7e67SChris Wilson put_task_struct(tsk); 600dc2f7e67SChris Wilson 601dc2f7e67SChris Wilson err: 602dc2f7e67SChris Wilson fence_chains_fini(&fc); 603dc2f7e67SChris Wilson return err; 604dc2f7e67SChris Wilson } 605dc2f7e67SChris Wilson 606dc2f7e67SChris Wilson static int wait_backward(void *arg) 607dc2f7e67SChris Wilson { 608dc2f7e67SChris Wilson struct fence_chains fc; 609dc2f7e67SChris Wilson struct task_struct *tsk; 610dc2f7e67SChris Wilson int err; 611dc2f7e67SChris Wilson int i; 612dc2f7e67SChris Wilson 613dc2f7e67SChris Wilson err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 614dc2f7e67SChris Wilson if (err) 615dc2f7e67SChris Wilson return err; 616dc2f7e67SChris Wilson 617dc2f7e67SChris Wilson tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 618dc2f7e67SChris Wilson if (IS_ERR(tsk)) { 619dc2f7e67SChris Wilson err = PTR_ERR(tsk); 620dc2f7e67SChris Wilson goto err; 621dc2f7e67SChris Wilson } 622dc2f7e67SChris Wilson get_task_struct(tsk); 623dc2f7e67SChris Wilson yield_to(tsk, true); 624dc2f7e67SChris Wilson 625dc2f7e67SChris Wilson for (i = fc.chain_length; i--; ) 626dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 627dc2f7e67SChris Wilson 628dc2f7e67SChris Wilson err = kthread_stop(tsk); 629dc2f7e67SChris Wilson put_task_struct(tsk); 630dc2f7e67SChris Wilson 631dc2f7e67SChris Wilson err: 632dc2f7e67SChris Wilson fence_chains_fini(&fc); 633dc2f7e67SChris Wilson return err; 634dc2f7e67SChris Wilson } 635dc2f7e67SChris Wilson 636dc2f7e67SChris Wilson static void randomise_fences(struct fence_chains *fc) 637dc2f7e67SChris Wilson { 638dc2f7e67SChris Wilson unsigned int count = fc->chain_length; 639dc2f7e67SChris Wilson 640dc2f7e67SChris Wilson /* Fisher-Yates shuffle courtesy of Knuth */ 641dc2f7e67SChris Wilson while (--count) { 642dc2f7e67SChris Wilson unsigned int swp; 643dc2f7e67SChris Wilson 644dc2f7e67SChris Wilson swp = prandom_u32_max(count + 1); 645dc2f7e67SChris Wilson if (swp == count) 646dc2f7e67SChris Wilson continue; 647dc2f7e67SChris Wilson 648dc2f7e67SChris Wilson swap(fc->fences[count], fc->fences[swp]); 649dc2f7e67SChris Wilson } 650dc2f7e67SChris Wilson } 651dc2f7e67SChris Wilson 652dc2f7e67SChris Wilson static int wait_random(void *arg) 653dc2f7e67SChris Wilson { 654dc2f7e67SChris Wilson struct fence_chains fc; 655dc2f7e67SChris Wilson struct task_struct *tsk; 656dc2f7e67SChris Wilson int err; 657dc2f7e67SChris Wilson int i; 658dc2f7e67SChris Wilson 659dc2f7e67SChris Wilson err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 660dc2f7e67SChris Wilson if (err) 661dc2f7e67SChris Wilson return err; 662dc2f7e67SChris Wilson 663dc2f7e67SChris Wilson randomise_fences(&fc); 664dc2f7e67SChris Wilson 665dc2f7e67SChris Wilson tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 666dc2f7e67SChris Wilson if (IS_ERR(tsk)) { 667dc2f7e67SChris Wilson err = PTR_ERR(tsk); 668dc2f7e67SChris Wilson goto err; 669dc2f7e67SChris Wilson } 670dc2f7e67SChris Wilson get_task_struct(tsk); 671dc2f7e67SChris Wilson yield_to(tsk, true); 672dc2f7e67SChris Wilson 673dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) 674dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 675dc2f7e67SChris Wilson 676dc2f7e67SChris Wilson err = kthread_stop(tsk); 677dc2f7e67SChris Wilson put_task_struct(tsk); 678dc2f7e67SChris Wilson 679dc2f7e67SChris Wilson err: 680dc2f7e67SChris Wilson fence_chains_fini(&fc); 681dc2f7e67SChris Wilson return err; 682dc2f7e67SChris Wilson } 683dc2f7e67SChris Wilson 684dc2f7e67SChris Wilson int dma_fence_chain(void) 685dc2f7e67SChris Wilson { 686dc2f7e67SChris Wilson static const struct subtest tests[] = { 687dc2f7e67SChris Wilson SUBTEST(sanitycheck), 688dc2f7e67SChris Wilson SUBTEST(find_seqno), 689dc2f7e67SChris Wilson SUBTEST(find_signaled), 690dc2f7e67SChris Wilson SUBTEST(find_out_of_order), 691dc2f7e67SChris Wilson SUBTEST(find_gap), 692dc2f7e67SChris Wilson SUBTEST(find_race), 693dc2f7e67SChris Wilson SUBTEST(signal_forward), 694dc2f7e67SChris Wilson SUBTEST(signal_backward), 695dc2f7e67SChris Wilson SUBTEST(wait_forward), 696dc2f7e67SChris Wilson SUBTEST(wait_backward), 697dc2f7e67SChris Wilson SUBTEST(wait_random), 698dc2f7e67SChris Wilson }; 699dc2f7e67SChris Wilson int ret; 700dc2f7e67SChris Wilson 701dc2f7e67SChris Wilson pr_info("sizeof(dma_fence_chain)=%zu\n", 702dc2f7e67SChris Wilson sizeof(struct dma_fence_chain)); 703dc2f7e67SChris Wilson 704dc2f7e67SChris Wilson slab_fences = KMEM_CACHE(mock_fence, 705dc2f7e67SChris Wilson SLAB_TYPESAFE_BY_RCU | 706dc2f7e67SChris Wilson SLAB_HWCACHE_ALIGN); 707dc2f7e67SChris Wilson if (!slab_fences) 708dc2f7e67SChris Wilson return -ENOMEM; 709dc2f7e67SChris Wilson 710dc2f7e67SChris Wilson ret = subtests(tests, NULL); 711dc2f7e67SChris Wilson 712dc2f7e67SChris Wilson kmem_cache_destroy(slab_fences); 713dc2f7e67SChris Wilson return ret; 714dc2f7e67SChris Wilson } 715