1dc2f7e67SChris Wilson // SPDX-License-Identifier: MIT 2dc2f7e67SChris Wilson 3dc2f7e67SChris Wilson /* 4dc2f7e67SChris Wilson * Copyright © 2019 Intel Corporation 5dc2f7e67SChris Wilson */ 6dc2f7e67SChris Wilson 7dc2f7e67SChris Wilson #include <linux/delay.h> 8dc2f7e67SChris Wilson #include <linux/dma-fence.h> 9dc2f7e67SChris Wilson #include <linux/dma-fence-chain.h> 10dc2f7e67SChris Wilson #include <linux/kernel.h> 11dc2f7e67SChris Wilson #include <linux/kthread.h> 12dc2f7e67SChris Wilson #include <linux/mm.h> 13dc2f7e67SChris Wilson #include <linux/sched/signal.h> 14dc2f7e67SChris Wilson #include <linux/slab.h> 15dc2f7e67SChris Wilson #include <linux/spinlock.h> 16dc2f7e67SChris Wilson #include <linux/random.h> 17dc2f7e67SChris Wilson 18dc2f7e67SChris Wilson #include "selftest.h" 19dc2f7e67SChris Wilson 20dc2f7e67SChris Wilson #define CHAIN_SZ (4 << 10) 21dc2f7e67SChris Wilson 22dc2f7e67SChris Wilson static struct kmem_cache *slab_fences; 23dc2f7e67SChris Wilson 24dc2f7e67SChris Wilson static inline struct mock_fence { 25dc2f7e67SChris Wilson struct dma_fence base; 26dc2f7e67SChris Wilson spinlock_t lock; 27dc2f7e67SChris Wilson } *to_mock_fence(struct dma_fence *f) { 28dc2f7e67SChris Wilson return container_of(f, struct mock_fence, base); 29dc2f7e67SChris Wilson } 30dc2f7e67SChris Wilson 31dc2f7e67SChris Wilson static const char *mock_name(struct dma_fence *f) 32dc2f7e67SChris Wilson { 33dc2f7e67SChris Wilson return "mock"; 34dc2f7e67SChris Wilson } 35dc2f7e67SChris Wilson 36dc2f7e67SChris Wilson static void mock_fence_release(struct dma_fence *f) 37dc2f7e67SChris Wilson { 38dc2f7e67SChris Wilson kmem_cache_free(slab_fences, to_mock_fence(f)); 39dc2f7e67SChris Wilson } 40dc2f7e67SChris Wilson 41dc2f7e67SChris Wilson static const struct dma_fence_ops mock_ops = { 42dc2f7e67SChris Wilson .get_driver_name = mock_name, 43dc2f7e67SChris Wilson .get_timeline_name = mock_name, 44dc2f7e67SChris Wilson .release = mock_fence_release, 45dc2f7e67SChris Wilson }; 46dc2f7e67SChris Wilson 47dc2f7e67SChris Wilson static struct dma_fence *mock_fence(void) 48dc2f7e67SChris Wilson { 49dc2f7e67SChris Wilson struct mock_fence *f; 50dc2f7e67SChris Wilson 51dc2f7e67SChris Wilson f = kmem_cache_alloc(slab_fences, GFP_KERNEL); 52dc2f7e67SChris Wilson if (!f) 53dc2f7e67SChris Wilson return NULL; 54dc2f7e67SChris Wilson 55dc2f7e67SChris Wilson spin_lock_init(&f->lock); 56dc2f7e67SChris Wilson dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0); 57dc2f7e67SChris Wilson 58dc2f7e67SChris Wilson return &f->base; 59dc2f7e67SChris Wilson } 60dc2f7e67SChris Wilson 61dc2f7e67SChris Wilson static struct dma_fence *mock_chain(struct dma_fence *prev, 62dc2f7e67SChris Wilson struct dma_fence *fence, 63dc2f7e67SChris Wilson u64 seqno) 64dc2f7e67SChris Wilson { 65*440d0f12SChristian König struct dma_fence_chain *f; 66dc2f7e67SChris Wilson 67*440d0f12SChristian König f = dma_fence_chain_alloc(); 68dc2f7e67SChris Wilson if (!f) 69dc2f7e67SChris Wilson return NULL; 70dc2f7e67SChris Wilson 71*440d0f12SChristian König dma_fence_chain_init(f, dma_fence_get(prev), dma_fence_get(fence), 72dc2f7e67SChris Wilson seqno); 73dc2f7e67SChris Wilson 74*440d0f12SChristian König return &f->base; 75dc2f7e67SChris Wilson } 76dc2f7e67SChris Wilson 77dc2f7e67SChris Wilson static int sanitycheck(void *arg) 78dc2f7e67SChris Wilson { 79dc2f7e67SChris Wilson struct dma_fence *f, *chain; 80dc2f7e67SChris Wilson int err = 0; 81dc2f7e67SChris Wilson 82dc2f7e67SChris Wilson f = mock_fence(); 83dc2f7e67SChris Wilson if (!f) 84dc2f7e67SChris Wilson return -ENOMEM; 85dc2f7e67SChris Wilson 86dc2f7e67SChris Wilson chain = mock_chain(NULL, f, 1); 87dc2f7e67SChris Wilson if (!chain) 88dc2f7e67SChris Wilson err = -ENOMEM; 89dc2f7e67SChris Wilson 90dc2f7e67SChris Wilson dma_fence_signal(f); 91dc2f7e67SChris Wilson dma_fence_put(f); 92dc2f7e67SChris Wilson 93dc2f7e67SChris Wilson dma_fence_put(chain); 94dc2f7e67SChris Wilson 95dc2f7e67SChris Wilson return err; 96dc2f7e67SChris Wilson } 97dc2f7e67SChris Wilson 98dc2f7e67SChris Wilson struct fence_chains { 99dc2f7e67SChris Wilson unsigned int chain_length; 100dc2f7e67SChris Wilson struct dma_fence **fences; 101dc2f7e67SChris Wilson struct dma_fence **chains; 102dc2f7e67SChris Wilson 103dc2f7e67SChris Wilson struct dma_fence *tail; 104dc2f7e67SChris Wilson }; 105dc2f7e67SChris Wilson 106dc2f7e67SChris Wilson static uint64_t seqno_inc(unsigned int i) 107dc2f7e67SChris Wilson { 108dc2f7e67SChris Wilson return i + 1; 109dc2f7e67SChris Wilson } 110dc2f7e67SChris Wilson 111dc2f7e67SChris Wilson static int fence_chains_init(struct fence_chains *fc, unsigned int count, 112dc2f7e67SChris Wilson uint64_t (*seqno_fn)(unsigned int)) 113dc2f7e67SChris Wilson { 114dc2f7e67SChris Wilson unsigned int i; 115dc2f7e67SChris Wilson int err = 0; 116dc2f7e67SChris Wilson 117dc2f7e67SChris Wilson fc->chains = kvmalloc_array(count, sizeof(*fc->chains), 118dc2f7e67SChris Wilson GFP_KERNEL | __GFP_ZERO); 119dc2f7e67SChris Wilson if (!fc->chains) 120dc2f7e67SChris Wilson return -ENOMEM; 121dc2f7e67SChris Wilson 122dc2f7e67SChris Wilson fc->fences = kvmalloc_array(count, sizeof(*fc->fences), 123dc2f7e67SChris Wilson GFP_KERNEL | __GFP_ZERO); 124dc2f7e67SChris Wilson if (!fc->fences) { 125dc2f7e67SChris Wilson err = -ENOMEM; 126dc2f7e67SChris Wilson goto err_chains; 127dc2f7e67SChris Wilson } 128dc2f7e67SChris Wilson 129dc2f7e67SChris Wilson fc->tail = NULL; 130dc2f7e67SChris Wilson for (i = 0; i < count; i++) { 131dc2f7e67SChris Wilson fc->fences[i] = mock_fence(); 132dc2f7e67SChris Wilson if (!fc->fences[i]) { 133dc2f7e67SChris Wilson err = -ENOMEM; 134dc2f7e67SChris Wilson goto unwind; 135dc2f7e67SChris Wilson } 136dc2f7e67SChris Wilson 137dc2f7e67SChris Wilson fc->chains[i] = mock_chain(fc->tail, 138dc2f7e67SChris Wilson fc->fences[i], 139dc2f7e67SChris Wilson seqno_fn(i)); 140dc2f7e67SChris Wilson if (!fc->chains[i]) { 141dc2f7e67SChris Wilson err = -ENOMEM; 142dc2f7e67SChris Wilson goto unwind; 143dc2f7e67SChris Wilson } 144dc2f7e67SChris Wilson 145dc2f7e67SChris Wilson fc->tail = fc->chains[i]; 146dc2f7e67SChris Wilson } 147dc2f7e67SChris Wilson 148dc2f7e67SChris Wilson fc->chain_length = i; 149dc2f7e67SChris Wilson return 0; 150dc2f7e67SChris Wilson 151dc2f7e67SChris Wilson unwind: 152dc2f7e67SChris Wilson for (i = 0; i < count; i++) { 153dc2f7e67SChris Wilson dma_fence_put(fc->fences[i]); 154dc2f7e67SChris Wilson dma_fence_put(fc->chains[i]); 155dc2f7e67SChris Wilson } 156dc2f7e67SChris Wilson kvfree(fc->fences); 157dc2f7e67SChris Wilson err_chains: 158dc2f7e67SChris Wilson kvfree(fc->chains); 159dc2f7e67SChris Wilson return err; 160dc2f7e67SChris Wilson } 161dc2f7e67SChris Wilson 162dc2f7e67SChris Wilson static void fence_chains_fini(struct fence_chains *fc) 163dc2f7e67SChris Wilson { 164dc2f7e67SChris Wilson unsigned int i; 165dc2f7e67SChris Wilson 166dc2f7e67SChris Wilson for (i = 0; i < fc->chain_length; i++) { 167dc2f7e67SChris Wilson dma_fence_signal(fc->fences[i]); 168dc2f7e67SChris Wilson dma_fence_put(fc->fences[i]); 169dc2f7e67SChris Wilson } 170dc2f7e67SChris Wilson kvfree(fc->fences); 171dc2f7e67SChris Wilson 172dc2f7e67SChris Wilson for (i = 0; i < fc->chain_length; i++) 173dc2f7e67SChris Wilson dma_fence_put(fc->chains[i]); 174dc2f7e67SChris Wilson kvfree(fc->chains); 175dc2f7e67SChris Wilson } 176dc2f7e67SChris Wilson 177dc2f7e67SChris Wilson static int find_seqno(void *arg) 178dc2f7e67SChris Wilson { 179dc2f7e67SChris Wilson struct fence_chains fc; 180dc2f7e67SChris Wilson struct dma_fence *fence; 181dc2f7e67SChris Wilson int err; 182dc2f7e67SChris Wilson int i; 183dc2f7e67SChris Wilson 184dc2f7e67SChris Wilson err = fence_chains_init(&fc, 64, seqno_inc); 185dc2f7e67SChris Wilson if (err) 186dc2f7e67SChris Wilson return err; 187dc2f7e67SChris Wilson 188dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 189dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 0); 190dc2f7e67SChris Wilson dma_fence_put(fence); 191dc2f7e67SChris Wilson if (err) { 192dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno(0)!\n", err); 193dc2f7e67SChris Wilson goto err; 194dc2f7e67SChris Wilson } 195dc2f7e67SChris Wilson 196dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) { 197dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 198dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, i + 1); 199dc2f7e67SChris Wilson dma_fence_put(fence); 200dc2f7e67SChris Wilson if (err) { 201dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno(%d:%d)!\n", 202dc2f7e67SChris Wilson err, fc.chain_length + 1, i + 1); 203dc2f7e67SChris Wilson goto err; 204dc2f7e67SChris Wilson } 205dc2f7e67SChris Wilson if (fence != fc.chains[i]) { 206dc2f7e67SChris Wilson pr_err("Incorrect fence reported by find_seqno(%d:%d)\n", 207dc2f7e67SChris Wilson fc.chain_length + 1, i + 1); 208dc2f7e67SChris Wilson err = -EINVAL; 209dc2f7e67SChris Wilson goto err; 210dc2f7e67SChris Wilson } 211dc2f7e67SChris Wilson 212dc2f7e67SChris Wilson dma_fence_get(fence); 213dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, i + 1); 214dc2f7e67SChris Wilson dma_fence_put(fence); 215dc2f7e67SChris Wilson if (err) { 216dc2f7e67SChris Wilson pr_err("Error reported for finding self\n"); 217dc2f7e67SChris Wilson goto err; 218dc2f7e67SChris Wilson } 219dc2f7e67SChris Wilson if (fence != fc.chains[i]) { 220dc2f7e67SChris Wilson pr_err("Incorrect fence reported by find self\n"); 221dc2f7e67SChris Wilson err = -EINVAL; 222dc2f7e67SChris Wilson goto err; 223dc2f7e67SChris Wilson } 224dc2f7e67SChris Wilson 225dc2f7e67SChris Wilson dma_fence_get(fence); 226dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, i + 2); 227dc2f7e67SChris Wilson dma_fence_put(fence); 228dc2f7e67SChris Wilson if (!err) { 229dc2f7e67SChris Wilson pr_err("Error not reported for future fence: find_seqno(%d:%d)!\n", 230dc2f7e67SChris Wilson i + 1, i + 2); 231dc2f7e67SChris Wilson err = -EINVAL; 232dc2f7e67SChris Wilson goto err; 233dc2f7e67SChris Wilson } 234dc2f7e67SChris Wilson 235dc2f7e67SChris Wilson dma_fence_get(fence); 236dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, i); 237dc2f7e67SChris Wilson dma_fence_put(fence); 238dc2f7e67SChris Wilson if (err) { 239dc2f7e67SChris Wilson pr_err("Error reported for previous fence!\n"); 240dc2f7e67SChris Wilson goto err; 241dc2f7e67SChris Wilson } 242dc2f7e67SChris Wilson if (i > 0 && fence != fc.chains[i - 1]) { 243dc2f7e67SChris Wilson pr_err("Incorrect fence reported by find_seqno(%d:%d)\n", 244dc2f7e67SChris Wilson i + 1, i); 245dc2f7e67SChris Wilson err = -EINVAL; 246dc2f7e67SChris Wilson goto err; 247dc2f7e67SChris Wilson } 248dc2f7e67SChris Wilson } 249dc2f7e67SChris Wilson 250dc2f7e67SChris Wilson err: 251dc2f7e67SChris Wilson fence_chains_fini(&fc); 252dc2f7e67SChris Wilson return err; 253dc2f7e67SChris Wilson } 254dc2f7e67SChris Wilson 255dc2f7e67SChris Wilson static int find_signaled(void *arg) 256dc2f7e67SChris Wilson { 257dc2f7e67SChris Wilson struct fence_chains fc; 258dc2f7e67SChris Wilson struct dma_fence *fence; 259dc2f7e67SChris Wilson int err; 260dc2f7e67SChris Wilson 261dc2f7e67SChris Wilson err = fence_chains_init(&fc, 2, seqno_inc); 262dc2f7e67SChris Wilson if (err) 263dc2f7e67SChris Wilson return err; 264dc2f7e67SChris Wilson 265dc2f7e67SChris Wilson dma_fence_signal(fc.fences[0]); 266dc2f7e67SChris Wilson 267dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 268dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 1); 269dc2f7e67SChris Wilson dma_fence_put(fence); 270dc2f7e67SChris Wilson if (err) { 271dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno()!\n", err); 272dc2f7e67SChris Wilson goto err; 273dc2f7e67SChris Wilson } 274dc2f7e67SChris Wilson 275dc2f7e67SChris Wilson if (fence && fence != fc.chains[0]) { 276dc2f7e67SChris Wilson pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:1\n", 277dc2f7e67SChris Wilson fence->seqno); 278dc2f7e67SChris Wilson 279dc2f7e67SChris Wilson dma_fence_get(fence); 280dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 1); 281dc2f7e67SChris Wilson dma_fence_put(fence); 282dc2f7e67SChris Wilson if (err) 283dc2f7e67SChris Wilson pr_err("Reported %d for finding self!\n", err); 284dc2f7e67SChris Wilson 285dc2f7e67SChris Wilson err = -EINVAL; 286dc2f7e67SChris Wilson } 287dc2f7e67SChris Wilson 288dc2f7e67SChris Wilson err: 289dc2f7e67SChris Wilson fence_chains_fini(&fc); 290dc2f7e67SChris Wilson return err; 291dc2f7e67SChris Wilson } 292dc2f7e67SChris Wilson 293dc2f7e67SChris Wilson static int find_out_of_order(void *arg) 294dc2f7e67SChris Wilson { 295dc2f7e67SChris Wilson struct fence_chains fc; 296dc2f7e67SChris Wilson struct dma_fence *fence; 297dc2f7e67SChris Wilson int err; 298dc2f7e67SChris Wilson 299dc2f7e67SChris Wilson err = fence_chains_init(&fc, 3, seqno_inc); 300dc2f7e67SChris Wilson if (err) 301dc2f7e67SChris Wilson return err; 302dc2f7e67SChris Wilson 303dc2f7e67SChris Wilson dma_fence_signal(fc.fences[1]); 304dc2f7e67SChris Wilson 305dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 306dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 2); 307dc2f7e67SChris Wilson dma_fence_put(fence); 308dc2f7e67SChris Wilson if (err) { 309dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno()!\n", err); 310dc2f7e67SChris Wilson goto err; 311dc2f7e67SChris Wilson } 312dc2f7e67SChris Wilson 3134cca2e64SLionel Landwerlin /* 3144cca2e64SLionel Landwerlin * We signaled the middle fence (2) of the 1-2-3 chain. The behavior 3154cca2e64SLionel Landwerlin * of the dma-fence-chain is to make us wait for all the fences up to 3164cca2e64SLionel Landwerlin * the point we want. Since fence 1 is still not signaled, this what 3174cca2e64SLionel Landwerlin * we should get as fence to wait upon (fence 2 being garbage 3184cca2e64SLionel Landwerlin * collected during the traversal of the chain). 3194cca2e64SLionel Landwerlin */ 3204cca2e64SLionel Landwerlin if (fence != fc.chains[0]) { 321dc2f7e67SChris Wilson pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:2\n", 3224cca2e64SLionel Landwerlin fence ? fence->seqno : 0); 323dc2f7e67SChris Wilson 324dc2f7e67SChris Wilson err = -EINVAL; 325dc2f7e67SChris Wilson } 326dc2f7e67SChris Wilson 327dc2f7e67SChris Wilson err: 328dc2f7e67SChris Wilson fence_chains_fini(&fc); 329dc2f7e67SChris Wilson return err; 330dc2f7e67SChris Wilson } 331dc2f7e67SChris Wilson 332dc2f7e67SChris Wilson static uint64_t seqno_inc2(unsigned int i) 333dc2f7e67SChris Wilson { 334dc2f7e67SChris Wilson return 2 * i + 2; 335dc2f7e67SChris Wilson } 336dc2f7e67SChris Wilson 337dc2f7e67SChris Wilson static int find_gap(void *arg) 338dc2f7e67SChris Wilson { 339dc2f7e67SChris Wilson struct fence_chains fc; 340dc2f7e67SChris Wilson struct dma_fence *fence; 341dc2f7e67SChris Wilson int err; 342dc2f7e67SChris Wilson int i; 343dc2f7e67SChris Wilson 344dc2f7e67SChris Wilson err = fence_chains_init(&fc, 64, seqno_inc2); 345dc2f7e67SChris Wilson if (err) 346dc2f7e67SChris Wilson return err; 347dc2f7e67SChris Wilson 348dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) { 349dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 350dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 2 * i + 1); 351dc2f7e67SChris Wilson dma_fence_put(fence); 352dc2f7e67SChris Wilson if (err) { 353dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno(%d:%d)!\n", 354dc2f7e67SChris Wilson err, fc.chain_length + 1, 2 * i + 1); 355dc2f7e67SChris Wilson goto err; 356dc2f7e67SChris Wilson } 357dc2f7e67SChris Wilson if (fence != fc.chains[i]) { 358dc2f7e67SChris Wilson pr_err("Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)\n", 359dc2f7e67SChris Wilson fence->seqno, 360dc2f7e67SChris Wilson fc.chain_length + 1, 361dc2f7e67SChris Wilson 2 * i + 1); 362dc2f7e67SChris Wilson err = -EINVAL; 363dc2f7e67SChris Wilson goto err; 364dc2f7e67SChris Wilson } 365dc2f7e67SChris Wilson 366dc2f7e67SChris Wilson dma_fence_get(fence); 367dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 2 * i + 2); 368dc2f7e67SChris Wilson dma_fence_put(fence); 369dc2f7e67SChris Wilson if (err) { 370dc2f7e67SChris Wilson pr_err("Error reported for finding self\n"); 371dc2f7e67SChris Wilson goto err; 372dc2f7e67SChris Wilson } 373dc2f7e67SChris Wilson if (fence != fc.chains[i]) { 374dc2f7e67SChris Wilson pr_err("Incorrect fence reported by find self\n"); 375dc2f7e67SChris Wilson err = -EINVAL; 376dc2f7e67SChris Wilson goto err; 377dc2f7e67SChris Wilson } 378dc2f7e67SChris Wilson } 379dc2f7e67SChris Wilson 380dc2f7e67SChris Wilson err: 381dc2f7e67SChris Wilson fence_chains_fini(&fc); 382dc2f7e67SChris Wilson return err; 383dc2f7e67SChris Wilson } 384dc2f7e67SChris Wilson 385dc2f7e67SChris Wilson struct find_race { 386dc2f7e67SChris Wilson struct fence_chains fc; 387dc2f7e67SChris Wilson atomic_t children; 388dc2f7e67SChris Wilson }; 389dc2f7e67SChris Wilson 390dc2f7e67SChris Wilson static int __find_race(void *arg) 391dc2f7e67SChris Wilson { 392dc2f7e67SChris Wilson struct find_race *data = arg; 393dc2f7e67SChris Wilson int err = 0; 394dc2f7e67SChris Wilson 395dc2f7e67SChris Wilson while (!kthread_should_stop()) { 396dc2f7e67SChris Wilson struct dma_fence *fence = dma_fence_get(data->fc.tail); 397dc2f7e67SChris Wilson int seqno; 398dc2f7e67SChris Wilson 399dc2f7e67SChris Wilson seqno = prandom_u32_max(data->fc.chain_length) + 1; 400dc2f7e67SChris Wilson 401dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, seqno); 402dc2f7e67SChris Wilson if (err) { 403dc2f7e67SChris Wilson pr_err("Failed to find fence seqno:%d\n", 404dc2f7e67SChris Wilson seqno); 405dc2f7e67SChris Wilson dma_fence_put(fence); 406dc2f7e67SChris Wilson break; 407dc2f7e67SChris Wilson } 408dc2f7e67SChris Wilson if (!fence) 409dc2f7e67SChris Wilson goto signal; 410dc2f7e67SChris Wilson 4114cca2e64SLionel Landwerlin /* 4124cca2e64SLionel Landwerlin * We can only find ourselves if we are on fence we were 4134cca2e64SLionel Landwerlin * looking for. 4144cca2e64SLionel Landwerlin */ 4154cca2e64SLionel Landwerlin if (fence->seqno == seqno) { 416dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, seqno); 417dc2f7e67SChris Wilson if (err) { 418dc2f7e67SChris Wilson pr_err("Reported an invalid fence for find-self:%d\n", 419dc2f7e67SChris Wilson seqno); 420dc2f7e67SChris Wilson dma_fence_put(fence); 421dc2f7e67SChris Wilson break; 422dc2f7e67SChris Wilson } 423dc2f7e67SChris Wilson } 424dc2f7e67SChris Wilson 425dc2f7e67SChris Wilson dma_fence_put(fence); 426dc2f7e67SChris Wilson 427dc2f7e67SChris Wilson signal: 428dc2f7e67SChris Wilson seqno = prandom_u32_max(data->fc.chain_length - 1); 429dc2f7e67SChris Wilson dma_fence_signal(data->fc.fences[seqno]); 430dc2f7e67SChris Wilson cond_resched(); 431dc2f7e67SChris Wilson } 432dc2f7e67SChris Wilson 433dc2f7e67SChris Wilson if (atomic_dec_and_test(&data->children)) 434dc2f7e67SChris Wilson wake_up_var(&data->children); 435dc2f7e67SChris Wilson return err; 436dc2f7e67SChris Wilson } 437dc2f7e67SChris Wilson 438dc2f7e67SChris Wilson static int find_race(void *arg) 439dc2f7e67SChris Wilson { 440dc2f7e67SChris Wilson struct find_race data; 441dc2f7e67SChris Wilson int ncpus = num_online_cpus(); 442dc2f7e67SChris Wilson struct task_struct **threads; 443dc2f7e67SChris Wilson unsigned long count; 444dc2f7e67SChris Wilson int err; 445dc2f7e67SChris Wilson int i; 446dc2f7e67SChris Wilson 447dc2f7e67SChris Wilson err = fence_chains_init(&data.fc, CHAIN_SZ, seqno_inc); 448dc2f7e67SChris Wilson if (err) 449dc2f7e67SChris Wilson return err; 450dc2f7e67SChris Wilson 451dc2f7e67SChris Wilson threads = kmalloc_array(ncpus, sizeof(*threads), GFP_KERNEL); 452dc2f7e67SChris Wilson if (!threads) { 453dc2f7e67SChris Wilson err = -ENOMEM; 454dc2f7e67SChris Wilson goto err; 455dc2f7e67SChris Wilson } 456dc2f7e67SChris Wilson 457dc2f7e67SChris Wilson atomic_set(&data.children, 0); 458dc2f7e67SChris Wilson for (i = 0; i < ncpus; i++) { 459dc2f7e67SChris Wilson threads[i] = kthread_run(__find_race, &data, "dmabuf/%d", i); 460dc2f7e67SChris Wilson if (IS_ERR(threads[i])) { 461dc2f7e67SChris Wilson ncpus = i; 462dc2f7e67SChris Wilson break; 463dc2f7e67SChris Wilson } 464dc2f7e67SChris Wilson atomic_inc(&data.children); 465dc2f7e67SChris Wilson get_task_struct(threads[i]); 466dc2f7e67SChris Wilson } 467dc2f7e67SChris Wilson 468dc2f7e67SChris Wilson wait_var_event_timeout(&data.children, 469dc2f7e67SChris Wilson !atomic_read(&data.children), 470dc2f7e67SChris Wilson 5 * HZ); 471dc2f7e67SChris Wilson 472dc2f7e67SChris Wilson for (i = 0; i < ncpus; i++) { 473dc2f7e67SChris Wilson int ret; 474dc2f7e67SChris Wilson 475dc2f7e67SChris Wilson ret = kthread_stop(threads[i]); 476dc2f7e67SChris Wilson if (ret && !err) 477dc2f7e67SChris Wilson err = ret; 478dc2f7e67SChris Wilson put_task_struct(threads[i]); 479dc2f7e67SChris Wilson } 480dc2f7e67SChris Wilson kfree(threads); 481dc2f7e67SChris Wilson 482dc2f7e67SChris Wilson count = 0; 483dc2f7e67SChris Wilson for (i = 0; i < data.fc.chain_length; i++) 484dc2f7e67SChris Wilson if (dma_fence_is_signaled(data.fc.fences[i])) 485dc2f7e67SChris Wilson count++; 486dc2f7e67SChris Wilson pr_info("Completed %lu cycles\n", count); 487dc2f7e67SChris Wilson 488dc2f7e67SChris Wilson err: 489dc2f7e67SChris Wilson fence_chains_fini(&data.fc); 490dc2f7e67SChris Wilson return err; 491dc2f7e67SChris Wilson } 492dc2f7e67SChris Wilson 493dc2f7e67SChris Wilson static int signal_forward(void *arg) 494dc2f7e67SChris Wilson { 495dc2f7e67SChris Wilson struct fence_chains fc; 496dc2f7e67SChris Wilson int err; 497dc2f7e67SChris Wilson int i; 498dc2f7e67SChris Wilson 499dc2f7e67SChris Wilson err = fence_chains_init(&fc, 64, seqno_inc); 500dc2f7e67SChris Wilson if (err) 501dc2f7e67SChris Wilson return err; 502dc2f7e67SChris Wilson 503dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) { 504dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 505dc2f7e67SChris Wilson 506dc2f7e67SChris Wilson if (!dma_fence_is_signaled(fc.chains[i])) { 507dc2f7e67SChris Wilson pr_err("chain[%d] not signaled!\n", i); 508dc2f7e67SChris Wilson err = -EINVAL; 509dc2f7e67SChris Wilson goto err; 510dc2f7e67SChris Wilson } 511dc2f7e67SChris Wilson 512dc2f7e67SChris Wilson if (i + 1 < fc.chain_length && 513dc2f7e67SChris Wilson dma_fence_is_signaled(fc.chains[i + 1])) { 514dc2f7e67SChris Wilson pr_err("chain[%d] is signaled!\n", i); 515dc2f7e67SChris Wilson err = -EINVAL; 516dc2f7e67SChris Wilson goto err; 517dc2f7e67SChris Wilson } 518dc2f7e67SChris Wilson } 519dc2f7e67SChris Wilson 520dc2f7e67SChris Wilson err: 521dc2f7e67SChris Wilson fence_chains_fini(&fc); 522dc2f7e67SChris Wilson return err; 523dc2f7e67SChris Wilson } 524dc2f7e67SChris Wilson 525dc2f7e67SChris Wilson static int signal_backward(void *arg) 526dc2f7e67SChris Wilson { 527dc2f7e67SChris Wilson struct fence_chains fc; 528dc2f7e67SChris Wilson int err; 529dc2f7e67SChris Wilson int i; 530dc2f7e67SChris Wilson 531dc2f7e67SChris Wilson err = fence_chains_init(&fc, 64, seqno_inc); 532dc2f7e67SChris Wilson if (err) 533dc2f7e67SChris Wilson return err; 534dc2f7e67SChris Wilson 535dc2f7e67SChris Wilson for (i = fc.chain_length; i--; ) { 536dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 537dc2f7e67SChris Wilson 538dc2f7e67SChris Wilson if (i > 0 && dma_fence_is_signaled(fc.chains[i])) { 539dc2f7e67SChris Wilson pr_err("chain[%d] is signaled!\n", i); 540dc2f7e67SChris Wilson err = -EINVAL; 541dc2f7e67SChris Wilson goto err; 542dc2f7e67SChris Wilson } 543dc2f7e67SChris Wilson } 544dc2f7e67SChris Wilson 545dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) { 546dc2f7e67SChris Wilson if (!dma_fence_is_signaled(fc.chains[i])) { 547dc2f7e67SChris Wilson pr_err("chain[%d] was not signaled!\n", i); 548dc2f7e67SChris Wilson err = -EINVAL; 549dc2f7e67SChris Wilson goto err; 550dc2f7e67SChris Wilson } 551dc2f7e67SChris Wilson } 552dc2f7e67SChris Wilson 553dc2f7e67SChris Wilson err: 554dc2f7e67SChris Wilson fence_chains_fini(&fc); 555dc2f7e67SChris Wilson return err; 556dc2f7e67SChris Wilson } 557dc2f7e67SChris Wilson 558dc2f7e67SChris Wilson static int __wait_fence_chains(void *arg) 559dc2f7e67SChris Wilson { 560dc2f7e67SChris Wilson struct fence_chains *fc = arg; 561dc2f7e67SChris Wilson 562dc2f7e67SChris Wilson if (dma_fence_wait(fc->tail, false)) 563dc2f7e67SChris Wilson return -EIO; 564dc2f7e67SChris Wilson 565dc2f7e67SChris Wilson return 0; 566dc2f7e67SChris Wilson } 567dc2f7e67SChris Wilson 568dc2f7e67SChris Wilson static int wait_forward(void *arg) 569dc2f7e67SChris Wilson { 570dc2f7e67SChris Wilson struct fence_chains fc; 571dc2f7e67SChris Wilson struct task_struct *tsk; 572dc2f7e67SChris Wilson int err; 573dc2f7e67SChris Wilson int i; 574dc2f7e67SChris Wilson 575dc2f7e67SChris Wilson err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 576dc2f7e67SChris Wilson if (err) 577dc2f7e67SChris Wilson return err; 578dc2f7e67SChris Wilson 579dc2f7e67SChris Wilson tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 580dc2f7e67SChris Wilson if (IS_ERR(tsk)) { 581dc2f7e67SChris Wilson err = PTR_ERR(tsk); 582dc2f7e67SChris Wilson goto err; 583dc2f7e67SChris Wilson } 584dc2f7e67SChris Wilson get_task_struct(tsk); 585dc2f7e67SChris Wilson yield_to(tsk, true); 586dc2f7e67SChris Wilson 587dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) 588dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 589dc2f7e67SChris Wilson 590dc2f7e67SChris Wilson err = kthread_stop(tsk); 591dc2f7e67SChris Wilson put_task_struct(tsk); 592dc2f7e67SChris Wilson 593dc2f7e67SChris Wilson err: 594dc2f7e67SChris Wilson fence_chains_fini(&fc); 595dc2f7e67SChris Wilson return err; 596dc2f7e67SChris Wilson } 597dc2f7e67SChris Wilson 598dc2f7e67SChris Wilson static int wait_backward(void *arg) 599dc2f7e67SChris Wilson { 600dc2f7e67SChris Wilson struct fence_chains fc; 601dc2f7e67SChris Wilson struct task_struct *tsk; 602dc2f7e67SChris Wilson int err; 603dc2f7e67SChris Wilson int i; 604dc2f7e67SChris Wilson 605dc2f7e67SChris Wilson err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 606dc2f7e67SChris Wilson if (err) 607dc2f7e67SChris Wilson return err; 608dc2f7e67SChris Wilson 609dc2f7e67SChris Wilson tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 610dc2f7e67SChris Wilson if (IS_ERR(tsk)) { 611dc2f7e67SChris Wilson err = PTR_ERR(tsk); 612dc2f7e67SChris Wilson goto err; 613dc2f7e67SChris Wilson } 614dc2f7e67SChris Wilson get_task_struct(tsk); 615dc2f7e67SChris Wilson yield_to(tsk, true); 616dc2f7e67SChris Wilson 617dc2f7e67SChris Wilson for (i = fc.chain_length; i--; ) 618dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 619dc2f7e67SChris Wilson 620dc2f7e67SChris Wilson err = kthread_stop(tsk); 621dc2f7e67SChris Wilson put_task_struct(tsk); 622dc2f7e67SChris Wilson 623dc2f7e67SChris Wilson err: 624dc2f7e67SChris Wilson fence_chains_fini(&fc); 625dc2f7e67SChris Wilson return err; 626dc2f7e67SChris Wilson } 627dc2f7e67SChris Wilson 628dc2f7e67SChris Wilson static void randomise_fences(struct fence_chains *fc) 629dc2f7e67SChris Wilson { 630dc2f7e67SChris Wilson unsigned int count = fc->chain_length; 631dc2f7e67SChris Wilson 632dc2f7e67SChris Wilson /* Fisher-Yates shuffle courtesy of Knuth */ 633dc2f7e67SChris Wilson while (--count) { 634dc2f7e67SChris Wilson unsigned int swp; 635dc2f7e67SChris Wilson 636dc2f7e67SChris Wilson swp = prandom_u32_max(count + 1); 637dc2f7e67SChris Wilson if (swp == count) 638dc2f7e67SChris Wilson continue; 639dc2f7e67SChris Wilson 640dc2f7e67SChris Wilson swap(fc->fences[count], fc->fences[swp]); 641dc2f7e67SChris Wilson } 642dc2f7e67SChris Wilson } 643dc2f7e67SChris Wilson 644dc2f7e67SChris Wilson static int wait_random(void *arg) 645dc2f7e67SChris Wilson { 646dc2f7e67SChris Wilson struct fence_chains fc; 647dc2f7e67SChris Wilson struct task_struct *tsk; 648dc2f7e67SChris Wilson int err; 649dc2f7e67SChris Wilson int i; 650dc2f7e67SChris Wilson 651dc2f7e67SChris Wilson err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 652dc2f7e67SChris Wilson if (err) 653dc2f7e67SChris Wilson return err; 654dc2f7e67SChris Wilson 655dc2f7e67SChris Wilson randomise_fences(&fc); 656dc2f7e67SChris Wilson 657dc2f7e67SChris Wilson tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 658dc2f7e67SChris Wilson if (IS_ERR(tsk)) { 659dc2f7e67SChris Wilson err = PTR_ERR(tsk); 660dc2f7e67SChris Wilson goto err; 661dc2f7e67SChris Wilson } 662dc2f7e67SChris Wilson get_task_struct(tsk); 663dc2f7e67SChris Wilson yield_to(tsk, true); 664dc2f7e67SChris Wilson 665dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) 666dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 667dc2f7e67SChris Wilson 668dc2f7e67SChris Wilson err = kthread_stop(tsk); 669dc2f7e67SChris Wilson put_task_struct(tsk); 670dc2f7e67SChris Wilson 671dc2f7e67SChris Wilson err: 672dc2f7e67SChris Wilson fence_chains_fini(&fc); 673dc2f7e67SChris Wilson return err; 674dc2f7e67SChris Wilson } 675dc2f7e67SChris Wilson 676dc2f7e67SChris Wilson int dma_fence_chain(void) 677dc2f7e67SChris Wilson { 678dc2f7e67SChris Wilson static const struct subtest tests[] = { 679dc2f7e67SChris Wilson SUBTEST(sanitycheck), 680dc2f7e67SChris Wilson SUBTEST(find_seqno), 681dc2f7e67SChris Wilson SUBTEST(find_signaled), 682dc2f7e67SChris Wilson SUBTEST(find_out_of_order), 683dc2f7e67SChris Wilson SUBTEST(find_gap), 684dc2f7e67SChris Wilson SUBTEST(find_race), 685dc2f7e67SChris Wilson SUBTEST(signal_forward), 686dc2f7e67SChris Wilson SUBTEST(signal_backward), 687dc2f7e67SChris Wilson SUBTEST(wait_forward), 688dc2f7e67SChris Wilson SUBTEST(wait_backward), 689dc2f7e67SChris Wilson SUBTEST(wait_random), 690dc2f7e67SChris Wilson }; 691dc2f7e67SChris Wilson int ret; 692dc2f7e67SChris Wilson 693dc2f7e67SChris Wilson pr_info("sizeof(dma_fence_chain)=%zu\n", 694dc2f7e67SChris Wilson sizeof(struct dma_fence_chain)); 695dc2f7e67SChris Wilson 696dc2f7e67SChris Wilson slab_fences = KMEM_CACHE(mock_fence, 697dc2f7e67SChris Wilson SLAB_TYPESAFE_BY_RCU | 698dc2f7e67SChris Wilson SLAB_HWCACHE_ALIGN); 699dc2f7e67SChris Wilson if (!slab_fences) 700dc2f7e67SChris Wilson return -ENOMEM; 701dc2f7e67SChris Wilson 702dc2f7e67SChris Wilson ret = subtests(tests, NULL); 703dc2f7e67SChris Wilson 704dc2f7e67SChris Wilson kmem_cache_destroy(slab_fences); 705dc2f7e67SChris Wilson return ret; 706dc2f7e67SChris Wilson } 707