1dc2f7e67SChris Wilson // SPDX-License-Identifier: MIT 2dc2f7e67SChris Wilson 3dc2f7e67SChris Wilson /* 4dc2f7e67SChris Wilson * Copyright © 2019 Intel Corporation 5dc2f7e67SChris Wilson */ 6dc2f7e67SChris Wilson 7dc2f7e67SChris Wilson #include <linux/delay.h> 8dc2f7e67SChris Wilson #include <linux/dma-fence.h> 9dc2f7e67SChris Wilson #include <linux/dma-fence-chain.h> 10dc2f7e67SChris Wilson #include <linux/kernel.h> 11dc2f7e67SChris Wilson #include <linux/kthread.h> 12dc2f7e67SChris Wilson #include <linux/mm.h> 13dc2f7e67SChris Wilson #include <linux/sched/signal.h> 14dc2f7e67SChris Wilson #include <linux/slab.h> 15dc2f7e67SChris Wilson #include <linux/spinlock.h> 16dc2f7e67SChris Wilson #include <linux/random.h> 17dc2f7e67SChris Wilson 18dc2f7e67SChris Wilson #include "selftest.h" 19dc2f7e67SChris Wilson 20dc2f7e67SChris Wilson #define CHAIN_SZ (4 << 10) 21dc2f7e67SChris Wilson 22dc2f7e67SChris Wilson static struct kmem_cache *slab_fences; 23dc2f7e67SChris Wilson 24dc2f7e67SChris Wilson static inline struct mock_fence { 25dc2f7e67SChris Wilson struct dma_fence base; 26dc2f7e67SChris Wilson spinlock_t lock; 27dc2f7e67SChris Wilson } *to_mock_fence(struct dma_fence *f) { 28dc2f7e67SChris Wilson return container_of(f, struct mock_fence, base); 29dc2f7e67SChris Wilson } 30dc2f7e67SChris Wilson 31dc2f7e67SChris Wilson static const char *mock_name(struct dma_fence *f) 32dc2f7e67SChris Wilson { 33dc2f7e67SChris Wilson return "mock"; 34dc2f7e67SChris Wilson } 35dc2f7e67SChris Wilson 36dc2f7e67SChris Wilson static void mock_fence_release(struct dma_fence *f) 37dc2f7e67SChris Wilson { 38dc2f7e67SChris Wilson kmem_cache_free(slab_fences, to_mock_fence(f)); 39dc2f7e67SChris Wilson } 40dc2f7e67SChris Wilson 41dc2f7e67SChris Wilson static const struct dma_fence_ops mock_ops = { 42dc2f7e67SChris Wilson .get_driver_name = mock_name, 43dc2f7e67SChris Wilson .get_timeline_name = mock_name, 44dc2f7e67SChris Wilson .release = mock_fence_release, 45dc2f7e67SChris Wilson }; 46dc2f7e67SChris Wilson 47dc2f7e67SChris Wilson static struct dma_fence *mock_fence(void) 48dc2f7e67SChris Wilson { 49dc2f7e67SChris Wilson struct mock_fence *f; 50dc2f7e67SChris Wilson 51dc2f7e67SChris Wilson f = kmem_cache_alloc(slab_fences, GFP_KERNEL); 52dc2f7e67SChris Wilson if (!f) 53dc2f7e67SChris Wilson return NULL; 54dc2f7e67SChris Wilson 55dc2f7e67SChris Wilson spin_lock_init(&f->lock); 56dc2f7e67SChris Wilson dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0); 57dc2f7e67SChris Wilson 58dc2f7e67SChris Wilson return &f->base; 59dc2f7e67SChris Wilson } 60dc2f7e67SChris Wilson 61dc2f7e67SChris Wilson static struct dma_fence *mock_chain(struct dma_fence *prev, 62dc2f7e67SChris Wilson struct dma_fence *fence, 63dc2f7e67SChris Wilson u64 seqno) 64dc2f7e67SChris Wilson { 65440d0f12SChristian König struct dma_fence_chain *f; 66dc2f7e67SChris Wilson 67440d0f12SChristian König f = dma_fence_chain_alloc(); 68dc2f7e67SChris Wilson if (!f) 69dc2f7e67SChris Wilson return NULL; 70dc2f7e67SChris Wilson 71440d0f12SChristian König dma_fence_chain_init(f, dma_fence_get(prev), dma_fence_get(fence), 72dc2f7e67SChris Wilson seqno); 73dc2f7e67SChris Wilson 74440d0f12SChristian König return &f->base; 75dc2f7e67SChris Wilson } 76dc2f7e67SChris Wilson 77dc2f7e67SChris Wilson static int sanitycheck(void *arg) 78dc2f7e67SChris Wilson { 79dc2f7e67SChris Wilson struct dma_fence *f, *chain; 80dc2f7e67SChris Wilson int err = 0; 81dc2f7e67SChris Wilson 82dc2f7e67SChris Wilson f = mock_fence(); 83dc2f7e67SChris Wilson if (!f) 84dc2f7e67SChris Wilson return -ENOMEM; 85dc2f7e67SChris Wilson 86dc2f7e67SChris Wilson chain = mock_chain(NULL, f, 1); 87dc2f7e67SChris Wilson if (!chain) 88dc2f7e67SChris Wilson err = -ENOMEM; 89dc2f7e67SChris Wilson 90d62c43a9SArvind Yadav dma_fence_enable_sw_signaling(chain); 91d62c43a9SArvind Yadav 92dc2f7e67SChris Wilson dma_fence_signal(f); 93dc2f7e67SChris Wilson dma_fence_put(f); 94dc2f7e67SChris Wilson 95dc2f7e67SChris Wilson dma_fence_put(chain); 96dc2f7e67SChris Wilson 97dc2f7e67SChris Wilson return err; 98dc2f7e67SChris Wilson } 99dc2f7e67SChris Wilson 100dc2f7e67SChris Wilson struct fence_chains { 101dc2f7e67SChris Wilson unsigned int chain_length; 102dc2f7e67SChris Wilson struct dma_fence **fences; 103dc2f7e67SChris Wilson struct dma_fence **chains; 104dc2f7e67SChris Wilson 105dc2f7e67SChris Wilson struct dma_fence *tail; 106dc2f7e67SChris Wilson }; 107dc2f7e67SChris Wilson 108dc2f7e67SChris Wilson static uint64_t seqno_inc(unsigned int i) 109dc2f7e67SChris Wilson { 110dc2f7e67SChris Wilson return i + 1; 111dc2f7e67SChris Wilson } 112dc2f7e67SChris Wilson 113dc2f7e67SChris Wilson static int fence_chains_init(struct fence_chains *fc, unsigned int count, 114dc2f7e67SChris Wilson uint64_t (*seqno_fn)(unsigned int)) 115dc2f7e67SChris Wilson { 116dc2f7e67SChris Wilson unsigned int i; 117dc2f7e67SChris Wilson int err = 0; 118dc2f7e67SChris Wilson 119dc2f7e67SChris Wilson fc->chains = kvmalloc_array(count, sizeof(*fc->chains), 120dc2f7e67SChris Wilson GFP_KERNEL | __GFP_ZERO); 121dc2f7e67SChris Wilson if (!fc->chains) 122dc2f7e67SChris Wilson return -ENOMEM; 123dc2f7e67SChris Wilson 124dc2f7e67SChris Wilson fc->fences = kvmalloc_array(count, sizeof(*fc->fences), 125dc2f7e67SChris Wilson GFP_KERNEL | __GFP_ZERO); 126dc2f7e67SChris Wilson if (!fc->fences) { 127dc2f7e67SChris Wilson err = -ENOMEM; 128dc2f7e67SChris Wilson goto err_chains; 129dc2f7e67SChris Wilson } 130dc2f7e67SChris Wilson 131dc2f7e67SChris Wilson fc->tail = NULL; 132dc2f7e67SChris Wilson for (i = 0; i < count; i++) { 133dc2f7e67SChris Wilson fc->fences[i] = mock_fence(); 134dc2f7e67SChris Wilson if (!fc->fences[i]) { 135dc2f7e67SChris Wilson err = -ENOMEM; 136dc2f7e67SChris Wilson goto unwind; 137dc2f7e67SChris Wilson } 138dc2f7e67SChris Wilson 139dc2f7e67SChris Wilson fc->chains[i] = mock_chain(fc->tail, 140dc2f7e67SChris Wilson fc->fences[i], 141dc2f7e67SChris Wilson seqno_fn(i)); 142dc2f7e67SChris Wilson if (!fc->chains[i]) { 143dc2f7e67SChris Wilson err = -ENOMEM; 144dc2f7e67SChris Wilson goto unwind; 145dc2f7e67SChris Wilson } 146dc2f7e67SChris Wilson 147dc2f7e67SChris Wilson fc->tail = fc->chains[i]; 148d62c43a9SArvind Yadav 149d62c43a9SArvind Yadav dma_fence_enable_sw_signaling(fc->chains[i]); 150dc2f7e67SChris Wilson } 151dc2f7e67SChris Wilson 152dc2f7e67SChris Wilson fc->chain_length = i; 153dc2f7e67SChris Wilson return 0; 154dc2f7e67SChris Wilson 155dc2f7e67SChris Wilson unwind: 156dc2f7e67SChris Wilson for (i = 0; i < count; i++) { 157dc2f7e67SChris Wilson dma_fence_put(fc->fences[i]); 158dc2f7e67SChris Wilson dma_fence_put(fc->chains[i]); 159dc2f7e67SChris Wilson } 160dc2f7e67SChris Wilson kvfree(fc->fences); 161dc2f7e67SChris Wilson err_chains: 162dc2f7e67SChris Wilson kvfree(fc->chains); 163dc2f7e67SChris Wilson return err; 164dc2f7e67SChris Wilson } 165dc2f7e67SChris Wilson 166dc2f7e67SChris Wilson static void fence_chains_fini(struct fence_chains *fc) 167dc2f7e67SChris Wilson { 168dc2f7e67SChris Wilson unsigned int i; 169dc2f7e67SChris Wilson 170dc2f7e67SChris Wilson for (i = 0; i < fc->chain_length; i++) { 171dc2f7e67SChris Wilson dma_fence_signal(fc->fences[i]); 172dc2f7e67SChris Wilson dma_fence_put(fc->fences[i]); 173dc2f7e67SChris Wilson } 174dc2f7e67SChris Wilson kvfree(fc->fences); 175dc2f7e67SChris Wilson 176dc2f7e67SChris Wilson for (i = 0; i < fc->chain_length; i++) 177dc2f7e67SChris Wilson dma_fence_put(fc->chains[i]); 178dc2f7e67SChris Wilson kvfree(fc->chains); 179dc2f7e67SChris Wilson } 180dc2f7e67SChris Wilson 181dc2f7e67SChris Wilson static int find_seqno(void *arg) 182dc2f7e67SChris Wilson { 183dc2f7e67SChris Wilson struct fence_chains fc; 184dc2f7e67SChris Wilson struct dma_fence *fence; 185dc2f7e67SChris Wilson int err; 186dc2f7e67SChris Wilson int i; 187dc2f7e67SChris Wilson 188dc2f7e67SChris Wilson err = fence_chains_init(&fc, 64, seqno_inc); 189dc2f7e67SChris Wilson if (err) 190dc2f7e67SChris Wilson return err; 191dc2f7e67SChris Wilson 192dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 193dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 0); 194dc2f7e67SChris Wilson dma_fence_put(fence); 195dc2f7e67SChris Wilson if (err) { 196dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno(0)!\n", err); 197dc2f7e67SChris Wilson goto err; 198dc2f7e67SChris Wilson } 199dc2f7e67SChris Wilson 200dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) { 201dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 202dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, i + 1); 203dc2f7e67SChris Wilson dma_fence_put(fence); 204dc2f7e67SChris Wilson if (err) { 205dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno(%d:%d)!\n", 206dc2f7e67SChris Wilson err, fc.chain_length + 1, i + 1); 207dc2f7e67SChris Wilson goto err; 208dc2f7e67SChris Wilson } 209dc2f7e67SChris Wilson if (fence != fc.chains[i]) { 210dc2f7e67SChris Wilson pr_err("Incorrect fence reported by find_seqno(%d:%d)\n", 211dc2f7e67SChris Wilson fc.chain_length + 1, i + 1); 212dc2f7e67SChris Wilson err = -EINVAL; 213dc2f7e67SChris Wilson goto err; 214dc2f7e67SChris Wilson } 215dc2f7e67SChris Wilson 216dc2f7e67SChris Wilson dma_fence_get(fence); 217dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, i + 1); 218dc2f7e67SChris Wilson dma_fence_put(fence); 219dc2f7e67SChris Wilson if (err) { 220dc2f7e67SChris Wilson pr_err("Error reported for finding self\n"); 221dc2f7e67SChris Wilson goto err; 222dc2f7e67SChris Wilson } 223dc2f7e67SChris Wilson if (fence != fc.chains[i]) { 224dc2f7e67SChris Wilson pr_err("Incorrect fence reported by find self\n"); 225dc2f7e67SChris Wilson err = -EINVAL; 226dc2f7e67SChris Wilson goto err; 227dc2f7e67SChris Wilson } 228dc2f7e67SChris Wilson 229dc2f7e67SChris Wilson dma_fence_get(fence); 230dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, i + 2); 231dc2f7e67SChris Wilson dma_fence_put(fence); 232dc2f7e67SChris Wilson if (!err) { 233dc2f7e67SChris Wilson pr_err("Error not reported for future fence: find_seqno(%d:%d)!\n", 234dc2f7e67SChris Wilson i + 1, i + 2); 235dc2f7e67SChris Wilson err = -EINVAL; 236dc2f7e67SChris Wilson goto err; 237dc2f7e67SChris Wilson } 238dc2f7e67SChris Wilson 239dc2f7e67SChris Wilson dma_fence_get(fence); 240dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, i); 241dc2f7e67SChris Wilson dma_fence_put(fence); 242dc2f7e67SChris Wilson if (err) { 243dc2f7e67SChris Wilson pr_err("Error reported for previous fence!\n"); 244dc2f7e67SChris Wilson goto err; 245dc2f7e67SChris Wilson } 246dc2f7e67SChris Wilson if (i > 0 && fence != fc.chains[i - 1]) { 247dc2f7e67SChris Wilson pr_err("Incorrect fence reported by find_seqno(%d:%d)\n", 248dc2f7e67SChris Wilson i + 1, i); 249dc2f7e67SChris Wilson err = -EINVAL; 250dc2f7e67SChris Wilson goto err; 251dc2f7e67SChris Wilson } 252dc2f7e67SChris Wilson } 253dc2f7e67SChris Wilson 254dc2f7e67SChris Wilson err: 255dc2f7e67SChris Wilson fence_chains_fini(&fc); 256dc2f7e67SChris Wilson return err; 257dc2f7e67SChris Wilson } 258dc2f7e67SChris Wilson 259dc2f7e67SChris Wilson static int find_signaled(void *arg) 260dc2f7e67SChris Wilson { 261dc2f7e67SChris Wilson struct fence_chains fc; 262dc2f7e67SChris Wilson struct dma_fence *fence; 263dc2f7e67SChris Wilson int err; 264dc2f7e67SChris Wilson 265dc2f7e67SChris Wilson err = fence_chains_init(&fc, 2, seqno_inc); 266dc2f7e67SChris Wilson if (err) 267dc2f7e67SChris Wilson return err; 268dc2f7e67SChris Wilson 269dc2f7e67SChris Wilson dma_fence_signal(fc.fences[0]); 270dc2f7e67SChris Wilson 271dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 272dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 1); 273dc2f7e67SChris Wilson dma_fence_put(fence); 274dc2f7e67SChris Wilson if (err) { 275dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno()!\n", err); 276dc2f7e67SChris Wilson goto err; 277dc2f7e67SChris Wilson } 278dc2f7e67SChris Wilson 279dc2f7e67SChris Wilson if (fence && fence != fc.chains[0]) { 280dc2f7e67SChris Wilson pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:1\n", 281dc2f7e67SChris Wilson fence->seqno); 282dc2f7e67SChris Wilson 283dc2f7e67SChris Wilson dma_fence_get(fence); 284dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 1); 285dc2f7e67SChris Wilson dma_fence_put(fence); 286dc2f7e67SChris Wilson if (err) 287dc2f7e67SChris Wilson pr_err("Reported %d for finding self!\n", err); 288dc2f7e67SChris Wilson 289dc2f7e67SChris Wilson err = -EINVAL; 290dc2f7e67SChris Wilson } 291dc2f7e67SChris Wilson 292dc2f7e67SChris Wilson err: 293dc2f7e67SChris Wilson fence_chains_fini(&fc); 294dc2f7e67SChris Wilson return err; 295dc2f7e67SChris Wilson } 296dc2f7e67SChris Wilson 297dc2f7e67SChris Wilson static int find_out_of_order(void *arg) 298dc2f7e67SChris Wilson { 299dc2f7e67SChris Wilson struct fence_chains fc; 300dc2f7e67SChris Wilson struct dma_fence *fence; 301dc2f7e67SChris Wilson int err; 302dc2f7e67SChris Wilson 303dc2f7e67SChris Wilson err = fence_chains_init(&fc, 3, seqno_inc); 304dc2f7e67SChris Wilson if (err) 305dc2f7e67SChris Wilson return err; 306dc2f7e67SChris Wilson 307dc2f7e67SChris Wilson dma_fence_signal(fc.fences[1]); 308dc2f7e67SChris Wilson 309dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 310dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 2); 311dc2f7e67SChris Wilson dma_fence_put(fence); 312dc2f7e67SChris Wilson if (err) { 313dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno()!\n", err); 314dc2f7e67SChris Wilson goto err; 315dc2f7e67SChris Wilson } 316dc2f7e67SChris Wilson 3174cca2e64SLionel Landwerlin /* 3184cca2e64SLionel Landwerlin * We signaled the middle fence (2) of the 1-2-3 chain. The behavior 3194cca2e64SLionel Landwerlin * of the dma-fence-chain is to make us wait for all the fences up to 3204cca2e64SLionel Landwerlin * the point we want. Since fence 1 is still not signaled, this what 3214cca2e64SLionel Landwerlin * we should get as fence to wait upon (fence 2 being garbage 3224cca2e64SLionel Landwerlin * collected during the traversal of the chain). 3234cca2e64SLionel Landwerlin */ 3244cca2e64SLionel Landwerlin if (fence != fc.chains[0]) { 325dc2f7e67SChris Wilson pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:2\n", 3264cca2e64SLionel Landwerlin fence ? fence->seqno : 0); 327dc2f7e67SChris Wilson 328dc2f7e67SChris Wilson err = -EINVAL; 329dc2f7e67SChris Wilson } 330dc2f7e67SChris Wilson 331dc2f7e67SChris Wilson err: 332dc2f7e67SChris Wilson fence_chains_fini(&fc); 333dc2f7e67SChris Wilson return err; 334dc2f7e67SChris Wilson } 335dc2f7e67SChris Wilson 336dc2f7e67SChris Wilson static uint64_t seqno_inc2(unsigned int i) 337dc2f7e67SChris Wilson { 338dc2f7e67SChris Wilson return 2 * i + 2; 339dc2f7e67SChris Wilson } 340dc2f7e67SChris Wilson 341dc2f7e67SChris Wilson static int find_gap(void *arg) 342dc2f7e67SChris Wilson { 343dc2f7e67SChris Wilson struct fence_chains fc; 344dc2f7e67SChris Wilson struct dma_fence *fence; 345dc2f7e67SChris Wilson int err; 346dc2f7e67SChris Wilson int i; 347dc2f7e67SChris Wilson 348dc2f7e67SChris Wilson err = fence_chains_init(&fc, 64, seqno_inc2); 349dc2f7e67SChris Wilson if (err) 350dc2f7e67SChris Wilson return err; 351dc2f7e67SChris Wilson 352dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) { 353dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 354dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 2 * i + 1); 355dc2f7e67SChris Wilson dma_fence_put(fence); 356dc2f7e67SChris Wilson if (err) { 357dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno(%d:%d)!\n", 358dc2f7e67SChris Wilson err, fc.chain_length + 1, 2 * i + 1); 359dc2f7e67SChris Wilson goto err; 360dc2f7e67SChris Wilson } 361dc2f7e67SChris Wilson if (fence != fc.chains[i]) { 362dc2f7e67SChris Wilson pr_err("Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)\n", 363dc2f7e67SChris Wilson fence->seqno, 364dc2f7e67SChris Wilson fc.chain_length + 1, 365dc2f7e67SChris Wilson 2 * i + 1); 366dc2f7e67SChris Wilson err = -EINVAL; 367dc2f7e67SChris Wilson goto err; 368dc2f7e67SChris Wilson } 369dc2f7e67SChris Wilson 370dc2f7e67SChris Wilson dma_fence_get(fence); 371dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 2 * i + 2); 372dc2f7e67SChris Wilson dma_fence_put(fence); 373dc2f7e67SChris Wilson if (err) { 374dc2f7e67SChris Wilson pr_err("Error reported for finding self\n"); 375dc2f7e67SChris Wilson goto err; 376dc2f7e67SChris Wilson } 377dc2f7e67SChris Wilson if (fence != fc.chains[i]) { 378dc2f7e67SChris Wilson pr_err("Incorrect fence reported by find self\n"); 379dc2f7e67SChris Wilson err = -EINVAL; 380dc2f7e67SChris Wilson goto err; 381dc2f7e67SChris Wilson } 382dc2f7e67SChris Wilson } 383dc2f7e67SChris Wilson 384dc2f7e67SChris Wilson err: 385dc2f7e67SChris Wilson fence_chains_fini(&fc); 386dc2f7e67SChris Wilson return err; 387dc2f7e67SChris Wilson } 388dc2f7e67SChris Wilson 389dc2f7e67SChris Wilson struct find_race { 390dc2f7e67SChris Wilson struct fence_chains fc; 391dc2f7e67SChris Wilson atomic_t children; 392dc2f7e67SChris Wilson }; 393dc2f7e67SChris Wilson 394dc2f7e67SChris Wilson static int __find_race(void *arg) 395dc2f7e67SChris Wilson { 396dc2f7e67SChris Wilson struct find_race *data = arg; 397dc2f7e67SChris Wilson int err = 0; 398dc2f7e67SChris Wilson 399dc2f7e67SChris Wilson while (!kthread_should_stop()) { 400dc2f7e67SChris Wilson struct dma_fence *fence = dma_fence_get(data->fc.tail); 401dc2f7e67SChris Wilson int seqno; 402dc2f7e67SChris Wilson 403*8032bf12SJason A. Donenfeld seqno = get_random_u32_below(data->fc.chain_length) + 1; 404dc2f7e67SChris Wilson 405dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, seqno); 406dc2f7e67SChris Wilson if (err) { 407dc2f7e67SChris Wilson pr_err("Failed to find fence seqno:%d\n", 408dc2f7e67SChris Wilson seqno); 409dc2f7e67SChris Wilson dma_fence_put(fence); 410dc2f7e67SChris Wilson break; 411dc2f7e67SChris Wilson } 412dc2f7e67SChris Wilson if (!fence) 413dc2f7e67SChris Wilson goto signal; 414dc2f7e67SChris Wilson 4154cca2e64SLionel Landwerlin /* 4164cca2e64SLionel Landwerlin * We can only find ourselves if we are on fence we were 4174cca2e64SLionel Landwerlin * looking for. 4184cca2e64SLionel Landwerlin */ 4194cca2e64SLionel Landwerlin if (fence->seqno == seqno) { 420dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, seqno); 421dc2f7e67SChris Wilson if (err) { 422dc2f7e67SChris Wilson pr_err("Reported an invalid fence for find-self:%d\n", 423dc2f7e67SChris Wilson seqno); 424dc2f7e67SChris Wilson dma_fence_put(fence); 425dc2f7e67SChris Wilson break; 426dc2f7e67SChris Wilson } 427dc2f7e67SChris Wilson } 428dc2f7e67SChris Wilson 429dc2f7e67SChris Wilson dma_fence_put(fence); 430dc2f7e67SChris Wilson 431dc2f7e67SChris Wilson signal: 432*8032bf12SJason A. Donenfeld seqno = get_random_u32_below(data->fc.chain_length - 1); 433dc2f7e67SChris Wilson dma_fence_signal(data->fc.fences[seqno]); 434dc2f7e67SChris Wilson cond_resched(); 435dc2f7e67SChris Wilson } 436dc2f7e67SChris Wilson 437dc2f7e67SChris Wilson if (atomic_dec_and_test(&data->children)) 438dc2f7e67SChris Wilson wake_up_var(&data->children); 439dc2f7e67SChris Wilson return err; 440dc2f7e67SChris Wilson } 441dc2f7e67SChris Wilson 442dc2f7e67SChris Wilson static int find_race(void *arg) 443dc2f7e67SChris Wilson { 444dc2f7e67SChris Wilson struct find_race data; 445dc2f7e67SChris Wilson int ncpus = num_online_cpus(); 446dc2f7e67SChris Wilson struct task_struct **threads; 447dc2f7e67SChris Wilson unsigned long count; 448dc2f7e67SChris Wilson int err; 449dc2f7e67SChris Wilson int i; 450dc2f7e67SChris Wilson 451dc2f7e67SChris Wilson err = fence_chains_init(&data.fc, CHAIN_SZ, seqno_inc); 452dc2f7e67SChris Wilson if (err) 453dc2f7e67SChris Wilson return err; 454dc2f7e67SChris Wilson 455dc2f7e67SChris Wilson threads = kmalloc_array(ncpus, sizeof(*threads), GFP_KERNEL); 456dc2f7e67SChris Wilson if (!threads) { 457dc2f7e67SChris Wilson err = -ENOMEM; 458dc2f7e67SChris Wilson goto err; 459dc2f7e67SChris Wilson } 460dc2f7e67SChris Wilson 461dc2f7e67SChris Wilson atomic_set(&data.children, 0); 462dc2f7e67SChris Wilson for (i = 0; i < ncpus; i++) { 463dc2f7e67SChris Wilson threads[i] = kthread_run(__find_race, &data, "dmabuf/%d", i); 464dc2f7e67SChris Wilson if (IS_ERR(threads[i])) { 465dc2f7e67SChris Wilson ncpus = i; 466dc2f7e67SChris Wilson break; 467dc2f7e67SChris Wilson } 468dc2f7e67SChris Wilson atomic_inc(&data.children); 469dc2f7e67SChris Wilson get_task_struct(threads[i]); 470dc2f7e67SChris Wilson } 471dc2f7e67SChris Wilson 472dc2f7e67SChris Wilson wait_var_event_timeout(&data.children, 473dc2f7e67SChris Wilson !atomic_read(&data.children), 474dc2f7e67SChris Wilson 5 * HZ); 475dc2f7e67SChris Wilson 476dc2f7e67SChris Wilson for (i = 0; i < ncpus; i++) { 477dc2f7e67SChris Wilson int ret; 478dc2f7e67SChris Wilson 479dc2f7e67SChris Wilson ret = kthread_stop(threads[i]); 480dc2f7e67SChris Wilson if (ret && !err) 481dc2f7e67SChris Wilson err = ret; 482dc2f7e67SChris Wilson put_task_struct(threads[i]); 483dc2f7e67SChris Wilson } 484dc2f7e67SChris Wilson kfree(threads); 485dc2f7e67SChris Wilson 486dc2f7e67SChris Wilson count = 0; 487dc2f7e67SChris Wilson for (i = 0; i < data.fc.chain_length; i++) 488dc2f7e67SChris Wilson if (dma_fence_is_signaled(data.fc.fences[i])) 489dc2f7e67SChris Wilson count++; 490dc2f7e67SChris Wilson pr_info("Completed %lu cycles\n", count); 491dc2f7e67SChris Wilson 492dc2f7e67SChris Wilson err: 493dc2f7e67SChris Wilson fence_chains_fini(&data.fc); 494dc2f7e67SChris Wilson return err; 495dc2f7e67SChris Wilson } 496dc2f7e67SChris Wilson 497dc2f7e67SChris Wilson static int signal_forward(void *arg) 498dc2f7e67SChris Wilson { 499dc2f7e67SChris Wilson struct fence_chains fc; 500dc2f7e67SChris Wilson int err; 501dc2f7e67SChris Wilson int i; 502dc2f7e67SChris Wilson 503dc2f7e67SChris Wilson err = fence_chains_init(&fc, 64, seqno_inc); 504dc2f7e67SChris Wilson if (err) 505dc2f7e67SChris Wilson return err; 506dc2f7e67SChris Wilson 507dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) { 508dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 509dc2f7e67SChris Wilson 510dc2f7e67SChris Wilson if (!dma_fence_is_signaled(fc.chains[i])) { 511dc2f7e67SChris Wilson pr_err("chain[%d] not signaled!\n", i); 512dc2f7e67SChris Wilson err = -EINVAL; 513dc2f7e67SChris Wilson goto err; 514dc2f7e67SChris Wilson } 515dc2f7e67SChris Wilson 516dc2f7e67SChris Wilson if (i + 1 < fc.chain_length && 517dc2f7e67SChris Wilson dma_fence_is_signaled(fc.chains[i + 1])) { 518dc2f7e67SChris Wilson pr_err("chain[%d] is signaled!\n", i); 519dc2f7e67SChris Wilson err = -EINVAL; 520dc2f7e67SChris Wilson goto err; 521dc2f7e67SChris Wilson } 522dc2f7e67SChris Wilson } 523dc2f7e67SChris Wilson 524dc2f7e67SChris Wilson err: 525dc2f7e67SChris Wilson fence_chains_fini(&fc); 526dc2f7e67SChris Wilson return err; 527dc2f7e67SChris Wilson } 528dc2f7e67SChris Wilson 529dc2f7e67SChris Wilson static int signal_backward(void *arg) 530dc2f7e67SChris Wilson { 531dc2f7e67SChris Wilson struct fence_chains fc; 532dc2f7e67SChris Wilson int err; 533dc2f7e67SChris Wilson int i; 534dc2f7e67SChris Wilson 535dc2f7e67SChris Wilson err = fence_chains_init(&fc, 64, seqno_inc); 536dc2f7e67SChris Wilson if (err) 537dc2f7e67SChris Wilson return err; 538dc2f7e67SChris Wilson 539dc2f7e67SChris Wilson for (i = fc.chain_length; i--; ) { 540dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 541dc2f7e67SChris Wilson 542dc2f7e67SChris Wilson if (i > 0 && dma_fence_is_signaled(fc.chains[i])) { 543dc2f7e67SChris Wilson pr_err("chain[%d] is signaled!\n", i); 544dc2f7e67SChris Wilson err = -EINVAL; 545dc2f7e67SChris Wilson goto err; 546dc2f7e67SChris Wilson } 547dc2f7e67SChris Wilson } 548dc2f7e67SChris Wilson 549dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) { 550dc2f7e67SChris Wilson if (!dma_fence_is_signaled(fc.chains[i])) { 551dc2f7e67SChris Wilson pr_err("chain[%d] was not signaled!\n", i); 552dc2f7e67SChris Wilson err = -EINVAL; 553dc2f7e67SChris Wilson goto err; 554dc2f7e67SChris Wilson } 555dc2f7e67SChris Wilson } 556dc2f7e67SChris Wilson 557dc2f7e67SChris Wilson err: 558dc2f7e67SChris Wilson fence_chains_fini(&fc); 559dc2f7e67SChris Wilson return err; 560dc2f7e67SChris Wilson } 561dc2f7e67SChris Wilson 562dc2f7e67SChris Wilson static int __wait_fence_chains(void *arg) 563dc2f7e67SChris Wilson { 564dc2f7e67SChris Wilson struct fence_chains *fc = arg; 565dc2f7e67SChris Wilson 566dc2f7e67SChris Wilson if (dma_fence_wait(fc->tail, false)) 567dc2f7e67SChris Wilson return -EIO; 568dc2f7e67SChris Wilson 569dc2f7e67SChris Wilson return 0; 570dc2f7e67SChris Wilson } 571dc2f7e67SChris Wilson 572dc2f7e67SChris Wilson static int wait_forward(void *arg) 573dc2f7e67SChris Wilson { 574dc2f7e67SChris Wilson struct fence_chains fc; 575dc2f7e67SChris Wilson struct task_struct *tsk; 576dc2f7e67SChris Wilson int err; 577dc2f7e67SChris Wilson int i; 578dc2f7e67SChris Wilson 579dc2f7e67SChris Wilson err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 580dc2f7e67SChris Wilson if (err) 581dc2f7e67SChris Wilson return err; 582dc2f7e67SChris Wilson 583dc2f7e67SChris Wilson tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 584dc2f7e67SChris Wilson if (IS_ERR(tsk)) { 585dc2f7e67SChris Wilson err = PTR_ERR(tsk); 586dc2f7e67SChris Wilson goto err; 587dc2f7e67SChris Wilson } 588dc2f7e67SChris Wilson get_task_struct(tsk); 589dc2f7e67SChris Wilson yield_to(tsk, true); 590dc2f7e67SChris Wilson 591dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) 592dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 593dc2f7e67SChris Wilson 594dc2f7e67SChris Wilson err = kthread_stop(tsk); 595dc2f7e67SChris Wilson put_task_struct(tsk); 596dc2f7e67SChris Wilson 597dc2f7e67SChris Wilson err: 598dc2f7e67SChris Wilson fence_chains_fini(&fc); 599dc2f7e67SChris Wilson return err; 600dc2f7e67SChris Wilson } 601dc2f7e67SChris Wilson 602dc2f7e67SChris Wilson static int wait_backward(void *arg) 603dc2f7e67SChris Wilson { 604dc2f7e67SChris Wilson struct fence_chains fc; 605dc2f7e67SChris Wilson struct task_struct *tsk; 606dc2f7e67SChris Wilson int err; 607dc2f7e67SChris Wilson int i; 608dc2f7e67SChris Wilson 609dc2f7e67SChris Wilson err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 610dc2f7e67SChris Wilson if (err) 611dc2f7e67SChris Wilson return err; 612dc2f7e67SChris Wilson 613dc2f7e67SChris Wilson tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 614dc2f7e67SChris Wilson if (IS_ERR(tsk)) { 615dc2f7e67SChris Wilson err = PTR_ERR(tsk); 616dc2f7e67SChris Wilson goto err; 617dc2f7e67SChris Wilson } 618dc2f7e67SChris Wilson get_task_struct(tsk); 619dc2f7e67SChris Wilson yield_to(tsk, true); 620dc2f7e67SChris Wilson 621dc2f7e67SChris Wilson for (i = fc.chain_length; i--; ) 622dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 623dc2f7e67SChris Wilson 624dc2f7e67SChris Wilson err = kthread_stop(tsk); 625dc2f7e67SChris Wilson put_task_struct(tsk); 626dc2f7e67SChris Wilson 627dc2f7e67SChris Wilson err: 628dc2f7e67SChris Wilson fence_chains_fini(&fc); 629dc2f7e67SChris Wilson return err; 630dc2f7e67SChris Wilson } 631dc2f7e67SChris Wilson 632dc2f7e67SChris Wilson static void randomise_fences(struct fence_chains *fc) 633dc2f7e67SChris Wilson { 634dc2f7e67SChris Wilson unsigned int count = fc->chain_length; 635dc2f7e67SChris Wilson 636dc2f7e67SChris Wilson /* Fisher-Yates shuffle courtesy of Knuth */ 637dc2f7e67SChris Wilson while (--count) { 638dc2f7e67SChris Wilson unsigned int swp; 639dc2f7e67SChris Wilson 640*8032bf12SJason A. Donenfeld swp = get_random_u32_below(count + 1); 641dc2f7e67SChris Wilson if (swp == count) 642dc2f7e67SChris Wilson continue; 643dc2f7e67SChris Wilson 644dc2f7e67SChris Wilson swap(fc->fences[count], fc->fences[swp]); 645dc2f7e67SChris Wilson } 646dc2f7e67SChris Wilson } 647dc2f7e67SChris Wilson 648dc2f7e67SChris Wilson static int wait_random(void *arg) 649dc2f7e67SChris Wilson { 650dc2f7e67SChris Wilson struct fence_chains fc; 651dc2f7e67SChris Wilson struct task_struct *tsk; 652dc2f7e67SChris Wilson int err; 653dc2f7e67SChris Wilson int i; 654dc2f7e67SChris Wilson 655dc2f7e67SChris Wilson err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 656dc2f7e67SChris Wilson if (err) 657dc2f7e67SChris Wilson return err; 658dc2f7e67SChris Wilson 659dc2f7e67SChris Wilson randomise_fences(&fc); 660dc2f7e67SChris Wilson 661dc2f7e67SChris Wilson tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 662dc2f7e67SChris Wilson if (IS_ERR(tsk)) { 663dc2f7e67SChris Wilson err = PTR_ERR(tsk); 664dc2f7e67SChris Wilson goto err; 665dc2f7e67SChris Wilson } 666dc2f7e67SChris Wilson get_task_struct(tsk); 667dc2f7e67SChris Wilson yield_to(tsk, true); 668dc2f7e67SChris Wilson 669dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) 670dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 671dc2f7e67SChris Wilson 672dc2f7e67SChris Wilson err = kthread_stop(tsk); 673dc2f7e67SChris Wilson put_task_struct(tsk); 674dc2f7e67SChris Wilson 675dc2f7e67SChris Wilson err: 676dc2f7e67SChris Wilson fence_chains_fini(&fc); 677dc2f7e67SChris Wilson return err; 678dc2f7e67SChris Wilson } 679dc2f7e67SChris Wilson 680dc2f7e67SChris Wilson int dma_fence_chain(void) 681dc2f7e67SChris Wilson { 682dc2f7e67SChris Wilson static const struct subtest tests[] = { 683dc2f7e67SChris Wilson SUBTEST(sanitycheck), 684dc2f7e67SChris Wilson SUBTEST(find_seqno), 685dc2f7e67SChris Wilson SUBTEST(find_signaled), 686dc2f7e67SChris Wilson SUBTEST(find_out_of_order), 687dc2f7e67SChris Wilson SUBTEST(find_gap), 688dc2f7e67SChris Wilson SUBTEST(find_race), 689dc2f7e67SChris Wilson SUBTEST(signal_forward), 690dc2f7e67SChris Wilson SUBTEST(signal_backward), 691dc2f7e67SChris Wilson SUBTEST(wait_forward), 692dc2f7e67SChris Wilson SUBTEST(wait_backward), 693dc2f7e67SChris Wilson SUBTEST(wait_random), 694dc2f7e67SChris Wilson }; 695dc2f7e67SChris Wilson int ret; 696dc2f7e67SChris Wilson 697dc2f7e67SChris Wilson pr_info("sizeof(dma_fence_chain)=%zu\n", 698dc2f7e67SChris Wilson sizeof(struct dma_fence_chain)); 699dc2f7e67SChris Wilson 700dc2f7e67SChris Wilson slab_fences = KMEM_CACHE(mock_fence, 701dc2f7e67SChris Wilson SLAB_TYPESAFE_BY_RCU | 702dc2f7e67SChris Wilson SLAB_HWCACHE_ALIGN); 703dc2f7e67SChris Wilson if (!slab_fences) 704dc2f7e67SChris Wilson return -ENOMEM; 705dc2f7e67SChris Wilson 706dc2f7e67SChris Wilson ret = subtests(tests, NULL); 707dc2f7e67SChris Wilson 708dc2f7e67SChris Wilson kmem_cache_destroy(slab_fences); 709dc2f7e67SChris Wilson return ret; 710dc2f7e67SChris Wilson } 711