1dc2f7e67SChris Wilson // SPDX-License-Identifier: MIT 2dc2f7e67SChris Wilson 3dc2f7e67SChris Wilson /* 4dc2f7e67SChris Wilson * Copyright © 2019 Intel Corporation 5dc2f7e67SChris Wilson */ 6dc2f7e67SChris Wilson 7dc2f7e67SChris Wilson #include <linux/delay.h> 8dc2f7e67SChris Wilson #include <linux/dma-fence.h> 9dc2f7e67SChris Wilson #include <linux/dma-fence-chain.h> 10dc2f7e67SChris Wilson #include <linux/kernel.h> 11dc2f7e67SChris Wilson #include <linux/kthread.h> 12dc2f7e67SChris Wilson #include <linux/mm.h> 13dc2f7e67SChris Wilson #include <linux/sched/signal.h> 14dc2f7e67SChris Wilson #include <linux/slab.h> 15dc2f7e67SChris Wilson #include <linux/spinlock.h> 16dc2f7e67SChris Wilson #include <linux/random.h> 17dc2f7e67SChris Wilson 18dc2f7e67SChris Wilson #include "selftest.h" 19dc2f7e67SChris Wilson 20dc2f7e67SChris Wilson #define CHAIN_SZ (4 << 10) 21dc2f7e67SChris Wilson 22dc2f7e67SChris Wilson static struct kmem_cache *slab_fences; 23dc2f7e67SChris Wilson 24dc2f7e67SChris Wilson static inline struct mock_fence { 25dc2f7e67SChris Wilson struct dma_fence base; 26dc2f7e67SChris Wilson spinlock_t lock; 27dc2f7e67SChris Wilson } *to_mock_fence(struct dma_fence *f) { 28dc2f7e67SChris Wilson return container_of(f, struct mock_fence, base); 29dc2f7e67SChris Wilson } 30dc2f7e67SChris Wilson 31dc2f7e67SChris Wilson static const char *mock_name(struct dma_fence *f) 32dc2f7e67SChris Wilson { 33dc2f7e67SChris Wilson return "mock"; 34dc2f7e67SChris Wilson } 35dc2f7e67SChris Wilson 36dc2f7e67SChris Wilson static void mock_fence_release(struct dma_fence *f) 37dc2f7e67SChris Wilson { 38dc2f7e67SChris Wilson kmem_cache_free(slab_fences, to_mock_fence(f)); 39dc2f7e67SChris Wilson } 40dc2f7e67SChris Wilson 41dc2f7e67SChris Wilson static const struct dma_fence_ops mock_ops = { 42dc2f7e67SChris Wilson .get_driver_name = mock_name, 43dc2f7e67SChris Wilson .get_timeline_name = mock_name, 44dc2f7e67SChris Wilson .release = mock_fence_release, 45dc2f7e67SChris Wilson }; 46dc2f7e67SChris Wilson 47dc2f7e67SChris Wilson static struct dma_fence *mock_fence(void) 48dc2f7e67SChris Wilson { 49dc2f7e67SChris Wilson struct mock_fence *f; 50dc2f7e67SChris Wilson 51dc2f7e67SChris Wilson f = kmem_cache_alloc(slab_fences, GFP_KERNEL); 52dc2f7e67SChris Wilson if (!f) 53dc2f7e67SChris Wilson return NULL; 54dc2f7e67SChris Wilson 55dc2f7e67SChris Wilson spin_lock_init(&f->lock); 56dc2f7e67SChris Wilson dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0); 57dc2f7e67SChris Wilson 58dc2f7e67SChris Wilson return &f->base; 59dc2f7e67SChris Wilson } 60dc2f7e67SChris Wilson 61dc2f7e67SChris Wilson static inline struct mock_chain { 62dc2f7e67SChris Wilson struct dma_fence_chain base; 63dc2f7e67SChris Wilson } *to_mock_chain(struct dma_fence *f) { 64dc2f7e67SChris Wilson return container_of(f, struct mock_chain, base.base); 65dc2f7e67SChris Wilson } 66dc2f7e67SChris Wilson 67dc2f7e67SChris Wilson static struct dma_fence *mock_chain(struct dma_fence *prev, 68dc2f7e67SChris Wilson struct dma_fence *fence, 69dc2f7e67SChris Wilson u64 seqno) 70dc2f7e67SChris Wilson { 71dc2f7e67SChris Wilson struct mock_chain *f; 72dc2f7e67SChris Wilson 73dc2f7e67SChris Wilson f = kmalloc(sizeof(*f), GFP_KERNEL); 74dc2f7e67SChris Wilson if (!f) 75dc2f7e67SChris Wilson return NULL; 76dc2f7e67SChris Wilson 77dc2f7e67SChris Wilson dma_fence_chain_init(&f->base, 78dc2f7e67SChris Wilson dma_fence_get(prev), 79dc2f7e67SChris Wilson dma_fence_get(fence), 80dc2f7e67SChris Wilson seqno); 81dc2f7e67SChris Wilson 82dc2f7e67SChris Wilson return &f->base.base; 83dc2f7e67SChris Wilson } 84dc2f7e67SChris Wilson 85dc2f7e67SChris Wilson static int sanitycheck(void *arg) 86dc2f7e67SChris Wilson { 87dc2f7e67SChris Wilson struct dma_fence *f, *chain; 88dc2f7e67SChris Wilson int err = 0; 89dc2f7e67SChris Wilson 90dc2f7e67SChris Wilson f = mock_fence(); 91dc2f7e67SChris Wilson if (!f) 92dc2f7e67SChris Wilson return -ENOMEM; 93dc2f7e67SChris Wilson 94dc2f7e67SChris Wilson chain = mock_chain(NULL, f, 1); 95dc2f7e67SChris Wilson if (!chain) 96dc2f7e67SChris Wilson err = -ENOMEM; 97dc2f7e67SChris Wilson 98dc2f7e67SChris Wilson dma_fence_signal(f); 99dc2f7e67SChris Wilson dma_fence_put(f); 100dc2f7e67SChris Wilson 101dc2f7e67SChris Wilson dma_fence_put(chain); 102dc2f7e67SChris Wilson 103dc2f7e67SChris Wilson return err; 104dc2f7e67SChris Wilson } 105dc2f7e67SChris Wilson 106dc2f7e67SChris Wilson struct fence_chains { 107dc2f7e67SChris Wilson unsigned int chain_length; 108dc2f7e67SChris Wilson struct dma_fence **fences; 109dc2f7e67SChris Wilson struct dma_fence **chains; 110dc2f7e67SChris Wilson 111dc2f7e67SChris Wilson struct dma_fence *tail; 112dc2f7e67SChris Wilson }; 113dc2f7e67SChris Wilson 114dc2f7e67SChris Wilson static uint64_t seqno_inc(unsigned int i) 115dc2f7e67SChris Wilson { 116dc2f7e67SChris Wilson return i + 1; 117dc2f7e67SChris Wilson } 118dc2f7e67SChris Wilson 119dc2f7e67SChris Wilson static int fence_chains_init(struct fence_chains *fc, unsigned int count, 120dc2f7e67SChris Wilson uint64_t (*seqno_fn)(unsigned int)) 121dc2f7e67SChris Wilson { 122dc2f7e67SChris Wilson unsigned int i; 123dc2f7e67SChris Wilson int err = 0; 124dc2f7e67SChris Wilson 125dc2f7e67SChris Wilson fc->chains = kvmalloc_array(count, sizeof(*fc->chains), 126dc2f7e67SChris Wilson GFP_KERNEL | __GFP_ZERO); 127dc2f7e67SChris Wilson if (!fc->chains) 128dc2f7e67SChris Wilson return -ENOMEM; 129dc2f7e67SChris Wilson 130dc2f7e67SChris Wilson fc->fences = kvmalloc_array(count, sizeof(*fc->fences), 131dc2f7e67SChris Wilson GFP_KERNEL | __GFP_ZERO); 132dc2f7e67SChris Wilson if (!fc->fences) { 133dc2f7e67SChris Wilson err = -ENOMEM; 134dc2f7e67SChris Wilson goto err_chains; 135dc2f7e67SChris Wilson } 136dc2f7e67SChris Wilson 137dc2f7e67SChris Wilson fc->tail = NULL; 138dc2f7e67SChris Wilson for (i = 0; i < count; i++) { 139dc2f7e67SChris Wilson fc->fences[i] = mock_fence(); 140dc2f7e67SChris Wilson if (!fc->fences[i]) { 141dc2f7e67SChris Wilson err = -ENOMEM; 142dc2f7e67SChris Wilson goto unwind; 143dc2f7e67SChris Wilson } 144dc2f7e67SChris Wilson 145dc2f7e67SChris Wilson fc->chains[i] = mock_chain(fc->tail, 146dc2f7e67SChris Wilson fc->fences[i], 147dc2f7e67SChris Wilson seqno_fn(i)); 148dc2f7e67SChris Wilson if (!fc->chains[i]) { 149dc2f7e67SChris Wilson err = -ENOMEM; 150dc2f7e67SChris Wilson goto unwind; 151dc2f7e67SChris Wilson } 152dc2f7e67SChris Wilson 153dc2f7e67SChris Wilson fc->tail = fc->chains[i]; 154dc2f7e67SChris Wilson } 155dc2f7e67SChris Wilson 156dc2f7e67SChris Wilson fc->chain_length = i; 157dc2f7e67SChris Wilson return 0; 158dc2f7e67SChris Wilson 159dc2f7e67SChris Wilson unwind: 160dc2f7e67SChris Wilson for (i = 0; i < count; i++) { 161dc2f7e67SChris Wilson dma_fence_put(fc->fences[i]); 162dc2f7e67SChris Wilson dma_fence_put(fc->chains[i]); 163dc2f7e67SChris Wilson } 164dc2f7e67SChris Wilson kvfree(fc->fences); 165dc2f7e67SChris Wilson err_chains: 166dc2f7e67SChris Wilson kvfree(fc->chains); 167dc2f7e67SChris Wilson return err; 168dc2f7e67SChris Wilson } 169dc2f7e67SChris Wilson 170dc2f7e67SChris Wilson static void fence_chains_fini(struct fence_chains *fc) 171dc2f7e67SChris Wilson { 172dc2f7e67SChris Wilson unsigned int i; 173dc2f7e67SChris Wilson 174dc2f7e67SChris Wilson for (i = 0; i < fc->chain_length; i++) { 175dc2f7e67SChris Wilson dma_fence_signal(fc->fences[i]); 176dc2f7e67SChris Wilson dma_fence_put(fc->fences[i]); 177dc2f7e67SChris Wilson } 178dc2f7e67SChris Wilson kvfree(fc->fences); 179dc2f7e67SChris Wilson 180dc2f7e67SChris Wilson for (i = 0; i < fc->chain_length; i++) 181dc2f7e67SChris Wilson dma_fence_put(fc->chains[i]); 182dc2f7e67SChris Wilson kvfree(fc->chains); 183dc2f7e67SChris Wilson } 184dc2f7e67SChris Wilson 185dc2f7e67SChris Wilson static int find_seqno(void *arg) 186dc2f7e67SChris Wilson { 187dc2f7e67SChris Wilson struct fence_chains fc; 188dc2f7e67SChris Wilson struct dma_fence *fence; 189dc2f7e67SChris Wilson int err; 190dc2f7e67SChris Wilson int i; 191dc2f7e67SChris Wilson 192dc2f7e67SChris Wilson err = fence_chains_init(&fc, 64, seqno_inc); 193dc2f7e67SChris Wilson if (err) 194dc2f7e67SChris Wilson return err; 195dc2f7e67SChris Wilson 196dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 197dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 0); 198dc2f7e67SChris Wilson dma_fence_put(fence); 199dc2f7e67SChris Wilson if (err) { 200dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno(0)!\n", err); 201dc2f7e67SChris Wilson goto err; 202dc2f7e67SChris Wilson } 203dc2f7e67SChris Wilson 204dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) { 205dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 206dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, i + 1); 207dc2f7e67SChris Wilson dma_fence_put(fence); 208dc2f7e67SChris Wilson if (err) { 209dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno(%d:%d)!\n", 210dc2f7e67SChris Wilson err, fc.chain_length + 1, i + 1); 211dc2f7e67SChris Wilson goto err; 212dc2f7e67SChris Wilson } 213dc2f7e67SChris Wilson if (fence != fc.chains[i]) { 214dc2f7e67SChris Wilson pr_err("Incorrect fence reported by find_seqno(%d:%d)\n", 215dc2f7e67SChris Wilson fc.chain_length + 1, i + 1); 216dc2f7e67SChris Wilson err = -EINVAL; 217dc2f7e67SChris Wilson goto err; 218dc2f7e67SChris Wilson } 219dc2f7e67SChris Wilson 220dc2f7e67SChris Wilson dma_fence_get(fence); 221dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, i + 1); 222dc2f7e67SChris Wilson dma_fence_put(fence); 223dc2f7e67SChris Wilson if (err) { 224dc2f7e67SChris Wilson pr_err("Error reported for finding self\n"); 225dc2f7e67SChris Wilson goto err; 226dc2f7e67SChris Wilson } 227dc2f7e67SChris Wilson if (fence != fc.chains[i]) { 228dc2f7e67SChris Wilson pr_err("Incorrect fence reported by find self\n"); 229dc2f7e67SChris Wilson err = -EINVAL; 230dc2f7e67SChris Wilson goto err; 231dc2f7e67SChris Wilson } 232dc2f7e67SChris Wilson 233dc2f7e67SChris Wilson dma_fence_get(fence); 234dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, i + 2); 235dc2f7e67SChris Wilson dma_fence_put(fence); 236dc2f7e67SChris Wilson if (!err) { 237dc2f7e67SChris Wilson pr_err("Error not reported for future fence: find_seqno(%d:%d)!\n", 238dc2f7e67SChris Wilson i + 1, i + 2); 239dc2f7e67SChris Wilson err = -EINVAL; 240dc2f7e67SChris Wilson goto err; 241dc2f7e67SChris Wilson } 242dc2f7e67SChris Wilson 243dc2f7e67SChris Wilson dma_fence_get(fence); 244dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, i); 245dc2f7e67SChris Wilson dma_fence_put(fence); 246dc2f7e67SChris Wilson if (err) { 247dc2f7e67SChris Wilson pr_err("Error reported for previous fence!\n"); 248dc2f7e67SChris Wilson goto err; 249dc2f7e67SChris Wilson } 250dc2f7e67SChris Wilson if (i > 0 && fence != fc.chains[i - 1]) { 251dc2f7e67SChris Wilson pr_err("Incorrect fence reported by find_seqno(%d:%d)\n", 252dc2f7e67SChris Wilson i + 1, i); 253dc2f7e67SChris Wilson err = -EINVAL; 254dc2f7e67SChris Wilson goto err; 255dc2f7e67SChris Wilson } 256dc2f7e67SChris Wilson } 257dc2f7e67SChris Wilson 258dc2f7e67SChris Wilson err: 259dc2f7e67SChris Wilson fence_chains_fini(&fc); 260dc2f7e67SChris Wilson return err; 261dc2f7e67SChris Wilson } 262dc2f7e67SChris Wilson 263dc2f7e67SChris Wilson static int find_signaled(void *arg) 264dc2f7e67SChris Wilson { 265dc2f7e67SChris Wilson struct fence_chains fc; 266dc2f7e67SChris Wilson struct dma_fence *fence; 267dc2f7e67SChris Wilson int err; 268dc2f7e67SChris Wilson 269dc2f7e67SChris Wilson err = fence_chains_init(&fc, 2, seqno_inc); 270dc2f7e67SChris Wilson if (err) 271dc2f7e67SChris Wilson return err; 272dc2f7e67SChris Wilson 273dc2f7e67SChris Wilson dma_fence_signal(fc.fences[0]); 274dc2f7e67SChris Wilson 275dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 276dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 1); 277dc2f7e67SChris Wilson dma_fence_put(fence); 278dc2f7e67SChris Wilson if (err) { 279dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno()!\n", err); 280dc2f7e67SChris Wilson goto err; 281dc2f7e67SChris Wilson } 282dc2f7e67SChris Wilson 283dc2f7e67SChris Wilson if (fence && fence != fc.chains[0]) { 284dc2f7e67SChris Wilson pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:1\n", 285dc2f7e67SChris Wilson fence->seqno); 286dc2f7e67SChris Wilson 287dc2f7e67SChris Wilson dma_fence_get(fence); 288dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 1); 289dc2f7e67SChris Wilson dma_fence_put(fence); 290dc2f7e67SChris Wilson if (err) 291dc2f7e67SChris Wilson pr_err("Reported %d for finding self!\n", err); 292dc2f7e67SChris Wilson 293dc2f7e67SChris Wilson err = -EINVAL; 294dc2f7e67SChris Wilson } 295dc2f7e67SChris Wilson 296dc2f7e67SChris Wilson err: 297dc2f7e67SChris Wilson fence_chains_fini(&fc); 298dc2f7e67SChris Wilson return err; 299dc2f7e67SChris Wilson } 300dc2f7e67SChris Wilson 301dc2f7e67SChris Wilson static int find_out_of_order(void *arg) 302dc2f7e67SChris Wilson { 303dc2f7e67SChris Wilson struct fence_chains fc; 304dc2f7e67SChris Wilson struct dma_fence *fence; 305dc2f7e67SChris Wilson int err; 306dc2f7e67SChris Wilson 307dc2f7e67SChris Wilson err = fence_chains_init(&fc, 3, seqno_inc); 308dc2f7e67SChris Wilson if (err) 309dc2f7e67SChris Wilson return err; 310dc2f7e67SChris Wilson 311dc2f7e67SChris Wilson dma_fence_signal(fc.fences[1]); 312dc2f7e67SChris Wilson 313dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 314dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 2); 315dc2f7e67SChris Wilson dma_fence_put(fence); 316dc2f7e67SChris Wilson if (err) { 317dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno()!\n", err); 318dc2f7e67SChris Wilson goto err; 319dc2f7e67SChris Wilson } 320dc2f7e67SChris Wilson 321dc2f7e67SChris Wilson if (fence && fence != fc.chains[1]) { 322dc2f7e67SChris Wilson pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:2\n", 323dc2f7e67SChris Wilson fence->seqno); 324dc2f7e67SChris Wilson 325dc2f7e67SChris Wilson dma_fence_get(fence); 326dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 2); 327dc2f7e67SChris Wilson dma_fence_put(fence); 328dc2f7e67SChris Wilson if (err) 329dc2f7e67SChris Wilson pr_err("Reported %d for finding self!\n", err); 330dc2f7e67SChris Wilson 331dc2f7e67SChris Wilson err = -EINVAL; 332dc2f7e67SChris Wilson } 333dc2f7e67SChris Wilson 334dc2f7e67SChris Wilson err: 335dc2f7e67SChris Wilson fence_chains_fini(&fc); 336dc2f7e67SChris Wilson return err; 337dc2f7e67SChris Wilson } 338dc2f7e67SChris Wilson 339dc2f7e67SChris Wilson static uint64_t seqno_inc2(unsigned int i) 340dc2f7e67SChris Wilson { 341dc2f7e67SChris Wilson return 2 * i + 2; 342dc2f7e67SChris Wilson } 343dc2f7e67SChris Wilson 344dc2f7e67SChris Wilson static int find_gap(void *arg) 345dc2f7e67SChris Wilson { 346dc2f7e67SChris Wilson struct fence_chains fc; 347dc2f7e67SChris Wilson struct dma_fence *fence; 348dc2f7e67SChris Wilson int err; 349dc2f7e67SChris Wilson int i; 350dc2f7e67SChris Wilson 351dc2f7e67SChris Wilson err = fence_chains_init(&fc, 64, seqno_inc2); 352dc2f7e67SChris Wilson if (err) 353dc2f7e67SChris Wilson return err; 354dc2f7e67SChris Wilson 355dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) { 356dc2f7e67SChris Wilson fence = dma_fence_get(fc.tail); 357dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 2 * i + 1); 358dc2f7e67SChris Wilson dma_fence_put(fence); 359dc2f7e67SChris Wilson if (err) { 360dc2f7e67SChris Wilson pr_err("Reported %d for find_seqno(%d:%d)!\n", 361dc2f7e67SChris Wilson err, fc.chain_length + 1, 2 * i + 1); 362dc2f7e67SChris Wilson goto err; 363dc2f7e67SChris Wilson } 364dc2f7e67SChris Wilson if (fence != fc.chains[i]) { 365dc2f7e67SChris Wilson pr_err("Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)\n", 366dc2f7e67SChris Wilson fence->seqno, 367dc2f7e67SChris Wilson fc.chain_length + 1, 368dc2f7e67SChris Wilson 2 * i + 1); 369dc2f7e67SChris Wilson err = -EINVAL; 370dc2f7e67SChris Wilson goto err; 371dc2f7e67SChris Wilson } 372dc2f7e67SChris Wilson 373dc2f7e67SChris Wilson dma_fence_get(fence); 374dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, 2 * i + 2); 375dc2f7e67SChris Wilson dma_fence_put(fence); 376dc2f7e67SChris Wilson if (err) { 377dc2f7e67SChris Wilson pr_err("Error reported for finding self\n"); 378dc2f7e67SChris Wilson goto err; 379dc2f7e67SChris Wilson } 380dc2f7e67SChris Wilson if (fence != fc.chains[i]) { 381dc2f7e67SChris Wilson pr_err("Incorrect fence reported by find self\n"); 382dc2f7e67SChris Wilson err = -EINVAL; 383dc2f7e67SChris Wilson goto err; 384dc2f7e67SChris Wilson } 385dc2f7e67SChris Wilson } 386dc2f7e67SChris Wilson 387dc2f7e67SChris Wilson err: 388dc2f7e67SChris Wilson fence_chains_fini(&fc); 389dc2f7e67SChris Wilson return err; 390dc2f7e67SChris Wilson } 391dc2f7e67SChris Wilson 392dc2f7e67SChris Wilson struct find_race { 393dc2f7e67SChris Wilson struct fence_chains fc; 394dc2f7e67SChris Wilson atomic_t children; 395dc2f7e67SChris Wilson }; 396dc2f7e67SChris Wilson 397dc2f7e67SChris Wilson static int __find_race(void *arg) 398dc2f7e67SChris Wilson { 399dc2f7e67SChris Wilson struct find_race *data = arg; 400dc2f7e67SChris Wilson int err = 0; 401dc2f7e67SChris Wilson 402dc2f7e67SChris Wilson while (!kthread_should_stop()) { 403dc2f7e67SChris Wilson struct dma_fence *fence = dma_fence_get(data->fc.tail); 404dc2f7e67SChris Wilson int seqno; 405dc2f7e67SChris Wilson 406dc2f7e67SChris Wilson seqno = prandom_u32_max(data->fc.chain_length) + 1; 407dc2f7e67SChris Wilson 408dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, seqno); 409dc2f7e67SChris Wilson if (err) { 410dc2f7e67SChris Wilson pr_err("Failed to find fence seqno:%d\n", 411dc2f7e67SChris Wilson seqno); 412dc2f7e67SChris Wilson dma_fence_put(fence); 413dc2f7e67SChris Wilson break; 414dc2f7e67SChris Wilson } 415dc2f7e67SChris Wilson if (!fence) 416dc2f7e67SChris Wilson goto signal; 417dc2f7e67SChris Wilson 418dc2f7e67SChris Wilson err = dma_fence_chain_find_seqno(&fence, seqno); 419dc2f7e67SChris Wilson if (err) { 420dc2f7e67SChris Wilson pr_err("Reported an invalid fence for find-self:%d\n", 421dc2f7e67SChris Wilson seqno); 422dc2f7e67SChris Wilson dma_fence_put(fence); 423dc2f7e67SChris Wilson break; 424dc2f7e67SChris Wilson } 425dc2f7e67SChris Wilson 426dc2f7e67SChris Wilson if (fence->seqno < seqno) { 427dc2f7e67SChris Wilson pr_err("Reported an earlier fence.seqno:%lld for seqno:%d\n", 428dc2f7e67SChris Wilson fence->seqno, seqno); 429dc2f7e67SChris Wilson err = -EINVAL; 430dc2f7e67SChris Wilson dma_fence_put(fence); 431dc2f7e67SChris Wilson break; 432dc2f7e67SChris Wilson } 433dc2f7e67SChris Wilson 434dc2f7e67SChris Wilson dma_fence_put(fence); 435dc2f7e67SChris Wilson 436dc2f7e67SChris Wilson signal: 437dc2f7e67SChris Wilson seqno = prandom_u32_max(data->fc.chain_length - 1); 438dc2f7e67SChris Wilson dma_fence_signal(data->fc.fences[seqno]); 439dc2f7e67SChris Wilson cond_resched(); 440dc2f7e67SChris Wilson } 441dc2f7e67SChris Wilson 442dc2f7e67SChris Wilson if (atomic_dec_and_test(&data->children)) 443dc2f7e67SChris Wilson wake_up_var(&data->children); 444dc2f7e67SChris Wilson return err; 445dc2f7e67SChris Wilson } 446dc2f7e67SChris Wilson 447dc2f7e67SChris Wilson static int find_race(void *arg) 448dc2f7e67SChris Wilson { 449dc2f7e67SChris Wilson struct find_race data; 450dc2f7e67SChris Wilson int ncpus = num_online_cpus(); 451dc2f7e67SChris Wilson struct task_struct **threads; 452dc2f7e67SChris Wilson unsigned long count; 453dc2f7e67SChris Wilson int err; 454dc2f7e67SChris Wilson int i; 455dc2f7e67SChris Wilson 456dc2f7e67SChris Wilson err = fence_chains_init(&data.fc, CHAIN_SZ, seqno_inc); 457dc2f7e67SChris Wilson if (err) 458dc2f7e67SChris Wilson return err; 459dc2f7e67SChris Wilson 460dc2f7e67SChris Wilson threads = kmalloc_array(ncpus, sizeof(*threads), GFP_KERNEL); 461dc2f7e67SChris Wilson if (!threads) { 462dc2f7e67SChris Wilson err = -ENOMEM; 463dc2f7e67SChris Wilson goto err; 464dc2f7e67SChris Wilson } 465dc2f7e67SChris Wilson 466dc2f7e67SChris Wilson atomic_set(&data.children, 0); 467dc2f7e67SChris Wilson for (i = 0; i < ncpus; i++) { 468dc2f7e67SChris Wilson threads[i] = kthread_run(__find_race, &data, "dmabuf/%d", i); 469dc2f7e67SChris Wilson if (IS_ERR(threads[i])) { 470dc2f7e67SChris Wilson ncpus = i; 471dc2f7e67SChris Wilson break; 472dc2f7e67SChris Wilson } 473dc2f7e67SChris Wilson atomic_inc(&data.children); 474dc2f7e67SChris Wilson get_task_struct(threads[i]); 475dc2f7e67SChris Wilson } 476dc2f7e67SChris Wilson 477dc2f7e67SChris Wilson wait_var_event_timeout(&data.children, 478dc2f7e67SChris Wilson !atomic_read(&data.children), 479dc2f7e67SChris Wilson 5 * HZ); 480dc2f7e67SChris Wilson 481dc2f7e67SChris Wilson for (i = 0; i < ncpus; i++) { 482dc2f7e67SChris Wilson int ret; 483dc2f7e67SChris Wilson 484dc2f7e67SChris Wilson ret = kthread_stop(threads[i]); 485dc2f7e67SChris Wilson if (ret && !err) 486dc2f7e67SChris Wilson err = ret; 487dc2f7e67SChris Wilson put_task_struct(threads[i]); 488dc2f7e67SChris Wilson } 489dc2f7e67SChris Wilson kfree(threads); 490dc2f7e67SChris Wilson 491dc2f7e67SChris Wilson count = 0; 492dc2f7e67SChris Wilson for (i = 0; i < data.fc.chain_length; i++) 493dc2f7e67SChris Wilson if (dma_fence_is_signaled(data.fc.fences[i])) 494dc2f7e67SChris Wilson count++; 495dc2f7e67SChris Wilson pr_info("Completed %lu cycles\n", count); 496dc2f7e67SChris Wilson 497dc2f7e67SChris Wilson err: 498dc2f7e67SChris Wilson fence_chains_fini(&data.fc); 499dc2f7e67SChris Wilson return err; 500dc2f7e67SChris Wilson } 501dc2f7e67SChris Wilson 502dc2f7e67SChris Wilson static int signal_forward(void *arg) 503dc2f7e67SChris Wilson { 504dc2f7e67SChris Wilson struct fence_chains fc; 505dc2f7e67SChris Wilson int err; 506dc2f7e67SChris Wilson int i; 507dc2f7e67SChris Wilson 508dc2f7e67SChris Wilson err = fence_chains_init(&fc, 64, seqno_inc); 509dc2f7e67SChris Wilson if (err) 510dc2f7e67SChris Wilson return err; 511dc2f7e67SChris Wilson 512dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) { 513dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 514dc2f7e67SChris Wilson 515dc2f7e67SChris Wilson if (!dma_fence_is_signaled(fc.chains[i])) { 516dc2f7e67SChris Wilson pr_err("chain[%d] not signaled!\n", i); 517dc2f7e67SChris Wilson err = -EINVAL; 518dc2f7e67SChris Wilson goto err; 519dc2f7e67SChris Wilson } 520dc2f7e67SChris Wilson 521dc2f7e67SChris Wilson if (i + 1 < fc.chain_length && 522dc2f7e67SChris Wilson dma_fence_is_signaled(fc.chains[i + 1])) { 523dc2f7e67SChris Wilson pr_err("chain[%d] is signaled!\n", i); 524dc2f7e67SChris Wilson err = -EINVAL; 525dc2f7e67SChris Wilson goto err; 526dc2f7e67SChris Wilson } 527dc2f7e67SChris Wilson } 528dc2f7e67SChris Wilson 529dc2f7e67SChris Wilson err: 530dc2f7e67SChris Wilson fence_chains_fini(&fc); 531dc2f7e67SChris Wilson return err; 532dc2f7e67SChris Wilson } 533dc2f7e67SChris Wilson 534dc2f7e67SChris Wilson static int signal_backward(void *arg) 535dc2f7e67SChris Wilson { 536dc2f7e67SChris Wilson struct fence_chains fc; 537dc2f7e67SChris Wilson int err; 538dc2f7e67SChris Wilson int i; 539dc2f7e67SChris Wilson 540dc2f7e67SChris Wilson err = fence_chains_init(&fc, 64, seqno_inc); 541dc2f7e67SChris Wilson if (err) 542dc2f7e67SChris Wilson return err; 543dc2f7e67SChris Wilson 544dc2f7e67SChris Wilson for (i = fc.chain_length; i--; ) { 545dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 546dc2f7e67SChris Wilson 547dc2f7e67SChris Wilson if (i > 0 && dma_fence_is_signaled(fc.chains[i])) { 548dc2f7e67SChris Wilson pr_err("chain[%d] is signaled!\n", i); 549dc2f7e67SChris Wilson err = -EINVAL; 550dc2f7e67SChris Wilson goto err; 551dc2f7e67SChris Wilson } 552dc2f7e67SChris Wilson } 553dc2f7e67SChris Wilson 554dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) { 555dc2f7e67SChris Wilson if (!dma_fence_is_signaled(fc.chains[i])) { 556dc2f7e67SChris Wilson pr_err("chain[%d] was not signaled!\n", i); 557dc2f7e67SChris Wilson err = -EINVAL; 558dc2f7e67SChris Wilson goto err; 559dc2f7e67SChris Wilson } 560dc2f7e67SChris Wilson } 561dc2f7e67SChris Wilson 562dc2f7e67SChris Wilson err: 563dc2f7e67SChris Wilson fence_chains_fini(&fc); 564dc2f7e67SChris Wilson return err; 565dc2f7e67SChris Wilson } 566dc2f7e67SChris Wilson 567dc2f7e67SChris Wilson static int __wait_fence_chains(void *arg) 568dc2f7e67SChris Wilson { 569dc2f7e67SChris Wilson struct fence_chains *fc = arg; 570dc2f7e67SChris Wilson 571dc2f7e67SChris Wilson if (dma_fence_wait(fc->tail, false)) 572dc2f7e67SChris Wilson return -EIO; 573dc2f7e67SChris Wilson 574dc2f7e67SChris Wilson return 0; 575dc2f7e67SChris Wilson } 576dc2f7e67SChris Wilson 577dc2f7e67SChris Wilson static int wait_forward(void *arg) 578dc2f7e67SChris Wilson { 579dc2f7e67SChris Wilson struct fence_chains fc; 580dc2f7e67SChris Wilson struct task_struct *tsk; 581dc2f7e67SChris Wilson int err; 582dc2f7e67SChris Wilson int i; 583dc2f7e67SChris Wilson 584dc2f7e67SChris Wilson err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 585dc2f7e67SChris Wilson if (err) 586dc2f7e67SChris Wilson return err; 587dc2f7e67SChris Wilson 588dc2f7e67SChris Wilson tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 589dc2f7e67SChris Wilson if (IS_ERR(tsk)) { 590dc2f7e67SChris Wilson err = PTR_ERR(tsk); 591dc2f7e67SChris Wilson goto err; 592dc2f7e67SChris Wilson } 593dc2f7e67SChris Wilson get_task_struct(tsk); 594dc2f7e67SChris Wilson yield_to(tsk, true); 595dc2f7e67SChris Wilson 596dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) 597dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 598dc2f7e67SChris Wilson 599dc2f7e67SChris Wilson err = kthread_stop(tsk); 600dc2f7e67SChris Wilson put_task_struct(tsk); 601dc2f7e67SChris Wilson 602dc2f7e67SChris Wilson err: 603dc2f7e67SChris Wilson fence_chains_fini(&fc); 604dc2f7e67SChris Wilson return err; 605dc2f7e67SChris Wilson } 606dc2f7e67SChris Wilson 607dc2f7e67SChris Wilson static int wait_backward(void *arg) 608dc2f7e67SChris Wilson { 609dc2f7e67SChris Wilson struct fence_chains fc; 610dc2f7e67SChris Wilson struct task_struct *tsk; 611dc2f7e67SChris Wilson int err; 612dc2f7e67SChris Wilson int i; 613dc2f7e67SChris Wilson 614dc2f7e67SChris Wilson err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 615dc2f7e67SChris Wilson if (err) 616dc2f7e67SChris Wilson return err; 617dc2f7e67SChris Wilson 618dc2f7e67SChris Wilson tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 619dc2f7e67SChris Wilson if (IS_ERR(tsk)) { 620dc2f7e67SChris Wilson err = PTR_ERR(tsk); 621dc2f7e67SChris Wilson goto err; 622dc2f7e67SChris Wilson } 623dc2f7e67SChris Wilson get_task_struct(tsk); 624dc2f7e67SChris Wilson yield_to(tsk, true); 625dc2f7e67SChris Wilson 626dc2f7e67SChris Wilson for (i = fc.chain_length; i--; ) 627dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 628dc2f7e67SChris Wilson 629dc2f7e67SChris Wilson err = kthread_stop(tsk); 630dc2f7e67SChris Wilson put_task_struct(tsk); 631dc2f7e67SChris Wilson 632dc2f7e67SChris Wilson err: 633dc2f7e67SChris Wilson fence_chains_fini(&fc); 634dc2f7e67SChris Wilson return err; 635dc2f7e67SChris Wilson } 636dc2f7e67SChris Wilson 637dc2f7e67SChris Wilson static void randomise_fences(struct fence_chains *fc) 638dc2f7e67SChris Wilson { 639dc2f7e67SChris Wilson unsigned int count = fc->chain_length; 640dc2f7e67SChris Wilson 641dc2f7e67SChris Wilson /* Fisher-Yates shuffle courtesy of Knuth */ 642dc2f7e67SChris Wilson while (--count) { 643dc2f7e67SChris Wilson unsigned int swp; 644dc2f7e67SChris Wilson 645dc2f7e67SChris Wilson swp = prandom_u32_max(count + 1); 646dc2f7e67SChris Wilson if (swp == count) 647dc2f7e67SChris Wilson continue; 648dc2f7e67SChris Wilson 649dc2f7e67SChris Wilson swap(fc->fences[count], fc->fences[swp]); 650dc2f7e67SChris Wilson } 651dc2f7e67SChris Wilson } 652dc2f7e67SChris Wilson 653dc2f7e67SChris Wilson static int wait_random(void *arg) 654dc2f7e67SChris Wilson { 655dc2f7e67SChris Wilson struct fence_chains fc; 656dc2f7e67SChris Wilson struct task_struct *tsk; 657dc2f7e67SChris Wilson int err; 658dc2f7e67SChris Wilson int i; 659dc2f7e67SChris Wilson 660dc2f7e67SChris Wilson err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); 661dc2f7e67SChris Wilson if (err) 662dc2f7e67SChris Wilson return err; 663dc2f7e67SChris Wilson 664dc2f7e67SChris Wilson randomise_fences(&fc); 665dc2f7e67SChris Wilson 666dc2f7e67SChris Wilson tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); 667dc2f7e67SChris Wilson if (IS_ERR(tsk)) { 668dc2f7e67SChris Wilson err = PTR_ERR(tsk); 669dc2f7e67SChris Wilson goto err; 670dc2f7e67SChris Wilson } 671dc2f7e67SChris Wilson get_task_struct(tsk); 672dc2f7e67SChris Wilson yield_to(tsk, true); 673dc2f7e67SChris Wilson 674dc2f7e67SChris Wilson for (i = 0; i < fc.chain_length; i++) 675dc2f7e67SChris Wilson dma_fence_signal(fc.fences[i]); 676dc2f7e67SChris Wilson 677dc2f7e67SChris Wilson err = kthread_stop(tsk); 678dc2f7e67SChris Wilson put_task_struct(tsk); 679dc2f7e67SChris Wilson 680dc2f7e67SChris Wilson err: 681dc2f7e67SChris Wilson fence_chains_fini(&fc); 682dc2f7e67SChris Wilson return err; 683dc2f7e67SChris Wilson } 684dc2f7e67SChris Wilson 685dc2f7e67SChris Wilson int dma_fence_chain(void) 686dc2f7e67SChris Wilson { 687dc2f7e67SChris Wilson static const struct subtest tests[] = { 688dc2f7e67SChris Wilson SUBTEST(sanitycheck), 689dc2f7e67SChris Wilson SUBTEST(find_seqno), 690dc2f7e67SChris Wilson SUBTEST(find_signaled), 691dc2f7e67SChris Wilson SUBTEST(find_out_of_order), 692dc2f7e67SChris Wilson SUBTEST(find_gap), 693dc2f7e67SChris Wilson SUBTEST(find_race), 694dc2f7e67SChris Wilson SUBTEST(signal_forward), 695dc2f7e67SChris Wilson SUBTEST(signal_backward), 696dc2f7e67SChris Wilson SUBTEST(wait_forward), 697dc2f7e67SChris Wilson SUBTEST(wait_backward), 698dc2f7e67SChris Wilson SUBTEST(wait_random), 699dc2f7e67SChris Wilson }; 700dc2f7e67SChris Wilson int ret; 701dc2f7e67SChris Wilson 702dc2f7e67SChris Wilson pr_info("sizeof(dma_fence_chain)=%zu\n", 703dc2f7e67SChris Wilson sizeof(struct dma_fence_chain)); 704dc2f7e67SChris Wilson 705dc2f7e67SChris Wilson slab_fences = KMEM_CACHE(mock_fence, 706dc2f7e67SChris Wilson SLAB_TYPESAFE_BY_RCU | 707dc2f7e67SChris Wilson SLAB_HWCACHE_ALIGN); 708dc2f7e67SChris Wilson if (!slab_fences) 709dc2f7e67SChris Wilson return -ENOMEM; 710dc2f7e67SChris Wilson 711dc2f7e67SChris Wilson ret = subtests(tests, NULL); 712dc2f7e67SChris Wilson 713dc2f7e67SChris Wilson kmem_cache_destroy(slab_fences); 714dc2f7e67SChris Wilson return ret; 715dc2f7e67SChris Wilson } 716