// SPDX-License-Identifier: MIT /* * Copyright © 2019 Intel Corporation */ #include <linux/delay.h> #include <linux/dma-fence.h> #include <linux/dma-fence-chain.h> #include <linux/kernel.h> #include <linux/kthread.h> #include <linux/mm.h> #include <linux/sched/signal.h> #include <linux/slab.h> #include <linux/spinlock.h> #include <linux/random.h> #include "selftest.h" #define CHAIN_SZ (4 << 10) static struct kmem_cache *slab_fences; static inline struct mock_fence { struct dma_fence base; spinlock_t lock; } *to_mock_fence(struct dma_fence *f) { return container_of(f, struct mock_fence, base); } static const char *mock_name(struct dma_fence *f) { return "mock"; } static void mock_fence_release(struct dma_fence *f) { kmem_cache_free(slab_fences, to_mock_fence(f)); } static const struct dma_fence_ops mock_ops = { .get_driver_name = mock_name, .get_timeline_name = mock_name, .release = mock_fence_release, }; static struct dma_fence *mock_fence(void) { struct mock_fence *f; f = kmem_cache_alloc(slab_fences, GFP_KERNEL); if (!f) return NULL; spin_lock_init(&f->lock); dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0); return &f->base; } static struct dma_fence *mock_chain(struct dma_fence *prev, struct dma_fence *fence, u64 seqno) { struct dma_fence_chain *f; f = dma_fence_chain_alloc(); if (!f) return NULL; dma_fence_chain_init(f, dma_fence_get(prev), dma_fence_get(fence), seqno); return &f->base; } static int sanitycheck(void *arg) { struct dma_fence *f, *chain; int err = 0; f = mock_fence(); if (!f) return -ENOMEM; chain = mock_chain(NULL, f, 1); if (!chain) err = -ENOMEM; dma_fence_enable_sw_signaling(chain); dma_fence_signal(f); dma_fence_put(f); dma_fence_put(chain); return err; } struct fence_chains { unsigned int chain_length; struct dma_fence **fences; struct dma_fence **chains; struct dma_fence *tail; }; static uint64_t seqno_inc(unsigned int i) { return i + 1; } static int fence_chains_init(struct fence_chains *fc, unsigned int count, uint64_t (*seqno_fn)(unsigned int)) { unsigned int i; int err = 0; fc->chains = kvmalloc_array(count, sizeof(*fc->chains), GFP_KERNEL | __GFP_ZERO); if (!fc->chains) return -ENOMEM; fc->fences = kvmalloc_array(count, sizeof(*fc->fences), GFP_KERNEL | __GFP_ZERO); if (!fc->fences) { err = -ENOMEM; goto err_chains; } fc->tail = NULL; for (i = 0; i < count; i++) { fc->fences[i] = mock_fence(); if (!fc->fences[i]) { err = -ENOMEM; goto unwind; } fc->chains[i] = mock_chain(fc->tail, fc->fences[i], seqno_fn(i)); if (!fc->chains[i]) { err = -ENOMEM; goto unwind; } fc->tail = fc->chains[i]; dma_fence_enable_sw_signaling(fc->chains[i]); } fc->chain_length = i; return 0; unwind: for (i = 0; i < count; i++) { dma_fence_put(fc->fences[i]); dma_fence_put(fc->chains[i]); } kvfree(fc->fences); err_chains: kvfree(fc->chains); return err; } static void fence_chains_fini(struct fence_chains *fc) { unsigned int i; for (i = 0; i < fc->chain_length; i++) { dma_fence_signal(fc->fences[i]); dma_fence_put(fc->fences[i]); } kvfree(fc->fences); for (i = 0; i < fc->chain_length; i++) dma_fence_put(fc->chains[i]); kvfree(fc->chains); } static int find_seqno(void *arg) { struct fence_chains fc; struct dma_fence *fence; int err; int i; err = fence_chains_init(&fc, 64, seqno_inc); if (err) return err; fence = dma_fence_get(fc.tail); err = dma_fence_chain_find_seqno(&fence, 0); dma_fence_put(fence); if (err) { pr_err("Reported %d for find_seqno(0)!\n", err); goto err; } for (i = 0; i < fc.chain_length; i++) { fence = dma_fence_get(fc.tail); err = dma_fence_chain_find_seqno(&fence, i + 1); dma_fence_put(fence); if (err) { pr_err("Reported %d for find_seqno(%d:%d)!\n", err, fc.chain_length + 1, i + 1); goto err; } if (fence != fc.chains[i]) { pr_err("Incorrect fence reported by find_seqno(%d:%d)\n", fc.chain_length + 1, i + 1); err = -EINVAL; goto err; } dma_fence_get(fence); err = dma_fence_chain_find_seqno(&fence, i + 1); dma_fence_put(fence); if (err) { pr_err("Error reported for finding self\n"); goto err; } if (fence != fc.chains[i]) { pr_err("Incorrect fence reported by find self\n"); err = -EINVAL; goto err; } dma_fence_get(fence); err = dma_fence_chain_find_seqno(&fence, i + 2); dma_fence_put(fence); if (!err) { pr_err("Error not reported for future fence: find_seqno(%d:%d)!\n", i + 1, i + 2); err = -EINVAL; goto err; } dma_fence_get(fence); err = dma_fence_chain_find_seqno(&fence, i); dma_fence_put(fence); if (err) { pr_err("Error reported for previous fence!\n"); goto err; } if (i > 0 && fence != fc.chains[i - 1]) { pr_err("Incorrect fence reported by find_seqno(%d:%d)\n", i + 1, i); err = -EINVAL; goto err; } } err: fence_chains_fini(&fc); return err; } static int find_signaled(void *arg) { struct fence_chains fc; struct dma_fence *fence; int err; err = fence_chains_init(&fc, 2, seqno_inc); if (err) return err; dma_fence_signal(fc.fences[0]); fence = dma_fence_get(fc.tail); err = dma_fence_chain_find_seqno(&fence, 1); dma_fence_put(fence); if (err) { pr_err("Reported %d for find_seqno()!\n", err); goto err; } if (fence && fence != fc.chains[0]) { pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:1\n", fence->seqno); dma_fence_get(fence); err = dma_fence_chain_find_seqno(&fence, 1); dma_fence_put(fence); if (err) pr_err("Reported %d for finding self!\n", err); err = -EINVAL; } err: fence_chains_fini(&fc); return err; } static int find_out_of_order(void *arg) { struct fence_chains fc; struct dma_fence *fence; int err; err = fence_chains_init(&fc, 3, seqno_inc); if (err) return err; dma_fence_signal(fc.fences[1]); fence = dma_fence_get(fc.tail); err = dma_fence_chain_find_seqno(&fence, 2); dma_fence_put(fence); if (err) { pr_err("Reported %d for find_seqno()!\n", err); goto err; } /* * We signaled the middle fence (2) of the 1-2-3 chain. The behavior * of the dma-fence-chain is to make us wait for all the fences up to * the point we want. Since fence 1 is still not signaled, this what * we should get as fence to wait upon (fence 2 being garbage * collected during the traversal of the chain). */ if (fence != fc.chains[0]) { pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:2\n", fence ? fence->seqno : 0); err = -EINVAL; } err: fence_chains_fini(&fc); return err; } static uint64_t seqno_inc2(unsigned int i) { return 2 * i + 2; } static int find_gap(void *arg) { struct fence_chains fc; struct dma_fence *fence; int err; int i; err = fence_chains_init(&fc, 64, seqno_inc2); if (err) return err; for (i = 0; i < fc.chain_length; i++) { fence = dma_fence_get(fc.tail); err = dma_fence_chain_find_seqno(&fence, 2 * i + 1); dma_fence_put(fence); if (err) { pr_err("Reported %d for find_seqno(%d:%d)!\n", err, fc.chain_length + 1, 2 * i + 1); goto err; } if (fence != fc.chains[i]) { pr_err("Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)\n", fence->seqno, fc.chain_length + 1, 2 * i + 1); err = -EINVAL; goto err; } dma_fence_get(fence); err = dma_fence_chain_find_seqno(&fence, 2 * i + 2); dma_fence_put(fence); if (err) { pr_err("Error reported for finding self\n"); goto err; } if (fence != fc.chains[i]) { pr_err("Incorrect fence reported by find self\n"); err = -EINVAL; goto err; } } err: fence_chains_fini(&fc); return err; } struct find_race { struct fence_chains fc; atomic_t children; }; static int __find_race(void *arg) { struct find_race *data = arg; int err = 0; while (!kthread_should_stop()) { struct dma_fence *fence = dma_fence_get(data->fc.tail); int seqno; seqno = get_random_u32_inclusive(1, data->fc.chain_length); err = dma_fence_chain_find_seqno(&fence, seqno); if (err) { pr_err("Failed to find fence seqno:%d\n", seqno); dma_fence_put(fence); break; } if (!fence) goto signal; /* * We can only find ourselves if we are on fence we were * looking for. */ if (fence->seqno == seqno) { err = dma_fence_chain_find_seqno(&fence, seqno); if (err) { pr_err("Reported an invalid fence for find-self:%d\n", seqno); dma_fence_put(fence); break; } } dma_fence_put(fence); signal: seqno = get_random_u32_below(data->fc.chain_length - 1); dma_fence_signal(data->fc.fences[seqno]); cond_resched(); } if (atomic_dec_and_test(&data->children)) wake_up_var(&data->children); return err; } static int find_race(void *arg) { struct find_race data; int ncpus = num_online_cpus(); struct task_struct **threads; unsigned long count; int err; int i; err = fence_chains_init(&data.fc, CHAIN_SZ, seqno_inc); if (err) return err; threads = kmalloc_array(ncpus, sizeof(*threads), GFP_KERNEL); if (!threads) { err = -ENOMEM; goto err; } atomic_set(&data.children, 0); for (i = 0; i < ncpus; i++) { threads[i] = kthread_run(__find_race, &data, "dmabuf/%d", i); if (IS_ERR(threads[i])) { ncpus = i; break; } atomic_inc(&data.children); get_task_struct(threads[i]); } wait_var_event_timeout(&data.children, !atomic_read(&data.children), 5 * HZ); for (i = 0; i < ncpus; i++) { int ret; ret = kthread_stop(threads[i]); if (ret && !err) err = ret; put_task_struct(threads[i]); } kfree(threads); count = 0; for (i = 0; i < data.fc.chain_length; i++) if (dma_fence_is_signaled(data.fc.fences[i])) count++; pr_info("Completed %lu cycles\n", count); err: fence_chains_fini(&data.fc); return err; } static int signal_forward(void *arg) { struct fence_chains fc; int err; int i; err = fence_chains_init(&fc, 64, seqno_inc); if (err) return err; for (i = 0; i < fc.chain_length; i++) { dma_fence_signal(fc.fences[i]); if (!dma_fence_is_signaled(fc.chains[i])) { pr_err("chain[%d] not signaled!\n", i); err = -EINVAL; goto err; } if (i + 1 < fc.chain_length && dma_fence_is_signaled(fc.chains[i + 1])) { pr_err("chain[%d] is signaled!\n", i); err = -EINVAL; goto err; } } err: fence_chains_fini(&fc); return err; } static int signal_backward(void *arg) { struct fence_chains fc; int err; int i; err = fence_chains_init(&fc, 64, seqno_inc); if (err) return err; for (i = fc.chain_length; i--; ) { dma_fence_signal(fc.fences[i]); if (i > 0 && dma_fence_is_signaled(fc.chains[i])) { pr_err("chain[%d] is signaled!\n", i); err = -EINVAL; goto err; } } for (i = 0; i < fc.chain_length; i++) { if (!dma_fence_is_signaled(fc.chains[i])) { pr_err("chain[%d] was not signaled!\n", i); err = -EINVAL; goto err; } } err: fence_chains_fini(&fc); return err; } static int __wait_fence_chains(void *arg) { struct fence_chains *fc = arg; if (dma_fence_wait(fc->tail, false)) return -EIO; return 0; } static int wait_forward(void *arg) { struct fence_chains fc; struct task_struct *tsk; int err; int i; err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); if (err) return err; tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); if (IS_ERR(tsk)) { err = PTR_ERR(tsk); goto err; } get_task_struct(tsk); yield_to(tsk, true); for (i = 0; i < fc.chain_length; i++) dma_fence_signal(fc.fences[i]); err = kthread_stop(tsk); put_task_struct(tsk); err: fence_chains_fini(&fc); return err; } static int wait_backward(void *arg) { struct fence_chains fc; struct task_struct *tsk; int err; int i; err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); if (err) return err; tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); if (IS_ERR(tsk)) { err = PTR_ERR(tsk); goto err; } get_task_struct(tsk); yield_to(tsk, true); for (i = fc.chain_length; i--; ) dma_fence_signal(fc.fences[i]); err = kthread_stop(tsk); put_task_struct(tsk); err: fence_chains_fini(&fc); return err; } static void randomise_fences(struct fence_chains *fc) { unsigned int count = fc->chain_length; /* Fisher-Yates shuffle courtesy of Knuth */ while (--count) { unsigned int swp; swp = get_random_u32_below(count + 1); if (swp == count) continue; swap(fc->fences[count], fc->fences[swp]); } } static int wait_random(void *arg) { struct fence_chains fc; struct task_struct *tsk; int err; int i; err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc); if (err) return err; randomise_fences(&fc); tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait"); if (IS_ERR(tsk)) { err = PTR_ERR(tsk); goto err; } get_task_struct(tsk); yield_to(tsk, true); for (i = 0; i < fc.chain_length; i++) dma_fence_signal(fc.fences[i]); err = kthread_stop(tsk); put_task_struct(tsk); err: fence_chains_fini(&fc); return err; } int dma_fence_chain(void) { static const struct subtest tests[] = { SUBTEST(sanitycheck), SUBTEST(find_seqno), SUBTEST(find_signaled), SUBTEST(find_out_of_order), SUBTEST(find_gap), SUBTEST(find_race), SUBTEST(signal_forward), SUBTEST(signal_backward), SUBTEST(wait_forward), SUBTEST(wait_backward), SUBTEST(wait_random), }; int ret; pr_info("sizeof(dma_fence_chain)=%zu\n", sizeof(struct dma_fence_chain)); slab_fences = KMEM_CACHE(mock_fence, SLAB_TYPESAFE_BY_RCU | SLAB_HWCACHE_ALIGN); if (!slab_fences) return -ENOMEM; ret = subtests(tests, NULL); kmem_cache_destroy(slab_fences); return ret; }