1f2a5fec1SChris Wilson /* 2f2a5fec1SChris Wilson * Module-based API test facility for ww_mutexes 3f2a5fec1SChris Wilson * 4f2a5fec1SChris Wilson * This program is free software; you can redistribute it and/or modify 5f2a5fec1SChris Wilson * it under the terms of the GNU General Public License as published by 6f2a5fec1SChris Wilson * the Free Software Foundation; either version 2 of the License, or 7f2a5fec1SChris Wilson * (at your option) any later version. 8f2a5fec1SChris Wilson * 9f2a5fec1SChris Wilson * This program is distributed in the hope that it will be useful, 10f2a5fec1SChris Wilson * but WITHOUT ANY WARRANTY; without even the implied warranty of 11f2a5fec1SChris Wilson * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12f2a5fec1SChris Wilson * GNU General Public License for more details. 13f2a5fec1SChris Wilson * 14f2a5fec1SChris Wilson * You should have received a copy of the GNU General Public License 15f2a5fec1SChris Wilson * along with this program; if not, you can access it online at 16f2a5fec1SChris Wilson * http://www.gnu.org/licenses/gpl-2.0.html. 17f2a5fec1SChris Wilson */ 18f2a5fec1SChris Wilson 19f2a5fec1SChris Wilson #include <linux/kernel.h> 20f2a5fec1SChris Wilson 21f2a5fec1SChris Wilson #include <linux/completion.h> 222a0c1128SChris Wilson #include <linux/delay.h> 23f2a5fec1SChris Wilson #include <linux/kthread.h> 24f2a5fec1SChris Wilson #include <linux/module.h> 252a0c1128SChris Wilson #include <linux/random.h> 26d1b42b80SChris Wilson #include <linux/slab.h> 27f2a5fec1SChris Wilson #include <linux/ww_mutex.h> 28f2a5fec1SChris Wilson 29f2a5fec1SChris Wilson static DEFINE_WW_CLASS(ww_class); 30d1b42b80SChris Wilson struct workqueue_struct *wq; 31f2a5fec1SChris Wilson 32f2a5fec1SChris Wilson struct test_mutex { 33f2a5fec1SChris Wilson struct work_struct work; 34f2a5fec1SChris Wilson struct ww_mutex mutex; 35f2a5fec1SChris Wilson struct completion ready, go, done; 36f2a5fec1SChris Wilson unsigned int flags; 37f2a5fec1SChris Wilson }; 38f2a5fec1SChris Wilson 39f2a5fec1SChris Wilson #define TEST_MTX_SPIN BIT(0) 40f2a5fec1SChris Wilson #define TEST_MTX_TRY BIT(1) 41f2a5fec1SChris Wilson #define TEST_MTX_CTX BIT(2) 42f2a5fec1SChris Wilson #define __TEST_MTX_LAST BIT(3) 43f2a5fec1SChris Wilson 44f2a5fec1SChris Wilson static void test_mutex_work(struct work_struct *work) 45f2a5fec1SChris Wilson { 46f2a5fec1SChris Wilson struct test_mutex *mtx = container_of(work, typeof(*mtx), work); 47f2a5fec1SChris Wilson 48f2a5fec1SChris Wilson complete(&mtx->ready); 49f2a5fec1SChris Wilson wait_for_completion(&mtx->go); 50f2a5fec1SChris Wilson 51f2a5fec1SChris Wilson if (mtx->flags & TEST_MTX_TRY) { 52f2a5fec1SChris Wilson while (!ww_mutex_trylock(&mtx->mutex)) 532b232e0cSChris Wilson cond_resched(); 54f2a5fec1SChris Wilson } else { 55f2a5fec1SChris Wilson ww_mutex_lock(&mtx->mutex, NULL); 56f2a5fec1SChris Wilson } 57f2a5fec1SChris Wilson complete(&mtx->done); 58f2a5fec1SChris Wilson ww_mutex_unlock(&mtx->mutex); 59f2a5fec1SChris Wilson } 60f2a5fec1SChris Wilson 61f2a5fec1SChris Wilson static int __test_mutex(unsigned int flags) 62f2a5fec1SChris Wilson { 63f2a5fec1SChris Wilson #define TIMEOUT (HZ / 16) 64f2a5fec1SChris Wilson struct test_mutex mtx; 65f2a5fec1SChris Wilson struct ww_acquire_ctx ctx; 66f2a5fec1SChris Wilson int ret; 67f2a5fec1SChris Wilson 68f2a5fec1SChris Wilson ww_mutex_init(&mtx.mutex, &ww_class); 69f2a5fec1SChris Wilson ww_acquire_init(&ctx, &ww_class); 70f2a5fec1SChris Wilson 71f2a5fec1SChris Wilson INIT_WORK_ONSTACK(&mtx.work, test_mutex_work); 72f2a5fec1SChris Wilson init_completion(&mtx.ready); 73f2a5fec1SChris Wilson init_completion(&mtx.go); 74f2a5fec1SChris Wilson init_completion(&mtx.done); 75f2a5fec1SChris Wilson mtx.flags = flags; 76f2a5fec1SChris Wilson 77f2a5fec1SChris Wilson schedule_work(&mtx.work); 78f2a5fec1SChris Wilson 79f2a5fec1SChris Wilson wait_for_completion(&mtx.ready); 80f2a5fec1SChris Wilson ww_mutex_lock(&mtx.mutex, (flags & TEST_MTX_CTX) ? &ctx : NULL); 81f2a5fec1SChris Wilson complete(&mtx.go); 82f2a5fec1SChris Wilson if (flags & TEST_MTX_SPIN) { 83f2a5fec1SChris Wilson unsigned long timeout = jiffies + TIMEOUT; 84f2a5fec1SChris Wilson 85f2a5fec1SChris Wilson ret = 0; 86f2a5fec1SChris Wilson do { 87f2a5fec1SChris Wilson if (completion_done(&mtx.done)) { 88f2a5fec1SChris Wilson ret = -EINVAL; 89f2a5fec1SChris Wilson break; 90f2a5fec1SChris Wilson } 912b232e0cSChris Wilson cond_resched(); 92f2a5fec1SChris Wilson } while (time_before(jiffies, timeout)); 93f2a5fec1SChris Wilson } else { 94f2a5fec1SChris Wilson ret = wait_for_completion_timeout(&mtx.done, TIMEOUT); 95f2a5fec1SChris Wilson } 96f2a5fec1SChris Wilson ww_mutex_unlock(&mtx.mutex); 97f2a5fec1SChris Wilson ww_acquire_fini(&ctx); 98f2a5fec1SChris Wilson 99f2a5fec1SChris Wilson if (ret) { 100f2a5fec1SChris Wilson pr_err("%s(flags=%x): mutual exclusion failure\n", 101f2a5fec1SChris Wilson __func__, flags); 102f2a5fec1SChris Wilson ret = -EINVAL; 103f2a5fec1SChris Wilson } 104f2a5fec1SChris Wilson 105f2a5fec1SChris Wilson flush_work(&mtx.work); 106f2a5fec1SChris Wilson destroy_work_on_stack(&mtx.work); 107f2a5fec1SChris Wilson return ret; 108f2a5fec1SChris Wilson #undef TIMEOUT 109f2a5fec1SChris Wilson } 110f2a5fec1SChris Wilson 111f2a5fec1SChris Wilson static int test_mutex(void) 112f2a5fec1SChris Wilson { 113f2a5fec1SChris Wilson int ret; 114f2a5fec1SChris Wilson int i; 115f2a5fec1SChris Wilson 116f2a5fec1SChris Wilson for (i = 0; i < __TEST_MTX_LAST; i++) { 117f2a5fec1SChris Wilson ret = __test_mutex(i); 118f2a5fec1SChris Wilson if (ret) 119f2a5fec1SChris Wilson return ret; 120f2a5fec1SChris Wilson } 121f2a5fec1SChris Wilson 122f2a5fec1SChris Wilson return 0; 123f2a5fec1SChris Wilson } 124f2a5fec1SChris Wilson 125c22fb380SChris Wilson static int test_aa(void) 126c22fb380SChris Wilson { 127c22fb380SChris Wilson struct ww_mutex mutex; 128c22fb380SChris Wilson struct ww_acquire_ctx ctx; 129c22fb380SChris Wilson int ret; 130c22fb380SChris Wilson 131c22fb380SChris Wilson ww_mutex_init(&mutex, &ww_class); 132c22fb380SChris Wilson ww_acquire_init(&ctx, &ww_class); 133c22fb380SChris Wilson 134c22fb380SChris Wilson ww_mutex_lock(&mutex, &ctx); 135c22fb380SChris Wilson 136c22fb380SChris Wilson if (ww_mutex_trylock(&mutex)) { 137c22fb380SChris Wilson pr_err("%s: trylocked itself!\n", __func__); 138c22fb380SChris Wilson ww_mutex_unlock(&mutex); 139c22fb380SChris Wilson ret = -EINVAL; 140c22fb380SChris Wilson goto out; 141c22fb380SChris Wilson } 142c22fb380SChris Wilson 143c22fb380SChris Wilson ret = ww_mutex_lock(&mutex, &ctx); 144c22fb380SChris Wilson if (ret != -EALREADY) { 145c22fb380SChris Wilson pr_err("%s: missed deadlock for recursing, ret=%d\n", 146c22fb380SChris Wilson __func__, ret); 147c22fb380SChris Wilson if (!ret) 148c22fb380SChris Wilson ww_mutex_unlock(&mutex); 149c22fb380SChris Wilson ret = -EINVAL; 150c22fb380SChris Wilson goto out; 151c22fb380SChris Wilson } 152c22fb380SChris Wilson 153c22fb380SChris Wilson ret = 0; 154c22fb380SChris Wilson out: 155c22fb380SChris Wilson ww_mutex_unlock(&mutex); 156c22fb380SChris Wilson ww_acquire_fini(&ctx); 157c22fb380SChris Wilson return ret; 158c22fb380SChris Wilson } 159c22fb380SChris Wilson 16070207686SChris Wilson struct test_abba { 16170207686SChris Wilson struct work_struct work; 16270207686SChris Wilson struct ww_mutex a_mutex; 16370207686SChris Wilson struct ww_mutex b_mutex; 16470207686SChris Wilson struct completion a_ready; 16570207686SChris Wilson struct completion b_ready; 16670207686SChris Wilson bool resolve; 16770207686SChris Wilson int result; 16870207686SChris Wilson }; 16970207686SChris Wilson 17070207686SChris Wilson static void test_abba_work(struct work_struct *work) 17170207686SChris Wilson { 17270207686SChris Wilson struct test_abba *abba = container_of(work, typeof(*abba), work); 17370207686SChris Wilson struct ww_acquire_ctx ctx; 17470207686SChris Wilson int err; 17570207686SChris Wilson 17670207686SChris Wilson ww_acquire_init(&ctx, &ww_class); 17770207686SChris Wilson ww_mutex_lock(&abba->b_mutex, &ctx); 17870207686SChris Wilson 17970207686SChris Wilson complete(&abba->b_ready); 18070207686SChris Wilson wait_for_completion(&abba->a_ready); 18170207686SChris Wilson 18270207686SChris Wilson err = ww_mutex_lock(&abba->a_mutex, &ctx); 18370207686SChris Wilson if (abba->resolve && err == -EDEADLK) { 18470207686SChris Wilson ww_mutex_unlock(&abba->b_mutex); 18570207686SChris Wilson ww_mutex_lock_slow(&abba->a_mutex, &ctx); 18670207686SChris Wilson err = ww_mutex_lock(&abba->b_mutex, &ctx); 18770207686SChris Wilson } 18870207686SChris Wilson 18970207686SChris Wilson if (!err) 19070207686SChris Wilson ww_mutex_unlock(&abba->a_mutex); 19170207686SChris Wilson ww_mutex_unlock(&abba->b_mutex); 19270207686SChris Wilson ww_acquire_fini(&ctx); 19370207686SChris Wilson 19470207686SChris Wilson abba->result = err; 19570207686SChris Wilson } 19670207686SChris Wilson 19770207686SChris Wilson static int test_abba(bool resolve) 19870207686SChris Wilson { 19970207686SChris Wilson struct test_abba abba; 20070207686SChris Wilson struct ww_acquire_ctx ctx; 20170207686SChris Wilson int err, ret; 20270207686SChris Wilson 20370207686SChris Wilson ww_mutex_init(&abba.a_mutex, &ww_class); 20470207686SChris Wilson ww_mutex_init(&abba.b_mutex, &ww_class); 20570207686SChris Wilson INIT_WORK_ONSTACK(&abba.work, test_abba_work); 20670207686SChris Wilson init_completion(&abba.a_ready); 20770207686SChris Wilson init_completion(&abba.b_ready); 20870207686SChris Wilson abba.resolve = resolve; 20970207686SChris Wilson 21070207686SChris Wilson schedule_work(&abba.work); 21170207686SChris Wilson 21270207686SChris Wilson ww_acquire_init(&ctx, &ww_class); 21370207686SChris Wilson ww_mutex_lock(&abba.a_mutex, &ctx); 21470207686SChris Wilson 21570207686SChris Wilson complete(&abba.a_ready); 21670207686SChris Wilson wait_for_completion(&abba.b_ready); 21770207686SChris Wilson 21870207686SChris Wilson err = ww_mutex_lock(&abba.b_mutex, &ctx); 21970207686SChris Wilson if (resolve && err == -EDEADLK) { 22070207686SChris Wilson ww_mutex_unlock(&abba.a_mutex); 22170207686SChris Wilson ww_mutex_lock_slow(&abba.b_mutex, &ctx); 22270207686SChris Wilson err = ww_mutex_lock(&abba.a_mutex, &ctx); 22370207686SChris Wilson } 22470207686SChris Wilson 22570207686SChris Wilson if (!err) 22670207686SChris Wilson ww_mutex_unlock(&abba.b_mutex); 22770207686SChris Wilson ww_mutex_unlock(&abba.a_mutex); 22870207686SChris Wilson ww_acquire_fini(&ctx); 22970207686SChris Wilson 23070207686SChris Wilson flush_work(&abba.work); 23170207686SChris Wilson destroy_work_on_stack(&abba.work); 23270207686SChris Wilson 23370207686SChris Wilson ret = 0; 23470207686SChris Wilson if (resolve) { 23570207686SChris Wilson if (err || abba.result) { 23670207686SChris Wilson pr_err("%s: failed to resolve ABBA deadlock, A err=%d, B err=%d\n", 23770207686SChris Wilson __func__, err, abba.result); 23870207686SChris Wilson ret = -EINVAL; 23970207686SChris Wilson } 24070207686SChris Wilson } else { 24170207686SChris Wilson if (err != -EDEADLK && abba.result != -EDEADLK) { 24270207686SChris Wilson pr_err("%s: missed ABBA deadlock, A err=%d, B err=%d\n", 24370207686SChris Wilson __func__, err, abba.result); 24470207686SChris Wilson ret = -EINVAL; 24570207686SChris Wilson } 24670207686SChris Wilson } 24770207686SChris Wilson return ret; 24870207686SChris Wilson } 24970207686SChris Wilson 250d1b42b80SChris Wilson struct test_cycle { 251d1b42b80SChris Wilson struct work_struct work; 252d1b42b80SChris Wilson struct ww_mutex a_mutex; 253d1b42b80SChris Wilson struct ww_mutex *b_mutex; 254d1b42b80SChris Wilson struct completion *a_signal; 255d1b42b80SChris Wilson struct completion b_signal; 256d1b42b80SChris Wilson int result; 257d1b42b80SChris Wilson }; 258d1b42b80SChris Wilson 259d1b42b80SChris Wilson static void test_cycle_work(struct work_struct *work) 260d1b42b80SChris Wilson { 261d1b42b80SChris Wilson struct test_cycle *cycle = container_of(work, typeof(*cycle), work); 262d1b42b80SChris Wilson struct ww_acquire_ctx ctx; 263d1b42b80SChris Wilson int err; 264d1b42b80SChris Wilson 265d1b42b80SChris Wilson ww_acquire_init(&ctx, &ww_class); 266d1b42b80SChris Wilson ww_mutex_lock(&cycle->a_mutex, &ctx); 267d1b42b80SChris Wilson 268d1b42b80SChris Wilson complete(cycle->a_signal); 269d1b42b80SChris Wilson wait_for_completion(&cycle->b_signal); 270d1b42b80SChris Wilson 271d1b42b80SChris Wilson err = ww_mutex_lock(cycle->b_mutex, &ctx); 272d1b42b80SChris Wilson if (err == -EDEADLK) { 273d1b42b80SChris Wilson ww_mutex_unlock(&cycle->a_mutex); 274d1b42b80SChris Wilson ww_mutex_lock_slow(cycle->b_mutex, &ctx); 275d1b42b80SChris Wilson err = ww_mutex_lock(&cycle->a_mutex, &ctx); 276d1b42b80SChris Wilson } 277d1b42b80SChris Wilson 278d1b42b80SChris Wilson if (!err) 279d1b42b80SChris Wilson ww_mutex_unlock(cycle->b_mutex); 280d1b42b80SChris Wilson ww_mutex_unlock(&cycle->a_mutex); 281d1b42b80SChris Wilson ww_acquire_fini(&ctx); 282d1b42b80SChris Wilson 283d1b42b80SChris Wilson cycle->result = err; 284d1b42b80SChris Wilson } 285d1b42b80SChris Wilson 286d1b42b80SChris Wilson static int __test_cycle(unsigned int nthreads) 287d1b42b80SChris Wilson { 288d1b42b80SChris Wilson struct test_cycle *cycles; 289d1b42b80SChris Wilson unsigned int n, last = nthreads - 1; 290d1b42b80SChris Wilson int ret; 291d1b42b80SChris Wilson 292d1b42b80SChris Wilson cycles = kmalloc_array(nthreads, sizeof(*cycles), GFP_KERNEL); 293d1b42b80SChris Wilson if (!cycles) 294d1b42b80SChris Wilson return -ENOMEM; 295d1b42b80SChris Wilson 296d1b42b80SChris Wilson for (n = 0; n < nthreads; n++) { 297d1b42b80SChris Wilson struct test_cycle *cycle = &cycles[n]; 298d1b42b80SChris Wilson 299d1b42b80SChris Wilson ww_mutex_init(&cycle->a_mutex, &ww_class); 300d1b42b80SChris Wilson if (n == last) 301d1b42b80SChris Wilson cycle->b_mutex = &cycles[0].a_mutex; 302d1b42b80SChris Wilson else 303d1b42b80SChris Wilson cycle->b_mutex = &cycles[n + 1].a_mutex; 304d1b42b80SChris Wilson 305d1b42b80SChris Wilson if (n == 0) 306d1b42b80SChris Wilson cycle->a_signal = &cycles[last].b_signal; 307d1b42b80SChris Wilson else 308d1b42b80SChris Wilson cycle->a_signal = &cycles[n - 1].b_signal; 309d1b42b80SChris Wilson init_completion(&cycle->b_signal); 310d1b42b80SChris Wilson 311d1b42b80SChris Wilson INIT_WORK(&cycle->work, test_cycle_work); 312d1b42b80SChris Wilson cycle->result = 0; 313d1b42b80SChris Wilson } 314d1b42b80SChris Wilson 315d1b42b80SChris Wilson for (n = 0; n < nthreads; n++) 316d1b42b80SChris Wilson queue_work(wq, &cycles[n].work); 317d1b42b80SChris Wilson 318d1b42b80SChris Wilson flush_workqueue(wq); 319d1b42b80SChris Wilson 320d1b42b80SChris Wilson ret = 0; 321d1b42b80SChris Wilson for (n = 0; n < nthreads; n++) { 322d1b42b80SChris Wilson struct test_cycle *cycle = &cycles[n]; 323d1b42b80SChris Wilson 324d1b42b80SChris Wilson if (!cycle->result) 325d1b42b80SChris Wilson continue; 326d1b42b80SChris Wilson 327d1b42b80SChris Wilson pr_err("cylic deadlock not resolved, ret[%d/%d] = %d\n", 328d1b42b80SChris Wilson n, nthreads, cycle->result); 329d1b42b80SChris Wilson ret = -EINVAL; 330d1b42b80SChris Wilson break; 331d1b42b80SChris Wilson } 332d1b42b80SChris Wilson 333d1b42b80SChris Wilson for (n = 0; n < nthreads; n++) 334d1b42b80SChris Wilson ww_mutex_destroy(&cycles[n].a_mutex); 335d1b42b80SChris Wilson kfree(cycles); 336d1b42b80SChris Wilson return ret; 337d1b42b80SChris Wilson } 338d1b42b80SChris Wilson 339d1b42b80SChris Wilson static int test_cycle(unsigned int ncpus) 340d1b42b80SChris Wilson { 341d1b42b80SChris Wilson unsigned int n; 342d1b42b80SChris Wilson int ret; 343d1b42b80SChris Wilson 344d1b42b80SChris Wilson for (n = 2; n <= ncpus + 1; n++) { 345d1b42b80SChris Wilson ret = __test_cycle(n); 346d1b42b80SChris Wilson if (ret) 347d1b42b80SChris Wilson return ret; 348d1b42b80SChris Wilson } 349d1b42b80SChris Wilson 350d1b42b80SChris Wilson return 0; 351d1b42b80SChris Wilson } 352d1b42b80SChris Wilson 3532a0c1128SChris Wilson struct stress { 3542a0c1128SChris Wilson struct work_struct work; 3552a0c1128SChris Wilson struct ww_mutex *locks; 3562a0c1128SChris Wilson int nlocks; 3572a0c1128SChris Wilson int nloops; 3582a0c1128SChris Wilson }; 3592a0c1128SChris Wilson 3602a0c1128SChris Wilson static int *get_random_order(int count) 3612a0c1128SChris Wilson { 3622a0c1128SChris Wilson int *order; 3632a0c1128SChris Wilson int n, r, tmp; 3642a0c1128SChris Wilson 3652a0c1128SChris Wilson order = kmalloc_array(count, sizeof(*order), GFP_TEMPORARY); 3662a0c1128SChris Wilson if (!order) 3672a0c1128SChris Wilson return order; 3682a0c1128SChris Wilson 3692a0c1128SChris Wilson for (n = 0; n < count; n++) 3702a0c1128SChris Wilson order[n] = n; 3712a0c1128SChris Wilson 3722a0c1128SChris Wilson for (n = count - 1; n > 1; n--) { 3732a0c1128SChris Wilson r = get_random_int() % (n + 1); 3742a0c1128SChris Wilson if (r != n) { 3752a0c1128SChris Wilson tmp = order[n]; 3762a0c1128SChris Wilson order[n] = order[r]; 3772a0c1128SChris Wilson order[r] = tmp; 3782a0c1128SChris Wilson } 3792a0c1128SChris Wilson } 3802a0c1128SChris Wilson 3812a0c1128SChris Wilson return order; 3822a0c1128SChris Wilson } 3832a0c1128SChris Wilson 3842a0c1128SChris Wilson static void dummy_load(struct stress *stress) 3852a0c1128SChris Wilson { 3862a0c1128SChris Wilson usleep_range(1000, 2000); 3872a0c1128SChris Wilson } 3882a0c1128SChris Wilson 3892a0c1128SChris Wilson static void stress_inorder_work(struct work_struct *work) 3902a0c1128SChris Wilson { 3912a0c1128SChris Wilson struct stress *stress = container_of(work, typeof(*stress), work); 3922a0c1128SChris Wilson const int nlocks = stress->nlocks; 3932a0c1128SChris Wilson struct ww_mutex *locks = stress->locks; 3942a0c1128SChris Wilson struct ww_acquire_ctx ctx; 3952a0c1128SChris Wilson int *order; 3962a0c1128SChris Wilson 3972a0c1128SChris Wilson order = get_random_order(nlocks); 3982a0c1128SChris Wilson if (!order) 3992a0c1128SChris Wilson return; 4002a0c1128SChris Wilson 4012a0c1128SChris Wilson ww_acquire_init(&ctx, &ww_class); 4022a0c1128SChris Wilson 4032a0c1128SChris Wilson do { 4042a0c1128SChris Wilson int contended = -1; 4052a0c1128SChris Wilson int n, err; 4062a0c1128SChris Wilson 4072a0c1128SChris Wilson retry: 4082a0c1128SChris Wilson err = 0; 4092a0c1128SChris Wilson for (n = 0; n < nlocks; n++) { 4102a0c1128SChris Wilson if (n == contended) 4112a0c1128SChris Wilson continue; 4122a0c1128SChris Wilson 4132a0c1128SChris Wilson err = ww_mutex_lock(&locks[order[n]], &ctx); 4142a0c1128SChris Wilson if (err < 0) 4152a0c1128SChris Wilson break; 4162a0c1128SChris Wilson } 4172a0c1128SChris Wilson if (!err) 4182a0c1128SChris Wilson dummy_load(stress); 4192a0c1128SChris Wilson 4202a0c1128SChris Wilson if (contended > n) 4212a0c1128SChris Wilson ww_mutex_unlock(&locks[order[contended]]); 4222a0c1128SChris Wilson contended = n; 4232a0c1128SChris Wilson while (n--) 4242a0c1128SChris Wilson ww_mutex_unlock(&locks[order[n]]); 4252a0c1128SChris Wilson 4262a0c1128SChris Wilson if (err == -EDEADLK) { 4272a0c1128SChris Wilson ww_mutex_lock_slow(&locks[order[contended]], &ctx); 4282a0c1128SChris Wilson goto retry; 4292a0c1128SChris Wilson } 4302a0c1128SChris Wilson 4312a0c1128SChris Wilson if (err) { 4322a0c1128SChris Wilson pr_err_once("stress (%s) failed with %d\n", 4332a0c1128SChris Wilson __func__, err); 4342a0c1128SChris Wilson break; 4352a0c1128SChris Wilson } 4362a0c1128SChris Wilson } while (--stress->nloops); 4372a0c1128SChris Wilson 4382a0c1128SChris Wilson ww_acquire_fini(&ctx); 4392a0c1128SChris Wilson 4402a0c1128SChris Wilson kfree(order); 4412a0c1128SChris Wilson kfree(stress); 4422a0c1128SChris Wilson } 4432a0c1128SChris Wilson 4442a0c1128SChris Wilson struct reorder_lock { 4452a0c1128SChris Wilson struct list_head link; 4462a0c1128SChris Wilson struct ww_mutex *lock; 4472a0c1128SChris Wilson }; 4482a0c1128SChris Wilson 4492a0c1128SChris Wilson static void stress_reorder_work(struct work_struct *work) 4502a0c1128SChris Wilson { 4512a0c1128SChris Wilson struct stress *stress = container_of(work, typeof(*stress), work); 4522a0c1128SChris Wilson LIST_HEAD(locks); 4532a0c1128SChris Wilson struct ww_acquire_ctx ctx; 4542a0c1128SChris Wilson struct reorder_lock *ll, *ln; 4552a0c1128SChris Wilson int *order; 4562a0c1128SChris Wilson int n, err; 4572a0c1128SChris Wilson 4582a0c1128SChris Wilson order = get_random_order(stress->nlocks); 4592a0c1128SChris Wilson if (!order) 4602a0c1128SChris Wilson return; 4612a0c1128SChris Wilson 4622a0c1128SChris Wilson for (n = 0; n < stress->nlocks; n++) { 4632a0c1128SChris Wilson ll = kmalloc(sizeof(*ll), GFP_KERNEL); 4642a0c1128SChris Wilson if (!ll) 4652a0c1128SChris Wilson goto out; 4662a0c1128SChris Wilson 4672a0c1128SChris Wilson ll->lock = &stress->locks[order[n]]; 4682a0c1128SChris Wilson list_add(&ll->link, &locks); 4692a0c1128SChris Wilson } 4702a0c1128SChris Wilson kfree(order); 4712a0c1128SChris Wilson order = NULL; 4722a0c1128SChris Wilson 4732a0c1128SChris Wilson ww_acquire_init(&ctx, &ww_class); 4742a0c1128SChris Wilson 4752a0c1128SChris Wilson do { 4762a0c1128SChris Wilson list_for_each_entry(ll, &locks, link) { 4772a0c1128SChris Wilson err = ww_mutex_lock(ll->lock, &ctx); 4782a0c1128SChris Wilson if (!err) 4792a0c1128SChris Wilson continue; 4802a0c1128SChris Wilson 4812a0c1128SChris Wilson ln = ll; 4822a0c1128SChris Wilson list_for_each_entry_continue_reverse(ln, &locks, link) 4832a0c1128SChris Wilson ww_mutex_unlock(ln->lock); 4842a0c1128SChris Wilson 4852a0c1128SChris Wilson if (err != -EDEADLK) { 4862a0c1128SChris Wilson pr_err_once("stress (%s) failed with %d\n", 4872a0c1128SChris Wilson __func__, err); 4882a0c1128SChris Wilson break; 4892a0c1128SChris Wilson } 4902a0c1128SChris Wilson 4912a0c1128SChris Wilson ww_mutex_lock_slow(ll->lock, &ctx); 4922a0c1128SChris Wilson list_move(&ll->link, &locks); /* restarts iteration */ 4932a0c1128SChris Wilson } 4942a0c1128SChris Wilson 4952a0c1128SChris Wilson dummy_load(stress); 4962a0c1128SChris Wilson list_for_each_entry(ll, &locks, link) 4972a0c1128SChris Wilson ww_mutex_unlock(ll->lock); 4982a0c1128SChris Wilson } while (--stress->nloops); 4992a0c1128SChris Wilson 5002a0c1128SChris Wilson ww_acquire_fini(&ctx); 5012a0c1128SChris Wilson 5022a0c1128SChris Wilson out: 5032a0c1128SChris Wilson list_for_each_entry_safe(ll, ln, &locks, link) 5042a0c1128SChris Wilson kfree(ll); 5052a0c1128SChris Wilson kfree(order); 5062a0c1128SChris Wilson kfree(stress); 5072a0c1128SChris Wilson } 5082a0c1128SChris Wilson 5092a0c1128SChris Wilson static void stress_one_work(struct work_struct *work) 5102a0c1128SChris Wilson { 5112a0c1128SChris Wilson struct stress *stress = container_of(work, typeof(*stress), work); 5122a0c1128SChris Wilson const int nlocks = stress->nlocks; 5132a0c1128SChris Wilson struct ww_mutex *lock = stress->locks + (get_random_int() % nlocks); 5142a0c1128SChris Wilson int err; 5152a0c1128SChris Wilson 5162a0c1128SChris Wilson do { 5172a0c1128SChris Wilson err = ww_mutex_lock(lock, NULL); 5182a0c1128SChris Wilson if (!err) { 5192a0c1128SChris Wilson dummy_load(stress); 5202a0c1128SChris Wilson ww_mutex_unlock(lock); 5212a0c1128SChris Wilson } else { 5222a0c1128SChris Wilson pr_err_once("stress (%s) failed with %d\n", 5232a0c1128SChris Wilson __func__, err); 5242a0c1128SChris Wilson break; 5252a0c1128SChris Wilson } 5262a0c1128SChris Wilson } while (--stress->nloops); 5272a0c1128SChris Wilson 5282a0c1128SChris Wilson kfree(stress); 5292a0c1128SChris Wilson } 5302a0c1128SChris Wilson 5312a0c1128SChris Wilson #define STRESS_INORDER BIT(0) 5322a0c1128SChris Wilson #define STRESS_REORDER BIT(1) 5332a0c1128SChris Wilson #define STRESS_ONE BIT(2) 5342a0c1128SChris Wilson #define STRESS_ALL (STRESS_INORDER | STRESS_REORDER | STRESS_ONE) 5352a0c1128SChris Wilson 5362a0c1128SChris Wilson static int stress(int nlocks, int nthreads, int nloops, unsigned int flags) 5372a0c1128SChris Wilson { 5382a0c1128SChris Wilson struct ww_mutex *locks; 5392a0c1128SChris Wilson int n; 5402a0c1128SChris Wilson 5412a0c1128SChris Wilson locks = kmalloc_array(nlocks, sizeof(*locks), GFP_KERNEL); 5422a0c1128SChris Wilson if (!locks) 5432a0c1128SChris Wilson return -ENOMEM; 5442a0c1128SChris Wilson 5452a0c1128SChris Wilson for (n = 0; n < nlocks; n++) 5462a0c1128SChris Wilson ww_mutex_init(&locks[n], &ww_class); 5472a0c1128SChris Wilson 5482a0c1128SChris Wilson for (n = 0; nthreads; n++) { 5492a0c1128SChris Wilson struct stress *stress; 5502a0c1128SChris Wilson void (*fn)(struct work_struct *work); 5512a0c1128SChris Wilson 5522a0c1128SChris Wilson fn = NULL; 5532a0c1128SChris Wilson switch (n & 3) { 5542a0c1128SChris Wilson case 0: 5552a0c1128SChris Wilson if (flags & STRESS_INORDER) 5562a0c1128SChris Wilson fn = stress_inorder_work; 5572a0c1128SChris Wilson break; 5582a0c1128SChris Wilson case 1: 5592a0c1128SChris Wilson if (flags & STRESS_REORDER) 5602a0c1128SChris Wilson fn = stress_reorder_work; 5612a0c1128SChris Wilson break; 5622a0c1128SChris Wilson case 2: 5632a0c1128SChris Wilson if (flags & STRESS_ONE) 5642a0c1128SChris Wilson fn = stress_one_work; 5652a0c1128SChris Wilson break; 5662a0c1128SChris Wilson } 5672a0c1128SChris Wilson 5682a0c1128SChris Wilson if (!fn) 5692a0c1128SChris Wilson continue; 5702a0c1128SChris Wilson 5712a0c1128SChris Wilson stress = kmalloc(sizeof(*stress), GFP_KERNEL); 5722a0c1128SChris Wilson if (!stress) 5732a0c1128SChris Wilson break; 5742a0c1128SChris Wilson 5752a0c1128SChris Wilson INIT_WORK(&stress->work, fn); 5762a0c1128SChris Wilson stress->locks = locks; 5772a0c1128SChris Wilson stress->nlocks = nlocks; 5782a0c1128SChris Wilson stress->nloops = nloops; 5792a0c1128SChris Wilson 5802a0c1128SChris Wilson queue_work(wq, &stress->work); 5812a0c1128SChris Wilson nthreads--; 5822a0c1128SChris Wilson } 5832a0c1128SChris Wilson 5842a0c1128SChris Wilson flush_workqueue(wq); 5852a0c1128SChris Wilson 5862a0c1128SChris Wilson for (n = 0; n < nlocks; n++) 5872a0c1128SChris Wilson ww_mutex_destroy(&locks[n]); 5882a0c1128SChris Wilson kfree(locks); 5892a0c1128SChris Wilson 5902a0c1128SChris Wilson return 0; 5912a0c1128SChris Wilson } 5922a0c1128SChris Wilson 593f2a5fec1SChris Wilson static int __init test_ww_mutex_init(void) 594f2a5fec1SChris Wilson { 595d1b42b80SChris Wilson int ncpus = num_online_cpus(); 596f2a5fec1SChris Wilson int ret; 597f2a5fec1SChris Wilson 598d1b42b80SChris Wilson wq = alloc_workqueue("test-ww_mutex", WQ_UNBOUND, 0); 599d1b42b80SChris Wilson if (!wq) 600d1b42b80SChris Wilson return -ENOMEM; 601d1b42b80SChris Wilson 602f2a5fec1SChris Wilson ret = test_mutex(); 603f2a5fec1SChris Wilson if (ret) 604f2a5fec1SChris Wilson return ret; 605f2a5fec1SChris Wilson 606c22fb380SChris Wilson ret = test_aa(); 607c22fb380SChris Wilson if (ret) 608c22fb380SChris Wilson return ret; 609c22fb380SChris Wilson 61070207686SChris Wilson ret = test_abba(false); 61170207686SChris Wilson if (ret) 61270207686SChris Wilson return ret; 61370207686SChris Wilson 61470207686SChris Wilson ret = test_abba(true); 61570207686SChris Wilson if (ret) 61670207686SChris Wilson return ret; 61770207686SChris Wilson 618d1b42b80SChris Wilson ret = test_cycle(ncpus); 619d1b42b80SChris Wilson if (ret) 620d1b42b80SChris Wilson return ret; 621d1b42b80SChris Wilson 6222a0c1128SChris Wilson ret = stress(16, 2*ncpus, 1<<10, STRESS_INORDER); 6232a0c1128SChris Wilson if (ret) 6242a0c1128SChris Wilson return ret; 6252a0c1128SChris Wilson 6262a0c1128SChris Wilson ret = stress(16, 2*ncpus, 1<<10, STRESS_REORDER); 6272a0c1128SChris Wilson if (ret) 6282a0c1128SChris Wilson return ret; 6292a0c1128SChris Wilson 6302a0c1128SChris Wilson ret = stress(4096, hweight32(STRESS_ALL)*ncpus, 1<<12, STRESS_ALL); 6312a0c1128SChris Wilson if (ret) 6322a0c1128SChris Wilson return ret; 6332a0c1128SChris Wilson 634f2a5fec1SChris Wilson return 0; 635f2a5fec1SChris Wilson } 636f2a5fec1SChris Wilson 637f2a5fec1SChris Wilson static void __exit test_ww_mutex_exit(void) 638f2a5fec1SChris Wilson { 639d1b42b80SChris Wilson destroy_workqueue(wq); 640f2a5fec1SChris Wilson } 641f2a5fec1SChris Wilson 642f2a5fec1SChris Wilson module_init(test_ww_mutex_init); 643f2a5fec1SChris Wilson module_exit(test_ww_mutex_exit); 644f2a5fec1SChris Wilson 645f2a5fec1SChris Wilson MODULE_LICENSE("GPL"); 646f2a5fec1SChris Wilson MODULE_AUTHOR("Intel Corporation"); 647