From 1de994452f44005e4b1f5c6c77eae4a26f86d484 Mon Sep 17 00:00:00 2001 From: Maarten Lankhorst Date: Thu, 20 Jun 2013 13:31:24 +0200 Subject: mutex: Add w/w tests to lib/locking-selftest.c This stresses the lockdep code in some ways specifically useful to ww_mutexes. It adds checks for most of the common locking errors. Signed-off-by: Maarten Lankhorst Acked-by: Peter Zijlstra Cc: dri-devel@lists.freedesktop.org Cc: linaro-mm-sig@lists.linaro.org Cc: robclark@gmail.com Cc: rostedt@goodmis.org Cc: daniel@ffwll.ch Cc: Linus Torvalds Cc: Andrew Morton Cc: Thomas Gleixner Link: http://lkml.kernel.org/r/20130620113124.4001.23186.stgit@patser Signed-off-by: Ingo Molnar --- lib/locking-selftest.c | 400 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 381 insertions(+), 19 deletions(-) (limited to 'lib/locking-selftest.c') diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index c3eb261a7df..996226224bc 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -26,6 +26,8 @@ */ static unsigned int debug_locks_verbose; +static DEFINE_WW_CLASS(ww_lockdep); + static int __init setup_debug_locks_verbose(char *str) { get_option(&str, &debug_locks_verbose); @@ -42,6 +44,10 @@ __setup("debug_locks_verbose=", setup_debug_locks_verbose); #define LOCKTYPE_RWLOCK 0x2 #define LOCKTYPE_MUTEX 0x4 #define LOCKTYPE_RWSEM 0x8 +#define LOCKTYPE_WW 0x10 + +static struct ww_acquire_ctx t, t2; +static struct ww_mutex o, o2; /* * Normal standalone locks, for the circular and irq-context @@ -193,6 +199,20 @@ static void init_shared_classes(void) #define RSU(x) up_read(&rwsem_##x) #define RWSI(x) init_rwsem(&rwsem_##x) +#ifndef CONFIG_DEBUG_WW_MUTEX_SLOWPATH +#define WWAI(x) ww_acquire_init(x, &ww_lockdep) +#else +#define WWAI(x) do { ww_acquire_init(x, &ww_lockdep); (x)->deadlock_inject_countdown = ~0U; } while (0) +#endif +#define WWAD(x) ww_acquire_done(x) +#define WWAF(x) ww_acquire_fini(x) + +#define WWL(x, c) ww_mutex_lock(x, c) +#define WWT(x) ww_mutex_trylock(x) +#define WWL1(x) ww_mutex_lock(x, NULL) +#define WWU(x) ww_mutex_unlock(x) + + #define LOCK_UNLOCK_2(x,y) LOCK(x); LOCK(y); UNLOCK(y); UNLOCK(x) /* @@ -894,11 +914,13 @@ GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft) # define I_RWLOCK(x) lockdep_reset_lock(&rwlock_##x.dep_map) # define I_MUTEX(x) lockdep_reset_lock(&mutex_##x.dep_map) # define I_RWSEM(x) lockdep_reset_lock(&rwsem_##x.dep_map) +# define I_WW(x) lockdep_reset_lock(&x.dep_map) #else # define I_SPINLOCK(x) # define I_RWLOCK(x) # define I_MUTEX(x) # define I_RWSEM(x) +# define I_WW(x) #endif #define I1(x) \ @@ -920,11 +942,20 @@ GENERATE_PERMUTATIONS_3_EVENTS(irq_read_recursion_soft) static void reset_locks(void) { local_irq_disable(); + lockdep_free_key_range(&ww_lockdep.acquire_key, 1); + lockdep_free_key_range(&ww_lockdep.mutex_key, 1); + I1(A); I1(B); I1(C); I1(D); I1(X1); I1(X2); I1(Y1); I1(Y2); I1(Z1); I1(Z2); + I_WW(t); I_WW(t2); I_WW(o.base); I_WW(o2.base); lockdep_reset(); I2(A); I2(B); I2(C); I2(D); init_shared_classes(); + + ww_mutex_init(&o, &ww_lockdep); ww_mutex_init(&o2, &ww_lockdep); + memset(&t, 0, sizeof(t)); memset(&t2, 0, sizeof(t2)); + memset(&ww_lockdep.acquire_key, 0, sizeof(ww_lockdep.acquire_key)); + memset(&ww_lockdep.mutex_key, 0, sizeof(ww_lockdep.mutex_key)); local_irq_enable(); } @@ -938,7 +969,6 @@ static int unexpected_testcase_failures; static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) { unsigned long saved_preempt_count = preempt_count(); - int expected_failure = 0; WARN_ON(irqs_disabled()); @@ -946,26 +976,16 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) /* * Filter out expected failures: */ + if (debug_locks != expected) { #ifndef CONFIG_PROVE_LOCKING - if ((lockclass_mask & LOCKTYPE_SPIN) && debug_locks != expected) - expected_failure = 1; - if ((lockclass_mask & LOCKTYPE_RWLOCK) && debug_locks != expected) - expected_failure = 1; - if ((lockclass_mask & LOCKTYPE_MUTEX) && debug_locks != expected) - expected_failure = 1; - if ((lockclass_mask & LOCKTYPE_RWSEM) && debug_locks != expected) - expected_failure = 1; + expected_testcase_failures++; + printk("failed|"); +#else + unexpected_testcase_failures++; + printk("FAILED|"); + + dump_stack(); #endif - if (debug_locks != expected) { - if (expected_failure) { - expected_testcase_failures++; - printk("failed|"); - } else { - unexpected_testcase_failures++; - - printk("FAILED|"); - dump_stack(); - } } else { testcase_successes++; printk(" ok |"); @@ -1108,6 +1128,346 @@ static inline void print_testname(const char *testname) DO_TESTCASE_6IRW(desc, name, 312); \ DO_TESTCASE_6IRW(desc, name, 321); +static void ww_test_fail_acquire(void) +{ + int ret; + + WWAI(&t); + t.stamp++; + + ret = WWL(&o, &t); + + if (WARN_ON(!o.ctx) || + WARN_ON(ret)) + return; + + /* No lockdep test, pure API */ + ret = WWL(&o, &t); + WARN_ON(ret != -EALREADY); + + ret = WWT(&o); + WARN_ON(ret); + + t2 = t; + t2.stamp++; + ret = WWL(&o, &t2); + WARN_ON(ret != -EDEADLK); + WWU(&o); + + if (WWT(&o)) + WWU(&o); +#ifdef CONFIG_DEBUG_LOCK_ALLOC + else + DEBUG_LOCKS_WARN_ON(1); +#endif +} + +static void ww_test_two_contexts(void) +{ + WWAI(&t); + WWAI(&t2); +} + +static void ww_test_diff_class(void) +{ + WWAI(&t); +#ifdef CONFIG_DEBUG_MUTEXES + t.ww_class = NULL; +#endif + WWL(&o, &t); +} + +static void ww_test_context_done_twice(void) +{ + WWAI(&t); + WWAD(&t); + WWAD(&t); + WWAF(&t); +} + +static void ww_test_context_unlock_twice(void) +{ + WWAI(&t); + WWAD(&t); + WWAF(&t); + WWAF(&t); +} + +static void ww_test_context_fini_early(void) +{ + WWAI(&t); + WWL(&o, &t); + WWAD(&t); + WWAF(&t); +} + +static void ww_test_context_lock_after_done(void) +{ + WWAI(&t); + WWAD(&t); + WWL(&o, &t); +} + +static void ww_test_object_unlock_twice(void) +{ + WWL1(&o); + WWU(&o); + WWU(&o); +} + +static void ww_test_object_lock_unbalanced(void) +{ + WWAI(&t); + WWL(&o, &t); + t.acquired = 0; + WWU(&o); + WWAF(&t); +} + +static void ww_test_object_lock_stale_context(void) +{ + WWAI(&t); + o.ctx = &t2; + WWL(&o, &t); +} + +static void ww_test_spin_nest_unlocked(void) +{ + raw_spin_lock_nest_lock(&lock_A, &o.base); + U(A); +} + +static void ww_test_unneeded_slow(void) +{ + WWAI(&t); + + ww_mutex_lock_slow(&o, &t); +} + +static void ww_test_context_block(void) +{ + int ret; + + WWAI(&t); + + ret = WWL(&o, &t); + WARN_ON(ret); + WWL1(&o2); +} + +static void ww_test_context_try(void) +{ + int ret; + + WWAI(&t); + + ret = WWL(&o, &t); + WARN_ON(ret); + + ret = WWT(&o2); + WARN_ON(!ret); + WWU(&o2); + WWU(&o); +} + +static void ww_test_context_context(void) +{ + int ret; + + WWAI(&t); + + ret = WWL(&o, &t); + WARN_ON(ret); + + ret = WWL(&o2, &t); + WARN_ON(ret); + + WWU(&o2); + WWU(&o); +} + +static void ww_test_try_block(void) +{ + bool ret; + + ret = WWT(&o); + WARN_ON(!ret); + + WWL1(&o2); + WWU(&o2); + WWU(&o); +} + +static void ww_test_try_try(void) +{ + bool ret; + + ret = WWT(&o); + WARN_ON(!ret); + ret = WWT(&o2); + WARN_ON(!ret); + WWU(&o2); + WWU(&o); +} + +static void ww_test_try_context(void) +{ + int ret; + + ret = WWT(&o); + WARN_ON(!ret); + + WWAI(&t); + + ret = WWL(&o2, &t); + WARN_ON(ret); +} + +static void ww_test_block_block(void) +{ + WWL1(&o); + WWL1(&o2); +} + +static void ww_test_block_try(void) +{ + bool ret; + + WWL1(&o); + ret = WWT(&o2); + WARN_ON(!ret); +} + +static void ww_test_block_context(void) +{ + int ret; + + WWL1(&o); + WWAI(&t); + + ret = WWL(&o2, &t); + WARN_ON(ret); +} + +static void ww_test_spin_block(void) +{ + L(A); + U(A); + + WWL1(&o); + L(A); + U(A); + WWU(&o); + + L(A); + WWL1(&o); + WWU(&o); + U(A); +} + +static void ww_test_spin_try(void) +{ + bool ret; + + L(A); + U(A); + + ret = WWT(&o); + WARN_ON(!ret); + L(A); + U(A); + WWU(&o); + + L(A); + ret = WWT(&o); + WARN_ON(!ret); + WWU(&o); + U(A); +} + +static void ww_test_spin_context(void) +{ + int ret; + + L(A); + U(A); + + WWAI(&t); + + ret = WWL(&o, &t); + WARN_ON(ret); + L(A); + U(A); + WWU(&o); + + L(A); + ret = WWL(&o, &t); + WARN_ON(ret); + WWU(&o); + U(A); +} + +static void ww_tests(void) +{ + printk(" --------------------------------------------------------------------------\n"); + printk(" | Wound/wait tests |\n"); + printk(" ---------------------\n"); + + print_testname("ww api failures"); + dotest(ww_test_fail_acquire, SUCCESS, LOCKTYPE_WW); + dotest(ww_test_unneeded_slow, FAILURE, LOCKTYPE_WW); + printk("\n"); + + print_testname("ww contexts mixing"); + dotest(ww_test_two_contexts, FAILURE, LOCKTYPE_WW); + dotest(ww_test_diff_class, FAILURE, LOCKTYPE_WW); + printk("\n"); + + print_testname("finishing ww context"); + dotest(ww_test_context_done_twice, FAILURE, LOCKTYPE_WW); + dotest(ww_test_context_unlock_twice, FAILURE, LOCKTYPE_WW); + dotest(ww_test_context_fini_early, FAILURE, LOCKTYPE_WW); + dotest(ww_test_context_lock_after_done, FAILURE, LOCKTYPE_WW); + printk("\n"); + + print_testname("locking mismatches"); + dotest(ww_test_object_unlock_twice, FAILURE, LOCKTYPE_WW); + dotest(ww_test_object_lock_unbalanced, FAILURE, LOCKTYPE_WW); + dotest(ww_test_object_lock_stale_context, FAILURE, LOCKTYPE_WW); + printk("\n"); + + print_testname("spinlock nest unlocked"); + dotest(ww_test_spin_nest_unlocked, FAILURE, LOCKTYPE_WW); + printk("\n"); + + printk(" -----------------------------------------------------\n"); + printk(" |block | try |context|\n"); + printk(" -----------------------------------------------------\n"); + + print_testname("context"); + dotest(ww_test_context_block, FAILURE, LOCKTYPE_WW); + dotest(ww_test_context_try, SUCCESS, LOCKTYPE_WW); + dotest(ww_test_context_context, SUCCESS, LOCKTYPE_WW); + printk("\n"); + + print_testname("try"); + dotest(ww_test_try_block, FAILURE, LOCKTYPE_WW); + dotest(ww_test_try_try, SUCCESS, LOCKTYPE_WW); + dotest(ww_test_try_context, FAILURE, LOCKTYPE_WW); + printk("\n"); + + print_testname("block"); + dotest(ww_test_block_block, FAILURE, LOCKTYPE_WW); + dotest(ww_test_block_try, SUCCESS, LOCKTYPE_WW); + dotest(ww_test_block_context, FAILURE, LOCKTYPE_WW); + printk("\n"); + + print_testname("spinlock"); + dotest(ww_test_spin_block, FAILURE, LOCKTYPE_WW); + dotest(ww_test_spin_try, SUCCESS, LOCKTYPE_WW); + dotest(ww_test_spin_context, FAILURE, LOCKTYPE_WW); + printk("\n"); +} void locking_selftest(void) { @@ -1188,6 +1548,8 @@ void locking_selftest(void) DO_TESTCASE_6x2("irq read-recursion", irq_read_recursion); // DO_TESTCASE_6x2B("irq read-recursion #2", irq_read_recursion2); + ww_tests(); + if (unexpected_testcase_failures) { printk("-----------------------------------------------------------------\n"); debug_locks = 0; -- cgit v1.2.3-70-g09d2 From 2fe3d4b149ccebbb384062fbbe6634439f2bf120 Mon Sep 17 00:00:00 2001 From: Maarten Lankhorst Date: Thu, 20 Jun 2013 13:31:30 +0200 Subject: mutex: Add more tests to lib/locking-selftest.c None of the ww_mutex codepaths should be taken in the 'normal' mutex calls. The easiest way to verify this is by using the normal mutex calls, and making sure o.ctx is unmodified. Signed-off-by: Maarten Lankhorst Acked-by: Peter Zijlstra Cc: dri-devel@lists.freedesktop.org Cc: linaro-mm-sig@lists.linaro.org Cc: robclark@gmail.com Cc: rostedt@goodmis.org Cc: daniel@ffwll.ch Cc: Linus Torvalds Cc: Andrew Morton Cc: Thomas Gleixner Link: http://lkml.kernel.org/r/20130620113130.4001.45423.stgit@patser Signed-off-by: Ingo Molnar --- lib/locking-selftest.c | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) (limited to 'lib/locking-selftest.c') diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index 996226224bc..37faefdbb67 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -1162,6 +1162,67 @@ static void ww_test_fail_acquire(void) #endif } +static void ww_test_normal(void) +{ + int ret; + + WWAI(&t); + + /* + * None of the ww_mutex codepaths should be taken in the 'normal' + * mutex calls. The easiest way to verify this is by using the + * normal mutex calls, and making sure o.ctx is unmodified. + */ + + /* mutex_lock (and indirectly, mutex_lock_nested) */ + o.ctx = (void *)~0UL; + mutex_lock(&o.base); + mutex_unlock(&o.base); + WARN_ON(o.ctx != (void *)~0UL); + + /* mutex_lock_interruptible (and *_nested) */ + o.ctx = (void *)~0UL; + ret = mutex_lock_interruptible(&o.base); + if (!ret) + mutex_unlock(&o.base); + else + WARN_ON(1); + WARN_ON(o.ctx != (void *)~0UL); + + /* mutex_lock_killable (and *_nested) */ + o.ctx = (void *)~0UL; + ret = mutex_lock_killable(&o.base); + if (!ret) + mutex_unlock(&o.base); + else + WARN_ON(1); + WARN_ON(o.ctx != (void *)~0UL); + + /* trylock, succeeding */ + o.ctx = (void *)~0UL; + ret = mutex_trylock(&o.base); + WARN_ON(!ret); + if (ret) + mutex_unlock(&o.base); + else + WARN_ON(1); + WARN_ON(o.ctx != (void *)~0UL); + + /* trylock, failing */ + o.ctx = (void *)~0UL; + mutex_lock(&o.base); + ret = mutex_trylock(&o.base); + WARN_ON(ret); + mutex_unlock(&o.base); + WARN_ON(o.ctx != (void *)~0UL); + + /* nest_lock */ + o.ctx = (void *)~0UL; + mutex_lock_nest_lock(&o.base, &t); + mutex_unlock(&o.base); + WARN_ON(o.ctx != (void *)~0UL); +} + static void ww_test_two_contexts(void) { WWAI(&t); @@ -1415,6 +1476,7 @@ static void ww_tests(void) print_testname("ww api failures"); dotest(ww_test_fail_acquire, SUCCESS, LOCKTYPE_WW); + dotest(ww_test_normal, SUCCESS, LOCKTYPE_WW); dotest(ww_test_unneeded_slow, FAILURE, LOCKTYPE_WW); printk("\n"); -- cgit v1.2.3-70-g09d2 From f3cf139efa4bc0fe4f032af6ca3e49e38a5d9ae5 Mon Sep 17 00:00:00 2001 From: Maarten Lankhorst Date: Thu, 20 Jun 2013 13:31:42 +0200 Subject: mutex: Add more w/w tests to test EDEADLK path handling Signed-off-by: Maarten Lankhorst Acked-by: Peter Zijlstra Cc: dri-devel@lists.freedesktop.org Cc: linaro-mm-sig@lists.linaro.org Cc: rostedt@goodmis.org Cc: daniel@ffwll.ch Cc: Linus Torvalds Cc: Andrew Morton Cc: Thomas Gleixner Link: http://lkml.kernel.org/r/20130620113141.4001.54331.stgit@patser Signed-off-by: Ingo Molnar --- lib/locking-selftest.c | 264 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 261 insertions(+), 3 deletions(-) (limited to 'lib/locking-selftest.c') diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index 37faefdbb67..d554f3fed84 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -47,7 +47,7 @@ __setup("debug_locks_verbose=", setup_debug_locks_verbose); #define LOCKTYPE_WW 0x10 static struct ww_acquire_ctx t, t2; -static struct ww_mutex o, o2; +static struct ww_mutex o, o2, o3; /* * Normal standalone locks, for the circular and irq-context @@ -947,12 +947,12 @@ static void reset_locks(void) I1(A); I1(B); I1(C); I1(D); I1(X1); I1(X2); I1(Y1); I1(Y2); I1(Z1); I1(Z2); - I_WW(t); I_WW(t2); I_WW(o.base); I_WW(o2.base); + I_WW(t); I_WW(t2); I_WW(o.base); I_WW(o2.base); I_WW(o3.base); lockdep_reset(); I2(A); I2(B); I2(C); I2(D); init_shared_classes(); - ww_mutex_init(&o, &ww_lockdep); ww_mutex_init(&o2, &ww_lockdep); + ww_mutex_init(&o, &ww_lockdep); ww_mutex_init(&o2, &ww_lockdep); ww_mutex_init(&o3, &ww_lockdep); memset(&t, 0, sizeof(t)); memset(&t2, 0, sizeof(t2)); memset(&ww_lockdep.acquire_key, 0, sizeof(ww_lockdep.acquire_key)); memset(&ww_lockdep.mutex_key, 0, sizeof(ww_lockdep.mutex_key)); @@ -1292,6 +1292,251 @@ static void ww_test_object_lock_stale_context(void) WWL(&o, &t); } +static void ww_test_edeadlk_normal(void) +{ + int ret; + + mutex_lock(&o2.base); + o2.ctx = &t2; + mutex_release(&o2.base.dep_map, 1, _THIS_IP_); + + WWAI(&t); + t2 = t; + t2.stamp--; + + ret = WWL(&o, &t); + WARN_ON(ret); + + ret = WWL(&o2, &t); + WARN_ON(ret != -EDEADLK); + + o2.ctx = NULL; + mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_); + mutex_unlock(&o2.base); + WWU(&o); + + WWL(&o2, &t); +} + +static void ww_test_edeadlk_normal_slow(void) +{ + int ret; + + mutex_lock(&o2.base); + mutex_release(&o2.base.dep_map, 1, _THIS_IP_); + o2.ctx = &t2; + + WWAI(&t); + t2 = t; + t2.stamp--; + + ret = WWL(&o, &t); + WARN_ON(ret); + + ret = WWL(&o2, &t); + WARN_ON(ret != -EDEADLK); + + o2.ctx = NULL; + mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_); + mutex_unlock(&o2.base); + WWU(&o); + + ww_mutex_lock_slow(&o2, &t); +} + +static void ww_test_edeadlk_no_unlock(void) +{ + int ret; + + mutex_lock(&o2.base); + o2.ctx = &t2; + mutex_release(&o2.base.dep_map, 1, _THIS_IP_); + + WWAI(&t); + t2 = t; + t2.stamp--; + + ret = WWL(&o, &t); + WARN_ON(ret); + + ret = WWL(&o2, &t); + WARN_ON(ret != -EDEADLK); + + o2.ctx = NULL; + mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_); + mutex_unlock(&o2.base); + + WWL(&o2, &t); +} + +static void ww_test_edeadlk_no_unlock_slow(void) +{ + int ret; + + mutex_lock(&o2.base); + mutex_release(&o2.base.dep_map, 1, _THIS_IP_); + o2.ctx = &t2; + + WWAI(&t); + t2 = t; + t2.stamp--; + + ret = WWL(&o, &t); + WARN_ON(ret); + + ret = WWL(&o2, &t); + WARN_ON(ret != -EDEADLK); + + o2.ctx = NULL; + mutex_acquire(&o2.base.dep_map, 0, 1, _THIS_IP_); + mutex_unlock(&o2.base); + + ww_mutex_lock_slow(&o2, &t); +} + +static void ww_test_edeadlk_acquire_more(void) +{ + int ret; + + mutex_lock(&o2.base); + mutex_release(&o2.base.dep_map, 1, _THIS_IP_); + o2.ctx = &t2; + + WWAI(&t); + t2 = t; + t2.stamp--; + + ret = WWL(&o, &t); + WARN_ON(ret); + + ret = WWL(&o2, &t); + WARN_ON(ret != -EDEADLK); + + ret = WWL(&o3, &t); +} + +static void ww_test_edeadlk_acquire_more_slow(void) +{ + int ret; + + mutex_lock(&o2.base); + mutex_release(&o2.base.dep_map, 1, _THIS_IP_); + o2.ctx = &t2; + + WWAI(&t); + t2 = t; + t2.stamp--; + + ret = WWL(&o, &t); + WARN_ON(ret); + + ret = WWL(&o2, &t); + WARN_ON(ret != -EDEADLK); + + ww_mutex_lock_slow(&o3, &t); +} + +static void ww_test_edeadlk_acquire_more_edeadlk(void) +{ + int ret; + + mutex_lock(&o2.base); + mutex_release(&o2.base.dep_map, 1, _THIS_IP_); + o2.ctx = &t2; + + mutex_lock(&o3.base); + mutex_release(&o3.base.dep_map, 1, _THIS_IP_); + o3.ctx = &t2; + + WWAI(&t); + t2 = t; + t2.stamp--; + + ret = WWL(&o, &t); + WARN_ON(ret); + + ret = WWL(&o2, &t); + WARN_ON(ret != -EDEADLK); + + ret = WWL(&o3, &t); + WARN_ON(ret != -EDEADLK); +} + +static void ww_test_edeadlk_acquire_more_edeadlk_slow(void) +{ + int ret; + + mutex_lock(&o2.base); + mutex_release(&o2.base.dep_map, 1, _THIS_IP_); + o2.ctx = &t2; + + mutex_lock(&o3.base); + mutex_release(&o3.base.dep_map, 1, _THIS_IP_); + o3.ctx = &t2; + + WWAI(&t); + t2 = t; + t2.stamp--; + + ret = WWL(&o, &t); + WARN_ON(ret); + + ret = WWL(&o2, &t); + WARN_ON(ret != -EDEADLK); + + ww_mutex_lock_slow(&o3, &t); +} + +static void ww_test_edeadlk_acquire_wrong(void) +{ + int ret; + + mutex_lock(&o2.base); + mutex_release(&o2.base.dep_map, 1, _THIS_IP_); + o2.ctx = &t2; + + WWAI(&t); + t2 = t; + t2.stamp--; + + ret = WWL(&o, &t); + WARN_ON(ret); + + ret = WWL(&o2, &t); + WARN_ON(ret != -EDEADLK); + if (!ret) + WWU(&o2); + + WWU(&o); + + ret = WWL(&o3, &t); +} + +static void ww_test_edeadlk_acquire_wrong_slow(void) +{ + int ret; + + mutex_lock(&o2.base); + mutex_release(&o2.base.dep_map, 1, _THIS_IP_); + o2.ctx = &t2; + + WWAI(&t); + t2 = t; + t2.stamp--; + + ret = WWL(&o, &t); + WARN_ON(ret); + + ret = WWL(&o2, &t); + WARN_ON(ret != -EDEADLK); + if (!ret) + WWU(&o2); + + WWU(&o); + + ww_mutex_lock_slow(&o3, &t); +} + static void ww_test_spin_nest_unlocked(void) { raw_spin_lock_nest_lock(&lock_A, &o.base); @@ -1498,6 +1743,19 @@ static void ww_tests(void) dotest(ww_test_object_lock_stale_context, FAILURE, LOCKTYPE_WW); printk("\n"); + print_testname("EDEADLK handling"); + dotest(ww_test_edeadlk_normal, SUCCESS, LOCKTYPE_WW); + dotest(ww_test_edeadlk_normal_slow, SUCCESS, LOCKTYPE_WW); + dotest(ww_test_edeadlk_no_unlock, FAILURE, LOCKTYPE_WW); + dotest(ww_test_edeadlk_no_unlock_slow, FAILURE, LOCKTYPE_WW); + dotest(ww_test_edeadlk_acquire_more, FAILURE, LOCKTYPE_WW); + dotest(ww_test_edeadlk_acquire_more_slow, FAILURE, LOCKTYPE_WW); + dotest(ww_test_edeadlk_acquire_more_edeadlk, FAILURE, LOCKTYPE_WW); + dotest(ww_test_edeadlk_acquire_more_edeadlk_slow, FAILURE, LOCKTYPE_WW); + dotest(ww_test_edeadlk_acquire_wrong, FAILURE, LOCKTYPE_WW); + dotest(ww_test_edeadlk_acquire_wrong_slow, FAILURE, LOCKTYPE_WW); + printk("\n"); + print_testname("spinlock nest unlocked"); dotest(ww_test_spin_nest_unlocked, FAILURE, LOCKTYPE_WW); printk("\n"); -- cgit v1.2.3-70-g09d2 From 166989e366ffa66108b2f37b870e66b85b2185ad Mon Sep 17 00:00:00 2001 From: Maarten Lankhorst Date: Thu, 20 Jun 2013 13:31:51 +0200 Subject: locking-selftests: Handle unexpected failures more strictly When CONFIG_PROVE_LOCKING is not enabled, more tests are expected to pass unexpectedly, but there no tests that should start to fail that pass with CONFIG_PROVE_LOCKING enabled. Signed-off-by: Maarten Lankhorst Acked-by: Peter Zijlstra Cc: dri-devel@lists.freedesktop.org Cc: linaro-mm-sig@lists.linaro.org Cc: rostedt@goodmis.org Cc: daniel@ffwll.ch Cc: Linus Torvalds Cc: Andrew Morton Cc: Thomas Gleixner Link: http://lkml.kernel.org/r/20130620113151.4001.77963.stgit@patser Signed-off-by: Ingo Molnar --- lib/locking-selftest.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'lib/locking-selftest.c') diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index d554f3fed84..aad024dde3c 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -976,16 +976,18 @@ static void dotest(void (*testcase_fn)(void), int expected, int lockclass_mask) /* * Filter out expected failures: */ - if (debug_locks != expected) { #ifndef CONFIG_PROVE_LOCKING + if (expected == FAILURE && debug_locks) { expected_testcase_failures++; printk("failed|"); -#else + } + else +#endif + if (debug_locks != expected) { unexpected_testcase_failures++; printk("FAILED|"); dump_stack(); -#endif } else { testcase_successes++; printk(" ok |"); -- cgit v1.2.3-70-g09d2 From 1b375dc30710180c4b88cc59caba6e3481ec5c8b Mon Sep 17 00:00:00 2001 From: Maarten Lankhorst Date: Fri, 5 Jul 2013 09:29:32 +0200 Subject: mutex: Move ww_mutex definitions to ww_mutex.h Move the definitions for wound/wait mutexes out to a separate header, ww_mutex.h. This reduces clutter in mutex.h, and increases readability. Suggested-by: Linus Torvalds Signed-off-by: Maarten Lankhorst Acked-by: Peter Zijlstra Acked-by: Rik van Riel Acked-by: Maarten Lankhorst Cc: Dave Airlie Link: http://lkml.kernel.org/r/51D675DC.3000907@canonical.com [ Tidied up the code a bit. ] Signed-off-by: Ingo Molnar --- include/linux/mutex.h | 358 ----------------------------------------- include/linux/reservation.h | 2 +- include/linux/ww_mutex.h | 378 ++++++++++++++++++++++++++++++++++++++++++++ kernel/mutex.c | 1 + lib/locking-selftest.c | 1 + 5 files changed, 381 insertions(+), 359 deletions(-) create mode 100644 include/linux/ww_mutex.h (limited to 'lib/locking-selftest.c') diff --git a/include/linux/mutex.h b/include/linux/mutex.h index 3793ed7feee..ccd4260834c 100644 --- a/include/linux/mutex.h +++ b/include/linux/mutex.h @@ -78,40 +78,6 @@ struct mutex_waiter { #endif }; -struct ww_class { - atomic_long_t stamp; - struct lock_class_key acquire_key; - struct lock_class_key mutex_key; - const char *acquire_name; - const char *mutex_name; -}; - -struct ww_acquire_ctx { - struct task_struct *task; - unsigned long stamp; - unsigned acquired; -#ifdef CONFIG_DEBUG_MUTEXES - unsigned done_acquire; - struct ww_class *ww_class; - struct ww_mutex *contending_lock; -#endif -#ifdef CONFIG_DEBUG_LOCK_ALLOC - struct lockdep_map dep_map; -#endif -#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH - unsigned deadlock_inject_interval; - unsigned deadlock_inject_countdown; -#endif -}; - -struct ww_mutex { - struct mutex base; - struct ww_acquire_ctx *ctx; -#ifdef CONFIG_DEBUG_MUTEXES - struct ww_class *ww_class; -#endif -}; - #ifdef CONFIG_DEBUG_MUTEXES # include #else @@ -136,11 +102,8 @@ static inline void mutex_destroy(struct mutex *lock) {} #ifdef CONFIG_DEBUG_LOCK_ALLOC # define __DEP_MAP_MUTEX_INITIALIZER(lockname) \ , .dep_map = { .name = #lockname } -# define __WW_CLASS_MUTEX_INITIALIZER(lockname, ww_class) \ - , .ww_class = &ww_class #else # define __DEP_MAP_MUTEX_INITIALIZER(lockname) -# define __WW_CLASS_MUTEX_INITIALIZER(lockname, ww_class) #endif #define __MUTEX_INITIALIZER(lockname) \ @@ -150,48 +113,12 @@ static inline void mutex_destroy(struct mutex *lock) {} __DEBUG_MUTEX_INITIALIZER(lockname) \ __DEP_MAP_MUTEX_INITIALIZER(lockname) } -#define __WW_CLASS_INITIALIZER(ww_class) \ - { .stamp = ATOMIC_LONG_INIT(0) \ - , .acquire_name = #ww_class "_acquire" \ - , .mutex_name = #ww_class "_mutex" } - -#define __WW_MUTEX_INITIALIZER(lockname, class) \ - { .base = { \__MUTEX_INITIALIZER(lockname) } \ - __WW_CLASS_MUTEX_INITIALIZER(lockname, class) } - #define DEFINE_MUTEX(mutexname) \ struct mutex mutexname = __MUTEX_INITIALIZER(mutexname) -#define DEFINE_WW_CLASS(classname) \ - struct ww_class classname = __WW_CLASS_INITIALIZER(classname) - -#define DEFINE_WW_MUTEX(mutexname, ww_class) \ - struct ww_mutex mutexname = __WW_MUTEX_INITIALIZER(mutexname, ww_class) - - extern void __mutex_init(struct mutex *lock, const char *name, struct lock_class_key *key); -/** - * ww_mutex_init - initialize the w/w mutex - * @lock: the mutex to be initialized - * @ww_class: the w/w class the mutex should belong to - * - * Initialize the w/w mutex to unlocked state and associate it with the given - * class. - * - * It is not allowed to initialize an already locked mutex. - */ -static inline void ww_mutex_init(struct ww_mutex *lock, - struct ww_class *ww_class) -{ - __mutex_init(&lock->base, ww_class->mutex_name, &ww_class->mutex_key); - lock->ctx = NULL; -#ifdef CONFIG_DEBUG_MUTEXES - lock->ww_class = ww_class; -#endif -} - /** * mutex_is_locked - is the mutex locked * @lock: the mutex to be queried @@ -246,291 +173,6 @@ extern int __must_check mutex_lock_killable(struct mutex *lock); extern int mutex_trylock(struct mutex *lock); extern void mutex_unlock(struct mutex *lock); -/** - * ww_acquire_init - initialize a w/w acquire context - * @ctx: w/w acquire context to initialize - * @ww_class: w/w class of the context - * - * Initializes an context to acquire multiple mutexes of the given w/w class. - * - * Context-based w/w mutex acquiring can be done in any order whatsoever within - * a given lock class. Deadlocks will be detected and handled with the - * wait/wound logic. - * - * Mixing of context-based w/w mutex acquiring and single w/w mutex locking can - * result in undetected deadlocks and is so forbidden. Mixing different contexts - * for the same w/w class when acquiring mutexes can also result in undetected - * deadlocks, and is hence also forbidden. Both types of abuse will be caught by - * enabling CONFIG_PROVE_LOCKING. - * - * Nesting of acquire contexts for _different_ w/w classes is possible, subject - * to the usual locking rules between different lock classes. - * - * An acquire context must be released with ww_acquire_fini by the same task - * before the memory is freed. It is recommended to allocate the context itself - * on the stack. - */ -static inline void ww_acquire_init(struct ww_acquire_ctx *ctx, - struct ww_class *ww_class) -{ - ctx->task = current; - ctx->stamp = atomic_long_inc_return(&ww_class->stamp); - ctx->acquired = 0; -#ifdef CONFIG_DEBUG_MUTEXES - ctx->ww_class = ww_class; - ctx->done_acquire = 0; - ctx->contending_lock = NULL; -#endif -#ifdef CONFIG_DEBUG_LOCK_ALLOC - debug_check_no_locks_freed((void *)ctx, sizeof(*ctx)); - lockdep_init_map(&ctx->dep_map, ww_class->acquire_name, - &ww_class->acquire_key, 0); - mutex_acquire(&ctx->dep_map, 0, 0, _RET_IP_); -#endif -#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH - ctx->deadlock_inject_interval = 1; - ctx->deadlock_inject_countdown = ctx->stamp & 0xf; -#endif -} - -/** - * ww_acquire_done - marks the end of the acquire phase - * @ctx: the acquire context - * - * Marks the end of the acquire phase, any further w/w mutex lock calls using - * this context are forbidden. - * - * Calling this function is optional, it is just useful to document w/w mutex - * code and clearly designated the acquire phase from actually using the locked - * data structures. - */ -static inline void ww_acquire_done(struct ww_acquire_ctx *ctx) -{ -#ifdef CONFIG_DEBUG_MUTEXES - lockdep_assert_held(ctx); - - DEBUG_LOCKS_WARN_ON(ctx->done_acquire); - ctx->done_acquire = 1; -#endif -} - -/** - * ww_acquire_fini - releases a w/w acquire context - * @ctx: the acquire context to free - * - * Releases a w/w acquire context. This must be called _after_ all acquired w/w - * mutexes have been released with ww_mutex_unlock. - */ -static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx) -{ -#ifdef CONFIG_DEBUG_MUTEXES - mutex_release(&ctx->dep_map, 0, _THIS_IP_); - - DEBUG_LOCKS_WARN_ON(ctx->acquired); - if (!config_enabled(CONFIG_PROVE_LOCKING)) - /* - * lockdep will normally handle this, - * but fail without anyway - */ - ctx->done_acquire = 1; - - if (!config_enabled(CONFIG_DEBUG_LOCK_ALLOC)) - /* ensure ww_acquire_fini will still fail if called twice */ - ctx->acquired = ~0U; -#endif -} - -extern int __must_check __ww_mutex_lock(struct ww_mutex *lock, - struct ww_acquire_ctx *ctx); -extern int __must_check __ww_mutex_lock_interruptible(struct ww_mutex *lock, - struct ww_acquire_ctx *ctx); - -/** - * ww_mutex_lock - acquire the w/w mutex - * @lock: the mutex to be acquired - * @ctx: w/w acquire context, or NULL to acquire only a single lock. - * - * Lock the w/w mutex exclusively for this task. - * - * Deadlocks within a given w/w class of locks are detected and handled with the - * wait/wound algorithm. If the lock isn't immediately avaiable this function - * will either sleep until it is (wait case). Or it selects the current context - * for backing off by returning -EDEADLK (wound case). Trying to acquire the - * same lock with the same context twice is also detected and signalled by - * returning -EALREADY. Returns 0 if the mutex was successfully acquired. - * - * In the wound case the caller must release all currently held w/w mutexes for - * the given context and then wait for this contending lock to be available by - * calling ww_mutex_lock_slow. Alternatively callers can opt to not acquire this - * lock and proceed with trying to acquire further w/w mutexes (e.g. when - * scanning through lru lists trying to free resources). - * - * The mutex must later on be released by the same task that - * acquired it. The task may not exit without first unlocking the mutex. Also, - * kernel memory where the mutex resides must not be freed with the mutex still - * locked. The mutex must first be initialized (or statically defined) before it - * can be locked. memset()-ing the mutex to 0 is not allowed. The mutex must be - * of the same w/w lock class as was used to initialize the acquire context. - * - * A mutex acquired with this function must be released with ww_mutex_unlock. - */ -static inline int ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) -{ - if (ctx) - return __ww_mutex_lock(lock, ctx); - else { - mutex_lock(&lock->base); - return 0; - } -} - -/** - * ww_mutex_lock_interruptible - acquire the w/w mutex, interruptible - * @lock: the mutex to be acquired - * @ctx: w/w acquire context - * - * Lock the w/w mutex exclusively for this task. - * - * Deadlocks within a given w/w class of locks are detected and handled with the - * wait/wound algorithm. If the lock isn't immediately avaiable this function - * will either sleep until it is (wait case). Or it selects the current context - * for backing off by returning -EDEADLK (wound case). Trying to acquire the - * same lock with the same context twice is also detected and signalled by - * returning -EALREADY. Returns 0 if the mutex was successfully acquired. If a - * signal arrives while waiting for the lock then this function returns -EINTR. - * - * In the wound case the caller must release all currently held w/w mutexes for - * the given context and then wait for this contending lock to be available by - * calling ww_mutex_lock_slow_interruptible. Alternatively callers can opt to - * not acquire this lock and proceed with trying to acquire further w/w mutexes - * (e.g. when scanning through lru lists trying to free resources). - * - * The mutex must later on be released by the same task that - * acquired it. The task may not exit without first unlocking the mutex. Also, - * kernel memory where the mutex resides must not be freed with the mutex still - * locked. The mutex must first be initialized (or statically defined) before it - * can be locked. memset()-ing the mutex to 0 is not allowed. The mutex must be - * of the same w/w lock class as was used to initialize the acquire context. - * - * A mutex acquired with this function must be released with ww_mutex_unlock. - */ -static inline int __must_check ww_mutex_lock_interruptible(struct ww_mutex *lock, - struct ww_acquire_ctx *ctx) -{ - if (ctx) - return __ww_mutex_lock_interruptible(lock, ctx); - else - return mutex_lock_interruptible(&lock->base); -} - -/** - * ww_mutex_lock_slow - slowpath acquiring of the w/w mutex - * @lock: the mutex to be acquired - * @ctx: w/w acquire context - * - * Acquires a w/w mutex with the given context after a wound case. This function - * will sleep until the lock becomes available. - * - * The caller must have released all w/w mutexes already acquired with the - * context and then call this function on the contended lock. - * - * Afterwards the caller may continue to (re)acquire the other w/w mutexes it - * needs with ww_mutex_lock. Note that the -EALREADY return code from - * ww_mutex_lock can be used to avoid locking this contended mutex twice. - * - * It is forbidden to call this function with any other w/w mutexes associated - * with the context held. It is forbidden to call this on anything else than the - * contending mutex. - * - * Note that the slowpath lock acquiring can also be done by calling - * ww_mutex_lock directly. This function here is simply to help w/w mutex - * locking code readability by clearly denoting the slowpath. - */ -static inline void -ww_mutex_lock_slow(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) -{ - int ret; -#ifdef CONFIG_DEBUG_MUTEXES - DEBUG_LOCKS_WARN_ON(!ctx->contending_lock); -#endif - ret = ww_mutex_lock(lock, ctx); - (void)ret; -} - -/** - * ww_mutex_lock_slow_interruptible - slowpath acquiring of the w/w mutex, - * interruptible - * @lock: the mutex to be acquired - * @ctx: w/w acquire context - * - * Acquires a w/w mutex with the given context after a wound case. This function - * will sleep until the lock becomes available and returns 0 when the lock has - * been acquired. If a signal arrives while waiting for the lock then this - * function returns -EINTR. - * - * The caller must have released all w/w mutexes already acquired with the - * context and then call this function on the contended lock. - * - * Afterwards the caller may continue to (re)acquire the other w/w mutexes it - * needs with ww_mutex_lock. Note that the -EALREADY return code from - * ww_mutex_lock can be used to avoid locking this contended mutex twice. - * - * It is forbidden to call this function with any other w/w mutexes associated - * with the given context held. It is forbidden to call this on anything else - * than the contending mutex. - * - * Note that the slowpath lock acquiring can also be done by calling - * ww_mutex_lock_interruptible directly. This function here is simply to help - * w/w mutex locking code readability by clearly denoting the slowpath. - */ -static inline int __must_check -ww_mutex_lock_slow_interruptible(struct ww_mutex *lock, - struct ww_acquire_ctx *ctx) -{ -#ifdef CONFIG_DEBUG_MUTEXES - DEBUG_LOCKS_WARN_ON(!ctx->contending_lock); -#endif - return ww_mutex_lock_interruptible(lock, ctx); -} - -extern void ww_mutex_unlock(struct ww_mutex *lock); - -/** - * ww_mutex_trylock - tries to acquire the w/w mutex without acquire context - * @lock: mutex to lock - * - * Trylocks a mutex without acquire context, so no deadlock detection is - * possible. Returns 1 if the mutex has been acquired successfully, 0 otherwise. - */ -static inline int __must_check ww_mutex_trylock(struct ww_mutex *lock) -{ - return mutex_trylock(&lock->base); -} - -/*** - * ww_mutex_destroy - mark a w/w mutex unusable - * @lock: the mutex to be destroyed - * - * This function marks the mutex uninitialized, and any subsequent - * use of the mutex is forbidden. The mutex must not be locked when - * this function is called. - */ -static inline void ww_mutex_destroy(struct ww_mutex *lock) -{ - mutex_destroy(&lock->base); -} - -/** - * ww_mutex_is_locked - is the w/w mutex locked - * @lock: the mutex to be queried - * - * Returns 1 if the mutex is locked, 0 if unlocked. - */ -static inline bool ww_mutex_is_locked(struct ww_mutex *lock) -{ - return mutex_is_locked(&lock->base); -} - extern int atomic_dec_and_mutex_lock(atomic_t *cnt, struct mutex *lock); #ifndef CONFIG_HAVE_ARCH_MUTEX_CPU_RELAX diff --git a/include/linux/reservation.h b/include/linux/reservation.h index e9ee806a9d7..813dae960eb 100644 --- a/include/linux/reservation.h +++ b/include/linux/reservation.h @@ -39,7 +39,7 @@ #ifndef _LINUX_RESERVATION_H #define _LINUX_RESERVATION_H -#include +#include extern struct ww_class reservation_ww_class; diff --git a/include/linux/ww_mutex.h b/include/linux/ww_mutex.h new file mode 100644 index 00000000000..760399a470b --- /dev/null +++ b/include/linux/ww_mutex.h @@ -0,0 +1,378 @@ +/* + * Wound/Wait Mutexes: blocking mutual exclusion locks with deadlock avoidance + * + * Original mutex implementation started by Ingo Molnar: + * + * Copyright (C) 2004, 2005, 2006 Red Hat, Inc., Ingo Molnar + * + * Wound/wait implementation: + * Copyright (C) 2013 Canonical Ltd. + * + * This file contains the main data structure and API definitions. + */ + +#ifndef __LINUX_WW_MUTEX_H +#define __LINUX_WW_MUTEX_H + +#include + +struct ww_class { + atomic_long_t stamp; + struct lock_class_key acquire_key; + struct lock_class_key mutex_key; + const char *acquire_name; + const char *mutex_name; +}; + +struct ww_acquire_ctx { + struct task_struct *task; + unsigned long stamp; + unsigned acquired; +#ifdef CONFIG_DEBUG_MUTEXES + unsigned done_acquire; + struct ww_class *ww_class; + struct ww_mutex *contending_lock; +#endif +#ifdef CONFIG_DEBUG_LOCK_ALLOC + struct lockdep_map dep_map; +#endif +#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH + unsigned deadlock_inject_interval; + unsigned deadlock_inject_countdown; +#endif +}; + +struct ww_mutex { + struct mutex base; + struct ww_acquire_ctx *ctx; +#ifdef CONFIG_DEBUG_MUTEXES + struct ww_class *ww_class; +#endif +}; + +#ifdef CONFIG_DEBUG_LOCK_ALLOC +# define __WW_CLASS_MUTEX_INITIALIZER(lockname, ww_class) \ + , .ww_class = &ww_class +#else +# define __WW_CLASS_MUTEX_INITIALIZER(lockname, ww_class) +#endif + +#define __WW_CLASS_INITIALIZER(ww_class) \ + { .stamp = ATOMIC_LONG_INIT(0) \ + , .acquire_name = #ww_class "_acquire" \ + , .mutex_name = #ww_class "_mutex" } + +#define __WW_MUTEX_INITIALIZER(lockname, class) \ + { .base = { \__MUTEX_INITIALIZER(lockname) } \ + __WW_CLASS_MUTEX_INITIALIZER(lockname, class) } + +#define DEFINE_WW_CLASS(classname) \ + struct ww_class classname = __WW_CLASS_INITIALIZER(classname) + +#define DEFINE_WW_MUTEX(mutexname, ww_class) \ + struct ww_mutex mutexname = __WW_MUTEX_INITIALIZER(mutexname, ww_class) + +/** + * ww_mutex_init - initialize the w/w mutex + * @lock: the mutex to be initialized + * @ww_class: the w/w class the mutex should belong to + * + * Initialize the w/w mutex to unlocked state and associate it with the given + * class. + * + * It is not allowed to initialize an already locked mutex. + */ +static inline void ww_mutex_init(struct ww_mutex *lock, + struct ww_class *ww_class) +{ + __mutex_init(&lock->base, ww_class->mutex_name, &ww_class->mutex_key); + lock->ctx = NULL; +#ifdef CONFIG_DEBUG_MUTEXES + lock->ww_class = ww_class; +#endif +} + +/** + * ww_acquire_init - initialize a w/w acquire context + * @ctx: w/w acquire context to initialize + * @ww_class: w/w class of the context + * + * Initializes an context to acquire multiple mutexes of the given w/w class. + * + * Context-based w/w mutex acquiring can be done in any order whatsoever within + * a given lock class. Deadlocks will be detected and handled with the + * wait/wound logic. + * + * Mixing of context-based w/w mutex acquiring and single w/w mutex locking can + * result in undetected deadlocks and is so forbidden. Mixing different contexts + * for the same w/w class when acquiring mutexes can also result in undetected + * deadlocks, and is hence also forbidden. Both types of abuse will be caught by + * enabling CONFIG_PROVE_LOCKING. + * + * Nesting of acquire contexts for _different_ w/w classes is possible, subject + * to the usual locking rules between different lock classes. + * + * An acquire context must be released with ww_acquire_fini by the same task + * before the memory is freed. It is recommended to allocate the context itself + * on the stack. + */ +static inline void ww_acquire_init(struct ww_acquire_ctx *ctx, + struct ww_class *ww_class) +{ + ctx->task = current; + ctx->stamp = atomic_long_inc_return(&ww_class->stamp); + ctx->acquired = 0; +#ifdef CONFIG_DEBUG_MUTEXES + ctx->ww_class = ww_class; + ctx->done_acquire = 0; + ctx->contending_lock = NULL; +#endif +#ifdef CONFIG_DEBUG_LOCK_ALLOC + debug_check_no_locks_freed((void *)ctx, sizeof(*ctx)); + lockdep_init_map(&ctx->dep_map, ww_class->acquire_name, + &ww_class->acquire_key, 0); + mutex_acquire(&ctx->dep_map, 0, 0, _RET_IP_); +#endif +#ifdef CONFIG_DEBUG_WW_MUTEX_SLOWPATH + ctx->deadlock_inject_interval = 1; + ctx->deadlock_inject_countdown = ctx->stamp & 0xf; +#endif +} + +/** + * ww_acquire_done - marks the end of the acquire phase + * @ctx: the acquire context + * + * Marks the end of the acquire phase, any further w/w mutex lock calls using + * this context are forbidden. + * + * Calling this function is optional, it is just useful to document w/w mutex + * code and clearly designated the acquire phase from actually using the locked + * data structures. + */ +static inline void ww_acquire_done(struct ww_acquire_ctx *ctx) +{ +#ifdef CONFIG_DEBUG_MUTEXES + lockdep_assert_held(ctx); + + DEBUG_LOCKS_WARN_ON(ctx->done_acquire); + ctx->done_acquire = 1; +#endif +} + +/** + * ww_acquire_fini - releases a w/w acquire context + * @ctx: the acquire context to free + * + * Releases a w/w acquire context. This must be called _after_ all acquired w/w + * mutexes have been released with ww_mutex_unlock. + */ +static inline void ww_acquire_fini(struct ww_acquire_ctx *ctx) +{ +#ifdef CONFIG_DEBUG_MUTEXES + mutex_release(&ctx->dep_map, 0, _THIS_IP_); + + DEBUG_LOCKS_WARN_ON(ctx->acquired); + if (!config_enabled(CONFIG_PROVE_LOCKING)) + /* + * lockdep will normally handle this, + * but fail without anyway + */ + ctx->done_acquire = 1; + + if (!config_enabled(CONFIG_DEBUG_LOCK_ALLOC)) + /* ensure ww_acquire_fini will still fail if called twice */ + ctx->acquired = ~0U; +#endif +} + +extern int __must_check __ww_mutex_lock(struct ww_mutex *lock, + struct ww_acquire_ctx *ctx); +extern int __must_check __ww_mutex_lock_interruptible(struct ww_mutex *lock, + struct ww_acquire_ctx *ctx); + +/** + * ww_mutex_lock - acquire the w/w mutex + * @lock: the mutex to be acquired + * @ctx: w/w acquire context, or NULL to acquire only a single lock. + * + * Lock the w/w mutex exclusively for this task. + * + * Deadlocks within a given w/w class of locks are detected and handled with the + * wait/wound algorithm. If the lock isn't immediately avaiable this function + * will either sleep until it is (wait case). Or it selects the current context + * for backing off by returning -EDEADLK (wound case). Trying to acquire the + * same lock with the same context twice is also detected and signalled by + * returning -EALREADY. Returns 0 if the mutex was successfully acquired. + * + * In the wound case the caller must release all currently held w/w mutexes for + * the given context and then wait for this contending lock to be available by + * calling ww_mutex_lock_slow. Alternatively callers can opt to not acquire this + * lock and proceed with trying to acquire further w/w mutexes (e.g. when + * scanning through lru lists trying to free resources). + * + * The mutex must later on be released by the same task that + * acquired it. The task may not exit without first unlocking the mutex. Also, + * kernel memory where the mutex resides must not be freed with the mutex still + * locked. The mutex must first be initialized (or statically defined) before it + * can be locked. memset()-ing the mutex to 0 is not allowed. The mutex must be + * of the same w/w lock class as was used to initialize the acquire context. + * + * A mutex acquired with this function must be released with ww_mutex_unlock. + */ +static inline int ww_mutex_lock(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) +{ + if (ctx) + return __ww_mutex_lock(lock, ctx); + + mutex_lock(&lock->base); + return 0; +} + +/** + * ww_mutex_lock_interruptible - acquire the w/w mutex, interruptible + * @lock: the mutex to be acquired + * @ctx: w/w acquire context + * + * Lock the w/w mutex exclusively for this task. + * + * Deadlocks within a given w/w class of locks are detected and handled with the + * wait/wound algorithm. If the lock isn't immediately avaiable this function + * will either sleep until it is (wait case). Or it selects the current context + * for backing off by returning -EDEADLK (wound case). Trying to acquire the + * same lock with the same context twice is also detected and signalled by + * returning -EALREADY. Returns 0 if the mutex was successfully acquired. If a + * signal arrives while waiting for the lock then this function returns -EINTR. + * + * In the wound case the caller must release all currently held w/w mutexes for + * the given context and then wait for this contending lock to be available by + * calling ww_mutex_lock_slow_interruptible. Alternatively callers can opt to + * not acquire this lock and proceed with trying to acquire further w/w mutexes + * (e.g. when scanning through lru lists trying to free resources). + * + * The mutex must later on be released by the same task that + * acquired it. The task may not exit without first unlocking the mutex. Also, + * kernel memory where the mutex resides must not be freed with the mutex still + * locked. The mutex must first be initialized (or statically defined) before it + * can be locked. memset()-ing the mutex to 0 is not allowed. The mutex must be + * of the same w/w lock class as was used to initialize the acquire context. + * + * A mutex acquired with this function must be released with ww_mutex_unlock. + */ +static inline int __must_check ww_mutex_lock_interruptible(struct ww_mutex *lock, + struct ww_acquire_ctx *ctx) +{ + if (ctx) + return __ww_mutex_lock_interruptible(lock, ctx); + else + return mutex_lock_interruptible(&lock->base); +} + +/** + * ww_mutex_lock_slow - slowpath acquiring of the w/w mutex + * @lock: the mutex to be acquired + * @ctx: w/w acquire context + * + * Acquires a w/w mutex with the given context after a wound case. This function + * will sleep until the lock becomes available. + * + * The caller must have released all w/w mutexes already acquired with the + * context and then call this function on the contended lock. + * + * Afterwards the caller may continue to (re)acquire the other w/w mutexes it + * needs with ww_mutex_lock. Note that the -EALREADY return code from + * ww_mutex_lock can be used to avoid locking this contended mutex twice. + * + * It is forbidden to call this function with any other w/w mutexes associated + * with the context held. It is forbidden to call this on anything else than the + * contending mutex. + * + * Note that the slowpath lock acquiring can also be done by calling + * ww_mutex_lock directly. This function here is simply to help w/w mutex + * locking code readability by clearly denoting the slowpath. + */ +static inline void +ww_mutex_lock_slow(struct ww_mutex *lock, struct ww_acquire_ctx *ctx) +{ + int ret; +#ifdef CONFIG_DEBUG_MUTEXES + DEBUG_LOCKS_WARN_ON(!ctx->contending_lock); +#endif + ret = ww_mutex_lock(lock, ctx); + (void)ret; +} + +/** + * ww_mutex_lock_slow_interruptible - slowpath acquiring of the w/w mutex, interruptible + * @lock: the mutex to be acquired + * @ctx: w/w acquire context + * + * Acquires a w/w mutex with the given context after a wound case. This function + * will sleep until the lock becomes available and returns 0 when the lock has + * been acquired. If a signal arrives while waiting for the lock then this + * function returns -EINTR. + * + * The caller must have released all w/w mutexes already acquired with the + * context and then call this function on the contended lock. + * + * Afterwards the caller may continue to (re)acquire the other w/w mutexes it + * needs with ww_mutex_lock. Note that the -EALREADY return code from + * ww_mutex_lock can be used to avoid locking this contended mutex twice. + * + * It is forbidden to call this function with any other w/w mutexes associated + * with the given context held. It is forbidden to call this on anything else + * than the contending mutex. + * + * Note that the slowpath lock acquiring can also be done by calling + * ww_mutex_lock_interruptible directly. This function here is simply to help + * w/w mutex locking code readability by clearly denoting the slowpath. + */ +static inline int __must_check +ww_mutex_lock_slow_interruptible(struct ww_mutex *lock, + struct ww_acquire_ctx *ctx) +{ +#ifdef CONFIG_DEBUG_MUTEXES + DEBUG_LOCKS_WARN_ON(!ctx->contending_lock); +#endif + return ww_mutex_lock_interruptible(lock, ctx); +} + +extern void ww_mutex_unlock(struct ww_mutex *lock); + +/** + * ww_mutex_trylock - tries to acquire the w/w mutex without acquire context + * @lock: mutex to lock + * + * Trylocks a mutex without acquire context, so no deadlock detection is + * possible. Returns 1 if the mutex has been acquired successfully, 0 otherwise. + */ +static inline int __must_check ww_mutex_trylock(struct ww_mutex *lock) +{ + return mutex_trylock(&lock->base); +} + +/*** + * ww_mutex_destroy - mark a w/w mutex unusable + * @lock: the mutex to be destroyed + * + * This function marks the mutex uninitialized, and any subsequent + * use of the mutex is forbidden. The mutex must not be locked when + * this function is called. + */ +static inline void ww_mutex_destroy(struct ww_mutex *lock) +{ + mutex_destroy(&lock->base); +} + +/** + * ww_mutex_is_locked - is the w/w mutex locked + * @lock: the mutex to be queried + * + * Returns 1 if the mutex is locked, 0 if unlocked. + */ +static inline bool ww_mutex_is_locked(struct ww_mutex *lock) +{ + return mutex_is_locked(&lock->base); +} + +#endif diff --git a/kernel/mutex.c b/kernel/mutex.c index e581ada5faf..ff05f4bd86e 100644 --- a/kernel/mutex.c +++ b/kernel/mutex.c @@ -18,6 +18,7 @@ * Also see Documentation/mutex-design.txt. */ #include +#include #include #include #include diff --git a/lib/locking-selftest.c b/lib/locking-selftest.c index aad024dde3c..6dc09d8f4c2 100644 --- a/lib/locking-selftest.c +++ b/lib/locking-selftest.c @@ -12,6 +12,7 @@ */ #include #include +#include #include #include #include -- cgit v1.2.3-70-g09d2