From d2212b4dce596fee83e5c523400bf084f4cc816c Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 26 Sep 2013 17:27:00 +0100 Subject: lockref: allow relaxed cmpxchg64 variant for lockless updates The 64-bit cmpxchg operation on the lockref is ordered by virtue of hazarding between the cmpxchg operation and the reference count manipulation. On weakly ordered memory architectures (such as ARM), it can be of great benefit to omit the barrier instructions where they are not needed. This patch moves the lockless lockref code over to a cmpxchg64_relaxed operation, which doesn't provide barrier semantics. If the operation isn't defined, we simply #define it as the usual 64-bit cmpxchg macro. Cc: Waiman Long Signed-off-by: Will Deacon Signed-off-by: Linus Torvalds --- lib/lockref.c | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) (limited to 'lib/lockref.c') diff --git a/lib/lockref.c b/lib/lockref.c index 677d036cf3c..e294ae445c9 100644 --- a/lib/lockref.c +++ b/lib/lockref.c @@ -3,6 +3,14 @@ #ifdef CONFIG_CMPXCHG_LOCKREF +/* + * Allow weakly-ordered memory architectures to provide barrier-less + * cmpxchg semantics for lockref updates. + */ +#ifndef cmpxchg64_relaxed +# define cmpxchg64_relaxed cmpxchg64 +#endif + /* * Note that the "cmpxchg()" reloads the "old" value for the * failure case. @@ -14,8 +22,9 @@ while (likely(arch_spin_value_unlocked(old.lock.rlock.raw_lock))) { \ struct lockref new = old, prev = old; \ CODE \ - old.lock_count = cmpxchg64(&lockref->lock_count, \ - old.lock_count, new.lock_count); \ + old.lock_count = cmpxchg64_relaxed(&lockref->lock_count, \ + old.lock_count, \ + new.lock_count); \ if (likely(old.lock_count == prev.lock_count)) { \ SUCCESS; \ } \ -- cgit v1.2.3-70-g09d2 From 491f6f8e5fd9a57aaf03b6d6e3e153f1c27d8a46 Mon Sep 17 00:00:00 2001 From: Heiko Carstens Date: Mon, 23 Sep 2013 12:59:56 +0200 Subject: lockref: use arch_mutex_cpu_relax() in CMPXCHG_LOOP() Make use of arch_mutex_cpu_relax() so architectures can override the default cpu_relax() semantics. This is especially useful for s390, where cpu_relax() means that we yield() the current (virtual) cpu and therefore is very expensive, and would contradict the whole purpose of the lockless cmpxchg loop. Signed-off-by: Heiko Carstens --- lib/lockref.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'lib/lockref.c') diff --git a/lib/lockref.c b/lib/lockref.c index e294ae445c9..6f9d434c152 100644 --- a/lib/lockref.c +++ b/lib/lockref.c @@ -11,6 +11,14 @@ # define cmpxchg64_relaxed cmpxchg64 #endif +/* + * Allow architectures to override the default cpu_relax() within CMPXCHG_LOOP. + * This is useful for architectures with an expensive cpu_relax(). + */ +#ifndef arch_mutex_cpu_relax +# define arch_mutex_cpu_relax() cpu_relax() +#endif + /* * Note that the "cmpxchg()" reloads the "old" value for the * failure case. @@ -28,7 +36,7 @@ if (likely(old.lock_count == prev.lock_count)) { \ SUCCESS; \ } \ - cpu_relax(); \ + arch_mutex_cpu_relax(); \ } \ } while (0) -- cgit v1.2.3-70-g09d2