From d630dc4c9adb41e5bd1e06df2dbeaf622469ddd5 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 28 Jun 2014 08:10:12 -0400 Subject: percpu-refcount: one bit is enough for REF_STATUS percpu-refcount currently reserves two lowest bits of its percpu pointer to indicate its state; however, only one bit is used for PCPU_REF_DEAD. Simplify it by removing PCPU_STATUS_BITS/MASK and testing PCPU_REF_DEAD directly. This also allows the compiler to choose a more efficient instruction depending on the architecture. Signed-off-by: Tejun Heo Cc: Kent Overstreet Cc: Christoph Lameter --- lib/percpu-refcount.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index 963b7034a51..17bce2bccc1 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -96,7 +96,7 @@ static void percpu_ref_kill_rcu(struct rcu_head *rcu) /* Mask out PCPU_REF_DEAD */ pcpu_count = (unsigned __percpu *) - (((unsigned long) pcpu_count) & ~PCPU_STATUS_MASK); + (((unsigned long) pcpu_count) & ~PCPU_REF_DEAD); for_each_possible_cpu(cpu) count += *per_cpu_ptr(pcpu_count, cpu); -- cgit v1.2.3-70-g09d2 From eae7975ddf031b3084f4a5f7d88f698aefad96fb Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 28 Jun 2014 08:10:13 -0400 Subject: percpu-refcount: add helpers for ->percpu_count accesses * All four percpu_ref_*() operations implemented in the header file perform the same operation to determine whether the percpu_ref is alive and extract the percpu pointer. Factor out the common logic into __pcpu_ref_alive(). This doesn't change the generated code. * There are a couple places in percpu-refcount.c which masks out PCPU_REF_DEAD to obtain the percpu pointer. Factor it out into pcpu_count_ptr(). * The above changes make the WARN_ON_ONCE() conditional at the top of percpu_ref_kill_and_confirm() the only user of REF_STATUS(). Test PCPU_REF_DEAD directly and remove REF_STATUS(). This patch doesn't introduce any functional change. Signed-off-by: Tejun Heo Cc: Kent Overstreet Cc: Christoph Lameter --- include/linux/percpu-refcount.h | 35 +++++++++++++++++++++-------------- lib/percpu-refcount.c | 17 +++++++++-------- 2 files changed, 30 insertions(+), 22 deletions(-) (limited to 'lib') diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h index bfdeb0d48e2..b62a4ee6d6a 100644 --- a/include/linux/percpu-refcount.h +++ b/include/linux/percpu-refcount.h @@ -88,10 +88,25 @@ static inline void percpu_ref_kill(struct percpu_ref *ref) return percpu_ref_kill_and_confirm(ref, NULL); } -#define PCPU_REF_PTR 0 #define PCPU_REF_DEAD 1 -#define REF_STATUS(count) (((unsigned long) count) & PCPU_REF_DEAD) +/* + * Internal helper. Don't use outside percpu-refcount proper. The + * function doesn't return the pointer and let the caller test it for NULL + * because doing so forces the compiler to generate two conditional + * branches as it can't assume that @ref->pcpu_count is not NULL. + */ +static inline bool __pcpu_ref_alive(struct percpu_ref *ref, + unsigned __percpu **pcpu_countp) +{ + unsigned long pcpu_ptr = (unsigned long)ACCESS_ONCE(ref->pcpu_count); + + if (unlikely(pcpu_ptr & PCPU_REF_DEAD)) + return false; + + *pcpu_countp = (unsigned __percpu *)pcpu_ptr; + return true; +} /** * percpu_ref_get - increment a percpu refcount @@ -105,9 +120,7 @@ static inline void percpu_ref_get(struct percpu_ref *ref) rcu_read_lock_sched(); - pcpu_count = ACCESS_ONCE(ref->pcpu_count); - - if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) + if (__pcpu_ref_alive(ref, &pcpu_count)) this_cpu_inc(*pcpu_count); else atomic_inc(&ref->count); @@ -131,9 +144,7 @@ static inline bool percpu_ref_tryget(struct percpu_ref *ref) rcu_read_lock_sched(); - pcpu_count = ACCESS_ONCE(ref->pcpu_count); - - if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) { + if (__pcpu_ref_alive(ref, &pcpu_count)) { this_cpu_inc(*pcpu_count); ret = true; } else { @@ -166,9 +177,7 @@ static inline bool percpu_ref_tryget_live(struct percpu_ref *ref) rcu_read_lock_sched(); - pcpu_count = ACCESS_ONCE(ref->pcpu_count); - - if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) { + if (__pcpu_ref_alive(ref, &pcpu_count)) { this_cpu_inc(*pcpu_count); ret = true; } @@ -191,9 +200,7 @@ static inline void percpu_ref_put(struct percpu_ref *ref) rcu_read_lock_sched(); - pcpu_count = ACCESS_ONCE(ref->pcpu_count); - - if (likely(REF_STATUS(pcpu_count) == PCPU_REF_PTR)) + if (__pcpu_ref_alive(ref, &pcpu_count)) this_cpu_dec(*pcpu_count); else if (unlikely(atomic_dec_and_test(&ref->count))) ref->release(ref); diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index 17bce2bccc1..087f1a04f9b 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -31,6 +31,11 @@ #define PCPU_COUNT_BIAS (1U << 31) +static unsigned __percpu *pcpu_count_ptr(struct percpu_ref *ref) +{ + return (unsigned __percpu *)((unsigned long)ref->pcpu_count & ~PCPU_REF_DEAD); +} + /** * percpu_ref_init - initialize a percpu refcount * @ref: percpu_ref to initialize @@ -74,7 +79,7 @@ EXPORT_SYMBOL_GPL(percpu_ref_init); */ void percpu_ref_cancel_init(struct percpu_ref *ref) { - unsigned __percpu *pcpu_count = ref->pcpu_count; + unsigned __percpu *pcpu_count = pcpu_count_ptr(ref); int cpu; WARN_ON_ONCE(atomic_read(&ref->count) != 1 + PCPU_COUNT_BIAS); @@ -82,7 +87,7 @@ void percpu_ref_cancel_init(struct percpu_ref *ref) if (pcpu_count) { for_each_possible_cpu(cpu) WARN_ON_ONCE(*per_cpu_ptr(pcpu_count, cpu)); - free_percpu(ref->pcpu_count); + free_percpu(pcpu_count); } } EXPORT_SYMBOL_GPL(percpu_ref_cancel_init); @@ -90,14 +95,10 @@ EXPORT_SYMBOL_GPL(percpu_ref_cancel_init); static void percpu_ref_kill_rcu(struct rcu_head *rcu) { struct percpu_ref *ref = container_of(rcu, struct percpu_ref, rcu); - unsigned __percpu *pcpu_count = ref->pcpu_count; + unsigned __percpu *pcpu_count = pcpu_count_ptr(ref); unsigned count = 0; int cpu; - /* Mask out PCPU_REF_DEAD */ - pcpu_count = (unsigned __percpu *) - (((unsigned long) pcpu_count) & ~PCPU_REF_DEAD); - for_each_possible_cpu(cpu) count += *per_cpu_ptr(pcpu_count, cpu); @@ -152,7 +153,7 @@ static void percpu_ref_kill_rcu(struct rcu_head *rcu) void percpu_ref_kill_and_confirm(struct percpu_ref *ref, percpu_ref_func_t *confirm_kill) { - WARN_ONCE(REF_STATUS(ref->pcpu_count) == PCPU_REF_DEAD, + WARN_ONCE((unsigned long)ref->pcpu_count & PCPU_REF_DEAD, "percpu_ref_kill() called more than once!\n"); ref->pcpu_count = (unsigned __percpu *) -- cgit v1.2.3-70-g09d2 From 7d742075120deb831c7b94c268ca20d409e91d60 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 28 Jun 2014 08:10:13 -0400 Subject: percpu-refcount: use unsigned long for pcpu_count pointer percpu_ref->pcpu_count is a percpu pointer with a status flag in its lowest bit. As such, it always goes through arithmetic operations which is very cumbersome to do on a pointer. It has to be first casted to unsigned long and then back. Let's just make the field unsigned long so that we can skip the first casts. While at it, rename it to pcpu_counter_ptr to clarify that it's a pointer value. Signed-off-by: Tejun Heo Cc: Kent Overstreet Cc: Christoph Lameter --- include/linux/percpu-refcount.h | 4 ++-- lib/percpu-refcount.c | 11 +++++------ 2 files changed, 7 insertions(+), 8 deletions(-) (limited to 'lib') diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h index b62a4ee6d6a..6f8cd4c0546 100644 --- a/include/linux/percpu-refcount.h +++ b/include/linux/percpu-refcount.h @@ -61,7 +61,7 @@ struct percpu_ref { * hack because we need to keep the pointer around for * percpu_ref_kill_rcu()) */ - unsigned __percpu *pcpu_count; + unsigned long pcpu_count_ptr; percpu_ref_func_t *release; percpu_ref_func_t *confirm_kill; struct rcu_head rcu; @@ -99,7 +99,7 @@ static inline void percpu_ref_kill(struct percpu_ref *ref) static inline bool __pcpu_ref_alive(struct percpu_ref *ref, unsigned __percpu **pcpu_countp) { - unsigned long pcpu_ptr = (unsigned long)ACCESS_ONCE(ref->pcpu_count); + unsigned long pcpu_ptr = ACCESS_ONCE(ref->pcpu_count_ptr); if (unlikely(pcpu_ptr & PCPU_REF_DEAD)) return false; diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index 087f1a04f9b..94e5b624de6 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -33,7 +33,7 @@ static unsigned __percpu *pcpu_count_ptr(struct percpu_ref *ref) { - return (unsigned __percpu *)((unsigned long)ref->pcpu_count & ~PCPU_REF_DEAD); + return (unsigned __percpu *)(ref->pcpu_count_ptr & ~PCPU_REF_DEAD); } /** @@ -51,8 +51,8 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release) { atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS); - ref->pcpu_count = alloc_percpu(unsigned); - if (!ref->pcpu_count) + ref->pcpu_count_ptr = (unsigned long)alloc_percpu(unsigned); + if (!ref->pcpu_count_ptr) return -ENOMEM; ref->release = release; @@ -153,11 +153,10 @@ static void percpu_ref_kill_rcu(struct rcu_head *rcu) void percpu_ref_kill_and_confirm(struct percpu_ref *ref, percpu_ref_func_t *confirm_kill) { - WARN_ONCE((unsigned long)ref->pcpu_count & PCPU_REF_DEAD, + WARN_ONCE(ref->pcpu_count_ptr & PCPU_REF_DEAD, "percpu_ref_kill() called more than once!\n"); - ref->pcpu_count = (unsigned __percpu *) - (((unsigned long) ref->pcpu_count)|PCPU_REF_DEAD); + ref->pcpu_count_ptr |= PCPU_REF_DEAD; ref->confirm_kill = confirm_kill; call_rcu_sched(&ref->rcu, percpu_ref_kill_rcu); -- cgit v1.2.3-70-g09d2 From 9a1049da9bd2cd83fe11d46433e603c193aa9c71 Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 28 Jun 2014 08:10:14 -0400 Subject: percpu-refcount: require percpu_ref to be exited explicitly Currently, a percpu_ref undoes percpu_ref_init() automatically by freeing the allocated percpu area when the percpu_ref is killed. While seemingly convenient, this has the following niggles. * It's impossible to re-init a released reference counter without going through re-allocation. * In the similar vein, it's impossible to initialize a percpu_ref count with static percpu variables. * We need and have an explicit destructor anyway for failure paths - percpu_ref_cancel_init(). This patch removes the automatic percpu counter freeing in percpu_ref_kill_rcu() and repurposes percpu_ref_cancel_init() into a generic destructor now named percpu_ref_exit(). percpu_ref_destroy() is considered but it gets confusing with percpu_ref_kill() while "exit" clearly indicates that it's the counterpart of percpu_ref_init(). All percpu_ref_cancel_init() users are updated to invoke percpu_ref_exit() instead and explicit percpu_ref_exit() calls are added to the destruction path of all percpu_ref users. Signed-off-by: Tejun Heo Acked-by: Benjamin LaHaise Cc: Kent Overstreet Cc: Christoph Lameter Cc: Benjamin LaHaise Cc: Nicholas A. Bellinger Cc: Li Zefan --- drivers/target/target_core_tpg.c | 4 +++- fs/aio.c | 6 ++++-- include/linux/percpu-refcount.h | 6 ++---- kernel/cgroup.c | 8 +++++--- lib/percpu-refcount.c | 33 ++++++++++----------------------- 5 files changed, 24 insertions(+), 33 deletions(-) (limited to 'lib') diff --git a/drivers/target/target_core_tpg.c b/drivers/target/target_core_tpg.c index c036595b17c..fddfae61222 100644 --- a/drivers/target/target_core_tpg.c +++ b/drivers/target/target_core_tpg.c @@ -825,7 +825,7 @@ int core_tpg_add_lun( ret = core_dev_export(dev, tpg, lun); if (ret < 0) { - percpu_ref_cancel_init(&lun->lun_ref); + percpu_ref_exit(&lun->lun_ref); return ret; } @@ -880,5 +880,7 @@ int core_tpg_post_dellun( lun->lun_status = TRANSPORT_LUN_STATUS_FREE; spin_unlock(&tpg->tpg_lun_lock); + percpu_ref_exit(&lun->lun_ref); + return 0; } diff --git a/fs/aio.c b/fs/aio.c index 5e0d7f9cb69..ea1bc2e8f4f 100644 --- a/fs/aio.c +++ b/fs/aio.c @@ -506,6 +506,8 @@ static void free_ioctx(struct work_struct *work) aio_free_ring(ctx); free_percpu(ctx->cpu); + percpu_ref_exit(&ctx->reqs); + percpu_ref_exit(&ctx->users); kmem_cache_free(kioctx_cachep, ctx); } @@ -715,8 +717,8 @@ err_ctx: err: mutex_unlock(&ctx->ring_lock); free_percpu(ctx->cpu); - percpu_ref_cancel_init(&ctx->reqs); - percpu_ref_cancel_init(&ctx->users); + percpu_ref_exit(&ctx->reqs); + percpu_ref_exit(&ctx->users); kmem_cache_free(kioctx_cachep, ctx); pr_debug("error allocating ioctx %d\n", err); return ERR_PTR(err); diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h index 6f8cd4c0546..0ddd2839ca8 100644 --- a/include/linux/percpu-refcount.h +++ b/include/linux/percpu-refcount.h @@ -57,9 +57,7 @@ struct percpu_ref { atomic_t count; /* * The low bit of the pointer indicates whether the ref is in percpu - * mode; if set, then get/put will manipulate the atomic_t (this is a - * hack because we need to keep the pointer around for - * percpu_ref_kill_rcu()) + * mode; if set, then get/put will manipulate the atomic_t. */ unsigned long pcpu_count_ptr; percpu_ref_func_t *release; @@ -69,7 +67,7 @@ struct percpu_ref { int __must_check percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release); -void percpu_ref_cancel_init(struct percpu_ref *ref); +void percpu_ref_exit(struct percpu_ref *ref); void percpu_ref_kill_and_confirm(struct percpu_ref *ref, percpu_ref_func_t *confirm_kill); diff --git a/kernel/cgroup.c b/kernel/cgroup.c index 7868fc3c0bc..c06aa5e257a 100644 --- a/kernel/cgroup.c +++ b/kernel/cgroup.c @@ -1638,7 +1638,7 @@ destroy_root: exit_root_id: cgroup_exit_root_id(root); cancel_ref: - percpu_ref_cancel_init(&root_cgrp->self.refcnt); + percpu_ref_exit(&root_cgrp->self.refcnt); out: free_cgrp_cset_links(&tmp_links); return ret; @@ -4133,6 +4133,8 @@ static void css_free_work_fn(struct work_struct *work) container_of(work, struct cgroup_subsys_state, destroy_work); struct cgroup *cgrp = css->cgroup; + percpu_ref_exit(&css->refcnt); + if (css->ss) { /* css free path */ if (css->parent) @@ -4330,7 +4332,7 @@ err_list_del: err_free_id: cgroup_idr_remove(&ss->css_idr, css->id); err_free_percpu_ref: - percpu_ref_cancel_init(&css->refcnt); + percpu_ref_exit(&css->refcnt); err_free_css: call_rcu(&css->rcu_head, css_free_rcu_fn); return err; @@ -4441,7 +4443,7 @@ static int cgroup_mkdir(struct kernfs_node *parent_kn, const char *name, out_free_id: cgroup_idr_remove(&root->cgroup_idr, cgrp->id); out_cancel_ref: - percpu_ref_cancel_init(&cgrp->self.refcnt); + percpu_ref_exit(&cgrp->self.refcnt); out_free_cgrp: kfree(cgrp); out_unlock: diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index 94e5b624de6..ac429912008 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -61,36 +61,25 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release) EXPORT_SYMBOL_GPL(percpu_ref_init); /** - * percpu_ref_cancel_init - cancel percpu_ref_init() - * @ref: percpu_ref to cancel init for + * percpu_ref_exit - undo percpu_ref_init() + * @ref: percpu_ref to exit * - * Once a percpu_ref is initialized, its destruction is initiated by - * percpu_ref_kill() and completes asynchronously, which can be painful to - * do when destroying a half-constructed object in init failure path. - * - * This function destroys @ref without invoking @ref->release and the - * memory area containing it can be freed immediately on return. To - * prevent accidental misuse, it's required that @ref has finished - * percpu_ref_init(), whether successful or not, but never used. - * - * The weird name and usage restriction are to prevent people from using - * this function by mistake for normal shutdown instead of - * percpu_ref_kill(). + * This function exits @ref. The caller is responsible for ensuring that + * @ref is no longer in active use. The usual places to invoke this + * function from are the @ref->release() callback or in init failure path + * where percpu_ref_init() succeeded but other parts of the initialization + * of the embedding object failed. */ -void percpu_ref_cancel_init(struct percpu_ref *ref) +void percpu_ref_exit(struct percpu_ref *ref) { unsigned __percpu *pcpu_count = pcpu_count_ptr(ref); - int cpu; - - WARN_ON_ONCE(atomic_read(&ref->count) != 1 + PCPU_COUNT_BIAS); if (pcpu_count) { - for_each_possible_cpu(cpu) - WARN_ON_ONCE(*per_cpu_ptr(pcpu_count, cpu)); free_percpu(pcpu_count); + ref->pcpu_count_ptr = PCPU_REF_DEAD; } } -EXPORT_SYMBOL_GPL(percpu_ref_cancel_init); +EXPORT_SYMBOL_GPL(percpu_ref_exit); static void percpu_ref_kill_rcu(struct rcu_head *rcu) { @@ -102,8 +91,6 @@ static void percpu_ref_kill_rcu(struct rcu_head *rcu) for_each_possible_cpu(cpu) count += *per_cpu_ptr(pcpu_count, cpu); - free_percpu(pcpu_count); - pr_debug("global %i pcpu %i", atomic_read(&ref->count), (int) count); /* -- cgit v1.2.3-70-g09d2 From 2d7227828e1475c7b272e55bd70c4cec8eea219a Mon Sep 17 00:00:00 2001 From: Tejun Heo Date: Sat, 28 Jun 2014 08:10:14 -0400 Subject: percpu-refcount: implement percpu_ref_reinit() and percpu_ref_is_zero() Now that explicit invocation of percpu_ref_exit() is necessary to free the percpu counter, we can implement percpu_ref_reinit() which reinitializes a released percpu_ref. This can be used implement scalable gating switch which can be drained and then re-opened without worrying about memory allocation failures. percpu_ref_is_zero() is added to be used in a sanity check in percpu_ref_exit(). As this function will be useful for other purposes too, make it a public interface. v2: Use smp_read_barrier_depends() instead of smp_load_acquire(). We only need data dep barrier and smp_load_acquire() is stronger and heavier on some archs. Spotted by Lai Jiangshan. Signed-off-by: Tejun Heo Cc: Kent Overstreet Cc: Christoph Lameter Cc: Lai Jiangshan --- include/linux/percpu-refcount.h | 19 +++++++++++++++++++ lib/percpu-refcount.c | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) (limited to 'lib') diff --git a/include/linux/percpu-refcount.h b/include/linux/percpu-refcount.h index 0ddd2839ca8..3dfbf237cd8 100644 --- a/include/linux/percpu-refcount.h +++ b/include/linux/percpu-refcount.h @@ -67,6 +67,7 @@ struct percpu_ref { int __must_check percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release); +void percpu_ref_reinit(struct percpu_ref *ref); void percpu_ref_exit(struct percpu_ref *ref); void percpu_ref_kill_and_confirm(struct percpu_ref *ref, percpu_ref_func_t *confirm_kill); @@ -99,6 +100,9 @@ static inline bool __pcpu_ref_alive(struct percpu_ref *ref, { unsigned long pcpu_ptr = ACCESS_ONCE(ref->pcpu_count_ptr); + /* paired with smp_store_release() in percpu_ref_reinit() */ + smp_read_barrier_depends(); + if (unlikely(pcpu_ptr & PCPU_REF_DEAD)) return false; @@ -206,4 +210,19 @@ static inline void percpu_ref_put(struct percpu_ref *ref) rcu_read_unlock_sched(); } +/** + * percpu_ref_is_zero - test whether a percpu refcount reached zero + * @ref: percpu_ref to test + * + * Returns %true if @ref reached zero. + */ +static inline bool percpu_ref_is_zero(struct percpu_ref *ref) +{ + unsigned __percpu *pcpu_count; + + if (__pcpu_ref_alive(ref, &pcpu_count)) + return false; + return !atomic_read(&ref->count); +} + #endif diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index ac429912008..fe5a3342e96 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -60,6 +60,41 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release) } EXPORT_SYMBOL_GPL(percpu_ref_init); +/** + * percpu_ref_reinit - re-initialize a percpu refcount + * @ref: perpcu_ref to re-initialize + * + * Re-initialize @ref so that it's in the same state as when it finished + * percpu_ref_init(). @ref must have been initialized successfully, killed + * and reached 0 but not exited. + * + * Note that percpu_ref_tryget[_live]() are safe to perform on @ref while + * this function is in progress. + */ +void percpu_ref_reinit(struct percpu_ref *ref) +{ + unsigned __percpu *pcpu_count = pcpu_count_ptr(ref); + int cpu; + + BUG_ON(!pcpu_count); + WARN_ON(!percpu_ref_is_zero(ref)); + + atomic_set(&ref->count, 1 + PCPU_COUNT_BIAS); + + /* + * Restore per-cpu operation. smp_store_release() is paired with + * smp_read_barrier_depends() in __pcpu_ref_alive() and guarantees + * that the zeroing is visible to all percpu accesses which can see + * the following PCPU_REF_DEAD clearing. + */ + for_each_possible_cpu(cpu) + *per_cpu_ptr(pcpu_count, cpu) = 0; + + smp_store_release(&ref->pcpu_count_ptr, + ref->pcpu_count_ptr & ~PCPU_REF_DEAD); +} +EXPORT_SYMBOL_GPL(percpu_ref_reinit); + /** * percpu_ref_exit - undo percpu_ref_init() * @ref: percpu_ref to exit -- cgit v1.2.3-70-g09d2