From 5fd77595ec62141fa71e575bdbf410e0192f87d0 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Mon, 24 Feb 2014 16:39:55 +0100 Subject: smp: Iterate functions through llist_for_each_entry_safe() The IPI function llist iteration is open coded. Lets simplify this with using an llist iterator. Also we want to keep the iteration safe against possible csd.llist->next value reuse from the IPI handler. At least the block subsystem used to do such things so lets stay careful and use llist_for_each_entry_safe(). Signed-off-by: Jan Kara Cc: Andrew Morton Cc: Christoph Hellwig Cc: Ingo Molnar Cc: Jens Axboe Signed-off-by: Frederic Weisbecker Signed-off-by: Jens Axboe --- kernel/smp.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) (limited to 'kernel/smp.c') diff --git a/kernel/smp.c b/kernel/smp.c index ffee35bef17..e3852de042a 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -151,7 +151,8 @@ static void generic_exec_single(int cpu, struct call_single_data *csd, int wait) */ void generic_smp_call_function_single_interrupt(void) { - struct llist_node *entry, *next; + struct llist_node *entry; + struct call_single_data *csd, *csd_next; /* * Shouldn't receive this interrupt on a cpu that is not yet online. @@ -161,16 +162,9 @@ void generic_smp_call_function_single_interrupt(void) entry = llist_del_all(&__get_cpu_var(call_single_queue)); entry = llist_reverse_order(entry); - while (entry) { - struct call_single_data *csd; - - next = entry->next; - - csd = llist_entry(entry, struct call_single_data, llist); + llist_for_each_entry_safe(csd, csd_next, entry, llist) { csd->func(csd->info); csd_unlock(csd); - - entry = next; } } -- cgit v1.2.3-70-g09d2 From 08eed44c7249d381a099bc55577e55c6bb533160 Mon Sep 17 00:00:00 2001 From: Jan Kara Date: Mon, 24 Feb 2014 16:39:57 +0100 Subject: smp: Teach __smp_call_function_single() to check for offline cpus Align __smp_call_function_single() with smp_call_function_single() so that it also checks whether requested cpu is still online. Signed-off-by: Jan Kara Cc: Andrew Morton Cc: Christoph Hellwig Cc: Ingo Molnar Cc: Jens Axboe Signed-off-by: Frederic Weisbecker Signed-off-by: Jens Axboe --- include/linux/smp.h | 3 +-- kernel/smp.c | 11 +++++++---- kernel/up.c | 5 +++-- 3 files changed, 11 insertions(+), 8 deletions(-) (limited to 'kernel/smp.c') diff --git a/include/linux/smp.h b/include/linux/smp.h index 4991c6b1283..c39074c794c 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -50,8 +50,7 @@ void on_each_cpu_cond(bool (*cond_func)(int cpu, void *info), smp_call_func_t func, void *info, bool wait, gfp_t gfp_flags); -void __smp_call_function_single(int cpuid, struct call_single_data *data, - int wait); +int __smp_call_function_single(int cpu, struct call_single_data *csd, int wait); #ifdef CONFIG_SMP diff --git a/kernel/smp.c b/kernel/smp.c index e3852de042a..5ff14e3739c 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -276,18 +276,18 @@ EXPORT_SYMBOL_GPL(smp_call_function_any); /** * __smp_call_function_single(): Run a function on a specific CPU * @cpu: The CPU to run on. - * @data: Pre-allocated and setup data structure + * @csd: Pre-allocated and setup data structure * @wait: If true, wait until function has completed on specified CPU. * * Like smp_call_function_single(), but allow caller to pass in a * pre-allocated data structure. Useful for embedding @data inside * other structures, for instance. */ -void __smp_call_function_single(int cpu, struct call_single_data *csd, - int wait) +int __smp_call_function_single(int cpu, struct call_single_data *csd, int wait) { unsigned int this_cpu; unsigned long flags; + int err = 0; this_cpu = get_cpu(); /* @@ -303,11 +303,14 @@ void __smp_call_function_single(int cpu, struct call_single_data *csd, local_irq_save(flags); csd->func(csd->info); local_irq_restore(flags); - } else { + } else if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) { csd_lock(csd); generic_exec_single(cpu, csd, wait); + } else { + err = -ENXIO; /* CPU not online */ } put_cpu(); + return err; } EXPORT_SYMBOL_GPL(__smp_call_function_single); diff --git a/kernel/up.c b/kernel/up.c index 509403e3fbc..cdf03d16840 100644 --- a/kernel/up.c +++ b/kernel/up.c @@ -22,14 +22,15 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info, } EXPORT_SYMBOL(smp_call_function_single); -void __smp_call_function_single(int cpu, struct call_single_data *csd, - int wait) +int __smp_call_function_single(int cpu, struct call_single_data *csd, + int wait) { unsigned long flags; local_irq_save(flags); csd->func(csd->info); local_irq_restore(flags); + return 0; } EXPORT_SYMBOL(__smp_call_function_single); -- cgit v1.2.3-70-g09d2 From 8b28499a71d3431c9128abc743e2d2bfbdae3ed4 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 24 Feb 2014 16:39:58 +0100 Subject: smp: Consolidate the various smp_call_function_single() declensions __smp_call_function_single() and smp_call_function_single() share some code that can be factorized: execute inline when the target is local, check if the target is online, lock the csd, call generic_exec_single(). Lets move the common parts to generic_exec_single(). Reviewed-by: Jan Kara Cc: Andrew Morton Cc: Christoph Hellwig Cc: Ingo Molnar Cc: Jan Kara Cc: Jens Axboe Signed-off-by: Frederic Weisbecker Signed-off-by: Jens Axboe --- kernel/smp.c | 80 +++++++++++++++++++++++++++++------------------------------- 1 file changed, 39 insertions(+), 41 deletions(-) (limited to 'kernel/smp.c') diff --git a/kernel/smp.c b/kernel/smp.c index 5ff14e3739c..64bb0d48e96 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -117,13 +117,43 @@ static void csd_unlock(struct call_single_data *csd) csd->flags &= ~CSD_FLAG_LOCK; } +static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_data, csd_data); + /* * Insert a previously allocated call_single_data element * for execution on the given CPU. data must already have * ->func, ->info, and ->flags set. */ -static void generic_exec_single(int cpu, struct call_single_data *csd, int wait) +static int generic_exec_single(int cpu, struct call_single_data *csd, + smp_call_func_t func, void *info, int wait) { + struct call_single_data csd_stack = { .flags = 0 }; + unsigned long flags; + + + if (cpu == smp_processor_id()) { + local_irq_save(flags); + func(info); + local_irq_restore(flags); + return 0; + } + + + if ((unsigned)cpu >= nr_cpu_ids || !cpu_online(cpu)) + return -ENXIO; + + + if (!csd) { + csd = &csd_stack; + if (!wait) + csd = &__get_cpu_var(csd_data); + } + + csd_lock(csd); + + csd->func = func; + csd->info = info; + if (wait) csd->flags |= CSD_FLAG_WAIT; @@ -143,6 +173,8 @@ static void generic_exec_single(int cpu, struct call_single_data *csd, int wait) if (wait) csd_lock_wait(csd); + + return 0; } /* @@ -168,8 +200,6 @@ void generic_smp_call_function_single_interrupt(void) } } -static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_data, csd_data); - /* * smp_call_function_single - Run a function on a specific CPU * @func: The function to run. This must be fast and non-blocking. @@ -181,12 +211,8 @@ static DEFINE_PER_CPU_SHARED_ALIGNED(struct call_single_data, csd_data); int smp_call_function_single(int cpu, smp_call_func_t func, void *info, int wait) { - struct call_single_data d = { - .flags = 0, - }; - unsigned long flags; int this_cpu; - int err = 0; + int err; /* * prevent preemption and reschedule on another processor, @@ -203,26 +229,7 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info, WARN_ON_ONCE(cpu_online(this_cpu) && irqs_disabled() && !oops_in_progress); - if (cpu == this_cpu) { - local_irq_save(flags); - func(info); - local_irq_restore(flags); - } else { - if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) { - struct call_single_data *csd = &d; - - if (!wait) - csd = &__get_cpu_var(csd_data); - - csd_lock(csd); - - csd->func = func; - csd->info = info; - generic_exec_single(cpu, csd, wait); - } else { - err = -ENXIO; /* CPU not online */ - } - } + err = generic_exec_single(cpu, NULL, func, info, wait); put_cpu(); @@ -285,9 +292,8 @@ EXPORT_SYMBOL_GPL(smp_call_function_any); */ int __smp_call_function_single(int cpu, struct call_single_data *csd, int wait) { - unsigned int this_cpu; - unsigned long flags; int err = 0; + int this_cpu; this_cpu = get_cpu(); /* @@ -296,20 +302,12 @@ int __smp_call_function_single(int cpu, struct call_single_data *csd, int wait) * send smp call function interrupt to this cpu and as such deadlocks * can't happen. */ - WARN_ON_ONCE(cpu_online(smp_processor_id()) && wait && irqs_disabled() + WARN_ON_ONCE(cpu_online(this_cpu) && wait && irqs_disabled() && !oops_in_progress); - if (cpu == this_cpu) { - local_irq_save(flags); - csd->func(csd->info); - local_irq_restore(flags); - } else if ((unsigned)cpu < nr_cpu_ids && cpu_online(cpu)) { - csd_lock(csd); - generic_exec_single(cpu, csd, wait); - } else { - err = -ENXIO; /* CPU not online */ - } + err = generic_exec_single(cpu, csd, csd->func, csd->info, wait); put_cpu(); + return err; } EXPORT_SYMBOL_GPL(__smp_call_function_single); -- cgit v1.2.3-70-g09d2 From d7877c03f1b62de06f9c00417952f39f56c1ab00 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 24 Feb 2014 16:39:59 +0100 Subject: smp: Move __smp_call_function_single() below its safe version Move this function closer to __smp_call_function_single(). These functions have very similar behavior and should be displayed in the same block for clarity. Reviewed-by: Jan Kara Cc: Andrew Morton Cc: Christoph Hellwig Cc: Ingo Molnar Cc: Jan Kara Cc: Jens Axboe Signed-off-by: Frederic Weisbecker Signed-off-by: Jens Axboe --- kernel/smp.c | 64 ++++++++++++++++++++++++++++++------------------------------ 1 file changed, 32 insertions(+), 32 deletions(-) (limited to 'kernel/smp.c') diff --git a/kernel/smp.c b/kernel/smp.c index 64bb0d48e96..fa04ab938e5 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -237,6 +237,38 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info, } EXPORT_SYMBOL(smp_call_function_single); +/** + * __smp_call_function_single(): Run a function on a specific CPU + * @cpu: The CPU to run on. + * @csd: Pre-allocated and setup data structure + * @wait: If true, wait until function has completed on specified CPU. + * + * Like smp_call_function_single(), but allow caller to pass in a + * pre-allocated data structure. Useful for embedding @data inside + * other structures, for instance. + */ +int __smp_call_function_single(int cpu, struct call_single_data *csd, int wait) +{ + int err = 0; + int this_cpu; + + this_cpu = get_cpu(); + /* + * Can deadlock when called with interrupts disabled. + * We allow cpu's that are not yet online though, as no one else can + * send smp call function interrupt to this cpu and as such deadlocks + * can't happen. + */ + WARN_ON_ONCE(cpu_online(this_cpu) && wait && irqs_disabled() + && !oops_in_progress); + + err = generic_exec_single(cpu, csd, csd->func, csd->info, wait); + put_cpu(); + + return err; +} +EXPORT_SYMBOL_GPL(__smp_call_function_single); + /* * smp_call_function_any - Run a function on any of the given cpus * @mask: The mask of cpus it can run on. @@ -280,38 +312,6 @@ call: } EXPORT_SYMBOL_GPL(smp_call_function_any); -/** - * __smp_call_function_single(): Run a function on a specific CPU - * @cpu: The CPU to run on. - * @csd: Pre-allocated and setup data structure - * @wait: If true, wait until function has completed on specified CPU. - * - * Like smp_call_function_single(), but allow caller to pass in a - * pre-allocated data structure. Useful for embedding @data inside - * other structures, for instance. - */ -int __smp_call_function_single(int cpu, struct call_single_data *csd, int wait) -{ - int err = 0; - int this_cpu; - - this_cpu = get_cpu(); - /* - * Can deadlock when called with interrupts disabled. - * We allow cpu's that are not yet online though, as no one else can - * send smp call function interrupt to this cpu and as such deadlocks - * can't happen. - */ - WARN_ON_ONCE(cpu_online(this_cpu) && wait && irqs_disabled() - && !oops_in_progress); - - err = generic_exec_single(cpu, csd, csd->func, csd->info, wait); - put_cpu(); - - return err; -} -EXPORT_SYMBOL_GPL(__smp_call_function_single); - /** * smp_call_function_many(): Run a function on a set of other CPUs. * @mask: The set of cpus to run on (only runs on online subset). -- cgit v1.2.3-70-g09d2 From fce8ad1568c57e7f334018dec4fa1744c926c135 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 24 Feb 2014 16:40:01 +0100 Subject: smp: Remove wait argument from __smp_call_function_single() The main point of calling __smp_call_function_single() is to send an IPI in a pure asynchronous way. By embedding a csd in an object, a caller can send the IPI without waiting for a previous one to complete as is required by smp_call_function_single() for example. As such, sending this kind of IPI can be safe even when irqs are disabled. This flexibility comes at the expense of the caller who then needs to synchronize the csd lifecycle by himself and make sure that IPIs on a single csd are serialized. This is how __smp_call_function_single() works when wait = 0 and this usecase is relevant. Now there don't seem to be any usecase with wait = 1 that can't be covered by smp_call_function_single() instead, which is safer. Lets look at the two possible scenario: 1) The user calls __smp_call_function_single(wait = 1) on a csd embedded in an object. It looks like a nice and convenient pattern at the first sight because we can then retrieve the object from the IPI handler easily. But actually it is a waste of memory space in the object since the csd can be allocated from the stack by smp_call_function_single(wait = 1) and the object can be passed an the IPI argument. Besides that, embedding the csd in an object is more error prone because the caller must take care of the serialization of the IPIs for this csd. 2) The user calls __smp_call_function_single(wait = 1) on a csd that is allocated on the stack. It's ok but smp_call_function_single() can do it as well and it already takes care of the allocation on the stack. Again it's more simple and less error prone. Therefore, using the underscore prepend API version with wait = 1 is a bad pattern and a sign that the caller can do safer and more simple. There was a single user of that which has just been converted. So lets remove this option to discourage further users. Cc: Andrew Morton Cc: Christoph Hellwig Cc: Ingo Molnar Cc: Jan Kara Cc: Jens Axboe Signed-off-by: Frederic Weisbecker Signed-off-by: Jens Axboe --- block/blk-mq.c | 2 +- block/blk-softirq.c | 2 +- drivers/cpuidle/coupled.c | 2 +- include/linux/smp.h | 2 +- kernel/sched/core.c | 2 +- kernel/smp.c | 19 ++++--------------- kernel/up.c | 3 +-- net/core/dev.c | 2 +- 8 files changed, 11 insertions(+), 23 deletions(-) (limited to 'kernel/smp.c') diff --git a/block/blk-mq.c b/block/blk-mq.c index 1fa9dd153fd..62154edf148 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -353,7 +353,7 @@ void __blk_mq_complete_request(struct request *rq) rq->csd.func = __blk_mq_complete_request_remote; rq->csd.info = rq; rq->csd.flags = 0; - __smp_call_function_single(ctx->cpu, &rq->csd, 0); + __smp_call_function_single(ctx->cpu, &rq->csd); } else { rq->q->softirq_done_fn(rq); } diff --git a/block/blk-softirq.c b/block/blk-softirq.c index b5c37d96cf0..6345b7ebd0d 100644 --- a/block/blk-softirq.c +++ b/block/blk-softirq.c @@ -70,7 +70,7 @@ static int raise_blk_irq(int cpu, struct request *rq) data->info = rq; data->flags = 0; - __smp_call_function_single(cpu, data, 0); + __smp_call_function_single(cpu, data); return 0; } diff --git a/drivers/cpuidle/coupled.c b/drivers/cpuidle/coupled.c index e952936418d..04115947acc 100644 --- a/drivers/cpuidle/coupled.c +++ b/drivers/cpuidle/coupled.c @@ -323,7 +323,7 @@ static void cpuidle_coupled_poke(int cpu) struct call_single_data *csd = &per_cpu(cpuidle_coupled_poke_cb, cpu); if (!cpumask_test_and_set_cpu(cpu, &cpuidle_coupled_poke_pending)) - __smp_call_function_single(cpu, csd, 0); + __smp_call_function_single(cpu, csd); } /** diff --git a/include/linux/smp.h b/include/linux/smp.h index c39074c794c..b410a1f2328 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -50,7 +50,7 @@ void on_each_cpu_cond(bool (*cond_func)(int cpu, void *info), smp_call_func_t func, void *info, bool wait, gfp_t gfp_flags); -int __smp_call_function_single(int cpu, struct call_single_data *csd, int wait); +int __smp_call_function_single(int cpu, struct call_single_data *csd); #ifdef CONFIG_SMP diff --git a/kernel/sched/core.c b/kernel/sched/core.c index b46131ef6aa..eba3d84765f 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -432,7 +432,7 @@ void hrtick_start(struct rq *rq, u64 delay) if (rq == this_rq()) { __hrtick_restart(rq); } else if (!rq->hrtick_csd_pending) { - __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd, 0); + __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd); rq->hrtick_csd_pending = 1; } } diff --git a/kernel/smp.c b/kernel/smp.c index fa04ab938e5..b7676318975 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -241,29 +241,18 @@ EXPORT_SYMBOL(smp_call_function_single); * __smp_call_function_single(): Run a function on a specific CPU * @cpu: The CPU to run on. * @csd: Pre-allocated and setup data structure - * @wait: If true, wait until function has completed on specified CPU. * * Like smp_call_function_single(), but allow caller to pass in a * pre-allocated data structure. Useful for embedding @data inside * other structures, for instance. */ -int __smp_call_function_single(int cpu, struct call_single_data *csd, int wait) +int __smp_call_function_single(int cpu, struct call_single_data *csd) { int err = 0; - int this_cpu; - this_cpu = get_cpu(); - /* - * Can deadlock when called with interrupts disabled. - * We allow cpu's that are not yet online though, as no one else can - * send smp call function interrupt to this cpu and as such deadlocks - * can't happen. - */ - WARN_ON_ONCE(cpu_online(this_cpu) && wait && irqs_disabled() - && !oops_in_progress); - - err = generic_exec_single(cpu, csd, csd->func, csd->info, wait); - put_cpu(); + preempt_disable(); + err = generic_exec_single(cpu, csd, csd->func, csd->info, 0); + preempt_enable(); return err; } diff --git a/kernel/up.c b/kernel/up.c index cdf03d16840..4e199d4cef8 100644 --- a/kernel/up.c +++ b/kernel/up.c @@ -22,8 +22,7 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info, } EXPORT_SYMBOL(smp_call_function_single); -int __smp_call_function_single(int cpu, struct call_single_data *csd, - int wait) +int __smp_call_function_single(int cpu, struct call_single_data *csd) { unsigned long flags; diff --git a/net/core/dev.c b/net/core/dev.c index 4ad1b78c9c7..d1298128bff 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4129,7 +4129,7 @@ static void net_rps_action_and_irq_enable(struct softnet_data *sd) if (cpu_online(remsd->cpu)) __smp_call_function_single(remsd->cpu, - &remsd->csd, 0); + &remsd->csd); remsd = next; } } else -- cgit v1.2.3-70-g09d2 From c46fff2a3b29794b35d717b5680a27f31a6a6bc0 Mon Sep 17 00:00:00 2001 From: Frederic Weisbecker Date: Mon, 24 Feb 2014 16:40:02 +0100 Subject: smp: Rename __smp_call_function_single() to smp_call_function_single_async() The name __smp_call_function_single() doesn't tell much about the properties of this function, especially when compared to smp_call_function_single(). The comments above the implementation are also misleading. The main point of this function is actually not to be able to embed the csd in an object. This is actually a requirement that result from the purpose of this function which is to raise an IPI asynchronously. As such it can be called with interrupts disabled. And this feature comes at the cost of the caller who then needs to serialize the IPIs on this csd. Lets rename the function and enhance the comments so that they reflect these properties. Suggested-by: Christoph Hellwig Cc: Andrew Morton Cc: Christoph Hellwig Cc: Ingo Molnar Cc: Jan Kara Cc: Jens Axboe Signed-off-by: Frederic Weisbecker Signed-off-by: Jens Axboe --- block/blk-mq.c | 2 +- block/blk-softirq.c | 2 +- drivers/cpuidle/coupled.c | 2 +- include/linux/smp.h | 2 +- kernel/sched/core.c | 2 +- kernel/smp.c | 19 +++++++++++++------ kernel/up.c | 4 ++-- net/core/dev.c | 2 +- 8 files changed, 21 insertions(+), 14 deletions(-) (limited to 'kernel/smp.c') diff --git a/block/blk-mq.c b/block/blk-mq.c index 62154edf148..6468a715a0e 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -353,7 +353,7 @@ void __blk_mq_complete_request(struct request *rq) rq->csd.func = __blk_mq_complete_request_remote; rq->csd.info = rq; rq->csd.flags = 0; - __smp_call_function_single(ctx->cpu, &rq->csd); + smp_call_function_single_async(ctx->cpu, &rq->csd); } else { rq->q->softirq_done_fn(rq); } diff --git a/block/blk-softirq.c b/block/blk-softirq.c index 6345b7ebd0d..ebd6b6f1bde 100644 --- a/block/blk-softirq.c +++ b/block/blk-softirq.c @@ -70,7 +70,7 @@ static int raise_blk_irq(int cpu, struct request *rq) data->info = rq; data->flags = 0; - __smp_call_function_single(cpu, data); + smp_call_function_single_async(cpu, data); return 0; } diff --git a/drivers/cpuidle/coupled.c b/drivers/cpuidle/coupled.c index 04115947acc..cb6654bfad7 100644 --- a/drivers/cpuidle/coupled.c +++ b/drivers/cpuidle/coupled.c @@ -323,7 +323,7 @@ static void cpuidle_coupled_poke(int cpu) struct call_single_data *csd = &per_cpu(cpuidle_coupled_poke_cb, cpu); if (!cpumask_test_and_set_cpu(cpu, &cpuidle_coupled_poke_pending)) - __smp_call_function_single(cpu, csd); + smp_call_function_single_async(cpu, csd); } /** diff --git a/include/linux/smp.h b/include/linux/smp.h index b410a1f2328..633f5edd747 100644 --- a/include/linux/smp.h +++ b/include/linux/smp.h @@ -50,7 +50,7 @@ void on_each_cpu_cond(bool (*cond_func)(int cpu, void *info), smp_call_func_t func, void *info, bool wait, gfp_t gfp_flags); -int __smp_call_function_single(int cpu, struct call_single_data *csd); +int smp_call_function_single_async(int cpu, struct call_single_data *csd); #ifdef CONFIG_SMP diff --git a/kernel/sched/core.c b/kernel/sched/core.c index eba3d84765f..0cca04a53de 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -432,7 +432,7 @@ void hrtick_start(struct rq *rq, u64 delay) if (rq == this_rq()) { __hrtick_restart(rq); } else if (!rq->hrtick_csd_pending) { - __smp_call_function_single(cpu_of(rq), &rq->hrtick_csd); + smp_call_function_single_async(cpu_of(rq), &rq->hrtick_csd); rq->hrtick_csd_pending = 1; } } diff --git a/kernel/smp.c b/kernel/smp.c index b7676318975..06d574e42c7 100644 --- a/kernel/smp.c +++ b/kernel/smp.c @@ -238,15 +238,22 @@ int smp_call_function_single(int cpu, smp_call_func_t func, void *info, EXPORT_SYMBOL(smp_call_function_single); /** - * __smp_call_function_single(): Run a function on a specific CPU + * smp_call_function_single_async(): Run an asynchronous function on a + * specific CPU. * @cpu: The CPU to run on. * @csd: Pre-allocated and setup data structure * - * Like smp_call_function_single(), but allow caller to pass in a - * pre-allocated data structure. Useful for embedding @data inside - * other structures, for instance. + * Like smp_call_function_single(), but the call is asynchonous and + * can thus be done from contexts with disabled interrupts. + * + * The caller passes his own pre-allocated data structure + * (ie: embedded in an object) and is responsible for synchronizing it + * such that the IPIs performed on the @csd are strictly serialized. + * + * NOTE: Be careful, there is unfortunately no current debugging facility to + * validate the correctness of this serialization. */ -int __smp_call_function_single(int cpu, struct call_single_data *csd) +int smp_call_function_single_async(int cpu, struct call_single_data *csd) { int err = 0; @@ -256,7 +263,7 @@ int __smp_call_function_single(int cpu, struct call_single_data *csd) return err; } -EXPORT_SYMBOL_GPL(__smp_call_function_single); +EXPORT_SYMBOL_GPL(smp_call_function_single_async); /* * smp_call_function_any - Run a function on any of the given cpus diff --git a/kernel/up.c b/kernel/up.c index 4e199d4cef8..1760bf3d146 100644 --- a/kernel/up.c +++ b/kernel/up.c @@ -22,7 +22,7 @@ int smp_call_function_single(int cpu, void (*func) (void *info), void *info, } EXPORT_SYMBOL(smp_call_function_single); -int __smp_call_function_single(int cpu, struct call_single_data *csd) +int smp_call_function_single_async(int cpu, struct call_single_data *csd) { unsigned long flags; @@ -31,7 +31,7 @@ int __smp_call_function_single(int cpu, struct call_single_data *csd) local_irq_restore(flags); return 0; } -EXPORT_SYMBOL(__smp_call_function_single); +EXPORT_SYMBOL(smp_call_function_single_async); int on_each_cpu(smp_call_func_t func, void *info, int wait) { diff --git a/net/core/dev.c b/net/core/dev.c index d1298128bff..ac7a2abb7f1 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4128,7 +4128,7 @@ static void net_rps_action_and_irq_enable(struct softnet_data *sd) struct softnet_data *next = remsd->rps_ipi_next; if (cpu_online(remsd->cpu)) - __smp_call_function_single(remsd->cpu, + smp_call_function_single_async(remsd->cpu, &remsd->csd); remsd = next; } -- cgit v1.2.3-70-g09d2