diff options
| -rw-r--r-- | include/linux/notifier.h | 43 | ||||
| -rw-r--r-- | include/linux/srcu.h | 6 | ||||
| -rw-r--r-- | kernel/sys.c | 124 |
3 files changed, 166 insertions, 7 deletions
diff --git a/include/linux/notifier.h b/include/linux/notifier.h index 7ff386a6ae8..10a43ed0527 100644 --- a/include/linux/notifier.h +++ b/include/linux/notifier.h @@ -12,9 +12,10 @@ #include <linux/errno.h> #include <linux/mutex.h> #include <linux/rwsem.h> +#include <linux/srcu.h> /* - * Notifier chains are of three types: + * Notifier chains are of four types: * * Atomic notifier chains: Chain callbacks run in interrupt/atomic * context. Callouts are not allowed to block. @@ -23,13 +24,27 @@ * Raw notifier chains: There are no restrictions on callbacks, * registration, or unregistration. All locking and protection * must be provided by the caller. + * SRCU notifier chains: A variant of blocking notifier chains, with + * the same restrictions. * * atomic_notifier_chain_register() may be called from an atomic context, - * but blocking_notifier_chain_register() must be called from a process - * context. Ditto for the corresponding _unregister() routines. + * but blocking_notifier_chain_register() and srcu_notifier_chain_register() + * must be called from a process context. Ditto for the corresponding + * _unregister() routines. * - * atomic_notifier_chain_unregister() and blocking_notifier_chain_unregister() - * _must not_ be called from within the call chain. + * atomic_notifier_chain_unregister(), blocking_notifier_chain_unregister(), + * and srcu_notifier_chain_unregister() _must not_ be called from within + * the call chain. + * + * SRCU notifier chains are an alternative form of blocking notifier chains. + * They use SRCU (Sleepable Read-Copy Update) instead of rw-semaphores for + * protection of the chain links. This means there is _very_ low overhead + * in srcu_notifier_call_chain(): no cache bounces and no memory barriers. + * As compensation, srcu_notifier_chain_unregister() is rather expensive. + * SRCU notifier chains should be used when the chain will be called very + * often but notifier_blocks will seldom be removed. Also, SRCU notifier + * chains are slightly more difficult to use because they require special + * runtime initialization. */ struct notifier_block { @@ -52,6 +67,12 @@ struct raw_notifier_head { struct notifier_block *head; }; +struct srcu_notifier_head { + struct mutex mutex; + struct srcu_struct srcu; + struct notifier_block *head; +}; + #define ATOMIC_INIT_NOTIFIER_HEAD(name) do { \ spin_lock_init(&(name)->lock); \ (name)->head = NULL; \ @@ -64,6 +85,11 @@ struct raw_notifier_head { (name)->head = NULL; \ } while (0) +/* srcu_notifier_heads must be initialized and cleaned up dynamically */ +extern void srcu_init_notifier_head(struct srcu_notifier_head *nh); +#define srcu_cleanup_notifier_head(name) \ + cleanup_srcu_struct(&(name)->srcu); + #define ATOMIC_NOTIFIER_INIT(name) { \ .lock = __SPIN_LOCK_UNLOCKED(name.lock), \ .head = NULL } @@ -72,6 +98,7 @@ struct raw_notifier_head { .head = NULL } #define RAW_NOTIFIER_INIT(name) { \ .head = NULL } +/* srcu_notifier_heads cannot be initialized statically */ #define ATOMIC_NOTIFIER_HEAD(name) \ struct atomic_notifier_head name = \ @@ -91,6 +118,8 @@ extern int blocking_notifier_chain_register(struct blocking_notifier_head *, struct notifier_block *); extern int raw_notifier_chain_register(struct raw_notifier_head *, struct notifier_block *); +extern int srcu_notifier_chain_register(struct srcu_notifier_head *, + struct notifier_block *); extern int atomic_notifier_chain_unregister(struct atomic_notifier_head *, struct notifier_block *); @@ -98,6 +127,8 @@ extern int blocking_notifier_chain_unregister(struct blocking_notifier_head *, struct notifier_block *); extern int raw_notifier_chain_unregister(struct raw_notifier_head *, struct notifier_block *); +extern int srcu_notifier_chain_unregister(struct srcu_notifier_head *, + struct notifier_block *); extern int atomic_notifier_call_chain(struct atomic_notifier_head *, unsigned long val, void *v); @@ -105,6 +136,8 @@ extern int blocking_notifier_call_chain(struct blocking_notifier_head *, unsigned long val, void *v); extern int raw_notifier_call_chain(struct raw_notifier_head *, unsigned long val, void *v); +extern int srcu_notifier_call_chain(struct srcu_notifier_head *, + unsigned long val, void *v); #define NOTIFY_DONE 0x0000 /* Don't care */ #define NOTIFY_OK 0x0001 /* Suits me */ diff --git a/include/linux/srcu.h b/include/linux/srcu.h index 947fdab2ddb..8a45367b5f3 100644 --- a/include/linux/srcu.h +++ b/include/linux/srcu.h @@ -24,6 +24,9 @@ * */ +#ifndef _LINUX_SRCU_H +#define _LINUX_SRCU_H + struct srcu_struct_array { int c[2]; }; @@ -46,4 +49,5 @@ int srcu_read_lock(struct srcu_struct *sp) __acquires(sp); void srcu_read_unlock(struct srcu_struct *sp, int idx) __releases(sp); void synchronize_srcu(struct srcu_struct *sp); long srcu_batches_completed(struct srcu_struct *sp); -void cleanup_srcu_struct(struct srcu_struct *sp); + +#endif diff --git a/kernel/sys.c b/kernel/sys.c index 2314867ae34..fd5c7100677 100644 --- a/kernel/sys.c +++ b/kernel/sys.c @@ -153,7 +153,7 @@ static int __kprobes notifier_call_chain(struct notifier_block **nl, /* * Atomic notifier chain routines. Registration and unregistration - * use a mutex, and call_chain is synchronized by RCU (no locks). + * use a spinlock, and call_chain is synchronized by RCU (no locks). */ /** @@ -401,6 +401,128 @@ int raw_notifier_call_chain(struct raw_notifier_head *nh, EXPORT_SYMBOL_GPL(raw_notifier_call_chain); +/* + * SRCU notifier chain routines. Registration and unregistration + * use a mutex, and call_chain is synchronized by SRCU (no locks). + */ + +/** + * srcu_notifier_chain_register - Add notifier to an SRCU notifier chain + * @nh: Pointer to head of the SRCU notifier chain + * @n: New entry in notifier chain + * + * Adds a notifier to an SRCU notifier chain. + * Must be called in process context. + * + * Currently always returns zero. + */ + +int srcu_notifier_chain_register(struct srcu_notifier_head *nh, + struct notifier_block *n) +{ + int ret; + + /* + * This code gets used during boot-up, when task switching is + * not yet working and interrupts must remain disabled. At + * such times we must not call mutex_lock(). + */ + if (unlikely(system_state == SYSTEM_BOOTING)) + return notifier_chain_register(&nh->head, n); + + mutex_lock(&nh->mutex); + ret = notifier_chain_register(&nh->head, n); + mutex_unlock(&nh->mutex); + return ret; +} + +EXPORT_SYMBOL_GPL(srcu_notifier_chain_register); + +/** + * srcu_notifier_chain_unregister - Remove notifier from an SRCU notifier chain + * @nh: Pointer to head of the SRCU notifier chain + * @n: Entry to remove from notifier chain + * + * Removes a notifier from an SRCU notifier chain. + * Must be called from process context. + * + * Returns zero on success or %-ENOENT on failure. + */ +int srcu_notifier_chain_unregister(struct srcu_notifier_head *nh, + struct notifier_block *n) +{ + int ret; + + /* + * This code gets used during boot-up, when task switching is + * not yet working and interrupts must remain disabled. At + * such times we must not call mutex_lock(). + */ + if (unlikely(system_state == SYSTEM_BOOTING)) + return notifier_chain_unregister(&nh->head, n); + + mutex_lock(&nh->mutex); + ret = notifier_chain_unregister(&nh->head, n); + mutex_unlock(&nh->mutex); + synchronize_srcu(&nh->srcu); + return ret; +} + +EXPORT_SYMBOL_GPL(srcu_notifier_chain_unregister); + +/** + * srcu_notifier_call_chain - Call functions in an SRCU notifier chain + * @nh: Pointer to head of the SRCU notifier chain + * @val: Value passed unmodified to notifier function + * @v: Pointer passed unmodified to notifier function + * + * Calls each function in a notifier chain in turn. The functions + * run in a process context, so they are allowed to block. + * + * If the return value of the notifier can be and'ed + * with %NOTIFY_STOP_MASK then srcu_notifier_call_chain + * will return immediately, with the return value of + * the notifier function which halted execution. + * Otherwise the return value is the return value + * of the last notifier function called. + */ + +int srcu_notifier_call_chain(struct srcu_notifier_head *nh, + unsigned long val, void *v) +{ + int ret; + int idx; + + idx = srcu_read_lock(&nh->srcu); + ret = notifier_call_chain(&nh->head, val, v); + srcu_read_unlock(&nh->srcu, idx); + return ret; +} + +EXPORT_SYMBOL_GPL(srcu_notifier_call_chain); + +/** + * srcu_init_notifier_head - Initialize an SRCU notifier head + * @nh: Pointer to head of the srcu notifier chain + * + * Unlike other sorts of notifier heads, SRCU notifier heads require + * dynamic initialization. Be sure to call this routine before + * calling any of the other SRCU notifier routines for this head. + * + * If an SRCU notifier head is deallocated, it must first be cleaned + * up by calling srcu_cleanup_notifier_head(). Otherwise the head's + * per-cpu data (used by the SRCU mechanism) will leak. + */ + +void srcu_init_notifier_head(struct srcu_notifier_head *nh) +{ + mutex_init(&nh->mutex); + init_srcu_struct(&nh->srcu); + nh->head = NULL; +} + +EXPORT_SYMBOL_GPL(srcu_init_notifier_head); + /** * register_reboot_notifier - Register function to be called at reboot time * @nb: Info about notifier function to be called |