From 37348804e0289087d21ae8bff4c0732030a3c6ac Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Thu, 29 Sep 2011 11:10:05 -0700 Subject: jump_label: if a key has already been initialized, don't nop it out If a key has been enabled before jump_label_init() is called, don't nop it out. This removes arch_jump_label_text_poke_early() (which can only nop out a site) and uses arch_jump_label_transform() instead. Signed-off-by: Jeremy Fitzhardinge Acked-by: Jason Baron Acked-by: Peter Zijlstra --- kernel/jump_label.c | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) (limited to 'kernel/jump_label.c') diff --git a/kernel/jump_label.c b/kernel/jump_label.c index a8ce45097f3..059202d5b77 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -121,13 +121,6 @@ static void __jump_label_update(struct jump_label_key *key, } } -/* - * Not all archs need this. - */ -void __weak arch_jump_label_text_poke_early(jump_label_t addr) -{ -} - static __init int jump_label_init(void) { struct jump_entry *iter_start = __start___jump_table; @@ -139,12 +132,15 @@ static __init int jump_label_init(void) jump_label_sort_entries(iter_start, iter_stop); for (iter = iter_start; iter < iter_stop; iter++) { - arch_jump_label_text_poke_early(iter->code); - if (iter->key == (jump_label_t)(unsigned long)key) + struct jump_label_key *iterk; + + iterk = (struct jump_label_key *)(unsigned long)iter->key; + arch_jump_label_transform(iter, jump_label_enabled(iterk) ? + JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE); + if (iterk == key) continue; - key = (struct jump_label_key *)(unsigned long)iter->key; - atomic_set(&key->enabled, 0); + key = iterk; key->entries = iter; #ifdef CONFIG_MODULES key->next = NULL; @@ -212,7 +208,7 @@ void jump_label_apply_nops(struct module *mod) return; for (iter = iter_start; iter < iter_stop; iter++) - arch_jump_label_text_poke_early(iter->code); + arch_jump_label_transform(iter, JUMP_LABEL_DISABLE); } static int jump_label_add_module(struct module *mod) -- cgit v1.2.3-70-g09d2 From 20284aa77c0f6227da4783a920b72dc61d4bcc09 Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Mon, 3 Oct 2011 11:01:46 -0700 Subject: jump_label: add arch_jump_label_transform_static() to optimise non-live code updates When updating a newly loaded module, the code is definitely not yet executing on any processor, so it can be updated with no need for any heavyweight synchronization. This patch adds arch_jump_label_static() which is implemented as arch_jump_label_transform() by default, but architectures can override it if it avoids, say, a call to stop_machine(). Signed-off-by: Jeremy Fitzhardinge Acked-by: Jason Baron Acked-by: Peter Zijlstra --- include/linux/jump_label.h | 2 ++ kernel/jump_label.c | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) (limited to 'kernel/jump_label.c') diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 12e804ea32a..56594e45b01 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -45,6 +45,8 @@ extern void jump_label_lock(void); extern void jump_label_unlock(void); extern void arch_jump_label_transform(struct jump_entry *entry, enum jump_label_type type); +extern void arch_jump_label_transform_static(struct jump_entry *entry, + enum jump_label_type type); extern int jump_label_text_reserved(void *start, void *end); extern void jump_label_inc(struct jump_label_key *key); extern void jump_label_dec(struct jump_label_key *key); diff --git a/kernel/jump_label.c b/kernel/jump_label.c index 059202d5b77..ff2028f35aa 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -104,6 +104,18 @@ static int __jump_label_text_reserved(struct jump_entry *iter_start, return 0; } +/* + * Update code which is definitely not currently executing. + * Architectures which need heavyweight synchronization to modify + * running code can override this to make the non-live update case + * cheaper. + */ +void __weak arch_jump_label_transform_static(struct jump_entry *entry, + enum jump_label_type type) +{ + arch_jump_label_transform(entry, type); +} + static void __jump_label_update(struct jump_label_key *key, struct jump_entry *entry, struct jump_entry *stop, int enable) @@ -135,8 +147,8 @@ static __init int jump_label_init(void) struct jump_label_key *iterk; iterk = (struct jump_label_key *)(unsigned long)iter->key; - arch_jump_label_transform(iter, jump_label_enabled(iterk) ? - JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE); + arch_jump_label_transform_static(iter, jump_label_enabled(iterk) ? + JUMP_LABEL_ENABLE : JUMP_LABEL_DISABLE); if (iterk == key) continue; @@ -208,7 +220,7 @@ void jump_label_apply_nops(struct module *mod) return; for (iter = iter_start; iter < iter_stop; iter++) - arch_jump_label_transform(iter, JUMP_LABEL_DISABLE); + arch_jump_label_transform_static(iter, JUMP_LABEL_DISABLE); } static int jump_label_add_module(struct module *mod) -- cgit v1.2.3-70-g09d2 From 97ce2c88f9ad42e3c60a9beb9fca87abf3639faa Mon Sep 17 00:00:00 2001 From: Jeremy Fitzhardinge Date: Wed, 12 Oct 2011 16:17:54 -0700 Subject: jump-label: initialize jump-label subsystem much earlier Initialize jump_labels much, much earlier, so they're available for use during system setup. Signed-off-by: Jeremy Fitzhardinge Acked-by: Peter Zijlstra --- include/linux/jump_label.h | 14 +++++++++----- init/main.c | 3 +++ kernel/jump_label.c | 5 +---- 3 files changed, 13 insertions(+), 9 deletions(-) (limited to 'kernel/jump_label.c') diff --git a/include/linux/jump_label.h b/include/linux/jump_label.h index 56594e45b01..388b0d425b5 100644 --- a/include/linux/jump_label.h +++ b/include/linux/jump_label.h @@ -16,7 +16,7 @@ struct jump_label_key { # include # define HAVE_JUMP_LABEL -#endif +#endif /* CC_HAVE_ASM_GOTO && CONFIG_JUMP_LABEL */ enum jump_label_type { JUMP_LABEL_DISABLE = 0, @@ -41,6 +41,7 @@ static __always_inline bool static_branch(struct jump_label_key *key) extern struct jump_entry __start___jump_table[]; extern struct jump_entry __stop___jump_table[]; +extern void jump_label_init(void); extern void jump_label_lock(void); extern void jump_label_unlock(void); extern void arch_jump_label_transform(struct jump_entry *entry, @@ -53,7 +54,7 @@ extern void jump_label_dec(struct jump_label_key *key); extern bool jump_label_enabled(struct jump_label_key *key); extern void jump_label_apply_nops(struct module *mod); -#else +#else /* !HAVE_JUMP_LABEL */ #include @@ -63,6 +64,10 @@ struct jump_label_key { atomic_t enabled; }; +static __always_inline void jump_label_init(void) +{ +} + static __always_inline bool static_branch(struct jump_label_key *key) { if (unlikely(atomic_read(&key->enabled))) @@ -97,7 +102,6 @@ static inline int jump_label_apply_nops(struct module *mod) { return 0; } +#endif /* HAVE_JUMP_LABEL */ -#endif - -#endif +#endif /* _LINUX_JUMP_LABEL_H */ diff --git a/init/main.c b/init/main.c index 2a9b88aa5e7..29d8d847de9 100644 --- a/init/main.c +++ b/init/main.c @@ -515,6 +515,9 @@ asmlinkage void __init start_kernel(void) parse_args("Booting kernel", static_command_line, __start___param, __stop___param - __start___param, &unknown_bootoption); + + jump_label_init(); + /* * These use large bootmem allocations and must precede * kmem_cache_init() diff --git a/kernel/jump_label.c b/kernel/jump_label.c index ff2028f35aa..bbdfe2a462a 100644 --- a/kernel/jump_label.c +++ b/kernel/jump_label.c @@ -133,7 +133,7 @@ static void __jump_label_update(struct jump_label_key *key, } } -static __init int jump_label_init(void) +void __init jump_label_init(void) { struct jump_entry *iter_start = __start___jump_table; struct jump_entry *iter_stop = __stop___jump_table; @@ -159,10 +159,7 @@ static __init int jump_label_init(void) #endif } jump_label_unlock(); - - return 0; } -early_initcall(jump_label_init); #ifdef CONFIG_MODULES -- cgit v1.2.3-70-g09d2