diff options
| -rw-r--r-- | arch/x86/include/asm/smp.h | 32 | ||||
| -rw-r--r-- | arch/x86/kernel/cpu/common.c | 26 | ||||
| -rw-r--r-- | arch/x86/kernel/setup_percpu.c | 25 | ||||
| -rw-r--r-- | arch/x86/kernel/smp.c | 17 | ||||
| -rw-r--r-- | arch/x86/kernel/smpboot.c | 128 |
5 files changed, 152 insertions, 76 deletions
diff --git a/arch/x86/include/asm/smp.h b/arch/x86/include/asm/smp.h index 830b9fcb642..19953df61c5 100644 --- a/arch/x86/include/asm/smp.h +++ b/arch/x86/include/asm/smp.h @@ -18,9 +18,26 @@ #include <asm/pda.h> #include <asm/thread_info.h> +#ifdef CONFIG_X86_64 + +extern cpumask_var_t cpu_callin_mask; +extern cpumask_var_t cpu_callout_mask; +extern cpumask_var_t cpu_initialized_mask; +extern cpumask_var_t cpu_sibling_setup_mask; + +#else /* CONFIG_X86_32 */ + +extern cpumask_t cpu_callin_map; extern cpumask_t cpu_callout_map; extern cpumask_t cpu_initialized; -extern cpumask_t cpu_callin_map; +extern cpumask_t cpu_sibling_setup_map; + +#define cpu_callin_mask ((struct cpumask *)&cpu_callin_map) +#define cpu_callout_mask ((struct cpumask *)&cpu_callout_map) +#define cpu_initialized_mask ((struct cpumask *)&cpu_initialized) +#define cpu_sibling_setup_mask ((struct cpumask *)&cpu_sibling_setup_map) + +#endif /* CONFIG_X86_32 */ extern void (*mtrr_hook)(void); extern void zap_low_mappings(void); @@ -29,7 +46,6 @@ extern int __cpuinit get_local_pda(int cpu); extern int smp_num_siblings; extern unsigned int num_processors; -extern cpumask_t cpu_initialized; DECLARE_PER_CPU(cpumask_t, cpu_sibling_map); DECLARE_PER_CPU(cpumask_t, cpu_core_map); @@ -38,6 +54,16 @@ DECLARE_PER_CPU(u16, cpu_llc_id); DECLARE_PER_CPU(int, cpu_number); #endif +static inline struct cpumask *cpu_sibling_mask(int cpu) +{ + return &per_cpu(cpu_sibling_map, cpu); +} + +static inline struct cpumask *cpu_core_mask(int cpu) +{ + return &per_cpu(cpu_core_map, cpu); +} + DECLARE_EARLY_PER_CPU(u16, x86_cpu_to_apicid); DECLARE_EARLY_PER_CPU(u16, x86_bios_cpu_apicid); @@ -149,7 +175,7 @@ void smp_store_cpu_info(int id); /* We don't mark CPUs online until __cpu_up(), so we need another measure */ static inline int num_booting_cpus(void) { - return cpus_weight(cpu_callout_map); + return cpumask_weight(cpu_callout_mask); } #else static inline void prefill_possible_map(void) diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c index 3f95a40f718..83492b1f93b 100644 --- a/arch/x86/kernel/cpu/common.c +++ b/arch/x86/kernel/cpu/common.c @@ -40,6 +40,26 @@ #include "cpu.h" +#ifdef CONFIG_X86_64 + +/* all of these masks are initialized in setup_cpu_local_masks() */ +cpumask_var_t cpu_callin_mask; +cpumask_var_t cpu_callout_mask; +cpumask_var_t cpu_initialized_mask; + +/* representing cpus for which sibling maps can be computed */ +cpumask_var_t cpu_sibling_setup_mask; + +#else /* CONFIG_X86_32 */ + +cpumask_t cpu_callin_map; +cpumask_t cpu_callout_map; +cpumask_t cpu_initialized; +cpumask_t cpu_sibling_setup_map; + +#endif /* CONFIG_X86_32 */ + + static struct cpu_dev *this_cpu __cpuinitdata; #ifdef CONFIG_X86_64 @@ -856,8 +876,6 @@ static __init int setup_disablecpuid(char *arg) } __setup("clearcpuid=", setup_disablecpuid); -cpumask_t cpu_initialized __cpuinitdata = CPU_MASK_NONE; - #ifdef CONFIG_X86_64 struct x8664_pda **_cpu_pda __read_mostly; EXPORT_SYMBOL(_cpu_pda); @@ -976,7 +994,7 @@ void __cpuinit cpu_init(void) me = current; - if (cpu_test_and_set(cpu, cpu_initialized)) + if (cpumask_test_and_set_cpu(cpu, cpu_initialized_mask)) panic("CPU#%d already initialized!\n", cpu); printk(KERN_INFO "Initializing CPU#%d\n", cpu); @@ -1085,7 +1103,7 @@ void __cpuinit cpu_init(void) struct tss_struct *t = &per_cpu(init_tss, cpu); struct thread_struct *thread = &curr->thread; - if (cpu_test_and_set(cpu, cpu_initialized)) { + if (cpumask_test_and_set_cpu(cpu, cpu_initialized_mask)) { printk(KERN_WARNING "CPU#%d already initialized!\n", cpu); for (;;) local_irq_enable(); } diff --git a/arch/x86/kernel/setup_percpu.c b/arch/x86/kernel/setup_percpu.c index a4b619c3310..aa55764602b 100644 --- a/arch/x86/kernel/setup_percpu.c +++ b/arch/x86/kernel/setup_percpu.c @@ -131,7 +131,27 @@ static void __init setup_cpu_pda_map(void) /* point to new pointer table */ _cpu_pda = new_cpu_pda; } -#endif + +#endif /* CONFIG_SMP && CONFIG_X86_64 */ + +#ifdef CONFIG_X86_64 + +/* correctly size the local cpu masks */ +static void setup_cpu_local_masks(void) +{ + alloc_bootmem_cpumask_var(&cpu_initialized_mask); + alloc_bootmem_cpumask_var(&cpu_callin_mask); + alloc_bootmem_cpumask_var(&cpu_callout_mask); + alloc_bootmem_cpumask_var(&cpu_sibling_setup_mask); +} + +#else /* CONFIG_X86_32 */ + +static inline void setup_cpu_local_masks(void) +{ +} + +#endif /* CONFIG_X86_32 */ /* * Great future plan: @@ -187,6 +207,9 @@ void __init setup_per_cpu_areas(void) /* Setup node to cpumask map */ setup_node_to_cpumask_map(); + + /* Setup cpu initialized, callin, callout masks */ + setup_cpu_local_masks(); } #endif diff --git a/arch/x86/kernel/smp.c b/arch/x86/kernel/smp.c index beea2649a24..182135ba1ea 100644 --- a/arch/x86/kernel/smp.c +++ b/arch/x86/kernel/smp.c @@ -128,16 +128,23 @@ void native_send_call_func_single_ipi(int cpu) void native_send_call_func_ipi(const struct cpumask *mask) { - cpumask_t allbutself; + cpumask_var_t allbutself; - allbutself = cpu_online_map; - cpu_clear(smp_processor_id(), allbutself); + if (!alloc_cpumask_var(&allbutself, GFP_ATOMIC)) { + send_IPI_mask(mask, CALL_FUNCTION_VECTOR); + return; + } - if (cpus_equal(*mask, allbutself) && - cpus_equal(cpu_online_map, cpu_callout_map)) + cpumask_copy(allbutself, cpu_online_mask); + cpumask_clear_cpu(smp_processor_id(), allbutself); + + if (cpumask_equal(mask, allbutself) && + cpumask_equal(cpu_online_mask, cpu_callout_mask)) send_IPI_allbutself(CALL_FUNCTION_VECTOR); else send_IPI_mask(mask, CALL_FUNCTION_VECTOR); + + free_cpumask_var(allbutself); } /* diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c index 6bd4d9b7387..00e17e58948 100644 --- a/arch/x86/kernel/smpboot.c +++ b/arch/x86/kernel/smpboot.c @@ -102,9 +102,6 @@ EXPORT_SYMBOL(smp_num_siblings); /* Last level cache ID of each logical CPU */ DEFINE_PER_CPU(u16, cpu_llc_id) = BAD_APICID; -cpumask_t cpu_callin_map; -cpumask_t cpu_callout_map; - /* representing HT siblings of each logical CPU */ DEFINE_PER_CPU(cpumask_t, cpu_sibling_map); EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); @@ -120,9 +117,6 @@ EXPORT_PER_CPU_SYMBOL(cpu_info); static atomic_t init_deasserted; -/* representing cpus for which sibling maps can be computed */ -static cpumask_t cpu_sibling_setup_map; - /* Set if we find a B stepping CPU */ static int __cpuinitdata smp_b_stepping; @@ -140,7 +134,7 @@ EXPORT_SYMBOL(cpu_to_node_map); static void map_cpu_to_node(int cpu, int node) { printk(KERN_INFO "Mapping cpu %d to node %d\n", cpu, node); - cpu_set(cpu, node_to_cpumask_map[node]); + cpumask_set_cpu(cpu, &node_to_cpumask_map[node]); cpu_to_node_map[cpu] = node; } @@ -151,7 +145,7 @@ static void unmap_cpu_to_node(int cpu) printk(KERN_INFO "Unmapping cpu %d from all nodes\n", cpu); for (node = 0; node < MAX_NUMNODES; node++) - cpu_clear(cpu, node_to_cpumask_map[node]); + cpumask_clear_cpu(cpu, &node_to_cpumask_map[node]); cpu_to_node_map[cpu] = 0; } #else /* !(CONFIG_NUMA && CONFIG_X86_32) */ @@ -209,7 +203,7 @@ static void __cpuinit smp_callin(void) */ phys_id = read_apic_id(); cpuid = smp_processor_id(); - if (cpu_isset(cpuid, cpu_callin_map)) { + if (cpumask_test_cpu(cpuid, cpu_callin_mask)) { panic("%s: phys CPU#%d, CPU#%d already present??\n", __func__, phys_id, cpuid); } @@ -231,7 +225,7 @@ static void __cpuinit smp_callin(void) /* * Has the boot CPU finished it's STARTUP sequence? */ - if (cpu_isset(cpuid, cpu_callout_map)) + if (cpumask_test_cpu(cpuid, cpu_callout_mask)) break; cpu_relax(); } @@ -274,7 +268,7 @@ static void __cpuinit smp_callin(void) /* * Allow the master to continue. */ - cpu_set(cpuid, cpu_callin_map); + cpumask_set_cpu(cpuid, cpu_callin_mask); } static int __cpuinitdata unsafe_smp; @@ -332,7 +326,7 @@ notrace static void __cpuinit start_secondary(void *unused) ipi_call_lock(); lock_vector_lock(); __setup_vector_irq(smp_processor_id()); - cpu_set(smp_processor_id(), cpu_online_map); + set_cpu_online(smp_processor_id(), true); unlock_vector_lock(); ipi_call_unlock(); per_cpu(cpu_state, smp_processor_id()) = CPU_ONLINE; @@ -438,50 +432,52 @@ void __cpuinit set_cpu_sibling_map(int cpu) int i; struct cpuinfo_x86 *c = &cpu_data(cpu); - cpu_set(cpu, cpu_sibling_setup_map); + cpumask_set_cpu(cpu, cpu_sibling_setup_mask); if (smp_num_siblings > 1) { - for_each_cpu_mask_nr(i, cpu_sibling_setup_map) { - if (c->phys_proc_id == cpu_data(i).phys_proc_id && - c->cpu_core_id == cpu_data(i).cpu_core_id) { - cpu_set(i, per_cpu(cpu_sibling_map, cpu)); - cpu_set(cpu, per_cpu(cpu_sibling_map, i)); - cpu_set(i, per_cpu(cpu_core_map, cpu)); - cpu_set(cpu, per_cpu(cpu_core_map, i)); - cpu_set(i, c->llc_shared_map); - cpu_set(cpu, cpu_data(i).llc_shared_map); + for_each_cpu(i, cpu_sibling_setup_mask) { + struct cpuinfo_x86 *o = &cpu_data(i); + + if (c->phys_proc_id == o->phys_proc_id && + c->cpu_core_id == o->cpu_core_id) { + cpumask_set_cpu(i, cpu_sibling_mask(cpu)); + cpumask_set_cpu(cpu, cpu_sibling_mask(i)); + cpumask_set_cpu(i, cpu_core_mask(cpu)); + cpumask_set_cpu(cpu, cpu_core_mask(i)); + cpumask_set_cpu(i, &c->llc_shared_map); + cpumask_set_cpu(cpu, &o->llc_shared_map); } } } else { - cpu_set(cpu, per_cpu(cpu_sibling_map, cpu)); + cpumask_set_cpu(cpu, cpu_sibling_mask(cpu)); } - cpu_set(cpu, c->llc_shared_map); + cpumask_set_cpu(cpu, &c->llc_shared_map); if (current_cpu_data.x86_max_cores == 1) { - per_cpu(cpu_core_map, cpu) = per_cpu(cpu_sibling_map, cpu); + cpumask_copy(cpu_core_mask(cpu), cpu_sibling_mask(cpu)); c->booted_cores = 1; return; } - for_each_cpu_mask_nr(i, cpu_sibling_setup_map) { + for_each_cpu(i, cpu_sibling_setup_mask) { if (per_cpu(cpu_llc_id, cpu) != BAD_APICID && per_cpu(cpu_llc_id, cpu) == per_cpu(cpu_llc_id, i)) { - cpu_set(i, c->llc_shared_map); - cpu_set(cpu, cpu_data(i).llc_shared_map); + cpumask_set_cpu(i, &c->llc_shared_map); + cpumask_set_cpu(cpu, &cpu_data(i).llc_shared_map); } if (c->phys_proc_id == cpu_data(i).phys_proc_id) { - cpu_set(i, per_cpu(cpu_core_map, cpu)); - cpu_set(cpu, per_cpu(cpu_core_map, i)); + cpumask_set_cpu(i, cpu_core_mask(cpu)); + cpumask_set_cpu(cpu, cpu_core_mask(i)); /* * Does this new cpu bringup a new core? */ - if (cpus_weight(per_cpu(cpu_sibling_map, cpu)) == 1) { + if (cpumask_weight(cpu_sibling_mask(cpu)) == 1) { /* * for each core in package, increment * the booted_cores for this new cpu */ - if (first_cpu(per_cpu(cpu_sibling_map, i)) == i) + if (cpumask_first(cpu_sibling_mask(i)) == i) c->booted_cores++; /* * increment the core count for all @@ -504,7 +500,7 @@ const struct cpumask *cpu_coregroup_mask(int cpu) * And for power savings, we return cpu_core_map */ if (sched_mc_power_savings || sched_smt_power_savings) - return &per_cpu(cpu_core_map, cpu); + return cpu_core_mask(cpu); else return &c->llc_shared_map; } @@ -523,7 +519,7 @@ static void impress_friends(void) */ pr_debug("Before bogomips.\n"); for_each_possible_cpu(cpu) - if (cpu_isset(cpu, cpu_callout_map)) + if (cpumask_test_cpu(cpu, cpu_callout_mask)) bogosum += cpu_data(cpu).loops_per_jiffy; printk(KERN_INFO "Total of %d processors activated (%lu.%02lu BogoMIPS).\n", @@ -904,19 +900,19 @@ do_rest: * allow APs to start initializing. */ pr_debug("Before Callout %d.\n", cpu); - cpu_set(cpu, cpu_callout_map); + cpumask_set_cpu(cpu, cpu_callout_mask); pr_debug("After Callout %d.\n", cpu); /* * Wait 5s total for a response */ for (timeout = 0; timeout < 50000; timeout++) { - if (cpu_isset(cpu, cpu_callin_map)) + if (cpumask_test_cpu(cpu, cpu_callin_mask)) break; /* It has booted */ udelay(100); } - if (cpu_isset(cpu, cpu_callin_map)) { + if (cpumask_test_cpu(cpu, cpu_callin_mask)) { /* number CPUs logically, starting from 1 (BSP is 0) */ pr_debug("OK.\n"); printk(KERN_INFO "CPU%d: ", cpu); @@ -941,9 +937,14 @@ restore_state: if (boot_error) { /* Try to put things back the way they were before ... */ numa_remove_cpu(cpu); /* was set by numa_add_cpu */ - cpu_clear(cpu, cpu_callout_map); /* was set by do_boot_cpu() */ - cpu_clear(cpu, cpu_initialized); /* was set by cpu_init() */ - cpu_clear(cpu, cpu_present_map); + + /* was set by do_boot_cpu() */ + cpumask_clear_cpu(cpu, cpu_callout_mask); + + /* was set by cpu_init() */ + cpumask_clear_cpu(cpu, cpu_initialized_mask); + + set_cpu_present(cpu, false); per_cpu(x86_cpu_to_apicid, cpu) = BAD_APICID; } @@ -977,7 +978,7 @@ int __cpuinit native_cpu_up(unsigned int cpu) /* * Already booted CPU? */ - if (cpu_isset(cpu, cpu_callin_map)) { + if (cpumask_test_cpu(cpu, cpu_callin_mask)) { pr_debug("do_boot_cpu %d Already started\n", cpu); return -ENOSYS; } @@ -1032,8 +1033,9 @@ int __cpuinit native_cpu_up(unsigned int cpu) */ static __init void disable_smp(void) { - cpu_present_map = cpumask_of_cpu(0); - cpu_possible_map = cpumask_of_cpu(0); + /* use the read/write pointers to the present and possible maps */ + cpumask_copy(&cpu_present_map, cpumask_of(0)); + cpumask_copy(&cpu_possible_map, cpumask_of(0)); smpboot_clear_io_apic_irqs(); if (smp_found_config) @@ -1041,8 +1043,8 @@ static __init void disable_smp(void) else physid_set_mask_of_physid(0, &phys_cpu_present_map); map_cpu_to_logical_apicid(); - cpu_set(0, per_cpu(cpu_sibling_map, 0)); - cpu_set(0, per_cpu(cpu_core_map, 0)); + cpumask_set_cpu(0, cpu_sibling_mask(0)); + cpumask_set_cpu(0, cpu_core_mask(0)); } /* @@ -1064,14 +1066,14 @@ static int __init smp_sanity_check(unsigned max_cpus) nr = 0; for_each_present_cpu(cpu) { if (nr >= 8) - cpu_clear(cpu, cpu_present_map); + set_cpu_present(cpu, false); nr++; } nr = 0; for_each_possible_cpu(cpu) { if (nr >= 8) - cpu_clear(cpu, cpu_possible_map); + set_cpu_possible(cpu, false); nr++; } @@ -1167,7 +1169,7 @@ void __init native_smp_prepare_cpus(unsigned int max_cpus) preempt_disable(); smp_cpu_index_default(); current_cpu_data = boot_cpu_data; - cpu_callin_map = cpumask_of_cpu(0); + cpumask_copy(cpu_callin_mask, cpumask_of(0)); mb(); /* * Setup boot CPU information @@ -1242,8 +1244,8 @@ void __init native_smp_prepare_boot_cpu(void) init_gdt(me); #endif switch_to_new_gdt(); - /* already set me in cpu_online_map in boot_cpu_init() */ - cpu_set(me, cpu_callout_map); + /* already set me in cpu_online_mask in boot_cpu_init() */ + cpumask_set_cpu(me, cpu_callout_mask); per_cpu(cpu_state, me) = CPU_ONLINE; } @@ -1311,7 +1313,7 @@ __init void prefill_possible_map(void) possible, max_t(int, possible - num_processors, 0)); for (i = 0; i < possible; i++) - cpu_set(i, cpu_possible_map); + set_cpu_possible(i, true); nr_cpu_ids = possible; } @@ -1323,31 +1325,31 @@ static void remove_siblinginfo(int cpu) int sibling; struct cpuinfo_x86 *c = &cpu_data(cpu); - for_each_cpu_mask_nr(sibling, per_cpu(cpu_core_map, cpu)) { - cpu_clear(cpu, per_cpu(cpu_core_map, sibling)); + for_each_cpu(sibling, cpu_core_mask(cpu)) { + cpumask_clear_cpu(cpu, cpu_core_mask(sibling)); /*/ * last thread sibling in this cpu core going down */ - if (cpus_weight(per_cpu(cpu_sibling_map, cpu)) == 1) + if (cpumask_weight(cpu_sibling_mask(cpu)) == 1) cpu_data(sibling).booted_cores--; } - for_each_cpu_mask_nr(sibling, per_cpu(cpu_sibling_map, cpu)) - cpu_clear(cpu, per_cpu(cpu_sibling_map, sibling)); - cpus_clear(per_cpu(cpu_sibling_map, cpu)); - cpus_clear(per_cpu(cpu_core_map, cpu)); + for_each_cpu(sibling, cpu_sibling_mask(cpu)) + cpumask_clear_cpu(cpu, cpu_sibling_mask(sibling)); + cpumask_clear(cpu_sibling_mask(cpu)); + cpumask_clear(cpu_core_mask(cpu)); c->phys_proc_id = 0; c->cpu_core_id = 0; - cpu_clear(cpu, cpu_sibling_setup_map); + cpumask_clear_cpu(cpu, cpu_sibling_setup_mask); } static void __ref remove_cpu_from_maps(int cpu) { - cpu_clear(cpu, cpu_online_map); - cpu_clear(cpu, cpu_callout_map); - cpu_clear(cpu, cpu_callin_map); + set_cpu_online(cpu, false); + cpumask_clear_cpu(cpu, cpu_callout_mask); + cpumask_clear_cpu(cpu, cpu_callin_mask); /* was set by cpu_init() */ - cpu_clear(cpu, cpu_initialized); + cpumask_clear_cpu(cpu, cpu_initialized_mask); numa_remove_cpu(cpu); } |