From 6b6309b4c7f6da467c5d5b7d18fa8cb79730f381 Mon Sep 17 00:00:00 2001 From: Mike Travis Date: Tue, 25 Mar 2008 15:06:56 -0700 Subject: x86: reduce memory and stack usage in intel_cacheinfo * Change the following static arrays sized by NR_CPUS to per_cpu data variables: _cpuid4_info *cpuid4_info[NR_CPUS]; _index_kobject *index_kobject[NR_CPUS]; kobject * cache_kobject[NR_CPUS]; * Remove the local NR_CPUS array with a kmalloc'd region in show_shared_cpu_map(). Also some minor complaints from checkpatch.pl fixed. Cc: H. Peter Anvin Cc: Andi Kleen Signed-off-by: Mike Travis Signed-off-by: Ingo Molnar --- arch/x86/kernel/cpu/intel_cacheinfo.c | 70 ++++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 30 deletions(-) (limited to 'arch/x86') diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c index 1b889860eb7..2e8b323b34e 100644 --- a/arch/x86/kernel/cpu/intel_cacheinfo.c +++ b/arch/x86/kernel/cpu/intel_cacheinfo.c @@ -129,7 +129,7 @@ struct _cpuid4_info { union _cpuid4_leaf_ebx ebx; union _cpuid4_leaf_ecx ecx; unsigned long size; - cpumask_t shared_cpu_map; + cpumask_t shared_cpu_map; /* future?: only cpus/node is needed */ }; unsigned short num_cache_leaves; @@ -451,8 +451,8 @@ unsigned int __cpuinit init_intel_cacheinfo(struct cpuinfo_x86 *c) } /* pointer to _cpuid4_info array (for each cache leaf) */ -static struct _cpuid4_info *cpuid4_info[NR_CPUS]; -#define CPUID4_INFO_IDX(x,y) (&((cpuid4_info[x])[y])) +static DEFINE_PER_CPU(struct _cpuid4_info *, cpuid4_info); +#define CPUID4_INFO_IDX(x, y) (&((per_cpu(cpuid4_info, x))[y])) #ifdef CONFIG_SMP static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index) @@ -474,7 +474,7 @@ static void __cpuinit cache_shared_cpu_map_setup(unsigned int cpu, int index) if (cpu_data(i).apicid >> index_msb == c->apicid >> index_msb) { cpu_set(i, this_leaf->shared_cpu_map); - if (i != cpu && cpuid4_info[i]) { + if (i != cpu && per_cpu(cpuid4_info, i)) { sibling_leaf = CPUID4_INFO_IDX(i, index); cpu_set(cpu, sibling_leaf->shared_cpu_map); } @@ -505,8 +505,8 @@ static void __cpuinit free_cache_attributes(unsigned int cpu) for (i = 0; i < num_cache_leaves; i++) cache_remove_shared_cpu_map(cpu, i); - kfree(cpuid4_info[cpu]); - cpuid4_info[cpu] = NULL; + kfree(per_cpu(cpuid4_info, cpu)); + per_cpu(cpuid4_info, cpu) = NULL; } static int __cpuinit detect_cache_attributes(unsigned int cpu) @@ -519,9 +519,9 @@ static int __cpuinit detect_cache_attributes(unsigned int cpu) if (num_cache_leaves == 0) return -ENOENT; - cpuid4_info[cpu] = kzalloc( + per_cpu(cpuid4_info, cpu) = kzalloc( sizeof(struct _cpuid4_info) * num_cache_leaves, GFP_KERNEL); - if (cpuid4_info[cpu] == NULL) + if (per_cpu(cpuid4_info, cpu) == NULL) return -ENOMEM; oldmask = current->cpus_allowed; @@ -546,8 +546,8 @@ static int __cpuinit detect_cache_attributes(unsigned int cpu) out: if (retval) { - kfree(cpuid4_info[cpu]); - cpuid4_info[cpu] = NULL; + kfree(per_cpu(cpuid4_info, cpu)); + per_cpu(cpuid4_info, cpu) = NULL; } return retval; @@ -561,7 +561,7 @@ out: extern struct sysdev_class cpu_sysdev_class; /* from drivers/base/cpu.c */ /* pointer to kobject for cpuX/cache */ -static struct kobject * cache_kobject[NR_CPUS]; +static DEFINE_PER_CPU(struct kobject *, cache_kobject); struct _index_kobject { struct kobject kobj; @@ -570,8 +570,8 @@ struct _index_kobject { }; /* pointer to array of kobjects for cpuX/cache/indexY */ -static struct _index_kobject *index_kobject[NR_CPUS]; -#define INDEX_KOBJECT_PTR(x,y) (&((index_kobject[x])[y])) +static DEFINE_PER_CPU(struct _index_kobject *, index_kobject); +#define INDEX_KOBJECT_PTR(x, y) (&((per_cpu(index_kobject, x))[y])) #define show_one_plus(file_name, object, val) \ static ssize_t show_##file_name \ @@ -593,9 +593,16 @@ static ssize_t show_size(struct _cpuid4_info *this_leaf, char *buf) static ssize_t show_shared_cpu_map(struct _cpuid4_info *this_leaf, char *buf) { - char mask_str[NR_CPUS]; - cpumask_scnprintf(mask_str, NR_CPUS, this_leaf->shared_cpu_map); - return sprintf(buf, "%s\n", mask_str); + int n = 0; + int len = cpumask_scnprintf_len(nr_cpu_ids); + char *mask_str = kmalloc(len, GFP_KERNEL); + + if (mask_str) { + cpumask_scnprintf(mask_str, len, this_leaf->shared_cpu_map); + n = sprintf(buf, "%s\n", mask_str); + kfree(mask_str); + } + return n; } static ssize_t show_type(struct _cpuid4_info *this_leaf, char *buf) { @@ -684,10 +691,10 @@ static struct kobj_type ktype_percpu_entry = { static void __cpuinit cpuid4_cache_sysfs_exit(unsigned int cpu) { - kfree(cache_kobject[cpu]); - kfree(index_kobject[cpu]); - cache_kobject[cpu] = NULL; - index_kobject[cpu] = NULL; + kfree(per_cpu(cache_kobject, cpu)); + kfree(per_cpu(index_kobject, cpu)); + per_cpu(cache_kobject, cpu) = NULL; + per_cpu(index_kobject, cpu) = NULL; free_cache_attributes(cpu); } @@ -703,13 +710,14 @@ static int __cpuinit cpuid4_cache_sysfs_init(unsigned int cpu) return err; /* Allocate all required memory */ - cache_kobject[cpu] = kzalloc(sizeof(struct kobject), GFP_KERNEL); - if (unlikely(cache_kobject[cpu] == NULL)) + per_cpu(cache_kobject, cpu) = + kzalloc(sizeof(struct kobject), GFP_KERNEL); + if (unlikely(per_cpu(cache_kobject, cpu) == NULL)) goto err_out; - index_kobject[cpu] = kzalloc( + per_cpu(index_kobject, cpu) = kzalloc( sizeof(struct _index_kobject ) * num_cache_leaves, GFP_KERNEL); - if (unlikely(index_kobject[cpu] == NULL)) + if (unlikely(per_cpu(index_kobject, cpu) == NULL)) goto err_out; return 0; @@ -733,7 +741,8 @@ static int __cpuinit cache_add_dev(struct sys_device * sys_dev) if (unlikely(retval < 0)) return retval; - retval = kobject_init_and_add(cache_kobject[cpu], &ktype_percpu_entry, + retval = kobject_init_and_add(per_cpu(cache_kobject, cpu), + &ktype_percpu_entry, &sys_dev->kobj, "%s", "cache"); if (retval < 0) { cpuid4_cache_sysfs_exit(cpu); @@ -745,13 +754,14 @@ static int __cpuinit cache_add_dev(struct sys_device * sys_dev) this_object->cpu = cpu; this_object->index = i; retval = kobject_init_and_add(&(this_object->kobj), - &ktype_cache, cache_kobject[cpu], + &ktype_cache, + per_cpu(cache_kobject, cpu), "index%1lu", i); if (unlikely(retval)) { for (j = 0; j < i; j++) { kobject_put(&(INDEX_KOBJECT_PTR(cpu,j)->kobj)); } - kobject_put(cache_kobject[cpu]); + kobject_put(per_cpu(cache_kobject, cpu)); cpuid4_cache_sysfs_exit(cpu); break; } @@ -760,7 +770,7 @@ static int __cpuinit cache_add_dev(struct sys_device * sys_dev) if (!retval) cpu_set(cpu, cache_dev_map); - kobject_uevent(cache_kobject[cpu], KOBJ_ADD); + kobject_uevent(per_cpu(cache_kobject, cpu), KOBJ_ADD); return retval; } @@ -769,7 +779,7 @@ static void __cpuinit cache_remove_dev(struct sys_device * sys_dev) unsigned int cpu = sys_dev->id; unsigned long i; - if (cpuid4_info[cpu] == NULL) + if (per_cpu(cpuid4_info, cpu) == NULL) return; if (!cpu_isset(cpu, cache_dev_map)) return; @@ -777,7 +787,7 @@ static void __cpuinit cache_remove_dev(struct sys_device * sys_dev) for (i = 0; i < num_cache_leaves; i++) kobject_put(&(INDEX_KOBJECT_PTR(cpu,i)->kobj)); - kobject_put(cache_kobject[cpu]); + kobject_put(per_cpu(cache_kobject, cpu)); cpuid4_cache_sysfs_exit(cpu); } -- cgit v1.2.3-70-g09d2 From d18d00f5dbcd1a95811617e9812cf0560bd465ee Mon Sep 17 00:00:00 2001 From: Mike Travis Date: Tue, 25 Mar 2008 15:06:59 -0700 Subject: x86: oprofile: remove NR_CPUS arrays in arch/x86/oprofile/nmi_int.c Change the following arrays sized by NR_CPUS to be PERCPU variables: static struct op_msrs cpu_msrs[NR_CPUS]; static unsigned long saved_lvtpc[NR_CPUS]; Also some minor complaints from checkpatch.pl fixed. Based on: git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux-2.6.git git://git.kernel.org/pub/scm/linux/kernel/git/x86/linux-2.6-x86.git All changes were transparent except for: static void nmi_shutdown(void) { + struct op_msrs *msrs = &__get_cpu_var(cpu_msrs); nmi_enabled = 0; on_each_cpu(nmi_cpu_shutdown, NULL, 0, 1); unregister_die_notifier(&profile_exceptions_nb); - model->shutdown(cpu_msrs); + model->shutdown(msrs); free_msrs(); } The existing code passed a reference to cpu 0's instance of struct op_msrs to model->shutdown, whilst the other functions are passed a reference to instance of a struct op_msrs. This seemed to be a bug to me even though as long as cpu 0 and are of the same type it would have the same effect...? Cc: Philippe Elie Signed-off-by: Mike Travis Signed-off-by: Ingo Molnar --- arch/x86/oprofile/nmi_int.c | 49 +++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 22 deletions(-) (limited to 'arch/x86') diff --git a/arch/x86/oprofile/nmi_int.c b/arch/x86/oprofile/nmi_int.c index 1f11cf0a307..cc48d3fde54 100644 --- a/arch/x86/oprofile/nmi_int.c +++ b/arch/x86/oprofile/nmi_int.c @@ -23,8 +23,8 @@ #include "op_x86_model.h" static struct op_x86_model_spec const *model; -static struct op_msrs cpu_msrs[NR_CPUS]; -static unsigned long saved_lvtpc[NR_CPUS]; +static DEFINE_PER_CPU(struct op_msrs, cpu_msrs); +static DEFINE_PER_CPU(unsigned long, saved_lvtpc); static int nmi_start(void); static void nmi_stop(void); @@ -89,7 +89,7 @@ static int profile_exceptions_notify(struct notifier_block *self, switch (val) { case DIE_NMI: - if (model->check_ctrs(args->regs, &cpu_msrs[cpu])) + if (model->check_ctrs(args->regs, &per_cpu(cpu_msrs, cpu))) ret = NOTIFY_STOP; break; default: @@ -126,7 +126,7 @@ static void nmi_cpu_save_registers(struct op_msrs *msrs) static void nmi_save_registers(void *dummy) { int cpu = smp_processor_id(); - struct op_msrs *msrs = &cpu_msrs[cpu]; + struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu); nmi_cpu_save_registers(msrs); } @@ -134,10 +134,10 @@ static void free_msrs(void) { int i; for_each_possible_cpu(i) { - kfree(cpu_msrs[i].counters); - cpu_msrs[i].counters = NULL; - kfree(cpu_msrs[i].controls); - cpu_msrs[i].controls = NULL; + kfree(per_cpu(cpu_msrs, i).counters); + per_cpu(cpu_msrs, i).counters = NULL; + kfree(per_cpu(cpu_msrs, i).controls); + per_cpu(cpu_msrs, i).controls = NULL; } } @@ -149,13 +149,15 @@ static int allocate_msrs(void) int i; for_each_possible_cpu(i) { - cpu_msrs[i].counters = kmalloc(counters_size, GFP_KERNEL); - if (!cpu_msrs[i].counters) { + per_cpu(cpu_msrs, i).counters = kmalloc(counters_size, + GFP_KERNEL); + if (!per_cpu(cpu_msrs, i).counters) { success = 0; break; } - cpu_msrs[i].controls = kmalloc(controls_size, GFP_KERNEL); - if (!cpu_msrs[i].controls) { + per_cpu(cpu_msrs, i).controls = kmalloc(controls_size, + GFP_KERNEL); + if (!per_cpu(cpu_msrs, i).controls) { success = 0; break; } @@ -170,11 +172,11 @@ static int allocate_msrs(void) static void nmi_cpu_setup(void *dummy) { int cpu = smp_processor_id(); - struct op_msrs *msrs = &cpu_msrs[cpu]; + struct op_msrs *msrs = &per_cpu(cpu_msrs, cpu); spin_lock(&oprofilefs_lock); model->setup_ctrs(msrs); spin_unlock(&oprofilefs_lock); - saved_lvtpc[cpu] = apic_read(APIC_LVTPC); + per_cpu(saved_lvtpc, cpu) = apic_read(APIC_LVTPC); apic_write(APIC_LVTPC, APIC_DM_NMI); } @@ -203,13 +205,15 @@ static int nmi_setup(void) */ /* Assume saved/restored counters are the same on all CPUs */ - model->fill_in_addresses(&cpu_msrs[0]); + model->fill_in_addresses(&per_cpu(cpu_msrs, 0)); for_each_possible_cpu(cpu) { if (cpu != 0) { - memcpy(cpu_msrs[cpu].counters, cpu_msrs[0].counters, + memcpy(per_cpu(cpu_msrs, cpu).counters, + per_cpu(cpu_msrs, 0).counters, sizeof(struct op_msr) * model->num_counters); - memcpy(cpu_msrs[cpu].controls, cpu_msrs[0].controls, + memcpy(per_cpu(cpu_msrs, cpu).controls, + per_cpu(cpu_msrs, 0).controls, sizeof(struct op_msr) * model->num_controls); } @@ -249,7 +253,7 @@ static void nmi_cpu_shutdown(void *dummy) { unsigned int v; int cpu = smp_processor_id(); - struct op_msrs *msrs = &cpu_msrs[cpu]; + struct op_msrs *msrs = &__get_cpu_var(cpu_msrs); /* restoring APIC_LVTPC can trigger an apic error because the delivery * mode and vector nr combination can be illegal. That's by design: on @@ -258,23 +262,24 @@ static void nmi_cpu_shutdown(void *dummy) */ v = apic_read(APIC_LVTERR); apic_write(APIC_LVTERR, v | APIC_LVT_MASKED); - apic_write(APIC_LVTPC, saved_lvtpc[cpu]); + apic_write(APIC_LVTPC, per_cpu(saved_lvtpc, cpu)); apic_write(APIC_LVTERR, v); nmi_restore_registers(msrs); } static void nmi_shutdown(void) { + struct op_msrs *msrs = &__get_cpu_var(cpu_msrs); nmi_enabled = 0; on_each_cpu(nmi_cpu_shutdown, NULL, 0, 1); unregister_die_notifier(&profile_exceptions_nb); - model->shutdown(cpu_msrs); + model->shutdown(msrs); free_msrs(); } static void nmi_cpu_start(void *dummy) { - struct op_msrs const *msrs = &cpu_msrs[smp_processor_id()]; + struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs); model->start(msrs); } @@ -286,7 +291,7 @@ static int nmi_start(void) static void nmi_cpu_stop(void *dummy) { - struct op_msrs const *msrs = &cpu_msrs[smp_processor_id()]; + struct op_msrs const *msrs = &__get_cpu_var(cpu_msrs); model->stop(msrs); } -- cgit v1.2.3-70-g09d2 From f46bdf2db25dfaff3b611c9711705645cdb03acc Mon Sep 17 00:00:00 2001 From: Mike Travis Date: Fri, 4 Apr 2008 18:11:09 -0700 Subject: numa: move large array from stack to _initdata section * Move large array "struct bootnode nodes" from stack to _initdata section to reduce amount of stack space required. Cc: H. Peter Anvin Signed-off-by: Mike Travis Signed-off-by: Ingo Molnar --- arch/x86/mm/numa_64.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'arch/x86') diff --git a/arch/x86/mm/numa_64.c b/arch/x86/mm/numa_64.c index 2ea56f48f29..3f6c53c0e00 100644 --- a/arch/x86/mm/numa_64.c +++ b/arch/x86/mm/numa_64.c @@ -380,9 +380,10 @@ static int __init split_nodes_by_size(struct bootnode *nodes, u64 *addr, * Sets up the system RAM area from start_pfn to end_pfn according to the * numa=fake command-line option. */ +static struct bootnode nodes[MAX_NUMNODES] __initdata; + static int __init numa_emulation(unsigned long start_pfn, unsigned long end_pfn) { - struct bootnode nodes[MAX_NUMNODES]; u64 size, addr = start_pfn << PAGE_SHIFT; u64 max_addr = end_pfn << PAGE_SHIFT; int num_nodes = 0, num = 0, coeff_flag, coeff = -1, i; -- cgit v1.2.3-70-g09d2 From d366f8cbc16882e93538d9a52423c2f50dad7c06 Mon Sep 17 00:00:00 2001 From: Mike Travis Date: Fri, 4 Apr 2008 18:11:12 -0700 Subject: cpumask: Cleanup more uses of CPU_MASK and NODE_MASK * Replace usages of CPU_MASK_NONE, CPU_MASK_ALL, NODE_MASK_NONE, NODE_MASK_ALL to reduce stack requirements for large NR_CPUS and MAXNODES counts. * In some cases, the cpumask variable was initialized but then overwritten with another value. This is the case for changes like this: - cpumask_t oldmask = CPU_MASK_ALL; + cpumask_t oldmask; Signed-off-by: Mike Travis Signed-off-by: Ingo Molnar --- arch/x86/kernel/io_apic_64.c | 2 +- kernel/irq/chip.c | 2 +- mm/allocpercpu.c | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) (limited to 'arch/x86') diff --git a/arch/x86/kernel/io_apic_64.c b/arch/x86/kernel/io_apic_64.c index b54464b2665..9ba11d07920 100644 --- a/arch/x86/kernel/io_apic_64.c +++ b/arch/x86/kernel/io_apic_64.c @@ -785,7 +785,7 @@ static void __clear_irq_vector(int irq) per_cpu(vector_irq, cpu)[vector] = -1; cfg->vector = 0; - cfg->domain = CPU_MASK_NONE; + cpus_clear(cfg->domain); } void __setup_vector_irq(int cpu) diff --git a/kernel/irq/chip.c b/kernel/irq/chip.c index fdb3fbe2b0c..964964baefa 100644 --- a/kernel/irq/chip.c +++ b/kernel/irq/chip.c @@ -47,7 +47,7 @@ void dynamic_irq_init(unsigned int irq) desc->irq_count = 0; desc->irqs_unhandled = 0; #ifdef CONFIG_SMP - desc->affinity = CPU_MASK_ALL; + cpus_setall(desc->affinity); #endif spin_unlock_irqrestore(&desc->lock, flags); } diff --git a/mm/allocpercpu.c b/mm/allocpercpu.c index b0012e27fea..f4026bae6ee 100644 --- a/mm/allocpercpu.c +++ b/mm/allocpercpu.c @@ -82,9 +82,10 @@ EXPORT_SYMBOL_GPL(percpu_populate); int __percpu_populate_mask(void *__pdata, size_t size, gfp_t gfp, cpumask_t *mask) { - cpumask_t populated = CPU_MASK_NONE; + cpumask_t populated; int cpu; + cpus_clear(populated); for_each_cpu_mask(cpu, *mask) if (unlikely(!percpu_populate(__pdata, size, gfp, cpu))) { __percpu_depopulate_mask(__pdata, &populated); -- cgit v1.2.3-70-g09d2 From fc0e474840d1fd96f28fbd76d4f36b80e7ad1cc3 Mon Sep 17 00:00:00 2001 From: Mike Travis Date: Fri, 4 Apr 2008 18:11:05 -0700 Subject: x86: use new set_cpus_allowed_ptr function * Use new set_cpus_allowed_ptr() function added by previous patch, which instead of passing the "newly allowed cpus" cpumask_t arg by value, pass it by pointer: -int set_cpus_allowed(struct task_struct *p, cpumask_t new_mask) +int set_cpus_allowed_ptr(struct task_struct *p, const cpumask_t *new_mask) * Cleanup uses of CPU_MASK_ALL. * Collapse other NR_CPUS changes to arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c Use pointers to cpumask_t arguments whenever possible. Depends on: [sched-devel]: sched: add new set_cpus_allowed_ptr function Cc: Len Brown Cc: Dave Jones Signed-off-by: Mike Travis Signed-off-by: Ingo Molnar --- arch/x86/kernel/acpi/cstate.c | 4 +-- arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c | 28 ++++++++++----------- arch/x86/kernel/cpu/cpufreq/powernow-k8.c | 32 +++++++++++++----------- arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c | 13 +++++----- arch/x86/kernel/cpu/cpufreq/speedstep-ich.c | 20 +++++++-------- arch/x86/kernel/cpu/intel_cacheinfo.c | 4 +-- arch/x86/kernel/microcode.c | 16 ++++++------ arch/x86/kernel/reboot.c | 2 +- 8 files changed, 61 insertions(+), 58 deletions(-) (limited to 'arch/x86') diff --git a/arch/x86/kernel/acpi/cstate.c b/arch/x86/kernel/acpi/cstate.c index 8ca3557a6d5..c6dc05af882 100644 --- a/arch/x86/kernel/acpi/cstate.c +++ b/arch/x86/kernel/acpi/cstate.c @@ -93,7 +93,7 @@ int acpi_processor_ffh_cstate_probe(unsigned int cpu, /* Make sure we are running on right CPU */ saved_mask = current->cpus_allowed; - retval = set_cpus_allowed(current, cpumask_of_cpu(cpu)); + retval = set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu)); if (retval) return -1; @@ -130,7 +130,7 @@ int acpi_processor_ffh_cstate_probe(unsigned int cpu, cx->address); out: - set_cpus_allowed(current, saved_mask); + set_cpus_allowed_ptr(current, &saved_mask); return retval; } EXPORT_SYMBOL_GPL(acpi_processor_ffh_cstate_probe); diff --git a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c index a962dcb9c40..e2d870de837 100644 --- a/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c +++ b/arch/x86/kernel/cpu/cpufreq/acpi-cpufreq.c @@ -192,9 +192,9 @@ static void drv_read(struct drv_cmd *cmd) cpumask_t saved_mask = current->cpus_allowed; cmd->val = 0; - set_cpus_allowed(current, cmd->mask); + set_cpus_allowed_ptr(current, &cmd->mask); do_drv_read(cmd); - set_cpus_allowed(current, saved_mask); + set_cpus_allowed_ptr(current, &saved_mask); } static void drv_write(struct drv_cmd *cmd) @@ -203,30 +203,30 @@ static void drv_write(struct drv_cmd *cmd) unsigned int i; for_each_cpu_mask(i, cmd->mask) { - set_cpus_allowed(current, cpumask_of_cpu(i)); + set_cpus_allowed_ptr(current, &cpumask_of_cpu(i)); do_drv_write(cmd); } - set_cpus_allowed(current, saved_mask); + set_cpus_allowed_ptr(current, &saved_mask); return; } -static u32 get_cur_val(cpumask_t mask) +static u32 get_cur_val(const cpumask_t *mask) { struct acpi_processor_performance *perf; struct drv_cmd cmd; - if (unlikely(cpus_empty(mask))) + if (unlikely(cpus_empty(*mask))) return 0; - switch (per_cpu(drv_data, first_cpu(mask))->cpu_feature) { + switch (per_cpu(drv_data, first_cpu(*mask))->cpu_feature) { case SYSTEM_INTEL_MSR_CAPABLE: cmd.type = SYSTEM_INTEL_MSR_CAPABLE; cmd.addr.msr.reg = MSR_IA32_PERF_STATUS; break; case SYSTEM_IO_CAPABLE: cmd.type = SYSTEM_IO_CAPABLE; - perf = per_cpu(drv_data, first_cpu(mask))->acpi_data; + perf = per_cpu(drv_data, first_cpu(*mask))->acpi_data; cmd.addr.io.port = perf->control_register.address; cmd.addr.io.bit_width = perf->control_register.bit_width; break; @@ -234,7 +234,7 @@ static u32 get_cur_val(cpumask_t mask) return 0; } - cmd.mask = mask; + cmd.mask = *mask; drv_read(&cmd); @@ -271,7 +271,7 @@ static unsigned int get_measured_perf(unsigned int cpu) unsigned int retval; saved_mask = current->cpus_allowed; - set_cpus_allowed(current, cpumask_of_cpu(cpu)); + set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu)); if (get_cpu() != cpu) { /* We were not able to run on requested processor */ put_cpu(); @@ -329,7 +329,7 @@ static unsigned int get_measured_perf(unsigned int cpu) retval = per_cpu(drv_data, cpu)->max_freq * perf_percent / 100; put_cpu(); - set_cpus_allowed(current, saved_mask); + set_cpus_allowed_ptr(current, &saved_mask); dprintk("cpu %d: performance percent %d\n", cpu, perf_percent); return retval; @@ -347,13 +347,13 @@ static unsigned int get_cur_freq_on_cpu(unsigned int cpu) return 0; } - freq = extract_freq(get_cur_val(cpumask_of_cpu(cpu)), data); + freq = extract_freq(get_cur_val(&cpumask_of_cpu(cpu)), data); dprintk("cur freq = %u\n", freq); return freq; } -static unsigned int check_freqs(cpumask_t mask, unsigned int freq, +static unsigned int check_freqs(const cpumask_t *mask, unsigned int freq, struct acpi_cpufreq_data *data) { unsigned int cur_freq; @@ -449,7 +449,7 @@ static int acpi_cpufreq_target(struct cpufreq_policy *policy, drv_write(&cmd); if (acpi_pstate_strict) { - if (!check_freqs(cmd.mask, freqs.new, data)) { + if (!check_freqs(&cmd.mask, freqs.new, data)) { dprintk("acpi_cpufreq_target failed (%d)\n", policy->cpu); return -EAGAIN; diff --git a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c index c99d59d8ef2..46d4034d9f3 100644 --- a/arch/x86/kernel/cpu/cpufreq/powernow-k8.c +++ b/arch/x86/kernel/cpu/cpufreq/powernow-k8.c @@ -478,12 +478,12 @@ static int core_voltage_post_transition(struct powernow_k8_data *data, u32 reqvi static int check_supported_cpu(unsigned int cpu) { - cpumask_t oldmask = CPU_MASK_ALL; + cpumask_t oldmask; u32 eax, ebx, ecx, edx; unsigned int rc = 0; oldmask = current->cpus_allowed; - set_cpus_allowed(current, cpumask_of_cpu(cpu)); + set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu)); if (smp_processor_id() != cpu) { printk(KERN_ERR PFX "limiting to cpu %u failed\n", cpu); @@ -528,7 +528,7 @@ static int check_supported_cpu(unsigned int cpu) rc = 1; out: - set_cpus_allowed(current, oldmask); + set_cpus_allowed_ptr(current, &oldmask); return rc; } @@ -1015,7 +1015,7 @@ static int transition_frequency_pstate(struct powernow_k8_data *data, unsigned i /* Driver entry point to switch to the target frequency */ static int powernowk8_target(struct cpufreq_policy *pol, unsigned targfreq, unsigned relation) { - cpumask_t oldmask = CPU_MASK_ALL; + cpumask_t oldmask; struct powernow_k8_data *data = per_cpu(powernow_data, pol->cpu); u32 checkfid; u32 checkvid; @@ -1030,7 +1030,7 @@ static int powernowk8_target(struct cpufreq_policy *pol, unsigned targfreq, unsi /* only run on specific CPU from here on */ oldmask = current->cpus_allowed; - set_cpus_allowed(current, cpumask_of_cpu(pol->cpu)); + set_cpus_allowed_ptr(current, &cpumask_of_cpu(pol->cpu)); if (smp_processor_id() != pol->cpu) { printk(KERN_ERR PFX "limiting to cpu %u failed\n", pol->cpu); @@ -1085,7 +1085,7 @@ static int powernowk8_target(struct cpufreq_policy *pol, unsigned targfreq, unsi ret = 0; err_out: - set_cpus_allowed(current, oldmask); + set_cpus_allowed_ptr(current, &oldmask); return ret; } @@ -1104,7 +1104,7 @@ static int powernowk8_verify(struct cpufreq_policy *pol) static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol) { struct powernow_k8_data *data; - cpumask_t oldmask = CPU_MASK_ALL; + cpumask_t oldmask; int rc; if (!cpu_online(pol->cpu)) @@ -1145,7 +1145,7 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol) /* only run on specific CPU from here on */ oldmask = current->cpus_allowed; - set_cpus_allowed(current, cpumask_of_cpu(pol->cpu)); + set_cpus_allowed_ptr(current, &cpumask_of_cpu(pol->cpu)); if (smp_processor_id() != pol->cpu) { printk(KERN_ERR PFX "limiting to cpu %u failed\n", pol->cpu); @@ -1164,7 +1164,7 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol) fidvid_msr_init(); /* run on any CPU again */ - set_cpus_allowed(current, oldmask); + set_cpus_allowed_ptr(current, &oldmask); if (cpu_family == CPU_HW_PSTATE) pol->cpus = cpumask_of_cpu(pol->cpu); @@ -1205,7 +1205,7 @@ static int __cpuinit powernowk8_cpu_init(struct cpufreq_policy *pol) return 0; err_out: - set_cpus_allowed(current, oldmask); + set_cpus_allowed_ptr(current, &oldmask); powernow_k8_cpu_exit_acpi(data); kfree(data); @@ -1242,10 +1242,11 @@ static unsigned int powernowk8_get (unsigned int cpu) if (!data) return -EINVAL; - set_cpus_allowed(current, cpumask_of_cpu(cpu)); + set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu)); if (smp_processor_id() != cpu) { - printk(KERN_ERR PFX "limiting to CPU %d failed in powernowk8_get\n", cpu); - set_cpus_allowed(current, oldmask); + printk(KERN_ERR PFX + "limiting to CPU %d failed in powernowk8_get\n", cpu); + set_cpus_allowed_ptr(current, &oldmask); return 0; } @@ -1253,13 +1254,14 @@ static unsigned int powernowk8_get (unsigned int cpu) goto out; if (cpu_family == CPU_HW_PSTATE) - khz = find_khz_freq_from_pstate(data->powernow_table, data->currpstate); + khz = find_khz_freq_from_pstate(data->powernow_table, + data->currpstate); else khz = find_khz_freq_from_fid(data->currfid); out: - set_cpus_allowed(current, oldmask); + set_cpus_allowed_ptr(current, &oldmask); return khz; } diff --git a/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c b/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c index 3031f119619..908dd347c67 100644 --- a/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c +++ b/arch/x86/kernel/cpu/cpufreq/speedstep-centrino.c @@ -315,7 +315,7 @@ static unsigned int get_cur_freq(unsigned int cpu) cpumask_t saved_mask; saved_mask = current->cpus_allowed; - set_cpus_allowed(current, cpumask_of_cpu(cpu)); + set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu)); if (smp_processor_id() != cpu) return 0; @@ -333,7 +333,7 @@ static unsigned int get_cur_freq(unsigned int cpu) clock_freq = extract_clock(l, cpu, 1); } - set_cpus_allowed(current, saved_mask); + set_cpus_allowed_ptr(current, &saved_mask); return clock_freq; } @@ -487,7 +487,7 @@ static int centrino_target (struct cpufreq_policy *policy, else cpu_set(j, set_mask); - set_cpus_allowed(current, set_mask); + set_cpus_allowed_ptr(current, &set_mask); preempt_disable(); if (unlikely(!cpu_isset(smp_processor_id(), set_mask))) { dprintk("couldn't limit to CPUs in this domain\n"); @@ -555,7 +555,8 @@ static int centrino_target (struct cpufreq_policy *policy, if (!cpus_empty(covered_cpus)) { for_each_cpu_mask(j, covered_cpus) { - set_cpus_allowed(current, cpumask_of_cpu(j)); + set_cpus_allowed_ptr(current, + &cpumask_of_cpu(j)); wrmsr(MSR_IA32_PERF_CTL, oldmsr, h); } } @@ -569,12 +570,12 @@ static int centrino_target (struct cpufreq_policy *policy, cpufreq_notify_transition(&freqs, CPUFREQ_POSTCHANGE); } } - set_cpus_allowed(current, saved_mask); + set_cpus_allowed_ptr(current, &saved_mask); return 0; migrate_end: preempt_enable(); - set_cpus_allowed(current, saved_mask); + set_cpus_allowed_ptr(current, &saved_mask); return 0; } diff --git a/arch/x86/kernel/cpu/cpufreq/speedstep-ich.c b/arch/x86/kernel/cpu/cpufreq/speedstep-ich.c index 14d68aa301e..1b50244b1fd 100644 --- a/arch/x86/kernel/cpu/cpufreq/speedstep-ich.c +++ b/arch/x86/kernel/cpu/cpufreq/speedstep-ich.c @@ -229,22 +229,22 @@ static unsigned int speedstep_detect_chipset (void) return 0; } -static unsigned int _speedstep_get(cpumask_t cpus) +static unsigned int _speedstep_get(const cpumask_t *cpus) { unsigned int speed; cpumask_t cpus_allowed; cpus_allowed = current->cpus_allowed; - set_cpus_allowed(current, cpus); + set_cpus_allowed_ptr(current, cpus); speed = speedstep_get_processor_frequency(speedstep_processor); - set_cpus_allowed(current, cpus_allowed); + set_cpus_allowed_ptr(current, &cpus_allowed); dprintk("detected %u kHz as current frequency\n", speed); return speed; } static unsigned int speedstep_get(unsigned int cpu) { - return _speedstep_get(cpumask_of_cpu(cpu)); + return _speedstep_get(&cpumask_of_cpu(cpu)); } /** @@ -267,7 +267,7 @@ static int speedstep_target (struct cpufreq_policy *policy, if (cpufreq_frequency_table_target(policy, &speedstep_freqs[0], target_freq, relation, &newstate)) return -EINVAL; - freqs.old = _speedstep_get(policy->cpus); + freqs.old = _speedstep_get(&policy->cpus); freqs.new = speedstep_freqs[newstate].frequency; freqs.cpu = policy->cpu; @@ -285,12 +285,12 @@ static int speedstep_target (struct cpufreq_policy *policy, } /* switch to physical CPU where state is to be changed */ - set_cpus_allowed(current, policy->cpus); + set_cpus_allowed_ptr(current, &policy->cpus); speedstep_set_state(newstate); /* allow to be run on all CPUs */ - set_cpus_allowed(current, cpus_allowed); + set_cpus_allowed_ptr(current, &cpus_allowed); for_each_cpu_mask(i, policy->cpus) { freqs.cpu = i; @@ -326,7 +326,7 @@ static int speedstep_cpu_init(struct cpufreq_policy *policy) #endif cpus_allowed = current->cpus_allowed; - set_cpus_allowed(current, policy->cpus); + set_cpus_allowed_ptr(current, &policy->cpus); /* detect low and high frequency and transition latency */ result = speedstep_get_freqs(speedstep_processor, @@ -334,12 +334,12 @@ static int speedstep_cpu_init(struct cpufreq_policy *policy) &speedstep_freqs[SPEEDSTEP_HIGH].frequency, &policy->cpuinfo.transition_latency, &speedstep_set_state); - set_cpus_allowed(current, cpus_allowed); + set_cpus_allowed_ptr(current, &cpus_allowed); if (result) return result; /* get current speed setting */ - speed = _speedstep_get(policy->cpus); + speed = _speedstep_get(&policy->cpus); if (!speed) return -EIO; diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c index 2e8b323b34e..e073a93ceb4 100644 --- a/arch/x86/kernel/cpu/intel_cacheinfo.c +++ b/arch/x86/kernel/cpu/intel_cacheinfo.c @@ -525,7 +525,7 @@ static int __cpuinit detect_cache_attributes(unsigned int cpu) return -ENOMEM; oldmask = current->cpus_allowed; - retval = set_cpus_allowed(current, cpumask_of_cpu(cpu)); + retval = set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu)); if (retval) goto out; @@ -542,7 +542,7 @@ static int __cpuinit detect_cache_attributes(unsigned int cpu) } cache_shared_cpu_map_setup(cpu, j); } - set_cpus_allowed(current, oldmask); + set_cpus_allowed_ptr(current, &oldmask); out: if (retval) { diff --git a/arch/x86/kernel/microcode.c b/arch/x86/kernel/microcode.c index 25cf6dee4e5..69729e38b78 100644 --- a/arch/x86/kernel/microcode.c +++ b/arch/x86/kernel/microcode.c @@ -402,7 +402,7 @@ static int do_microcode_update (void) if (!uci->valid) continue; - set_cpus_allowed(current, cpumask_of_cpu(cpu)); + set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu)); error = get_maching_microcode(new_mc, cpu); if (error < 0) goto out; @@ -416,7 +416,7 @@ out: vfree(new_mc); if (cursor < 0) error = cursor; - set_cpus_allowed(current, old); + set_cpus_allowed_ptr(current, &old); return error; } @@ -579,7 +579,7 @@ static int apply_microcode_check_cpu(int cpu) return 0; old = current->cpus_allowed; - set_cpus_allowed(current, cpumask_of_cpu(cpu)); + set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu)); /* Check if the microcode we have in memory matches the CPU */ if (c->x86_vendor != X86_VENDOR_INTEL || c->x86 < 6 || @@ -610,7 +610,7 @@ static int apply_microcode_check_cpu(int cpu) " sig=0x%x, pf=0x%x, rev=0x%x\n", cpu, uci->sig, uci->pf, uci->rev); - set_cpus_allowed(current, old); + set_cpus_allowed_ptr(current, &old); return err; } @@ -621,13 +621,13 @@ static void microcode_init_cpu(int cpu, int resume) old = current->cpus_allowed; - set_cpus_allowed(current, cpumask_of_cpu(cpu)); + set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu)); mutex_lock(µcode_mutex); collect_cpu_info(cpu); if (uci->valid && system_state == SYSTEM_RUNNING && !resume) cpu_request_microcode(cpu); mutex_unlock(µcode_mutex); - set_cpus_allowed(current, old); + set_cpus_allowed_ptr(current, &old); } static void microcode_fini_cpu(int cpu) @@ -657,14 +657,14 @@ static ssize_t reload_store(struct sys_device *dev, const char *buf, size_t sz) old = current->cpus_allowed; get_online_cpus(); - set_cpus_allowed(current, cpumask_of_cpu(cpu)); + set_cpus_allowed_ptr(current, &cpumask_of_cpu(cpu)); mutex_lock(µcode_mutex); if (uci->valid) err = cpu_request_microcode(cpu); mutex_unlock(µcode_mutex); put_online_cpus(); - set_cpus_allowed(current, old); + set_cpus_allowed_ptr(current, &old); } if (err) return err; diff --git a/arch/x86/kernel/reboot.c b/arch/x86/kernel/reboot.c index 9692202d3bf..19c9386ac11 100644 --- a/arch/x86/kernel/reboot.c +++ b/arch/x86/kernel/reboot.c @@ -420,7 +420,7 @@ static void native_machine_shutdown(void) reboot_cpu_id = smp_processor_id(); /* Make certain I only run on the appropriate processor */ - set_cpus_allowed(current, cpumask_of_cpu(reboot_cpu_id)); + set_cpus_allowed_ptr(current, &cpumask_of_cpu(reboot_cpu_id)); /* O.K Now that I'm on the appropriate processor, * stop all of the others. -- cgit v1.2.3-70-g09d2 From b53e921ba1cff8453dc9a87a84052fa12d5b30bd Mon Sep 17 00:00:00 2001 From: Mike Travis Date: Fri, 4 Apr 2008 18:11:08 -0700 Subject: generic: reduce stack pressure in sched_affinity * Modify sched_affinity functions to pass cpumask_t variables by reference instead of by value. * Use new set_cpus_allowed_ptr function. Depends on: [sched-devel]: sched: add new set_cpus_allowed_ptr function Cc: Paul Jackson Cc: Cliff Wickman Signed-off-by: Mike Travis Signed-off-by: Ingo Molnar --- arch/x86/kernel/cpu/mcheck/mce_amd_64.c | 46 ++++++++++++++++----------------- include/linux/sched.h | 2 +- kernel/compat.c | 2 +- kernel/rcupreempt.c | 4 +-- kernel/sched.c | 5 ++-- 5 files changed, 30 insertions(+), 29 deletions(-) (limited to 'arch/x86') diff --git a/arch/x86/kernel/cpu/mcheck/mce_amd_64.c b/arch/x86/kernel/cpu/mcheck/mce_amd_64.c index 32671da8184..7c9a813e119 100644 --- a/arch/x86/kernel/cpu/mcheck/mce_amd_64.c +++ b/arch/x86/kernel/cpu/mcheck/mce_amd_64.c @@ -251,18 +251,18 @@ struct threshold_attr { ssize_t(*store) (struct threshold_block *, const char *, size_t count); }; -static cpumask_t affinity_set(unsigned int cpu) +static void affinity_set(unsigned int cpu, cpumask_t *oldmask, + cpumask_t *newmask) { - cpumask_t oldmask = current->cpus_allowed; - cpumask_t newmask = CPU_MASK_NONE; - cpu_set(cpu, newmask); - set_cpus_allowed(current, newmask); - return oldmask; + *oldmask = current->cpus_allowed; + cpus_clear(*newmask); + cpu_set(cpu, *newmask); + set_cpus_allowed_ptr(current, newmask); } -static void affinity_restore(cpumask_t oldmask) +static void affinity_restore(const cpumask_t *oldmask) { - set_cpus_allowed(current, oldmask); + set_cpus_allowed_ptr(current, oldmask); } #define SHOW_FIELDS(name) \ @@ -277,15 +277,15 @@ static ssize_t store_interrupt_enable(struct threshold_block *b, const char *buf, size_t count) { char *end; - cpumask_t oldmask; + cpumask_t oldmask, newmask; unsigned long new = simple_strtoul(buf, &end, 0); if (end == buf) return -EINVAL; b->interrupt_enable = !!new; - oldmask = affinity_set(b->cpu); + affinity_set(b->cpu, &oldmask, &newmask); threshold_restart_bank(b, 0, 0); - affinity_restore(oldmask); + affinity_restore(&oldmask); return end - buf; } @@ -294,7 +294,7 @@ static ssize_t store_threshold_limit(struct threshold_block *b, const char *buf, size_t count) { char *end; - cpumask_t oldmask; + cpumask_t oldmask, newmask; u16 old; unsigned long new = simple_strtoul(buf, &end, 0); if (end == buf) @@ -306,9 +306,9 @@ static ssize_t store_threshold_limit(struct threshold_block *b, old = b->threshold_limit; b->threshold_limit = new; - oldmask = affinity_set(b->cpu); + affinity_set(b->cpu, &oldmask, &newmask); threshold_restart_bank(b, 0, old); - affinity_restore(oldmask); + affinity_restore(&oldmask); return end - buf; } @@ -316,10 +316,10 @@ static ssize_t store_threshold_limit(struct threshold_block *b, static ssize_t show_error_count(struct threshold_block *b, char *buf) { u32 high, low; - cpumask_t oldmask; - oldmask = affinity_set(b->cpu); + cpumask_t oldmask, newmask; + affinity_set(b->cpu, &oldmask, &newmask); rdmsr(b->address, low, high); - affinity_restore(oldmask); + affinity_restore(&oldmask); return sprintf(buf, "%x\n", (high & 0xFFF) - (THRESHOLD_MAX - b->threshold_limit)); } @@ -327,10 +327,10 @@ static ssize_t show_error_count(struct threshold_block *b, char *buf) static ssize_t store_error_count(struct threshold_block *b, const char *buf, size_t count) { - cpumask_t oldmask; - oldmask = affinity_set(b->cpu); + cpumask_t oldmask, newmask; + affinity_set(b->cpu, &oldmask, &newmask); threshold_restart_bank(b, 1, 0); - affinity_restore(oldmask); + affinity_restore(&oldmask); return 1; } @@ -468,7 +468,7 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank) { int i, err = 0; struct threshold_bank *b = NULL; - cpumask_t oldmask = CPU_MASK_NONE; + cpumask_t oldmask, newmask; char name[32]; sprintf(name, "threshold_bank%i", bank); @@ -519,10 +519,10 @@ static __cpuinit int threshold_create_bank(unsigned int cpu, unsigned int bank) per_cpu(threshold_banks, cpu)[bank] = b; - oldmask = affinity_set(cpu); + affinity_set(cpu, &oldmask, &newmask); err = allocate_threshold_blocks(cpu, bank, 0, MSR_IA32_MC0_MISC + bank * 4); - affinity_restore(oldmask); + affinity_restore(&oldmask); if (err) goto out_free; diff --git a/include/linux/sched.h b/include/linux/sched.h index be5d31752db..383502dfda1 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2034,7 +2034,7 @@ static inline void arch_pick_mmap_layout(struct mm_struct *mm) } #endif -extern long sched_setaffinity(pid_t pid, cpumask_t new_mask); +extern long sched_setaffinity(pid_t pid, const cpumask_t *new_mask); extern long sched_getaffinity(pid_t pid, cpumask_t *mask); extern int sched_mc_power_savings, sched_smt_power_savings; diff --git a/kernel/compat.c b/kernel/compat.c index 9c48abfcd4a..e1ef04870c2 100644 --- a/kernel/compat.c +++ b/kernel/compat.c @@ -445,7 +445,7 @@ asmlinkage long compat_sys_sched_setaffinity(compat_pid_t pid, if (retval) return retval; - return sched_setaffinity(pid, new_mask); + return sched_setaffinity(pid, &new_mask); } asmlinkage long compat_sys_sched_getaffinity(compat_pid_t pid, unsigned int len, diff --git a/kernel/rcupreempt.c b/kernel/rcupreempt.c index e9517014b57..e1cdf196a51 100644 --- a/kernel/rcupreempt.c +++ b/kernel/rcupreempt.c @@ -1007,10 +1007,10 @@ void __synchronize_sched(void) if (sched_getaffinity(0, &oldmask) < 0) oldmask = cpu_possible_map; for_each_online_cpu(cpu) { - sched_setaffinity(0, cpumask_of_cpu(cpu)); + sched_setaffinity(0, &cpumask_of_cpu(cpu)); schedule(); } - sched_setaffinity(0, oldmask); + sched_setaffinity(0, &oldmask); } EXPORT_SYMBOL_GPL(__synchronize_sched); diff --git a/kernel/sched.c b/kernel/sched.c index ccc23a9cd26..1a8252385c4 100644 --- a/kernel/sched.c +++ b/kernel/sched.c @@ -4908,9 +4908,10 @@ out_unlock: return retval; } -long sched_setaffinity(pid_t pid, cpumask_t new_mask) +long sched_setaffinity(pid_t pid, const cpumask_t *in_mask) { cpumask_t cpus_allowed; + cpumask_t new_mask = *in_mask; struct task_struct *p; int retval; @@ -4991,7 +4992,7 @@ asmlinkage long sys_sched_setaffinity(pid_t pid, unsigned int len, if (retval) return retval; - return sched_setaffinity(pid, new_mask); + return sched_setaffinity(pid, &new_mask); } /* -- cgit v1.2.3-70-g09d2 From 9f0e8d0400d925c3acd5f4e01dbeb736e4011882 Mon Sep 17 00:00:00 2001 From: Mike Travis Date: Fri, 4 Apr 2008 18:11:01 -0700 Subject: x86: convert cpumask_of_cpu macro to allocated array * Here is a simple patch to use an allocated array of cpumasks to represent cpumask_of_cpu() instead of constructing one on the stack. It's based on the Kconfig option "HAVE_CPUMASK_OF_CPU_MAP" which is currently only set for x86_64 SMP. Otherwise the the existing cpumask_of_cpu() is used but has been changed to produce an lvalue so a pointer to it can be used. Cc: H. Peter Anvin Signed-off-by: Christoph Lameter Signed-off-by: Mike Travis Signed-off-by: Ingo Molnar --- arch/x86/Kconfig | 3 +++ arch/x86/kernel/setup.c | 28 +++++++++++++++++++++++++++- include/linux/cpumask.h | 12 +++++++++--- 3 files changed, 39 insertions(+), 4 deletions(-) (limited to 'arch/x86') diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig index 2a59dbb2824..7f30b754bfc 100644 --- a/arch/x86/Kconfig +++ b/arch/x86/Kconfig @@ -117,6 +117,9 @@ config ARCH_HAS_CPU_RELAX config HAVE_SETUP_PER_CPU_AREA def_bool X86_64 || (X86_SMP && !X86_VOYAGER) +config HAVE_CPUMASK_OF_CPU_MAP + def_bool X86_64_SMP + config ARCH_HIBERNATION_POSSIBLE def_bool y depends on !SMP || !X86_VOYAGER diff --git a/arch/x86/kernel/setup.c b/arch/x86/kernel/setup.c index ed157c90412..0d1f44ae6ee 100644 --- a/arch/x86/kernel/setup.c +++ b/arch/x86/kernel/setup.c @@ -54,6 +54,24 @@ static void __init setup_per_cpu_maps(void) #endif } +#ifdef CONFIG_HAVE_CPUMASK_OF_CPU_MAP +cpumask_t *cpumask_of_cpu_map __read_mostly; +EXPORT_SYMBOL(cpumask_of_cpu_map); + +/* requires nr_cpu_ids to be initialized */ +static void __init setup_cpumask_of_cpu(void) +{ + int i; + + /* alloc_bootmem zeroes memory */ + cpumask_of_cpu_map = alloc_bootmem_low(sizeof(cpumask_t) * nr_cpu_ids); + for (i = 0; i < nr_cpu_ids; i++) + cpu_set(i, cpumask_of_cpu_map[i]); +} +#else +static inline void setup_cpumask_of_cpu(void) { } +#endif + #ifdef CONFIG_X86_32 /* * Great future not-so-futuristic plan: make i386 and x86_64 do it @@ -70,7 +88,7 @@ EXPORT_SYMBOL(__per_cpu_offset); */ void __init setup_per_cpu_areas(void) { - int i; + int i, highest_cpu = 0; unsigned long size; #ifdef CONFIG_HOTPLUG_CPU @@ -104,10 +122,18 @@ void __init setup_per_cpu_areas(void) __per_cpu_offset[i] = ptr - __per_cpu_start; #endif memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start); + + highest_cpu = i; } + nr_cpu_ids = highest_cpu + 1; + printk(KERN_DEBUG "NR_CPUS: %d, nr_cpu_ids: %d\n", NR_CPUS, nr_cpu_ids); + /* Setup percpu data maps */ setup_per_cpu_maps(); + + /* Setup cpumask_of_cpu map */ + setup_cpumask_of_cpu(); } #endif diff --git a/include/linux/cpumask.h b/include/linux/cpumask.h index 629102feaa6..259c8051155 100644 --- a/include/linux/cpumask.h +++ b/include/linux/cpumask.h @@ -222,8 +222,13 @@ int __next_cpu(int n, const cpumask_t *srcp); #define next_cpu(n, src) ({ (void)(src); 1; }) #endif +#ifdef CONFIG_HAVE_CPUMASK_OF_CPU_MAP +extern cpumask_t *cpumask_of_cpu_map; +#define cpumask_of_cpu(cpu) (cpumask_of_cpu_map[cpu]) + +#else #define cpumask_of_cpu(cpu) \ -({ \ +(*({ \ typeof(_unused_cpumask_arg_) m; \ if (sizeof(m) == sizeof(unsigned long)) { \ m.bits[0] = 1UL<<(cpu); \ @@ -231,8 +236,9 @@ int __next_cpu(int n, const cpumask_t *srcp); cpus_clear(m); \ cpu_set((cpu), m); \ } \ - m; \ -}) + &m; \ +})) +#endif #define CPU_MASK_LAST_WORD BITMAP_LAST_WORD_MASK(NR_CPUS) -- cgit v1.2.3-70-g09d2 From fb0f330e62d71f7c535251438068199af320cf73 Mon Sep 17 00:00:00 2001 From: Mike Travis Date: Tue, 8 Apr 2008 11:43:02 -0700 Subject: x86: modify show_shared_cpu_map in intel_cacheinfo * Removed kmalloc (or local array) in show_shared_cpu_map(). * Added show_shared_cpu_list() function. Signed-off-by: Mike Travis Signed-off-by: Ingo Molnar --- arch/x86/kernel/cpu/intel_cacheinfo.c | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) (limited to 'arch/x86') diff --git a/arch/x86/kernel/cpu/intel_cacheinfo.c b/arch/x86/kernel/cpu/intel_cacheinfo.c index e073a93ceb4..26d615dcb14 100644 --- a/arch/x86/kernel/cpu/intel_cacheinfo.c +++ b/arch/x86/kernel/cpu/intel_cacheinfo.c @@ -591,20 +591,34 @@ static ssize_t show_size(struct _cpuid4_info *this_leaf, char *buf) return sprintf (buf, "%luK\n", this_leaf->size / 1024); } -static ssize_t show_shared_cpu_map(struct _cpuid4_info *this_leaf, char *buf) +static ssize_t show_shared_cpu_map_func(struct _cpuid4_info *this_leaf, + int type, char *buf) { + ptrdiff_t len = PTR_ALIGN(buf + PAGE_SIZE - 1, PAGE_SIZE) - buf; int n = 0; - int len = cpumask_scnprintf_len(nr_cpu_ids); - char *mask_str = kmalloc(len, GFP_KERNEL); - if (mask_str) { - cpumask_scnprintf(mask_str, len, this_leaf->shared_cpu_map); - n = sprintf(buf, "%s\n", mask_str); - kfree(mask_str); + if (len > 1) { + cpumask_t *mask = &this_leaf->shared_cpu_map; + + n = type? + cpulist_scnprintf(buf, len-2, *mask): + cpumask_scnprintf(buf, len-2, *mask); + buf[n++] = '\n'; + buf[n] = '\0'; } return n; } +static inline ssize_t show_shared_cpu_map(struct _cpuid4_info *leaf, char *buf) +{ + return show_shared_cpu_map_func(leaf, 0, buf); +} + +static inline ssize_t show_shared_cpu_list(struct _cpuid4_info *leaf, char *buf) +{ + return show_shared_cpu_map_func(leaf, 1, buf); +} + static ssize_t show_type(struct _cpuid4_info *this_leaf, char *buf) { switch(this_leaf->eax.split.type) { case CACHE_TYPE_DATA: @@ -640,6 +654,7 @@ define_one_ro(ways_of_associativity); define_one_ro(number_of_sets); define_one_ro(size); define_one_ro(shared_cpu_map); +define_one_ro(shared_cpu_list); static struct attribute * default_attrs[] = { &type.attr, @@ -650,6 +665,7 @@ static struct attribute * default_attrs[] = { &number_of_sets.attr, &size.attr, &shared_cpu_map.attr, + &shared_cpu_list.attr, NULL }; -- cgit v1.2.3-70-g09d2