From 13870b657578bcce167978ee93dc02bf54e3beb0 Mon Sep 17 00:00:00 2001 From: Anton Blanchard Date: Fri, 13 Feb 2009 11:57:30 +0000 Subject: powerpc/mm: Reduce hashtable size when using 64kB pages At the moment we size the hashtable based on 4kB pages / 2, even on a 64kB kernel. This results in a hashtable that is much larger than it needs to be. Grab the real page size and size the hashtable based on that Note: This only has effect on non hypervisor machines. Signed-off-by: Anton Blanchard Signed-off-by: Benjamin Herrenschmidt --- arch/powerpc/mm/hash_utils_64.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'arch/powerpc/mm/hash_utils_64.c') diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index 8d5b4758c13..f5bc1b213f2 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -516,7 +516,7 @@ static int __init htab_dt_scan_pftsize(unsigned long node, static unsigned long __init htab_get_table_size(void) { - unsigned long mem_size, rnd_mem_size, pteg_count; + unsigned long mem_size, rnd_mem_size, pteg_count, psize; /* If hash size isn't already provided by the platform, we try to * retrieve it from the device-tree. If it's not there neither, we @@ -534,7 +534,8 @@ static unsigned long __init htab_get_table_size(void) rnd_mem_size <<= 1; /* # pages / 2 */ - pteg_count = max(rnd_mem_size >> (12 + 1), 1UL << 11); + psize = mmu_psize_defs[mmu_virtual_psize].shift; + pteg_count = max(rnd_mem_size >> (psize + 1), 1UL << 11); return pteg_count << 7; } -- cgit v1.2.3-70-g09d2 From 56aa4129e87be43676c6e3eac41a6aa553877802 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Sun, 15 Mar 2009 18:16:43 +0000 Subject: cpumask: Use mm_cpumask() wrapper instead of cpu_vm_mask Makes code futureproof against the impending change to mm->cpu_vm_mask. It's also a chance to use the new cpumask_ ops which take a pointer (the older ones are deprecated, but there's no hurry for arch code). Signed-off-by: Rusty Russell Signed-off-by: Benjamin Herrenschmidt --- arch/powerpc/include/asm/mmu_context.h | 2 +- arch/powerpc/mm/hash_utils_64.c | 10 ++++------ arch/powerpc/mm/mmu_context_nohash.c | 2 +- arch/powerpc/mm/pgtable.c | 3 +-- arch/powerpc/mm/tlb_hash64.c | 6 +++--- arch/powerpc/mm/tlb_nohash.c | 18 +++++++++--------- arch/powerpc/platforms/cell/spu_base.c | 2 +- 7 files changed, 20 insertions(+), 23 deletions(-) (limited to 'arch/powerpc/mm/hash_utils_64.c') diff --git a/arch/powerpc/include/asm/mmu_context.h b/arch/powerpc/include/asm/mmu_context.h index ab4f19263c4..b7063669f97 100644 --- a/arch/powerpc/include/asm/mmu_context.h +++ b/arch/powerpc/include/asm/mmu_context.h @@ -31,7 +31,7 @@ static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next, struct task_struct *tsk) { /* Mark this context has been used on the new CPU */ - cpu_set(smp_processor_id(), next->cpu_vm_mask); + cpumask_set_cpu(smp_processor_id(), mm_cpumask(next)); /* 32-bit keeps track of the current PGDIR in the thread struct */ #ifdef CONFIG_PPC32 diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index f5bc1b213f2..86c00c885e6 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -859,7 +859,7 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap) unsigned long vsid; struct mm_struct *mm; pte_t *ptep; - cpumask_t tmp; + const struct cpumask *tmp; int rc, user_region = 0, local = 0; int psize, ssize; @@ -907,8 +907,8 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap) return 1; /* Check CPU locality */ - tmp = cpumask_of_cpu(smp_processor_id()); - if (user_region && cpus_equal(mm->cpu_vm_mask, tmp)) + tmp = cpumask_of(smp_processor_id()); + if (user_region && cpumask_equal(mm_cpumask(mm), tmp)) local = 1; #ifdef CONFIG_HUGETLB_PAGE @@ -1024,7 +1024,6 @@ void hash_preload(struct mm_struct *mm, unsigned long ea, unsigned long vsid; void *pgdir; pte_t *ptep; - cpumask_t mask; unsigned long flags; int local = 0; int ssize; @@ -1067,8 +1066,7 @@ void hash_preload(struct mm_struct *mm, unsigned long ea, local_irq_save(flags); /* Is that local to this CPU ? */ - mask = cpumask_of_cpu(smp_processor_id()); - if (cpus_equal(mm->cpu_vm_mask, mask)) + if (cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) local = 1; /* Hash it in */ diff --git a/arch/powerpc/mm/mmu_context_nohash.c b/arch/powerpc/mm/mmu_context_nohash.c index 52a0cfc38b6..ac4cb04ceb6 100644 --- a/arch/powerpc/mm/mmu_context_nohash.c +++ b/arch/powerpc/mm/mmu_context_nohash.c @@ -97,7 +97,7 @@ static unsigned int steal_context_smp(unsigned int id) mm->context.id = MMU_NO_CONTEXT; /* Mark it stale on all CPUs that used this mm */ - for_each_cpu_mask_nr(cpu, mm->cpu_vm_mask) + for_each_cpu(cpu, mm_cpumask(mm)) __set_bit(id, stale_map[cpu]); return id; } diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c index a27ded3adac..f5c6fd42265 100644 --- a/arch/powerpc/mm/pgtable.c +++ b/arch/powerpc/mm/pgtable.c @@ -82,11 +82,10 @@ static void pte_free_submit(struct pte_freelist_batch *batch) void pgtable_free_tlb(struct mmu_gather *tlb, pgtable_free_t pgf) { /* This is safe since tlb_gather_mmu has disabled preemption */ - cpumask_t local_cpumask = cpumask_of_cpu(smp_processor_id()); struct pte_freelist_batch **batchp = &__get_cpu_var(pte_freelist_cur); if (atomic_read(&tlb->mm->mm_users) < 2 || - cpus_equal(tlb->mm->cpu_vm_mask, local_cpumask)) { + cpumask_equal(mm_cpumask(tlb->mm), cpumask_of(smp_processor_id()))){ pgtable_free(pgf); return; } diff --git a/arch/powerpc/mm/tlb_hash64.c b/arch/powerpc/mm/tlb_hash64.c index c931bc7d107..1be1b5e5979 100644 --- a/arch/powerpc/mm/tlb_hash64.c +++ b/arch/powerpc/mm/tlb_hash64.c @@ -139,12 +139,12 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr, */ void __flush_tlb_pending(struct ppc64_tlb_batch *batch) { - cpumask_t tmp; + const struct cpumask *tmp; int i, local = 0; i = batch->index; - tmp = cpumask_of_cpu(smp_processor_id()); - if (cpus_equal(batch->mm->cpu_vm_mask, tmp)) + tmp = cpumask_of(smp_processor_id()); + if (cpumask_equal(mm_cpumask(batch->mm), tmp)) local = 1; if (i == 1) flush_hash_page(batch->vaddr[0], batch->pte[0], diff --git a/arch/powerpc/mm/tlb_nohash.c b/arch/powerpc/mm/tlb_nohash.c index 39ac22b13c7..7af72970fae 100644 --- a/arch/powerpc/mm/tlb_nohash.c +++ b/arch/powerpc/mm/tlb_nohash.c @@ -132,11 +132,11 @@ void flush_tlb_mm(struct mm_struct *mm) pid = mm->context.id; if (unlikely(pid == MMU_NO_CONTEXT)) goto no_context; - cpu_mask = mm->cpu_vm_mask; - cpu_clear(smp_processor_id(), cpu_mask); - if (!cpus_empty(cpu_mask)) { + if (!cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) { struct tlb_flush_param p = { .pid = pid }; - smp_call_function_mask(cpu_mask, do_flush_tlb_mm_ipi, &p, 1); + /* Ignores smp_processor_id() even if set. */ + smp_call_function_many(mm_cpumask(mm), + do_flush_tlb_mm_ipi, &p, 1); } _tlbil_pid(pid); no_context: @@ -146,16 +146,15 @@ EXPORT_SYMBOL(flush_tlb_mm); void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) { - cpumask_t cpu_mask; + struct cpumask *cpu_mask; unsigned int pid; preempt_disable(); pid = vma ? vma->vm_mm->context.id : 0; if (unlikely(pid == MMU_NO_CONTEXT)) goto bail; - cpu_mask = vma->vm_mm->cpu_vm_mask; - cpu_clear(smp_processor_id(), cpu_mask); - if (!cpus_empty(cpu_mask)) { + cpu_mask = mm_cpumask(vma->vm_mm); + if (!cpumask_equal(cpu_mask, cpumask_of(smp_processor_id()))) { /* If broadcast tlbivax is supported, use it */ if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) { int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL); @@ -167,7 +166,8 @@ void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr) goto bail; } else { struct tlb_flush_param p = { .pid = pid, .addr = vmaddr }; - smp_call_function_mask(cpu_mask, + /* Ignores smp_processor_id() even if set in cpu_mask */ + smp_call_function_many(cpu_mask, do_flush_tlb_page_ipi, &p, 1); } } diff --git a/arch/powerpc/platforms/cell/spu_base.c b/arch/powerpc/platforms/cell/spu_base.c index e487ad68ac1..9abd210d87c 100644 --- a/arch/powerpc/platforms/cell/spu_base.c +++ b/arch/powerpc/platforms/cell/spu_base.c @@ -114,7 +114,7 @@ static inline void mm_needs_global_tlbie(struct mm_struct *mm) int nr = (NR_CPUS > 1) ? NR_CPUS : NR_CPUS + 1; /* Global TLBIE broadcast required with SPEs. */ - __cpus_setall(&mm->cpu_vm_mask, nr); + bitmap_fill(cpumask_bits(mm_cpumask(mm)), nr); } void spu_associate_mm(struct spu *spu, struct mm_struct *mm) -- cgit v1.2.3-70-g09d2 From 757c74d298dc8438760b8dea275c4c6e0ac8a77f Mon Sep 17 00:00:00 2001 From: Benjamin Herrenschmidt Date: Thu, 19 Mar 2009 19:34:16 +0000 Subject: powerpc/mm: Introduce early_init_mmu() on 64-bit This moves some MMU related init code out of setup_64.c into hash_utils_64.c and calls it early_init_mmu() and early_init_mmu_secondary(). This will make it easier to plug in a new MMU type. Signed-off-by: Benjamin Herrenschmidt --- arch/powerpc/include/asm/mmu-hash64.h | 2 -- arch/powerpc/include/asm/mmu.h | 4 ++++ arch/powerpc/kernel/setup_64.c | 35 +++++----------------------------- arch/powerpc/mm/hash_utils_64.c | 36 +++++++++++++++++++++++++++++++++-- 4 files changed, 43 insertions(+), 34 deletions(-) (limited to 'arch/powerpc/mm/hash_utils_64.c') diff --git a/arch/powerpc/include/asm/mmu-hash64.h b/arch/powerpc/include/asm/mmu-hash64.h index 68b75262680..98c104a0996 100644 --- a/arch/powerpc/include/asm/mmu-hash64.h +++ b/arch/powerpc/include/asm/mmu-hash64.h @@ -284,8 +284,6 @@ extern void add_gpage(unsigned long addr, unsigned long page_size, unsigned long number_of_pages); extern void demote_segment_4k(struct mm_struct *mm, unsigned long addr); -extern void htab_initialize(void); -extern void htab_initialize_secondary(void); extern void hpte_init_native(void); extern void hpte_init_lpar(void); extern void hpte_init_iSeries(void); diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h index c073de4af84..cbf15438709 100644 --- a/arch/powerpc/include/asm/mmu.h +++ b/arch/powerpc/include/asm/mmu.h @@ -62,6 +62,10 @@ static inline int mmu_has_feature(unsigned long feature) extern unsigned int __start___mmu_ftr_fixup, __stop___mmu_ftr_fixup; +/* MMU initialization (64-bit only fo now) */ +extern void early_init_mmu(void); +extern void early_init_mmu_secondary(void); + #endif /* !__ASSEMBLY__ */ diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index 73e16e298e2..c410c606955 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -202,8 +202,6 @@ void __init early_setup(unsigned long dt_ptr) /* Fix up paca fields required for the boot cpu */ get_paca()->cpu_start = 1; - get_paca()->stab_real = __pa((u64)&initial_stab); - get_paca()->stab_addr = (u64)&initial_stab; /* Probe the machine type */ probe_machine(); @@ -212,20 +210,8 @@ void __init early_setup(unsigned long dt_ptr) DBG("Found, Initializing memory management...\n"); - /* - * Initialize the MMU Hash table and create the linear mapping - * of memory. Has to be done before stab/slb initialization as - * this is currently where the page size encoding is obtained - */ - htab_initialize(); - - /* - * Initialize stab / SLB management except on iSeries - */ - if (cpu_has_feature(CPU_FTR_SLB)) - slb_initialize(); - else if (!firmware_has_feature(FW_FEATURE_ISERIES)) - stab_initialize(get_paca()->stab_real); + /* Initialize the hash table or TLB handling */ + early_init_mmu(); DBG(" <- early_setup()\n"); } @@ -233,22 +219,11 @@ void __init early_setup(unsigned long dt_ptr) #ifdef CONFIG_SMP void early_setup_secondary(void) { - struct paca_struct *lpaca = get_paca(); - /* Mark interrupts enabled in PACA */ - lpaca->soft_enabled = 0; + get_paca()->soft_enabled = 0; - /* Initialize hash table for that CPU */ - htab_initialize_secondary(); - - /* Initialize STAB/SLB. We use a virtual address as it works - * in real mode on pSeries and we want a virutal address on - * iSeries anyway - */ - if (cpu_has_feature(CPU_FTR_SLB)) - slb_initialize(); - else - stab_initialize(lpaca->stab_addr); + /* Initialize the hash table or TLB handling */ + early_init_mmu_secondary(); } #endif /* CONFIG_SMP */ diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index 86c00c885e6..db556d25c3a 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -590,7 +590,7 @@ static void __init htab_finish_init(void) make_bl(htab_call_hpte_updatepp, ppc_md.hpte_updatepp); } -void __init htab_initialize(void) +static void __init htab_initialize(void) { unsigned long table; unsigned long pteg_count; @@ -732,11 +732,43 @@ void __init htab_initialize(void) #undef KB #undef MB -void htab_initialize_secondary(void) +void __init early_init_mmu(void) { + /* Setup initial STAB address in the PACA */ + get_paca()->stab_real = __pa((u64)&initial_stab); + get_paca()->stab_addr = (u64)&initial_stab; + + /* Initialize the MMU Hash table and create the linear mapping + * of memory. Has to be done before stab/slb initialization as + * this is currently where the page size encoding is obtained + */ + htab_initialize(); + + /* Initialize stab / SLB management except on iSeries + */ + if (cpu_has_feature(CPU_FTR_SLB)) + slb_initialize(); + else if (!firmware_has_feature(FW_FEATURE_ISERIES)) + stab_initialize(get_paca()->stab_real); +} + +#ifdef CONFIG_SMP +void __init early_init_mmu_secondary(void) +{ + /* Initialize hash table for that CPU */ if (!firmware_has_feature(FW_FEATURE_LPAR)) mtspr(SPRN_SDR1, _SDR1); + + /* Initialize STAB/SLB. We use a virtual address as it works + * in real mode on pSeries and we want a virutal address on + * iSeries anyway + */ + if (cpu_has_feature(CPU_FTR_SLB)) + slb_initialize(); + else + stab_initialize(get_paca()->stab_addr); } +#endif /* CONFIG_SMP */ /* * Called by asm hashtable.S for doing lazy icache flush -- cgit v1.2.3-70-g09d2