diff options
Diffstat (limited to 'arch/powerpc/mm')
-rw-r--r-- | arch/powerpc/mm/Makefile | 10 | ||||
-rw-r--r-- | arch/powerpc/mm/fault.c | 12 | ||||
-rw-r--r-- | arch/powerpc/mm/gup.c | 10 | ||||
-rw-r--r-- | arch/powerpc/mm/hash_native_64.c | 13 | ||||
-rw-r--r-- | arch/powerpc/mm/highmem.c | 77 | ||||
-rw-r--r-- | arch/powerpc/mm/hugetlbpage.c | 4 | ||||
-rw-r--r-- | arch/powerpc/mm/init_64.c | 2 | ||||
-rw-r--r-- | arch/powerpc/mm/mmu_context_nohash.c | 34 | ||||
-rw-r--r-- | arch/powerpc/mm/numa.c | 2 | ||||
-rw-r--r-- | arch/powerpc/mm/pgtable.c | 4 | ||||
-rw-r--r-- | arch/powerpc/mm/slb.c | 15 | ||||
-rw-r--r-- | arch/powerpc/mm/tlb_hash64.c | 2 |
12 files changed, 143 insertions, 42 deletions
diff --git a/arch/powerpc/mm/Makefile b/arch/powerpc/mm/Makefile index b746f4ca420..3e68363405b 100644 --- a/arch/powerpc/mm/Makefile +++ b/arch/powerpc/mm/Makefile @@ -2,6 +2,8 @@ # Makefile for the linux ppc-specific parts of the memory manager. # +subdir-ccflags-$(CONFIG_PPC_WERROR) := -Werror + ifeq ($(CONFIG_PPC64),y) EXTRA_CFLAGS += -mno-minimal-toc endif @@ -11,10 +13,11 @@ obj-y := fault.o mem.o pgtable.o gup.o \ pgtable_$(CONFIG_WORD_SIZE).o obj-$(CONFIG_PPC_MMU_NOHASH) += mmu_context_nohash.o tlb_nohash.o \ tlb_nohash_low.o -hash-$(CONFIG_PPC_NATIVE) := hash_native_64.o -obj-$(CONFIG_PPC64) += hash_utils_64.o \ +obj-$(CONFIG_PPC64) += mmap_64.o +hash64-$(CONFIG_PPC_NATIVE) := hash_native_64.o +obj-$(CONFIG_PPC_STD_MMU_64) += hash_utils_64.o \ slb_low.o slb.o stab.o \ - mmap_64.o $(hash-y) + mmap_64.o $(hash64-y) obj-$(CONFIG_PPC_STD_MMU_32) += ppc_mmu_32.o obj-$(CONFIG_PPC_STD_MMU) += hash_low_$(CONFIG_WORD_SIZE).o \ tlb_hash$(CONFIG_WORD_SIZE).o \ @@ -27,3 +30,4 @@ obj-$(CONFIG_PPC_MM_SLICES) += slice.o obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o obj-$(CONFIG_PPC_SUBPAGE_PROT) += subpage-prot.o obj-$(CONFIG_NOT_COHERENT_CACHE) += dma-noncoherent.o +obj-$(CONFIG_HIGHMEM) += highmem.o diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index 76993941cac..830bef0a113 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -29,6 +29,7 @@ #include <linux/module.h> #include <linux/kprobes.h> #include <linux/kdebug.h> +#include <linux/perf_counter.h> #include <asm/firmware.h> #include <asm/page.h> @@ -170,6 +171,8 @@ int __kprobes do_page_fault(struct pt_regs *regs, unsigned long address, die("Weird page fault", regs, SIGSEGV); } + perf_swcounter_event(PERF_COUNT_SW_PAGE_FAULTS, 1, 0, regs, address); + /* When running in the kernel we expect faults to occur only to * addresses in user space. All other faults represent errors in the * kernel and should generate an OOPS. Unfortunately, in the case of an @@ -299,7 +302,7 @@ good_area: * the fault. */ survive: - ret = handle_mm_fault(mm, vma, address, is_write); + ret = handle_mm_fault(mm, vma, address, is_write ? FAULT_FLAG_WRITE : 0); if (unlikely(ret & VM_FAULT_ERROR)) { if (ret & VM_FAULT_OOM) goto out_of_memory; @@ -309,6 +312,8 @@ good_area: } if (ret & VM_FAULT_MAJOR) { current->maj_flt++; + perf_swcounter_event(PERF_COUNT_SW_PAGE_FAULTS_MAJ, 1, 0, + regs, address); #ifdef CONFIG_PPC_SMLPAR if (firmware_has_feature(FW_FEATURE_CMO)) { preempt_disable(); @@ -316,8 +321,11 @@ good_area: preempt_enable(); } #endif - } else + } else { current->min_flt++; + perf_swcounter_event(PERF_COUNT_SW_PAGE_FAULTS_MIN, 1, 0, + regs, address); + } up_read(&mm->mmap_sem); return 0; diff --git a/arch/powerpc/mm/gup.c b/arch/powerpc/mm/gup.c index bc400c78c97..bc122a120bf 100644 --- a/arch/powerpc/mm/gup.c +++ b/arch/powerpc/mm/gup.c @@ -159,7 +159,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, int psize; #endif - pr_debug("%s(%lx,%x,%s)\n", __func__, start, nr_pages, write ? "write" : "read"); + pr_devel("%s(%lx,%x,%s)\n", __func__, start, nr_pages, write ? "write" : "read"); start &= PAGE_MASK; addr = start; @@ -170,7 +170,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, start, len))) goto slow_irqon; - pr_debug(" aligned: %lx .. %lx\n", start, end); + pr_devel(" aligned: %lx .. %lx\n", start, end); #ifdef CONFIG_HUGETLB_PAGE /* We bail out on slice boundary crossing when hugetlb is @@ -234,7 +234,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, do { VM_BUG_ON(shift != mmu_psize_defs[get_slice_psize(mm, a)].shift); ptep = huge_pte_offset(mm, a); - pr_debug(" %016lx: huge ptep %p\n", a, ptep); + pr_devel(" %016lx: huge ptep %p\n", a, ptep); if (!ptep || !gup_huge_pte(ptep, hstate, &a, end, write, pages, &nr)) goto slow; @@ -249,7 +249,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, #ifdef CONFIG_PPC64 VM_BUG_ON(shift != mmu_psize_defs[get_slice_psize(mm, addr)].shift); #endif - pr_debug(" %016lx: normal pgd %p\n", addr, + pr_devel(" %016lx: normal pgd %p\n", addr, (void *)pgd_val(pgd)); next = pgd_addr_end(addr, end); if (pgd_none(pgd)) @@ -269,7 +269,7 @@ int get_user_pages_fast(unsigned long start, int nr_pages, int write, slow: local_irq_enable(); slow_irqon: - pr_debug(" slow path ! nr = %d\n", nr); + pr_devel(" slow path ! nr = %d\n", nr); /* Try to get the remaining pages with get_user_pages */ start += nr << PAGE_SHIFT; diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c index 34e5c0b219b..056d23a1b10 100644 --- a/arch/powerpc/mm/hash_native_64.c +++ b/arch/powerpc/mm/hash_native_64.c @@ -27,6 +27,7 @@ #include <asm/cputable.h> #include <asm/udbg.h> #include <asm/kexec.h> +#include <asm/ppc-opcode.h> #ifdef DEBUG_LOW #define DBG_LOW(fmt...) udbg_printf(fmt) @@ -49,14 +50,21 @@ static inline void __tlbie(unsigned long va, int psize, int ssize) case MMU_PAGE_4K: va &= ~0xffful; va |= ssize << 8; - asm volatile("tlbie %0,0" : : "r" (va) : "memory"); + asm volatile(ASM_MMU_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), + %2) + : : "r" (va), "r"(0), "i" (MMU_FTR_TLBIE_206) + : "memory"); break; default: penc = mmu_psize_defs[psize].penc; va &= ~((1ul << mmu_psize_defs[psize].shift) - 1); va |= penc << 12; va |= ssize << 8; - asm volatile("tlbie %0,1" : : "r" (va) : "memory"); + va |= 1; /* L */ + asm volatile(ASM_MMU_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), + %2) + : : "r" (va), "r"(0), "i" (MMU_FTR_TLBIE_206) + : "memory"); break; } } @@ -80,6 +88,7 @@ static inline void __tlbiel(unsigned long va, int psize, int ssize) va &= ~((1ul << mmu_psize_defs[psize].shift) - 1); va |= penc << 12; va |= ssize << 8; + va |= 1; /* L */ asm volatile(".long 0x7c000224 | (%0 << 11) | (1 << 21)" : : "r"(va) : "memory"); break; diff --git a/arch/powerpc/mm/highmem.c b/arch/powerpc/mm/highmem.c new file mode 100644 index 00000000000..c2186c74c85 --- /dev/null +++ b/arch/powerpc/mm/highmem.c @@ -0,0 +1,77 @@ +/* + * highmem.c: virtual kernel memory mappings for high memory + * + * PowerPC version, stolen from the i386 version. + * + * Used in CONFIG_HIGHMEM systems for memory pages which + * are not addressable by direct kernel virtual addresses. + * + * Copyright (C) 1999 Gerhard Wichert, Siemens AG + * Gerhard.Wichert@pdb.siemens.de + * + * + * Redesigned the x86 32-bit VM architecture to deal with + * up to 16 Terrabyte physical memory. With current x86 CPUs + * we now support up to 64 Gigabytes physical RAM. + * + * Copyright (C) 1999 Ingo Molnar <mingo@redhat.com> + * + * Reworked for PowerPC by various contributors. Moved from + * highmem.h by Benjamin Herrenschmidt (c) 2009 IBM Corp. + */ + +#include <linux/highmem.h> +#include <linux/module.h> + +/* + * The use of kmap_atomic/kunmap_atomic is discouraged - kmap/kunmap + * gives a more generic (and caching) interface. But kmap_atomic can + * be used in IRQ contexts, so in some (very limited) cases we need + * it. + */ +void *kmap_atomic_prot(struct page *page, enum km_type type, pgprot_t prot) +{ + unsigned int idx; + unsigned long vaddr; + + /* even !CONFIG_PREEMPT needs this, for in_atomic in do_page_fault */ + pagefault_disable(); + if (!PageHighMem(page)) + return page_address(page); + + debug_kmap_atomic(type); + idx = type + KM_TYPE_NR*smp_processor_id(); + vaddr = __fix_to_virt(FIX_KMAP_BEGIN + idx); +#ifdef CONFIG_DEBUG_HIGHMEM + BUG_ON(!pte_none(*(kmap_pte-idx))); +#endif + __set_pte_at(&init_mm, vaddr, kmap_pte-idx, mk_pte(page, prot), 1); + local_flush_tlb_page(NULL, vaddr); + + return (void*) vaddr; +} +EXPORT_SYMBOL(kmap_atomic_prot); + +void kunmap_atomic(void *kvaddr, enum km_type type) +{ +#ifdef CONFIG_DEBUG_HIGHMEM + unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK; + enum fixed_addresses idx = type + KM_TYPE_NR*smp_processor_id(); + + if (vaddr < __fix_to_virt(FIX_KMAP_END)) { + pagefault_enable(); + return; + } + + BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx)); + + /* + * force other mappings to Oops if they'll try to access + * this pte without first remap it + */ + pte_clear(&init_mm, vaddr, kmap_pte-idx); + local_flush_tlb_page(NULL, vaddr); +#endif + pagefault_enable(); +} +EXPORT_SYMBOL(kunmap_atomic); diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 9920d6a7cf2..c46ef2ffa3d 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -305,7 +305,7 @@ static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud, pmd = pmd_offset(pud, start); pud_clear(pud); - pmd_free_tlb(tlb, pmd); + pmd_free_tlb(tlb, pmd, start); } static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, @@ -348,7 +348,7 @@ static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, pud = pud_offset(pgd, start); pgd_clear(pgd); - pud_free_tlb(tlb, pud); + pud_free_tlb(tlb, pud, start); } /* diff --git a/arch/powerpc/mm/init_64.c b/arch/powerpc/mm/init_64.c index 3e6a6543f53..68a821add28 100644 --- a/arch/powerpc/mm/init_64.c +++ b/arch/powerpc/mm/init_64.c @@ -66,6 +66,7 @@ #include "mmu_decl.h" +#ifdef CONFIG_PPC_STD_MMU_64 #if PGTABLE_RANGE > USER_VSID_RANGE #warning Limited user VSID range means pagetable space is wasted #endif @@ -73,6 +74,7 @@ #if (TASK_SIZE_USER64 < PGTABLE_RANGE) && (TASK_SIZE_USER64 < USER_VSID_RANGE) #warning TASK_SIZE is smaller than it needs to be. #endif +#endif /* CONFIG_PPC_STD_MMU_64 */ phys_addr_t memstart_addr = ~0; phys_addr_t kernstart_addr; diff --git a/arch/powerpc/mm/mmu_context_nohash.c b/arch/powerpc/mm/mmu_context_nohash.c index 030d0005b4d..b1a727def15 100644 --- a/arch/powerpc/mm/mmu_context_nohash.c +++ b/arch/powerpc/mm/mmu_context_nohash.c @@ -46,7 +46,7 @@ static unsigned int next_context, nr_free_contexts; static unsigned long *context_map; static unsigned long *stale_map[NR_CPUS]; static struct mm_struct **context_mm; -static spinlock_t context_lock = SPIN_LOCK_UNLOCKED; +static DEFINE_SPINLOCK(context_lock); #define CTX_MAP_SIZE \ (sizeof(unsigned long) * (last_context / BITS_PER_LONG + 1)) @@ -73,7 +73,6 @@ static unsigned int steal_context_smp(unsigned int id) struct mm_struct *mm; unsigned int cpu, max; - again: max = last_context - first_context; /* Attempt to free next_context first and then loop until we manage */ @@ -90,7 +89,7 @@ static unsigned int steal_context_smp(unsigned int id) id = first_context; continue; } - pr_debug("[%d] steal context %d from mm @%p\n", + pr_devel("[%d] steal context %d from mm @%p\n", smp_processor_id(), id, mm); /* Mark this mm has having no context anymore */ @@ -108,7 +107,9 @@ static unsigned int steal_context_smp(unsigned int id) spin_unlock(&context_lock); cpu_relax(); spin_lock(&context_lock); - goto again; + + /* This will cause the caller to try again */ + return MMU_NO_CONTEXT; } #endif /* CONFIG_SMP */ @@ -125,7 +126,7 @@ static unsigned int steal_context_up(unsigned int id) /* Pick up the victim mm */ mm = context_mm[id]; - pr_debug("[%d] steal context %d from mm @%p\n", cpu, id, mm); + pr_devel("[%d] steal context %d from mm @%p\n", cpu, id, mm); /* Flush the TLB for that context */ local_flush_tlb_mm(mm); @@ -179,7 +180,7 @@ void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next) spin_lock(&context_lock); #ifndef DEBUG_STEAL_ONLY - pr_debug("[%d] activating context for mm @%p, active=%d, id=%d\n", + pr_devel("[%d] activating context for mm @%p, active=%d, id=%d\n", cpu, next, next->context.active, next->context.id); #endif @@ -188,12 +189,14 @@ void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next) next->context.active++; if (prev) { #ifndef DEBUG_STEAL_ONLY - pr_debug(" old context %p active was: %d\n", + pr_devel(" old context %p active was: %d\n", prev, prev->context.active); #endif WARN_ON(prev->context.active < 1); prev->context.active--; } + + again: #endif /* CONFIG_SMP */ /* If we already have a valid assigned context, skip all that */ @@ -212,6 +215,8 @@ void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next) #ifdef CONFIG_SMP if (num_online_cpus() > 1) { id = steal_context_smp(id); + if (id == MMU_NO_CONTEXT) + goto again; goto stolen; } #endif /* CONFIG_SMP */ @@ -232,7 +237,7 @@ void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next) next->context.id = id; #ifndef DEBUG_STEAL_ONLY - pr_debug("[%d] picked up new id %d, nrf is now %d\n", + pr_devel("[%d] picked up new id %d, nrf is now %d\n", cpu, id, nr_free_contexts); #endif @@ -243,7 +248,7 @@ void switch_mmu_context(struct mm_struct *prev, struct mm_struct *next) * local TLB for it and unmark it before we use it */ if (test_bit(id, stale_map[cpu])) { - pr_debug("[%d] flushing stale context %d for mm @%p !\n", + pr_devel("[%d] flushing stale context %d for mm @%p !\n", cpu, id, next); local_flush_tlb_mm(next); @@ -272,6 +277,7 @@ int init_new_context(struct task_struct *t, struct mm_struct *mm) */ void destroy_context(struct mm_struct *mm) { + unsigned long flags; unsigned int id; if (mm->context.id == MMU_NO_CONTEXT) @@ -279,18 +285,18 @@ void destroy_context(struct mm_struct *mm) WARN_ON(mm->context.active != 0); - spin_lock(&context_lock); + spin_lock_irqsave(&context_lock, flags); id = mm->context.id; if (id != MMU_NO_CONTEXT) { __clear_bit(id, context_map); mm->context.id = MMU_NO_CONTEXT; #ifdef DEBUG_MAP_CONSISTENCY mm->context.active = 0; - context_mm[id] = NULL; #endif + context_mm[id] = NULL; nr_free_contexts++; } - spin_unlock(&context_lock); + spin_unlock_irqrestore(&context_lock, flags); } #ifdef CONFIG_SMP @@ -309,13 +315,13 @@ static int __cpuinit mmu_context_cpu_notify(struct notifier_block *self, switch (action) { case CPU_ONLINE: case CPU_ONLINE_FROZEN: - pr_debug("MMU: Allocating stale context map for CPU %d\n", cpu); + pr_devel("MMU: Allocating stale context map for CPU %d\n", cpu); stale_map[cpu] = kzalloc(CTX_MAP_SIZE, GFP_KERNEL); break; #ifdef CONFIG_HOTPLUG_CPU case CPU_DEAD: case CPU_DEAD_FROZEN: - pr_debug("MMU: Freeing stale context map for CPU %d\n", cpu); + pr_devel("MMU: Freeing stale context map for CPU %d\n", cpu); kfree(stale_map[cpu]); stale_map[cpu] = NULL; break; diff --git a/arch/powerpc/mm/numa.c b/arch/powerpc/mm/numa.c index 9047145095a..b037d95eead 100644 --- a/arch/powerpc/mm/numa.c +++ b/arch/powerpc/mm/numa.c @@ -981,6 +981,8 @@ void __init do_init_bootmem(void) mark_reserved_regions_for_nid(nid); sparse_memory_present_with_active_regions(nid); } + + init_bootmem_done = 1; } void __init paging_init(void) diff --git a/arch/powerpc/mm/pgtable.c b/arch/powerpc/mm/pgtable.c index ae1d67cc090..627767d6169 100644 --- a/arch/powerpc/mm/pgtable.c +++ b/arch/powerpc/mm/pgtable.c @@ -129,12 +129,12 @@ static pte_t do_dcache_icache_coherency(pte_t pte) page = pfn_to_page(pfn); if (!PageReserved(page) && !test_bit(PG_arch_1, &page->flags)) { - pr_debug("do_dcache_icache_coherency... flushing\n"); + pr_devel("do_dcache_icache_coherency... flushing\n"); flush_dcache_icache_page(page); set_bit(PG_arch_1, &page->flags); } else - pr_debug("do_dcache_icache_coherency... already clean\n"); + pr_devel("do_dcache_icache_coherency... already clean\n"); return __pte(pte_val(pte) | _PAGE_HWEXEC); } diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c index 89497fb0428..5b7038f248b 100644 --- a/arch/powerpc/mm/slb.c +++ b/arch/powerpc/mm/slb.c @@ -2,7 +2,7 @@ * PowerPC64 SLB support. * * Copyright (C) 2004 David Gibson <dwg@au.ibm.com>, IBM - * Based on earlier code writteh by: + * Based on earlier code written by: * Dave Engebretsen and Mike Corrigan {engebret|mikejc}@us.ibm.com * Copyright (c) 2001 Dave Engebretsen * Copyright (C) 2002 Anton Blanchard <anton@au.ibm.com>, IBM @@ -14,8 +14,6 @@ * 2 of the License, or (at your option) any later version. */ -#undef DEBUG - #include <asm/pgtable.h> #include <asm/mmu.h> #include <asm/mmu_context.h> @@ -27,11 +25,6 @@ #include <linux/compiler.h> #include <asm/udbg.h> -#ifdef DEBUG -#define DBG(fmt...) printk(fmt) -#else -#define DBG pr_debug -#endif extern void slb_allocate_realmode(unsigned long ea); extern void slb_allocate_user(unsigned long ea); @@ -285,13 +278,13 @@ void slb_initialize(void) patch_slb_encoding(slb_compare_rr_to_size, mmu_slb_size); - DBG("SLB: linear LLP = %04lx\n", linear_llp); - DBG("SLB: io LLP = %04lx\n", io_llp); + pr_devel("SLB: linear LLP = %04lx\n", linear_llp); + pr_devel("SLB: io LLP = %04lx\n", io_llp); #ifdef CONFIG_SPARSEMEM_VMEMMAP patch_slb_encoding(slb_miss_kernel_load_vmemmap, SLB_VSID_KERNEL | vmemmap_llp); - DBG("SLB: vmemmap LLP = %04lx\n", vmemmap_llp); + pr_devel("SLB: vmemmap LLP = %04lx\n", vmemmap_llp); #endif } diff --git a/arch/powerpc/mm/tlb_hash64.c b/arch/powerpc/mm/tlb_hash64.c index 1be1b5e5979..937eb90677d 100644 --- a/arch/powerpc/mm/tlb_hash64.c +++ b/arch/powerpc/mm/tlb_hash64.c @@ -72,7 +72,7 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr, */ if (huge) { #ifdef CONFIG_HUGETLB_PAGE - psize = get_slice_psize(mm, addr);; + psize = get_slice_psize(mm, addr); #else BUG(); psize = pte_pagesize_index(mm, addr, pte); /* shutup gcc */ |