diff options
author | H. Peter Anvin <hpa@zytor.com> | 2010-04-29 16:53:17 -0700 |
---|---|---|
committer | H. Peter Anvin <hpa@zytor.com> | 2010-04-29 16:53:17 -0700 |
commit | d9c5841e22231e4e49fd0a1004164e6fce59b7a6 (patch) | |
tree | e1f589c46b3ff79bbe7b1b2469f6362f94576da6 /arch/sh/mm/pmb.c | |
parent | b701a47ba48b698976fb2fe05fb285b0edc1d26a (diff) | |
parent | 5967ed87ade85a421ef814296c3c7f182b08c225 (diff) |
Merge branch 'x86/asm' into x86/atomic
Merge reason:
Conflict between LOCK_PREFIX_HERE and relative alternatives
pointers
Resolved Conflicts:
arch/x86/include/asm/alternative.h
arch/x86/kernel/alternative.c
Signed-off-by: H. Peter Anvin <hpa@zytor.com>
Diffstat (limited to 'arch/sh/mm/pmb.c')
-rw-r--r-- | arch/sh/mm/pmb.c | 843 |
1 files changed, 648 insertions, 195 deletions
diff --git a/arch/sh/mm/pmb.c b/arch/sh/mm/pmb.c index 280f6a16603..e43ec600afc 100644 --- a/arch/sh/mm/pmb.c +++ b/arch/sh/mm/pmb.c @@ -3,11 +3,8 @@ * * Privileged Space Mapping Buffer (PMB) Support. * - * Copyright (C) 2005, 2006, 2007 Paul Mundt - * - * P1/P2 Section mapping definitions from map32.h, which was: - * - * Copyright 2003 (c) Lineo Solutions,Inc. + * Copyright (C) 2005 - 2010 Paul Mundt + * Copyright (C) 2010 Matt Fleming * * This file is subject to the terms and conditions of the GNU General Public * License. See the file "COPYING" in the main directory of this archive @@ -18,53 +15,221 @@ #include <linux/sysdev.h> #include <linux/cpu.h> #include <linux/module.h> -#include <linux/slab.h> #include <linux/bitops.h> #include <linux/debugfs.h> #include <linux/fs.h> #include <linux/seq_file.h> #include <linux/err.h> +#include <linux/io.h> +#include <linux/spinlock.h> +#include <linux/vmalloc.h> +#include <asm/cacheflush.h> +#include <asm/sizes.h> #include <asm/system.h> #include <asm/uaccess.h> #include <asm/pgtable.h> +#include <asm/page.h> #include <asm/mmu.h> -#include <asm/io.h> #include <asm/mmu_context.h> -#define NR_PMB_ENTRIES 16 +struct pmb_entry; + +struct pmb_entry { + unsigned long vpn; + unsigned long ppn; + unsigned long flags; + unsigned long size; + + spinlock_t lock; -static void __pmb_unmap(struct pmb_entry *); + /* + * 0 .. NR_PMB_ENTRIES for specific entry selection, or + * PMB_NO_ENTRY to search for a free one + */ + int entry; + /* Adjacent entry link for contiguous multi-entry mappings */ + struct pmb_entry *link; +}; + +static struct { + unsigned long size; + int flag; +} pmb_sizes[] = { + { .size = SZ_512M, .flag = PMB_SZ_512M, }, + { .size = SZ_128M, .flag = PMB_SZ_128M, }, + { .size = SZ_64M, .flag = PMB_SZ_64M, }, + { .size = SZ_16M, .flag = PMB_SZ_16M, }, +}; + +static void pmb_unmap_entry(struct pmb_entry *, int depth); + +static DEFINE_RWLOCK(pmb_rwlock); static struct pmb_entry pmb_entry_list[NR_PMB_ENTRIES]; -static unsigned long pmb_map; +static DECLARE_BITMAP(pmb_map, NR_PMB_ENTRIES); + +static unsigned int pmb_iomapping_enabled; -static inline unsigned long mk_pmb_entry(unsigned int entry) +static __always_inline unsigned long mk_pmb_entry(unsigned int entry) { return (entry & PMB_E_MASK) << PMB_E_SHIFT; } -static inline unsigned long mk_pmb_addr(unsigned int entry) +static __always_inline unsigned long mk_pmb_addr(unsigned int entry) { return mk_pmb_entry(entry) | PMB_ADDR; } -static inline unsigned long mk_pmb_data(unsigned int entry) +static __always_inline unsigned long mk_pmb_data(unsigned int entry) { return mk_pmb_entry(entry) | PMB_DATA; } -static int pmb_alloc_entry(void) +static __always_inline unsigned int pmb_ppn_in_range(unsigned long ppn) +{ + return ppn >= __pa(memory_start) && ppn < __pa(memory_end); +} + +/* + * Ensure that the PMB entries match our cache configuration. + * + * When we are in 32-bit address extended mode, CCR.CB becomes + * invalid, so care must be taken to manually adjust cacheable + * translations. + */ +static __always_inline unsigned long pmb_cache_flags(void) +{ + unsigned long flags = 0; + +#if defined(CONFIG_CACHE_OFF) + flags |= PMB_WT | PMB_UB; +#elif defined(CONFIG_CACHE_WRITETHROUGH) + flags |= PMB_C | PMB_WT | PMB_UB; +#elif defined(CONFIG_CACHE_WRITEBACK) + flags |= PMB_C; +#endif + + return flags; +} + +/* + * Convert typical pgprot value to the PMB equivalent + */ +static inline unsigned long pgprot_to_pmb_flags(pgprot_t prot) +{ + unsigned long pmb_flags = 0; + u64 flags = pgprot_val(prot); + + if (flags & _PAGE_CACHABLE) + pmb_flags |= PMB_C; + if (flags & _PAGE_WT) + pmb_flags |= PMB_WT | PMB_UB; + + return pmb_flags; +} + +static inline bool pmb_can_merge(struct pmb_entry *a, struct pmb_entry *b) +{ + return (b->vpn == (a->vpn + a->size)) && + (b->ppn == (a->ppn + a->size)) && + (b->flags == a->flags); +} + +static bool pmb_mapping_exists(unsigned long vaddr, phys_addr_t phys, + unsigned long size) +{ + int i; + + read_lock(&pmb_rwlock); + + for (i = 0; i < ARRAY_SIZE(pmb_entry_list); i++) { + struct pmb_entry *pmbe, *iter; + unsigned long span; + + if (!test_bit(i, pmb_map)) + continue; + + pmbe = &pmb_entry_list[i]; + + /* + * See if VPN and PPN are bounded by an existing mapping. + */ + if ((vaddr < pmbe->vpn) || (vaddr >= (pmbe->vpn + pmbe->size))) + continue; + if ((phys < pmbe->ppn) || (phys >= (pmbe->ppn + pmbe->size))) + continue; + + /* + * Now see if we're in range of a simple mapping. + */ + if (size <= pmbe->size) { + read_unlock(&pmb_rwlock); + return true; + } + + span = pmbe->size; + + /* + * Finally for sizes that involve compound mappings, walk + * the chain. + */ + for (iter = pmbe->link; iter; iter = iter->link) + span += iter->size; + + /* + * Nothing else to do if the range requirements are met. + */ + if (size <= span) { + read_unlock(&pmb_rwlock); + return true; + } + } + + read_unlock(&pmb_rwlock); + return false; +} + +static bool pmb_size_valid(unsigned long size) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(pmb_sizes); i++) + if (pmb_sizes[i].size == size) + return true; + + return false; +} + +static inline bool pmb_addr_valid(unsigned long addr, unsigned long size) { - unsigned int pos; + return (addr >= P1SEG && (addr + size - 1) < P3SEG); +} -repeat: - pos = find_first_zero_bit(&pmb_map, NR_PMB_ENTRIES); +static inline bool pmb_prot_valid(pgprot_t prot) +{ + return (pgprot_val(prot) & _PAGE_USER) == 0; +} + +static int pmb_size_to_flags(unsigned long size) +{ + int i; - if (unlikely(pos > NR_PMB_ENTRIES)) - return -ENOSPC; + for (i = 0; i < ARRAY_SIZE(pmb_sizes); i++) + if (pmb_sizes[i].size == size) + return pmb_sizes[i].flag; - if (test_and_set_bit(pos, &pmb_map)) - goto repeat; + return 0; +} + +static int pmb_alloc_entry(void) +{ + int pos; + + pos = find_first_zero_bit(pmb_map, NR_PMB_ENTRIES); + if (pos >= 0 && pos < NR_PMB_ENTRIES) + __set_bit(pos, pmb_map); + else + pos = -ENOSPC; return pos; } @@ -73,21 +238,34 @@ static struct pmb_entry *pmb_alloc(unsigned long vpn, unsigned long ppn, unsigned long flags, int entry) { struct pmb_entry *pmbe; + unsigned long irqflags; + void *ret = NULL; int pos; + write_lock_irqsave(&pmb_rwlock, irqflags); + if (entry == PMB_NO_ENTRY) { pos = pmb_alloc_entry(); - if (pos < 0) - return ERR_PTR(pos); + if (unlikely(pos < 0)) { + ret = ERR_PTR(pos); + goto out; + } } else { - if (test_bit(entry, &pmb_map)) - return ERR_PTR(-ENOSPC); + if (__test_and_set_bit(entry, pmb_map)) { + ret = ERR_PTR(-ENOSPC); + goto out; + } + pos = entry; } + write_unlock_irqrestore(&pmb_rwlock, irqflags); + pmbe = &pmb_entry_list[pos]; - if (!pmbe) - return ERR_PTR(-ENOMEM); + + memset(pmbe, 0, sizeof(struct pmb_entry)); + + spin_lock_init(&pmbe->lock); pmbe->vpn = vpn; pmbe->ppn = ppn; @@ -95,169 +273,219 @@ static struct pmb_entry *pmb_alloc(unsigned long vpn, unsigned long ppn, pmbe->entry = pos; return pmbe; + +out: + write_unlock_irqrestore(&pmb_rwlock, irqflags); + return ret; } static void pmb_free(struct pmb_entry *pmbe) { - int pos = pmbe->entry; + __clear_bit(pmbe->entry, pmb_map); - pmbe->vpn = 0; - pmbe->ppn = 0; - pmbe->flags = 0; - pmbe->entry = 0; - - clear_bit(pos, &pmb_map); + pmbe->entry = PMB_NO_ENTRY; + pmbe->link = NULL; } /* - * Must be in P2 for __set_pmb_entry() + * Must be run uncached. */ -static void __set_pmb_entry(unsigned long vpn, unsigned long ppn, - unsigned long flags, int pos) +static void __set_pmb_entry(struct pmb_entry *pmbe) { - ctrl_outl(vpn | PMB_V, mk_pmb_addr(pos)); - -#ifdef CONFIG_CACHE_WRITETHROUGH - /* - * When we are in 32-bit address extended mode, CCR.CB becomes - * invalid, so care must be taken to manually adjust cacheable - * translations. - */ - if (likely(flags & PMB_C)) - flags |= PMB_WT; -#endif + unsigned long addr, data; - ctrl_outl(ppn | flags | PMB_V, mk_pmb_data(pos)); -} + addr = mk_pmb_addr(pmbe->entry); + data = mk_pmb_data(pmbe->entry); -static void __uses_jump_to_uncached set_pmb_entry(struct pmb_entry *pmbe) -{ jump_to_uncached(); - __set_pmb_entry(pmbe->vpn, pmbe->ppn, pmbe->flags, pmbe->entry); + + /* Set V-bit */ + __raw_writel(pmbe->vpn | PMB_V, addr); + __raw_writel(pmbe->ppn | pmbe->flags | PMB_V, data); + back_to_cached(); } -static void __uses_jump_to_uncached clear_pmb_entry(struct pmb_entry *pmbe) +static void __clear_pmb_entry(struct pmb_entry *pmbe) { - unsigned int entry = pmbe->entry; - unsigned long addr; + unsigned long addr, data; + unsigned long addr_val, data_val; - if (unlikely(entry >= NR_PMB_ENTRIES)) - return; + addr = mk_pmb_addr(pmbe->entry); + data = mk_pmb_data(pmbe->entry); - jump_to_uncached(); + addr_val = __raw_readl(addr); + data_val = __raw_readl(data); /* Clear V-bit */ - addr = mk_pmb_addr(entry); - ctrl_outl(ctrl_inl(addr) & ~PMB_V, addr); - - addr = mk_pmb_data(entry); - ctrl_outl(ctrl_inl(addr) & ~PMB_V, addr); - - back_to_cached(); + writel_uncached(addr_val & ~PMB_V, addr); + writel_uncached(data_val & ~PMB_V, data); } +#ifdef CONFIG_PM +static void set_pmb_entry(struct pmb_entry *pmbe) +{ + unsigned long flags; -static struct { - unsigned long size; - int flag; -} pmb_sizes[] = { - { .size = 0x20000000, .flag = PMB_SZ_512M, }, - { .size = 0x08000000, .flag = PMB_SZ_128M, }, - { .size = 0x04000000, .flag = PMB_SZ_64M, }, - { .size = 0x01000000, .flag = PMB_SZ_16M, }, -}; + spin_lock_irqsave(&pmbe->lock, flags); + __set_pmb_entry(pmbe); + spin_unlock_irqrestore(&pmbe->lock, flags); +} +#endif /* CONFIG_PM */ -long pmb_remap(unsigned long vaddr, unsigned long phys, - unsigned long size, unsigned long flags) +int pmb_bolt_mapping(unsigned long vaddr, phys_addr_t phys, + unsigned long size, pgprot_t prot) { struct pmb_entry *pmbp, *pmbe; - unsigned long wanted; - int pmb_flags, i; - long err; - - /* Convert typical pgprot value to the PMB equivalent */ - if (flags & _PAGE_CACHABLE) { - if (flags & _PAGE_WT) - pmb_flags = PMB_WT; - else - pmb_flags = PMB_C; - } else - pmb_flags = PMB_WT | PMB_UB; + unsigned long orig_addr, orig_size; + unsigned long flags, pmb_flags; + int i, mapped; + if (!pmb_addr_valid(vaddr, size)) + return -EFAULT; + if (pmb_mapping_exists(vaddr, phys, size)) + return 0; + + orig_addr = vaddr; + orig_size = size; + + flush_tlb_kernel_range(vaddr, vaddr + size); + + pmb_flags = pgprot_to_pmb_flags(prot); pmbp = NULL; - wanted = size; -again: - for (i = 0; i < ARRAY_SIZE(pmb_sizes); i++) { - if (size < pmb_sizes[i].size) - continue; + do { + for (i = mapped = 0; i < ARRAY_SIZE(pmb_sizes); i++) { + if (size < pmb_sizes[i].size) + continue; + + pmbe = pmb_alloc(vaddr, phys, pmb_flags | + pmb_sizes[i].flag, PMB_NO_ENTRY); + if (IS_ERR(pmbe)) { + pmb_unmap_entry(pmbp, mapped); + return PTR_ERR(pmbe); + } - pmbe = pmb_alloc(vaddr, phys, pmb_flags | pmb_sizes[i].flag, - PMB_NO_ENTRY); - if (IS_ERR(pmbe)) { - err = PTR_ERR(pmbe); - goto out; + spin_lock_irqsave(&pmbe->lock, flags); + + pmbe->size = pmb_sizes[i].size; + + __set_pmb_entry(pmbe); + + phys += pmbe->size; + vaddr += pmbe->size; + size -= pmbe->size; + + /* + * Link adjacent entries that span multiple PMB + * entries for easier tear-down. + */ + if (likely(pmbp)) { + spin_lock(&pmbp->lock); + pmbp->link = pmbe; + spin_unlock(&pmbp->lock); + } + + pmbp = pmbe; + + /* + * Instead of trying smaller sizes on every + * iteration (even if we succeed in allocating + * space), try using pmb_sizes[i].size again. + */ + i--; + mapped++; + + spin_unlock_irqrestore(&pmbe->lock, flags); } + } while (size >= SZ_16M); - set_pmb_entry(pmbe); + flush_cache_vmap(orig_addr, orig_addr + orig_size); - phys += pmb_sizes[i].size; - vaddr += pmb_sizes[i].size; - size -= pmb_sizes[i].size; + return 0; +} - /* - * Link adjacent entries that span multiple PMB entries - * for easier tear-down. - */ - if (likely(pmbp)) - pmbp->link = pmbe; +void __iomem *pmb_remap_caller(phys_addr_t phys, unsigned long size, + pgprot_t prot, void *caller) +{ + unsigned long vaddr; + phys_addr_t offset, last_addr; + phys_addr_t align_mask; + unsigned long aligned; + struct vm_struct *area; + int i, ret; - pmbp = pmbe; + if (!pmb_iomapping_enabled) + return NULL; - /* - * Instead of trying smaller sizes on every iteration - * (even if we succeed in allocating space), try using - * pmb_sizes[i].size again. - */ - i--; - } + /* + * Small mappings need to go through the TLB. + */ + if (size < SZ_16M) + return ERR_PTR(-EINVAL); + if (!pmb_prot_valid(prot)) + return ERR_PTR(-EINVAL); - if (size >= 0x1000000) - goto again; + for (i = 0; i < ARRAY_SIZE(pmb_sizes); i++) + if (size >= pmb_sizes[i].size) + break; - return wanted - size; + last_addr = phys + size; + align_mask = ~(pmb_sizes[i].size - 1); + offset = phys & ~align_mask; + phys &= align_mask; + aligned = ALIGN(last_addr, pmb_sizes[i].size) - phys; -out: - if (pmbp) - __pmb_unmap(pmbp); + /* + * XXX: This should really start from uncached_end, but this + * causes the MMU to reset, so for now we restrict it to the + * 0xb000...0xc000 range. + */ + area = __get_vm_area_caller(aligned, VM_IOREMAP, 0xb0000000, + P3SEG, caller); + if (!area) + return NULL; + + area->phys_addr = phys; + vaddr = (unsigned long)area->addr; - return err; + ret = pmb_bolt_mapping(vaddr, phys, size, prot); + if (unlikely(ret != 0)) + return ERR_PTR(ret); + + return (void __iomem *)(offset + (char *)vaddr); } -void pmb_unmap(unsigned long addr) +int pmb_unmap(void __iomem *addr) { struct pmb_entry *pmbe = NULL; - int i; + unsigned long vaddr = (unsigned long __force)addr; + int i, found = 0; + + read_lock(&pmb_rwlock); for (i = 0; i < ARRAY_SIZE(pmb_entry_list); i++) { - if (test_bit(i, &pmb_map)) { + if (test_bit(i, pmb_map)) { pmbe = &pmb_entry_list[i]; - if (pmbe->vpn == addr) + if (pmbe->vpn == vaddr) { + found = 1; break; + } } } - if (unlikely(!pmbe)) - return; + read_unlock(&pmb_rwlock); - __pmb_unmap(pmbe); + if (found) { + pmb_unmap_entry(pmbe, NR_PMB_ENTRIES); + return 0; + } + + return -EINVAL; } -static void __pmb_unmap(struct pmb_entry *pmbe) +static void __pmb_unmap_entry(struct pmb_entry *pmbe, int depth) { - BUG_ON(!test_bit(pmbe->entry, &pmb_map)); - do { struct pmb_entry *pmblink = pmbe; @@ -268,102 +496,321 @@ static void __pmb_unmap(struct pmb_entry *pmbe) * this entry in pmb_alloc() (even if we haven't filled * it yet). * - * Therefore, calling clear_pmb_entry() is safe as no + * Therefore, calling __clear_pmb_entry() is safe as no * other mapping can be using that slot. */ - clear_pmb_entry(pmbe); + __clear_pmb_entry(pmbe); + + flush_cache_vunmap(pmbe->vpn, pmbe->vpn + pmbe->size); pmbe = pmblink->link; pmb_free(pmblink); - } while (pmbe); + } while (pmbe && --depth); } -#ifdef CONFIG_PMB -int __uses_jump_to_uncached pmb_init(void) +static void pmb_unmap_entry(struct pmb_entry *pmbe, int depth) { - unsigned int i; - long size, ret; + unsigned long flags; - jump_to_uncached(); + if (unlikely(!pmbe)) + return; + + write_lock_irqsave(&pmb_rwlock, flags); + __pmb_unmap_entry(pmbe, depth); + write_unlock_irqrestore(&pmb_rwlock, flags); +} + +static void __init pmb_notify(void) +{ + int i; + + pr_info("PMB: boot mappings:\n"); + + read_lock(&pmb_rwlock); + + for (i = 0; i < ARRAY_SIZE(pmb_entry_list); i++) { + struct pmb_entry *pmbe; + + if (!test_bit(i, pmb_map)) + continue; + + pmbe = &pmb_entry_list[i]; + + pr_info(" 0x%08lx -> 0x%08lx [ %4ldMB %2scached ]\n", + pmbe->vpn >> PAGE_SHIFT, pmbe->ppn >> PAGE_SHIFT, + pmbe->size >> 20, (pmbe->flags & PMB_C) ? "" : "un"); + } + + read_unlock(&pmb_rwlock); +} + +/* + * Sync our software copy of the PMB mappings with those in hardware. The + * mappings in the hardware PMB were either set up by the bootloader or + * very early on by the kernel. + */ +static void __init pmb_synchronize(void) +{ + struct pmb_entry *pmbp = NULL; + int i, j; /* - * Insert PMB entries for the P1 and P2 areas so that, after - * we've switched the MMU to 32-bit mode, the semantics of P1 - * and P2 are the same as in 29-bit mode, e.g. + * Run through the initial boot mappings, log the established + * ones, and blow away anything that falls outside of the valid + * PPN range. Specifically, we only care about existing mappings + * that impact the cached/uncached sections. + * + * Note that touching these can be a bit of a minefield; the boot + * loader can establish multi-page mappings with the same caching + * attributes, so we need to ensure that we aren't modifying a + * mapping that we're presently executing from, or may execute + * from in the case of straddling page boundaries. * - * P1 - provides a cached window onto physical memory - * P2 - provides an uncached window onto physical memory + * In the future we will have to tidy up after the boot loader by + * jumping between the cached and uncached mappings and tearing + * down alternating mappings while executing from the other. */ - size = __MEMORY_START + __MEMORY_SIZE; + for (i = 0; i < NR_PMB_ENTRIES; i++) { + unsigned long addr, data; + unsigned long addr_val, data_val; + unsigned long ppn, vpn, flags; + unsigned long irqflags; + unsigned int size; + struct pmb_entry *pmbe; - ret = pmb_remap(P1SEG, 0x00000000, size, PMB_C); - BUG_ON(ret != size); + addr = mk_pmb_addr(i); + data = mk_pmb_data(i); - ret = pmb_remap(P2SEG, 0x00000000, size, PMB_WT | PMB_UB); - BUG_ON(ret != size); + addr_val = __raw_readl(addr); + data_val = __raw_readl(data); - ctrl_outl(0, PMB_IRMCR); + /* + * Skip over any bogus entries + */ + if (!(data_val & PMB_V) || !(addr_val & PMB_V)) + continue; - /* PMB.SE and UB[7] */ - ctrl_outl(PASCR_SE | (1 << 7), PMB_PASCR); + ppn = data_val & PMB_PFN_MASK; + vpn = addr_val & PMB_PFN_MASK; - /* Flush out the TLB */ - i = ctrl_inl(MMUCR); - i |= MMUCR_TI; - ctrl_outl(i, MMUCR); + /* + * Only preserve in-range mappings. + */ + if (!pmb_ppn_in_range(ppn)) { + /* + * Invalidate anything out of bounds. + */ + writel_uncached(addr_val & ~PMB_V, addr); + writel_uncached(data_val & ~PMB_V, data); + continue; + } - back_to_cached(); + /* + * Update the caching attributes if necessary + */ + if (data_val & PMB_C) { + data_val &= ~PMB_CACHE_MASK; + data_val |= pmb_cache_flags(); - return 0; + writel_uncached(data_val, data); + } + + size = data_val & PMB_SZ_MASK; + flags = size | (data_val & PMB_CACHE_MASK); + + pmbe = pmb_alloc(vpn, ppn, flags, i); + if (IS_ERR(pmbe)) { + WARN_ON_ONCE(1); + continue; + } + + spin_lock_irqsave(&pmbe->lock, irqflags); + + for (j = 0; j < ARRAY_SIZE(pmb_sizes); j++) + if (pmb_sizes[j].flag == size) + pmbe->size = pmb_sizes[j].size; + + if (pmbp) { + spin_lock(&pmbp->lock); + + /* + * Compare the previous entry against the current one to + * see if the entries span a contiguous mapping. If so, + * setup the entry links accordingly. Compound mappings + * are later coalesced. + */ + if (pmb_can_merge(pmbp, pmbe)) + pmbp->link = pmbe; + + spin_unlock(&pmbp->lock); + } + + pmbp = pmbe; + + spin_unlock_irqrestore(&pmbe->lock, irqflags); + } } -#else -int __uses_jump_to_uncached pmb_init(void) + +static void __init pmb_merge(struct pmb_entry *head) { + unsigned long span, newsize; + struct pmb_entry *tail; + int i = 1, depth = 0; + + span = newsize = head->size; + + tail = head->link; + while (tail) { + span += tail->size; + + if (pmb_size_valid(span)) { + newsize = span; + depth = i; + } + + /* This is the end of the line.. */ + if (!tail->link) + break; + + tail = tail->link; + i++; + } + + /* + * The merged page size must be valid. + */ + if (!pmb_size_valid(newsize)) + return; + + head->flags &= ~PMB_SZ_MASK; + head->flags |= pmb_size_to_flags(newsize); + + head->size = newsize; + + __pmb_unmap_entry(head->link, depth); + __set_pmb_entry(head); +} + +static void __init pmb_coalesce(void) +{ + unsigned long flags; int i; - unsigned long addr, data; - jump_to_uncached(); + write_lock_irqsave(&pmb_rwlock, flags); - for (i = 0; i < PMB_ENTRY_MAX; i++) { + for (i = 0; i < ARRAY_SIZE(pmb_entry_list); i++) { struct pmb_entry *pmbe; - unsigned long vpn, ppn, flags; - addr = PMB_DATA + (i << PMB_E_SHIFT); - data = ctrl_inl(addr); - if (!(data & PMB_V)) + if (!test_bit(i, pmb_map)) continue; - if (data & PMB_C) { -#if defined(CONFIG_CACHE_WRITETHROUGH) - data |= PMB_WT; -#elif defined(CONFIG_CACHE_WRITEBACK) - data &= ~PMB_WT; -#else - data &= ~(PMB_C | PMB_WT); -#endif - } - ctrl_outl(data, addr); + pmbe = &pmb_entry_list[i]; - ppn = data & PMB_PFN_MASK; + /* + * We're only interested in compound mappings + */ + if (!pmbe->link) + continue; - flags = data & (PMB_C | PMB_WT | PMB_UB); - flags |= data & PMB_SZ_MASK; + /* + * Nothing to do if it already uses the largest possible + * page size. + */ + if (pmbe->size == SZ_512M) + continue; - addr = PMB_ADDR + (i << PMB_E_SHIFT); - data = ctrl_inl(addr); + pmb_merge(pmbe); + } - vpn = data & PMB_PFN_MASK; + write_unlock_irqrestore(&pmb_rwlock, flags); +} - pmbe = pmb_alloc(vpn, ppn, flags, i); - WARN_ON(IS_ERR(pmbe)); +#ifdef CONFIG_UNCACHED_MAPPING +static void __init pmb_resize(void) +{ + int i; + + /* + * If the uncached mapping was constructed by the kernel, it will + * already be a reasonable size. + */ + if (uncached_size == SZ_16M) + return; + + read_lock(&pmb_rwlock); + + for (i = 0; i < ARRAY_SIZE(pmb_entry_list); i++) { + struct pmb_entry *pmbe; + unsigned long flags; + + if (!test_bit(i, pmb_map)) + continue; + + pmbe = &pmb_entry_list[i]; + + if (pmbe->vpn != uncached_start) + continue; + + /* + * Found it, now resize it. + */ + spin_lock_irqsave(&pmbe->lock, flags); + + pmbe->size = SZ_16M; + pmbe->flags &= ~PMB_SZ_MASK; + pmbe->flags |= pmb_size_to_flags(pmbe->size); + + uncached_resize(pmbe->size); + + __set_pmb_entry(pmbe); + + spin_unlock_irqrestore(&pmbe->lock, flags); } - back_to_cached(); + read_lock(&pmb_rwlock); +} +#endif + +static int __init early_pmb(char *p) +{ + if (!p) + return 0; + + if (strstr(p, "iomap")) + pmb_iomapping_enabled = 1; return 0; } -#endif /* CONFIG_PMB */ +early_param("pmb", early_pmb); + +void __init pmb_init(void) +{ + /* Synchronize software state */ + pmb_synchronize(); + + /* Attempt to combine compound mappings */ + pmb_coalesce(); + +#ifdef CONFIG_UNCACHED_MAPPING + /* Resize initial mappings, if necessary */ + pmb_resize(); +#endif + + /* Log them */ + pmb_notify(); + + writel_uncached(0, PMB_IRMCR); + + /* Flush out the TLB */ + local_flush_tlb_all(); + ctrl_barrier(); +} + +bool __in_29bit_mode(void) +{ + return (__raw_readl(PMB_PASCR) & PASCR_SE) == 0; +} static int pmb_seq_show(struct seq_file *file, void *iter) { @@ -378,8 +825,8 @@ static int pmb_seq_show(struct seq_file *file, void *iter) unsigned int size; char *sz_str = NULL; - addr = ctrl_inl(mk_pmb_addr(i)); - data = ctrl_inl(mk_pmb_data(i)); + addr = __raw_readl(mk_pmb_addr(i)); + data = __raw_readl(mk_pmb_data(i)); size = data & PMB_SZ_MASK; sz_str = (size == PMB_SZ_16M) ? " 16MB": @@ -425,7 +872,7 @@ static int __init pmb_debugfs_init(void) return 0; } -postcore_initcall(pmb_debugfs_init); +subsys_initcall(pmb_debugfs_init); #ifdef CONFIG_PM static int pmb_sysdev_suspend(struct sys_device *dev, pm_message_t state) @@ -437,14 +884,21 @@ static int pmb_sysdev_suspend(struct sys_device *dev, pm_message_t state) if (state.event == PM_EVENT_ON && prev_state.event == PM_EVENT_FREEZE) { struct pmb_entry *pmbe; + + read_lock(&pmb_rwlock); + for (i = 0; i < ARRAY_SIZE(pmb_entry_list); i++) { - if (test_bit(i, &pmb_map)) { + if (test_bit(i, pmb_map)) { pmbe = &pmb_entry_list[i]; set_pmb_entry(pmbe); } } + + read_unlock(&pmb_rwlock); } + prev_state = state; + return 0; } @@ -462,6 +916,5 @@ static int __init pmb_sysdev_init(void) { return sysdev_driver_register(&cpu_sysdev_class, &pmb_sysdev_driver); } - subsys_initcall(pmb_sysdev_init); #endif |