summaryrefslogtreecommitdiffstats
path: root/arch/mips/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/mips/mm')
-rw-r--r--arch/mips/mm/c-r4k.c22
-rw-r--r--arch/mips/mm/c-sb1.c42
-rw-r--r--arch/mips/mm/fault.c4
-rw-r--r--arch/mips/mm/init.c209
-rw-r--r--arch/mips/mm/ioremap.c2
-rw-r--r--arch/mips/mm/pg-r4k.c30
-rw-r--r--arch/mips/mm/pgtable-32.c7
-rw-r--r--arch/mips/mm/pgtable-64.c14
-rw-r--r--arch/mips/mm/tlbex.c68
9 files changed, 341 insertions, 57 deletions
diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c
index cc895dad71d..df04a315d83 100644
--- a/arch/mips/mm/c-r4k.c
+++ b/arch/mips/mm/c-r4k.c
@@ -323,7 +323,6 @@ static void __init r4k_blast_scache_setup(void)
static inline void local_r4k_flush_cache_all(void * args)
{
r4k_blast_dcache();
- r4k_blast_icache();
}
static void r4k_flush_cache_all(void)
@@ -359,21 +358,19 @@ static void r4k___flush_cache_all(void)
static inline void local_r4k_flush_cache_range(void * args)
{
struct vm_area_struct *vma = args;
- int exec;
if (!(cpu_context(smp_processor_id(), vma->vm_mm)))
return;
- exec = vma->vm_flags & VM_EXEC;
- if (cpu_has_dc_aliases || exec)
- r4k_blast_dcache();
- if (exec)
- r4k_blast_icache();
+ r4k_blast_dcache();
}
static void r4k_flush_cache_range(struct vm_area_struct *vma,
unsigned long start, unsigned long end)
{
+ if (!cpu_has_dc_aliases)
+ return;
+
r4k_on_each_cpu(local_r4k_flush_cache_range, vma, 1, 1);
}
@@ -384,18 +381,21 @@ static inline void local_r4k_flush_cache_mm(void * args)
if (!cpu_context(smp_processor_id(), mm))
return;
- r4k_blast_dcache();
- r4k_blast_icache();
-
/*
* Kludge alert. For obscure reasons R4000SC and R4400SC go nuts if we
* only flush the primary caches but R10000 and R12000 behave sane ...
+ * R4000SC and R4400SC indexed S-cache ops also invalidate primary
+ * caches, so we can bail out early.
*/
if (current_cpu_data.cputype == CPU_R4000SC ||
current_cpu_data.cputype == CPU_R4000MC ||
current_cpu_data.cputype == CPU_R4400SC ||
- current_cpu_data.cputype == CPU_R4400MC)
+ current_cpu_data.cputype == CPU_R4400MC) {
r4k_blast_scache();
+ return;
+ }
+
+ r4k_blast_dcache();
}
static void r4k_flush_cache_mm(struct mm_struct *mm)
diff --git a/arch/mips/mm/c-sb1.c b/arch/mips/mm/c-sb1.c
index 5537558f19f..3a8afd47fea 100644
--- a/arch/mips/mm/c-sb1.c
+++ b/arch/mips/mm/c-sb1.c
@@ -19,6 +19,7 @@
* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
*/
#include <linux/init.h>
+#include <linux/hardirq.h>
#include <asm/asm.h>
#include <asm/bootinfo.h>
@@ -49,6 +50,15 @@ static unsigned short dcache_sets;
static unsigned int icache_range_cutoff;
static unsigned int dcache_range_cutoff;
+static inline void sb1_on_each_cpu(void (*func) (void *info), void *info,
+ int retry, int wait)
+{
+ preempt_disable();
+ smp_call_function(func, info, retry, wait);
+ func(info);
+ preempt_enable();
+}
+
/*
* The dcache is fully coherent to the system, with one
* big caveat: the instruction stream. In other words,
@@ -226,13 +236,32 @@ static void sb1_flush_cache_page(struct vm_area_struct *vma, unsigned long addr,
args.vma = vma;
args.addr = addr;
args.pfn = pfn;
- on_each_cpu(sb1_flush_cache_page_ipi, (void *) &args, 1, 1);
+ sb1_on_each_cpu(sb1_flush_cache_page_ipi, (void *) &args, 1, 1);
}
#else
void sb1_flush_cache_page(struct vm_area_struct *vma, unsigned long addr, unsigned long pfn)
__attribute__((alias("local_sb1_flush_cache_page")));
#endif
+#ifdef CONFIG_SMP
+static void sb1_flush_cache_data_page_ipi(void *info)
+{
+ unsigned long start = (unsigned long)info;
+
+ __sb1_writeback_inv_dcache_range(start, start + PAGE_SIZE);
+}
+
+static void sb1_flush_cache_data_page(unsigned long addr)
+{
+ if (in_atomic())
+ __sb1_writeback_inv_dcache_range(addr, addr + PAGE_SIZE);
+ else
+ on_each_cpu(sb1_flush_cache_data_page_ipi, (void *) addr, 1, 1);
+}
+#else
+void sb1_flush_cache_data_page(unsigned long)
+ __attribute__((alias("local_sb1_flush_cache_data_page")));
+#endif
/*
* Invalidate all caches on this CPU
@@ -249,7 +278,7 @@ void sb1___flush_cache_all_ipi(void *ignored)
static void sb1___flush_cache_all(void)
{
- on_each_cpu(sb1___flush_cache_all_ipi, 0, 1, 1);
+ sb1_on_each_cpu(sb1___flush_cache_all_ipi, 0, 1, 1);
}
#else
void sb1___flush_cache_all(void)
@@ -299,7 +328,7 @@ void sb1_flush_icache_range(unsigned long start, unsigned long end)
args.start = start;
args.end = end;
- on_each_cpu(sb1_flush_icache_range_ipi, &args, 1, 1);
+ sb1_on_each_cpu(sb1_flush_icache_range_ipi, &args, 1, 1);
}
#else
void sb1_flush_icache_range(unsigned long start, unsigned long end)
@@ -326,7 +355,7 @@ static void sb1_flush_cache_sigtramp_ipi(void *info)
static void sb1_flush_cache_sigtramp(unsigned long addr)
{
- on_each_cpu(sb1_flush_cache_sigtramp_ipi, (void *) addr, 1, 1);
+ sb1_on_each_cpu(sb1_flush_cache_sigtramp_ipi, (void *) addr, 1, 1);
}
#else
void sb1_flush_cache_sigtramp(unsigned long addr)
@@ -444,7 +473,6 @@ static __init void probe_cache_sizes(void)
void sb1_cache_init(void)
{
extern char except_vec2_sb1;
- extern char handle_vec2_sb1;
/* Special cache error handler for SB1 */
set_uncached_handler (0x100, &except_vec2_sb1, 0x80);
@@ -473,7 +501,7 @@ void sb1_cache_init(void)
flush_cache_sigtramp = sb1_flush_cache_sigtramp;
local_flush_data_cache_page = (void *) sb1_nop;
- flush_data_cache_page = (void *) sb1_nop;
+ flush_data_cache_page = sb1_flush_cache_data_page;
/* Full flush */
__flush_cache_all = sb1___flush_cache_all;
@@ -497,5 +525,5 @@ void sb1_cache_init(void)
:
: "memory");
- flush_cache_all();
+ local_sb1___flush_cache_all();
}
diff --git a/arch/mips/mm/fault.c b/arch/mips/mm/fault.c
index 8423d859077..6f90e7ef66a 100644
--- a/arch/mips/mm/fault.c
+++ b/arch/mips/mm/fault.c
@@ -60,6 +60,10 @@ asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long write,
*/
if (unlikely(address >= VMALLOC_START && address <= VMALLOC_END))
goto vmalloc_fault;
+#ifdef MODULE_START
+ if (unlikely(address >= MODULE_START && address < MODULE_END))
+ goto vmalloc_fault;
+#endif
/*
* If we're in an interrupt or have no user
diff --git a/arch/mips/mm/init.c b/arch/mips/mm/init.c
index 88b72c9a849..9e29ba9205f 100644
--- a/arch/mips/mm/init.c
+++ b/arch/mips/mm/init.c
@@ -30,11 +30,34 @@
#include <asm/cachectl.h>
#include <asm/cpu.h>
#include <asm/dma.h>
+#include <asm/kmap_types.h>
#include <asm/mmu_context.h>
#include <asm/sections.h>
#include <asm/pgtable.h>
#include <asm/pgalloc.h>
#include <asm/tlb.h>
+#include <asm/fixmap.h>
+
+/* Atomicity and interruptability */
+#ifdef CONFIG_MIPS_MT_SMTC
+
+#include <asm/mipsmtregs.h>
+
+#define ENTER_CRITICAL(flags) \
+ { \
+ unsigned int mvpflags; \
+ local_irq_save(flags);\
+ mvpflags = dvpe()
+#define EXIT_CRITICAL(flags) \
+ evpe(mvpflags); \
+ local_irq_restore(flags); \
+ }
+#else
+
+#define ENTER_CRITICAL(flags) local_irq_save(flags)
+#define EXIT_CRITICAL(flags) local_irq_restore(flags)
+
+#endif /* CONFIG_MIPS_MT_SMTC */
DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
@@ -67,9 +90,9 @@ unsigned long setup_zero_pages(void)
if (!empty_zero_page)
panic("Oh boy, that early out of memory?");
- page = virt_to_page(empty_zero_page);
+ page = virt_to_page((void *)empty_zero_page);
split_page(page, order);
- while (page < virt_to_page(empty_zero_page + (PAGE_SIZE << order))) {
+ while (page < virt_to_page((void *)(empty_zero_page + (PAGE_SIZE << order)))) {
SetPageReserved(page);
page++;
}
@@ -80,13 +103,142 @@ unsigned long setup_zero_pages(void)
return 1UL << order;
}
-#ifdef CONFIG_HIGHMEM
-pte_t *kmap_pte;
-pgprot_t kmap_prot;
+/*
+ * These are almost like kmap_atomic / kunmap_atmic except they take an
+ * additional address argument as the hint.
+ */
#define kmap_get_fixmap_pte(vaddr) \
pte_offset_kernel(pmd_offset(pud_offset(pgd_offset_k(vaddr), (vaddr)), (vaddr)), (vaddr))
+#ifdef CONFIG_MIPS_MT_SMTC
+static pte_t *kmap_coherent_pte;
+static void __init kmap_coherent_init(void)
+{
+ unsigned long vaddr;
+
+ /* cache the first coherent kmap pte */
+ vaddr = __fix_to_virt(FIX_CMAP_BEGIN);
+ kmap_coherent_pte = kmap_get_fixmap_pte(vaddr);
+}
+#else
+static inline void kmap_coherent_init(void) {}
+#endif
+
+static inline void *kmap_coherent(struct page *page, unsigned long addr)
+{
+ enum fixed_addresses idx;
+ unsigned long vaddr, flags, entrylo;
+ unsigned long old_ctx;
+ pte_t pte;
+ int tlbidx;
+
+ inc_preempt_count();
+ idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
+#ifdef CONFIG_MIPS_MT_SMTC
+ idx += FIX_N_COLOURS * smp_processor_id();
+#endif
+ vaddr = __fix_to_virt(FIX_CMAP_END - idx);
+ pte = mk_pte(page, PAGE_KERNEL);
+#if defined(CONFIG_64BIT_PHYS_ADDR) && defined(CONFIG_CPU_MIPS32_R1)
+ entrylo = pte.pte_high;
+#else
+ entrylo = pte_val(pte) >> 6;
+#endif
+
+ ENTER_CRITICAL(flags);
+ old_ctx = read_c0_entryhi();
+ write_c0_entryhi(vaddr & (PAGE_MASK << 1));
+ write_c0_entrylo0(entrylo);
+ write_c0_entrylo1(entrylo);
+#ifdef CONFIG_MIPS_MT_SMTC
+ set_pte(kmap_coherent_pte - (FIX_CMAP_END - idx), pte);
+ /* preload TLB instead of local_flush_tlb_one() */
+ mtc0_tlbw_hazard();
+ tlb_probe();
+ tlb_probe_hazard();
+ tlbidx = read_c0_index();
+ mtc0_tlbw_hazard();
+ if (tlbidx < 0)
+ tlb_write_random();
+ else
+ tlb_write_indexed();
+#else
+ tlbidx = read_c0_wired();
+ write_c0_wired(tlbidx + 1);
+ write_c0_index(tlbidx);
+ mtc0_tlbw_hazard();
+ tlb_write_indexed();
+#endif
+ tlbw_use_hazard();
+ write_c0_entryhi(old_ctx);
+ EXIT_CRITICAL(flags);
+
+ return (void*) vaddr;
+}
+
+#define UNIQUE_ENTRYHI(idx) (CKSEG0 + ((idx) << (PAGE_SHIFT + 1)))
+
+static inline void kunmap_coherent(struct page *page)
+{
+#ifndef CONFIG_MIPS_MT_SMTC
+ unsigned int wired;
+ unsigned long flags, old_ctx;
+
+ ENTER_CRITICAL(flags);
+ old_ctx = read_c0_entryhi();
+ wired = read_c0_wired() - 1;
+ write_c0_wired(wired);
+ write_c0_index(wired);
+ write_c0_entryhi(UNIQUE_ENTRYHI(wired));
+ write_c0_entrylo0(0);
+ write_c0_entrylo1(0);
+ mtc0_tlbw_hazard();
+ tlb_write_indexed();
+ tlbw_use_hazard();
+ write_c0_entryhi(old_ctx);
+ EXIT_CRITICAL(flags);
+#endif
+ dec_preempt_count();
+ preempt_check_resched();
+}
+
+void copy_to_user_page(struct vm_area_struct *vma,
+ struct page *page, unsigned long vaddr, void *dst, const void *src,
+ unsigned long len)
+{
+ if (cpu_has_dc_aliases) {
+ void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
+ memcpy(vto, src, len);
+ kunmap_coherent(page);
+ } else
+ memcpy(dst, src, len);
+ if ((vma->vm_flags & VM_EXEC) && !cpu_has_ic_fills_f_dc)
+ flush_cache_page(vma, vaddr, page_to_pfn(page));
+}
+
+EXPORT_SYMBOL(copy_to_user_page);
+
+void copy_from_user_page(struct vm_area_struct *vma,
+ struct page *page, unsigned long vaddr, void *dst, const void *src,
+ unsigned long len)
+{
+ if (cpu_has_dc_aliases) {
+ void *vfrom =
+ kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
+ memcpy(dst, vfrom, len);
+ kunmap_coherent(page);
+ } else
+ memcpy(dst, src, len);
+}
+
+EXPORT_SYMBOL(copy_from_user_page);
+
+
+#ifdef CONFIG_HIGHMEM
+pte_t *kmap_pte;
+pgprot_t kmap_prot;
+
static void __init kmap_init(void)
{
unsigned long kmap_vstart;
@@ -97,11 +249,12 @@ static void __init kmap_init(void)
kmap_prot = PAGE_KERNEL;
}
+#endif /* CONFIG_HIGHMEM */
-#ifdef CONFIG_32BIT
void __init fixrange_init(unsigned long start, unsigned long end,
pgd_t *pgd_base)
{
+#if defined(CONFIG_HIGHMEM) || defined(CONFIG_MIPS_MT_SMTC)
pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
@@ -122,7 +275,7 @@ void __init fixrange_init(unsigned long start, unsigned long end,
for (; (k < PTRS_PER_PMD) && (vaddr != end); pmd++, k++) {
if (pmd_none(*pmd)) {
pte = (pte_t *) alloc_bootmem_low_pages(PAGE_SIZE);
- set_pmd(pmd, __pmd(pte));
+ set_pmd(pmd, __pmd((unsigned long)pte));
if (pte != pte_offset_kernel(pmd, 0))
BUG();
}
@@ -132,9 +285,8 @@ void __init fixrange_init(unsigned long start, unsigned long end,
}
j = 0;
}
+#endif
}
-#endif /* CONFIG_32BIT */
-#endif /* CONFIG_HIGHMEM */
#ifndef CONFIG_NEED_MULTIPLE_NODES
extern void pagetable_init(void);
@@ -175,6 +327,7 @@ void __init paging_init(void)
#ifdef CONFIG_HIGHMEM
kmap_init();
#endif
+ kmap_coherent_init();
max_dma = virt_to_phys((char *)MAX_DMA_ADDRESS) >> PAGE_SHIFT;
low = max_low_pfn;
@@ -290,15 +443,18 @@ void __init mem_init(void)
}
#endif /* !CONFIG_NEED_MULTIPLE_NODES */
-void free_init_pages(char *what, unsigned long begin, unsigned long end)
+static void free_init_pages(char *what, unsigned long begin, unsigned long end)
{
- unsigned long addr;
+ unsigned long pfn;
+
+ for (pfn = PFN_UP(begin); pfn < PFN_DOWN(end); pfn++) {
+ struct page *page = pfn_to_page(pfn);
+ void *addr = phys_to_virt(PFN_PHYS(pfn));
- for (addr = begin; addr < end; addr += PAGE_SIZE) {
- ClearPageReserved(virt_to_page(addr));
- init_page_count(virt_to_page(addr));
- memset((void *)addr, 0xcc, PAGE_SIZE);
- free_page(addr);
+ ClearPageReserved(page);
+ init_page_count(page);
+ memset(addr, POISON_FREE_INITMEM, PAGE_SIZE);
+ __free_page(page);
totalram_pages++;
}
printk(KERN_INFO "Freeing %s: %ldk freed\n", what, (end - begin) >> 10);
@@ -307,12 +463,9 @@ void free_init_pages(char *what, unsigned long begin, unsigned long end)
#ifdef CONFIG_BLK_DEV_INITRD
void free_initrd_mem(unsigned long start, unsigned long end)
{
-#ifdef CONFIG_64BIT
- /* Switch from KSEG0 to XKPHYS addresses */
- start = (unsigned long)phys_to_virt(CPHYSADDR(start));
- end = (unsigned long)phys_to_virt(CPHYSADDR(end));
-#endif
- free_init_pages("initrd memory", start, end);
+ free_init_pages("initrd memory",
+ virt_to_phys((void *)start),
+ virt_to_phys((void *)end));
}
#endif
@@ -320,17 +473,13 @@ extern unsigned long prom_free_prom_memory(void);
void free_initmem(void)
{
- unsigned long start, end, freed;
+ unsigned long freed;
freed = prom_free_prom_memory();
if (freed)
printk(KERN_INFO "Freeing firmware memory: %ldk freed\n",freed);
- start = (unsigned long)(&__init_begin);
- end = (unsigned long)(&__init_end);
-#ifdef CONFIG_64BIT
- start = PAGE_OFFSET | CPHYSADDR(start);
- end = PAGE_OFFSET | CPHYSADDR(end);
-#endif
- free_init_pages("unused kernel memory", start, end);
+ free_init_pages("unused kernel memory",
+ __pa_symbol(&__init_begin),
+ __pa_symbol(&__init_end));
}
diff --git a/arch/mips/mm/ioremap.c b/arch/mips/mm/ioremap.c
index 3101d1db559..cea7d0ea36e 100644
--- a/arch/mips/mm/ioremap.c
+++ b/arch/mips/mm/ioremap.c
@@ -176,7 +176,7 @@ void __iomem * __ioremap(phys_t phys_addr, phys_t size, unsigned long flags)
#define IS_KSEG1(addr) (((unsigned long)(addr) & ~0x1fffffffUL) == CKSEG1)
-void __iounmap(volatile void __iomem *addr)
+void __iounmap(const volatile void __iomem *addr)
{
struct vm_struct *p;
diff --git a/arch/mips/mm/pg-r4k.c b/arch/mips/mm/pg-r4k.c
index b7c749232ff..d41fc5885e8 100644
--- a/arch/mips/mm/pg-r4k.c
+++ b/arch/mips/mm/pg-r4k.c
@@ -270,6 +270,20 @@ static inline void build_addiu_a2_a0(unsigned long offset)
emit_instruction(mi);
}
+static inline void build_addiu_a2(unsigned long offset)
+{
+ union mips_instruction mi;
+
+ BUG_ON(offset > 0x7fff);
+
+ mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op;
+ mi.i_format.rs = 6; /* $a2 */
+ mi.i_format.rt = 6; /* $a2 */
+ mi.i_format.simmediate = offset;
+
+ emit_instruction(mi);
+}
+
static inline void build_addiu_a1(unsigned long offset)
{
union mips_instruction mi;
@@ -333,6 +347,7 @@ static inline void build_jr_ra(void)
void __init build_clear_page(void)
{
unsigned int loop_start;
+ unsigned long off;
epc = (unsigned int *) &clear_page_array;
instruction_pending = 0;
@@ -369,7 +384,12 @@ void __init build_clear_page(void)
}
}
- build_addiu_a2_a0(PAGE_SIZE - (cpu_has_prefetch ? pref_offset_clear : 0));
+ off = PAGE_SIZE - (cpu_has_prefetch ? pref_offset_clear : 0);
+ if (off > 0x7fff) {
+ build_addiu_a2_a0(off >> 1);
+ build_addiu_a2(off >> 1);
+ } else
+ build_addiu_a2_a0(off);
if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
build_insn_word(0x3c01a000); /* lui $at, 0xa000 */
@@ -420,12 +440,18 @@ dest = label();
void __init build_copy_page(void)
{
unsigned int loop_start;
+ unsigned long off;
epc = (unsigned int *) &copy_page_array;
store_offset = load_offset = 0;
instruction_pending = 0;
- build_addiu_a2_a0(PAGE_SIZE - (cpu_has_prefetch ? pref_offset_copy : 0));
+ off = PAGE_SIZE - (cpu_has_prefetch ? pref_offset_copy : 0);
+ if (off > 0x7fff) {
+ build_addiu_a2_a0(off >> 1);
+ build_addiu_a2(off >> 1);
+ } else
+ build_addiu_a2_a0(off);
if (R4600_V2_HIT_CACHEOP_WAR && cpu_is_r4600_v2_x())
build_insn_word(0x3c01a000); /* lui $at, 0xa000 */
diff --git a/arch/mips/mm/pgtable-32.c b/arch/mips/mm/pgtable-32.c
index 4bdaa05f485..4a61e624b0e 100644
--- a/arch/mips/mm/pgtable-32.c
+++ b/arch/mips/mm/pgtable-32.c
@@ -31,9 +31,10 @@ void pgd_init(unsigned long page)
void __init pagetable_init(void)
{
-#ifdef CONFIG_HIGHMEM
unsigned long vaddr;
- pgd_t *pgd, *pgd_base;
+ pgd_t *pgd_base;
+#ifdef CONFIG_HIGHMEM
+ pgd_t *pgd;
pud_t *pud;
pmd_t *pmd;
pte_t *pte;
@@ -44,7 +45,6 @@ void __init pagetable_init(void)
pgd_init((unsigned long)swapper_pg_dir
+ sizeof(pgd_t) * USER_PTRS_PER_PGD);
-#ifdef CONFIG_HIGHMEM
pgd_base = swapper_pg_dir;
/*
@@ -53,6 +53,7 @@ void __init pagetable_init(void)
vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
fixrange_init(vaddr, 0, pgd_base);
+#ifdef CONFIG_HIGHMEM
/*
* Permanent kmaps:
*/
diff --git a/arch/mips/mm/pgtable-64.c b/arch/mips/mm/pgtable-64.c
index 44b5e97fff6..c46eb651bf0 100644
--- a/arch/mips/mm/pgtable-64.c
+++ b/arch/mips/mm/pgtable-64.c
@@ -8,6 +8,7 @@
*/
#include <linux/init.h>
#include <linux/mm.h>
+#include <asm/fixmap.h>
#include <asm/pgtable.h>
void pgd_init(unsigned long page)
@@ -52,7 +53,20 @@ void pmd_init(unsigned long addr, unsigned long pagetable)
void __init pagetable_init(void)
{
+ unsigned long vaddr;
+ pgd_t *pgd_base;
+
/* Initialize the entire pgd. */
pgd_init((unsigned long)swapper_pg_dir);
+#ifdef MODULE_START
+ pgd_init((unsigned long)module_pg_dir);
+#endif
pmd_init((unsigned long)invalid_pmd_table, (unsigned long)invalid_pte_table);
+
+ pgd_base = swapper_pg_dir;
+ /*
+ * Fixed mappings:
+ */
+ vaddr = __fix_to_virt(__end_of_fixed_addresses - 1) & PMD_MASK;
+ fixrange_init(vaddr, 0, pgd_base);
}
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index 6f8b25cfa6f..492c518e7ba 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -102,7 +102,7 @@ enum opcode {
insn_addu, insn_addiu, insn_and, insn_andi, insn_beq,
insn_beql, insn_bgez, insn_bgezl, insn_bltz, insn_bltzl,
insn_bne, insn_daddu, insn_daddiu, insn_dmfc0, insn_dmtc0,
- insn_dsll, insn_dsll32, insn_dsra, insn_dsrl,
+ insn_dsll, insn_dsll32, insn_dsra, insn_dsrl, insn_dsrl32,
insn_dsubu, insn_eret, insn_j, insn_jal, insn_jr, insn_ld,
insn_ll, insn_lld, insn_lui, insn_lw, insn_mfc0, insn_mtc0,
insn_ori, insn_rfe, insn_sc, insn_scd, insn_sd, insn_sll,
@@ -145,6 +145,7 @@ static __initdata struct insn insn_table[] = {
{ insn_dsll32, M(spec_op,0,0,0,0,dsll32_op), RT | RD | RE },
{ insn_dsra, M(spec_op,0,0,0,0,dsra_op), RT | RD | RE },
{ insn_dsrl, M(spec_op,0,0,0,0,dsrl_op), RT | RD | RE },
+ { insn_dsrl32, M(spec_op,0,0,0,0,dsrl32_op), RT | RD | RE },
{ insn_dsubu, M(spec_op,0,0,0,0,dsubu_op), RS | RT | RD },
{ insn_eret, M(cop0_op,cop_op,0,0,0,eret_op), 0 },
{ insn_j, M(j_op,0,0,0,0,0), JIMM },
@@ -385,6 +386,7 @@ I_u2u1u3(_dsll);
I_u2u1u3(_dsll32);
I_u2u1u3(_dsra);
I_u2u1u3(_dsrl);
+I_u2u1u3(_dsrl32);
I_u3u1u2(_dsubu);
I_0(_eret);
I_u1(_j);
@@ -421,6 +423,9 @@ enum label_id {
label_invalid,
label_second_part,
label_leave,
+#ifdef MODULE_START
+ label_module_alloc,
+#endif
label_vmalloc,
label_vmalloc_done,
label_tlbw_hazard,
@@ -453,6 +458,9 @@ static __init void build_label(struct label **lab, u32 *addr,
L_LA(_second_part)
L_LA(_leave)
+#ifdef MODULE_START
+L_LA(_module_alloc)
+#endif
L_LA(_vmalloc)
L_LA(_vmalloc_done)
L_LA(_tlbw_hazard)
@@ -684,6 +692,13 @@ static void __init il_bgezl(u32 **p, struct reloc **r, unsigned int reg,
i_bgezl(p, reg, 0);
}
+static void __init __attribute__((unused))
+il_bgez(u32 **p, struct reloc **r, unsigned int reg, enum label_id l)
+{
+ r_mips_pc16(r, *p, l);
+ i_bgez(p, reg, 0);
+}
+
/* The only general purpose registers allowed in TLB handlers. */
#define K0 26
#define K1 27
@@ -968,7 +983,11 @@ build_get_pmde64(u32 **p, struct label **l, struct reloc **r,
* The vmalloc handling is not in the hotpath.
*/
i_dmfc0(p, tmp, C0_BADVADDR);
+#ifdef MODULE_START
+ il_bltz(p, r, tmp, label_module_alloc);
+#else
il_bltz(p, r, tmp, label_vmalloc);
+#endif
/* No i_nop needed here, since the next insn doesn't touch TMP. */
#ifdef CONFIG_SMP
@@ -996,7 +1015,12 @@ build_get_pmde64(u32 **p, struct label **l, struct reloc **r,
#endif
l_vmalloc_done(l, *p);
- i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3); /* get pgd offset in bytes */
+
+ if (PGDIR_SHIFT - 3 < 32) /* get pgd offset in bytes */
+ i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3);
+ else
+ i_dsrl32(p, tmp, tmp, PGDIR_SHIFT - 3 - 32);
+
i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
@@ -1016,8 +1040,46 @@ build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r,
{
long swpd = (long)swapper_pg_dir;
+#ifdef MODULE_START
+ long modd = (long)module_pg_dir;
+
+ l_module_alloc(l, *p);
+ /*
+ * Assumption:
+ * VMALLOC_START >= 0xc000000000000000UL
+ * MODULE_START >= 0xe000000000000000UL
+ */
+ i_SLL(p, ptr, bvaddr, 2);
+ il_bgez(p, r, ptr, label_vmalloc);
+
+ if (in_compat_space_p(MODULE_START) && !rel_lo(MODULE_START)) {
+ i_lui(p, ptr, rel_hi(MODULE_START)); /* delay slot */
+ } else {
+ /* unlikely configuration */
+ i_nop(p); /* delay slot */
+ i_LA(p, ptr, MODULE_START);
+ }
+ i_dsubu(p, bvaddr, bvaddr, ptr);
+
+ if (in_compat_space_p(modd) && !rel_lo(modd)) {
+ il_b(p, r, label_vmalloc_done);
+ i_lui(p, ptr, rel_hi(modd));
+ } else {
+ i_LA_mostly(p, ptr, modd);
+ il_b(p, r, label_vmalloc_done);
+ i_daddiu(p, ptr, ptr, rel_lo(modd));
+ }
+
+ l_vmalloc(l, *p);
+ if (in_compat_space_p(MODULE_START) && !rel_lo(MODULE_START) &&
+ MODULE_START << 32 == VMALLOC_START)
+ i_dsll32(p, ptr, ptr, 0); /* typical case */
+ else
+ i_LA(p, ptr, VMALLOC_START);
+#else
l_vmalloc(l, *p);
i_LA(p, ptr, VMALLOC_START);
+#endif
i_dsubu(p, bvaddr, bvaddr, ptr);
if (in_compat_space_p(swpd) && !rel_lo(swpd)) {
@@ -1073,7 +1135,7 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr)
static __init void build_adjust_context(u32 **p, unsigned int ctx)
{
- unsigned int shift = 4 - (PTE_T_LOG2 + 1);
+ unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12;
unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1);
switch (current_cpu_data.cputype) {
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2024-11-29 09:33:12 +0000