diff options
Diffstat (limited to 'arch/mips/mm/tlbex.c')
| -rw-r--r-- | arch/mips/mm/tlbex.c | 590 |
1 files changed, 503 insertions, 87 deletions
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index 93816f3bca6..083d3412d0b 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -26,8 +26,10 @@ #include <linux/smp.h> #include <linux/string.h> #include <linux/init.h> +#include <linux/cache.h> -#include <asm/mmu_context.h> +#include <asm/cacheflush.h> +#include <asm/pgtable.h> #include <asm/war.h> #include <asm/uasm.h> @@ -63,6 +65,52 @@ static inline int __maybe_unused r10000_llsc_war(void) return R10000_LLSC_WAR; } +static int use_bbit_insns(void) +{ + switch (current_cpu_type()) { + case CPU_CAVIUM_OCTEON: + case CPU_CAVIUM_OCTEON_PLUS: + case CPU_CAVIUM_OCTEON2: + return 1; + default: + return 0; + } +} + +static int use_lwx_insns(void) +{ + switch (current_cpu_type()) { + case CPU_CAVIUM_OCTEON2: + return 1; + default: + return 0; + } +} +#if defined(CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE) && \ + CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE > 0 +static bool scratchpad_available(void) +{ + return true; +} +static int scratchpad_offset(int i) +{ + /* + * CVMSEG starts at address -32768 and extends for + * CAVIUM_OCTEON_CVMSEG_SIZE 128 byte cache lines. + */ + i += 1; /* Kernel use starts at the top and works down. */ + return CONFIG_CAVIUM_OCTEON_CVMSEG_SIZE * 128 - (8 * i) - 32768; +} +#else +static bool scratchpad_available(void) +{ + return false; +} +static int scratchpad_offset(int i) +{ + BUG(); +} +#endif /* * Found by experiment: At least some revisions of the 4kc throw under * some circumstances a machine check exception, triggered by invalid @@ -173,11 +221,41 @@ static struct uasm_reloc relocs[128] __cpuinitdata; static int check_for_high_segbits __cpuinitdata; #endif +static int check_for_high_segbits __cpuinitdata; + +static unsigned int kscratch_used_mask __cpuinitdata; + +static int __cpuinit allocate_kscratch(void) +{ + int r; + unsigned int a = cpu_data[0].kscratch_mask & ~kscratch_used_mask; + + r = ffs(a); + + if (r == 0) + return -1; + + r--; /* make it zero based */ + + kscratch_used_mask |= (1 << r); + + return r; +} + +static int scratch_reg __cpuinitdata; +static int pgd_reg __cpuinitdata; +enum vmalloc64_mode {not_refill, refill_scratch, refill_noscratch}; + #ifndef CONFIG_MIPS_PGD_C0_CONTEXT + /* * CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current, * we cannot do r3000 under these circumstances. + * + * Declare pgd_current here instead of including mmu_context.h to avoid type + * conflicts for tlbmiss_handler_setup_pgd */ +extern unsigned long pgd_current[]; /* * The R3000 TLB handler is simple. @@ -440,21 +518,43 @@ static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p, static __cpuinit void build_restore_pagemask(u32 **p, struct uasm_reloc **r, unsigned int tmp, - enum label_id lid) + enum label_id lid, + int restore_scratch) { - /* Reset default page size */ - if (PM_DEFAULT_MASK >> 16) { - uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); - uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); - uasm_il_b(p, r, lid); - uasm_i_mtc0(p, tmp, C0_PAGEMASK); - } else if (PM_DEFAULT_MASK) { - uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); - uasm_il_b(p, r, lid); - uasm_i_mtc0(p, tmp, C0_PAGEMASK); + if (restore_scratch) { + /* Reset default page size */ + if (PM_DEFAULT_MASK >> 16) { + uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); + uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); + uasm_i_mtc0(p, tmp, C0_PAGEMASK); + uasm_il_b(p, r, lid); + } else if (PM_DEFAULT_MASK) { + uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); + uasm_i_mtc0(p, tmp, C0_PAGEMASK); + uasm_il_b(p, r, lid); + } else { + uasm_i_mtc0(p, 0, C0_PAGEMASK); + uasm_il_b(p, r, lid); + } + if (scratch_reg > 0) + UASM_i_MFC0(p, 1, 31, scratch_reg); + else + UASM_i_LW(p, 1, scratchpad_offset(0), 0); } else { - uasm_il_b(p, r, lid); - uasm_i_mtc0(p, 0, C0_PAGEMASK); + /* Reset default page size */ + if (PM_DEFAULT_MASK >> 16) { + uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16); + uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff); + uasm_il_b(p, r, lid); + uasm_i_mtc0(p, tmp, C0_PAGEMASK); + } else if (PM_DEFAULT_MASK) { + uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK); + uasm_il_b(p, r, lid); + uasm_i_mtc0(p, tmp, C0_PAGEMASK); + } else { + uasm_il_b(p, r, lid); + uasm_i_mtc0(p, 0, C0_PAGEMASK); + } } } @@ -462,7 +562,8 @@ static __cpuinit void build_huge_tlb_write_entry(u32 **p, struct uasm_label **l, struct uasm_reloc **r, unsigned int tmp, - enum tlb_write_entry wmode) + enum tlb_write_entry wmode, + int restore_scratch) { /* Set huge page tlb entry size */ uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16); @@ -471,7 +572,7 @@ static __cpuinit void build_huge_tlb_write_entry(u32 **p, build_tlb_write_entry(p, l, r, wmode); - build_restore_pagemask(p, r, tmp, label_leave); + build_restore_pagemask(p, r, tmp, label_leave, restore_scratch); } /* @@ -482,8 +583,12 @@ build_is_huge_pte(u32 **p, struct uasm_reloc **r, unsigned int tmp, unsigned int pmd, int lid) { UASM_i_LW(p, tmp, 0, pmd); - uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); - uasm_il_bnez(p, r, tmp, lid); + if (use_bbit_insns()) { + uasm_il_bbit1(p, r, tmp, ilog2(_PAGE_HUGE), lid); + } else { + uasm_i_andi(p, tmp, tmp, _PAGE_HUGE); + uasm_il_bnez(p, r, tmp, lid); + } } static __cpuinit void build_huge_update_entries(u32 **p, @@ -532,7 +637,7 @@ static __cpuinit void build_huge_handler_tail(u32 **p, UASM_i_SW(p, pte, 0, ptr); #endif build_huge_update_entries(p, pte, ptr); - build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed); + build_huge_tlb_write_entry(p, l, r, pte, tlb_indexed, 0); } #endif /* CONFIG_HUGETLB_PAGE */ @@ -573,13 +678,22 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, /* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */ #ifdef CONFIG_MIPS_PGD_C0_CONTEXT - /* - * &pgd << 11 stored in CONTEXT [23..63]. - */ - UASM_i_MFC0(p, ptr, C0_CONTEXT); - uasm_i_dins(p, ptr, 0, 0, 23); /* Clear lower 23 bits of context. */ - uasm_i_ori(p, ptr, ptr, 0x540); /* 1 0 1 0 1 << 6 xkphys cached */ - uasm_i_drotr(p, ptr, ptr, 11); + if (pgd_reg != -1) { + /* pgd is in pgd_reg */ + UASM_i_MFC0(p, ptr, 31, pgd_reg); + } else { + /* + * &pgd << 11 stored in CONTEXT [23..63]. + */ + UASM_i_MFC0(p, ptr, C0_CONTEXT); + + /* Clear lower 23 bits of context. */ + uasm_i_dins(p, ptr, 0, 0, 23); + + /* 1 0 1 0 1 << 6 xkphys cached */ + uasm_i_ori(p, ptr, ptr, 0x540); + uasm_i_drotr(p, ptr, ptr, 11); + } #elif defined(CONFIG_SMP) # ifdef CONFIG_MIPS_MT_SMTC /* @@ -620,7 +734,6 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, #endif } -enum vmalloc64_mode {not_refill, refill}; /* * BVADDR is the faulting address, PTR is scratch. * PTR will hold the pgd for vmalloc. @@ -638,7 +751,7 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, uasm_l_vmalloc(l, *p); - if (mode == refill && check_for_high_segbits) { + if (mode != not_refill && check_for_high_segbits) { if (single_insn_swpd) { uasm_il_bltz(p, r, bvaddr, label_vmalloc_done); uasm_i_lui(p, ptr, uasm_rel_hi(swpd)); @@ -661,7 +774,7 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd)); } } - if (mode == refill && check_for_high_segbits) { + if (mode != not_refill && check_for_high_segbits) { uasm_l_large_segbits_fault(l, *p); /* * We get here if we are an xsseg address, or if we are @@ -677,7 +790,15 @@ build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r, */ UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0); uasm_i_jr(p, ptr); - uasm_i_nop(p); + + if (mode == refill_scratch) { + if (scratch_reg > 0) + UASM_i_MFC0(p, 1, 31, scratch_reg); + else + UASM_i_LW(p, 1, scratchpad_offset(0), 0); + } else { + uasm_i_nop(p); + } } } @@ -834,6 +955,185 @@ static void __cpuinit build_update_entries(u32 **p, unsigned int tmp, #endif } +struct mips_huge_tlb_info { + int huge_pte; + int restore_scratch; +}; + +static struct mips_huge_tlb_info __cpuinit +build_fast_tlb_refill_handler (u32 **p, struct uasm_label **l, + struct uasm_reloc **r, unsigned int tmp, + unsigned int ptr, int c0_scratch) +{ + struct mips_huge_tlb_info rv; + unsigned int even, odd; + int vmalloc_branch_delay_filled = 0; + const int scratch = 1; /* Our extra working register */ + + rv.huge_pte = scratch; + rv.restore_scratch = 0; + + if (check_for_high_segbits) { + UASM_i_MFC0(p, tmp, C0_BADVADDR); + + if (pgd_reg != -1) + UASM_i_MFC0(p, ptr, 31, pgd_reg); + else + UASM_i_MFC0(p, ptr, C0_CONTEXT); + + if (c0_scratch >= 0) + UASM_i_MTC0(p, scratch, 31, c0_scratch); + else + UASM_i_SW(p, scratch, scratchpad_offset(0), 0); + + uasm_i_dsrl_safe(p, scratch, tmp, + PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3); + uasm_il_bnez(p, r, scratch, label_vmalloc); + + if (pgd_reg == -1) { + vmalloc_branch_delay_filled = 1; + /* Clear lower 23 bits of context. */ + uasm_i_dins(p, ptr, 0, 0, 23); + } + } else { + if (pgd_reg != -1) + UASM_i_MFC0(p, ptr, 31, pgd_reg); + else + UASM_i_MFC0(p, ptr, C0_CONTEXT); + + UASM_i_MFC0(p, tmp, C0_BADVADDR); + + if (c0_scratch >= 0) + UASM_i_MTC0(p, scratch, 31, c0_scratch); + else + UASM_i_SW(p, scratch, scratchpad_offset(0), 0); + + if (pgd_reg == -1) + /* Clear lower 23 bits of context. */ + uasm_i_dins(p, ptr, 0, 0, 23); + + uasm_il_bltz(p, r, tmp, label_vmalloc); + } + + if (pgd_reg == -1) { + vmalloc_branch_delay_filled = 1; + /* 1 0 1 0 1 << 6 xkphys cached */ + uasm_i_ori(p, ptr, ptr, 0x540); + uasm_i_drotr(p, ptr, ptr, 11); + } + +#ifdef __PAGETABLE_PMD_FOLDED +#define LOC_PTEP scratch +#else +#define LOC_PTEP ptr +#endif + + if (!vmalloc_branch_delay_filled) + /* get pgd offset in bytes */ + uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); + + uasm_l_vmalloc_done(l, *p); + + /* + * tmp ptr + * fall-through case = badvaddr *pgd_current + * vmalloc case = badvaddr swapper_pg_dir + */ + + if (vmalloc_branch_delay_filled) + /* get pgd offset in bytes */ + uasm_i_dsrl_safe(p, scratch, tmp, PGDIR_SHIFT - 3); + +#ifdef __PAGETABLE_PMD_FOLDED + GET_CONTEXT(p, tmp); /* get context reg */ +#endif + uasm_i_andi(p, scratch, scratch, (PTRS_PER_PGD - 1) << 3); + + if (use_lwx_insns()) { + UASM_i_LWX(p, LOC_PTEP, scratch, ptr); + } else { + uasm_i_daddu(p, ptr, ptr, scratch); /* add in pgd offset */ + uasm_i_ld(p, LOC_PTEP, 0, ptr); /* get pmd pointer */ + } + +#ifndef __PAGETABLE_PMD_FOLDED + /* get pmd offset in bytes */ + uasm_i_dsrl_safe(p, scratch, tmp, PMD_SHIFT - 3); + uasm_i_andi(p, scratch, scratch, (PTRS_PER_PMD - 1) << 3); + GET_CONTEXT(p, tmp); /* get context reg */ + + if (use_lwx_insns()) { + UASM_i_LWX(p, scratch, scratch, ptr); + } else { + uasm_i_daddu(p, ptr, ptr, scratch); /* add in pmd offset */ + UASM_i_LW(p, scratch, 0, ptr); + } +#endif + /* Adjust the context during the load latency. */ + build_adjust_context(p, tmp); + +#ifdef CONFIG_HUGETLB_PAGE + uasm_il_bbit1(p, r, scratch, ilog2(_PAGE_HUGE), label_tlb_huge_update); + /* + * The in the LWX case we don't want to do the load in the + * delay slot. It cannot issue in the same cycle and may be + * speculative and unneeded. + */ + if (use_lwx_insns()) + uasm_i_nop(p); +#endif /* CONFIG_HUGETLB_PAGE */ + + + /* build_update_entries */ + if (use_lwx_insns()) { + even = ptr; + odd = tmp; + UASM_i_LWX(p, even, scratch, tmp); + UASM_i_ADDIU(p, tmp, tmp, sizeof(pte_t)); + UASM_i_LWX(p, odd, scratch, tmp); + } else { + UASM_i_ADDU(p, ptr, scratch, tmp); /* add in offset */ + even = tmp; + odd = ptr; + UASM_i_LW(p, even, 0, ptr); /* get even pte */ + UASM_i_LW(p, odd, sizeof(pte_t), ptr); /* get odd pte */ + } + if (kernel_uses_smartmips_rixi) { + uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_NO_EXEC)); + uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_NO_EXEC)); + uasm_i_drotr(p, even, even, + ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); + UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ + uasm_i_drotr(p, odd, odd, + ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC)); + } else { + uasm_i_dsrl_safe(p, even, even, ilog2(_PAGE_GLOBAL)); + UASM_i_MTC0(p, even, C0_ENTRYLO0); /* load it */ + uasm_i_dsrl_safe(p, odd, odd, ilog2(_PAGE_GLOBAL)); + } + UASM_i_MTC0(p, odd, C0_ENTRYLO1); /* load it */ + + if (c0_scratch >= 0) { + UASM_i_MFC0(p, scratch, 31, c0_scratch); + build_tlb_write_entry(p, l, r, tlb_random); + uasm_l_leave(l, *p); + rv.restore_scratch = 1; + } else if (PAGE_SHIFT == 14 || PAGE_SHIFT == 13) { + build_tlb_write_entry(p, l, r, tlb_random); + uasm_l_leave(l, *p); + UASM_i_LW(p, scratch, scratchpad_offset(0), 0); + } else { + UASM_i_LW(p, scratch, scratchpad_offset(0), 0); + build_tlb_write_entry(p, l, r, tlb_random); + uasm_l_leave(l, *p); + rv.restore_scratch = 1; + } + + uasm_i_eret(p); /* return from trap */ + + return rv; +} + /* * For a 64-bit kernel, we are using the 64-bit XTLB refill exception * because EXL == 0. If we wrap, we can also use the 32 instruction @@ -849,54 +1149,67 @@ static void __cpuinit build_r4000_tlb_refill_handler(void) struct uasm_reloc *r = relocs; u32 *f; unsigned int final_len; + struct mips_huge_tlb_info htlb_info; + enum vmalloc64_mode vmalloc_mode; memset(tlb_handler, 0, sizeof(tlb_handler)); memset(labels, 0, sizeof(labels)); memset(relocs, 0, sizeof(relocs)); memset(final_handler, 0, sizeof(final_handler)); - /* - * create the plain linear handler - */ - if (bcm1250_m3_war()) { - unsigned int segbits = 44; + if (scratch_reg == 0) + scratch_reg = allocate_kscratch(); - uasm_i_dmfc0(&p, K0, C0_BADVADDR); - uasm_i_dmfc0(&p, K1, C0_ENTRYHI); - uasm_i_xor(&p, K0, K0, K1); - uasm_i_dsrl_safe(&p, K1, K0, 62); - uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); - uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); - uasm_i_or(&p, K0, K0, K1); - uasm_il_bnez(&p, &r, K0, label_leave); - /* No need for uasm_i_nop */ - } + if ((scratch_reg > 0 || scratchpad_available()) && use_bbit_insns()) { + htlb_info = build_fast_tlb_refill_handler(&p, &l, &r, K0, K1, + scratch_reg); + vmalloc_mode = refill_scratch; + } else { + htlb_info.huge_pte = K0; + htlb_info.restore_scratch = 0; + vmalloc_mode = refill_noscratch; + /* + * create the plain linear handler + */ + if (bcm1250_m3_war()) { + unsigned int segbits = 44; + + uasm_i_dmfc0(&p, K0, C0_BADVADDR); + uasm_i_dmfc0(&p, K1, C0_ENTRYHI); + uasm_i_xor(&p, K0, K0, K1); + uasm_i_dsrl_safe(&p, K1, K0, 62); + uasm_i_dsrl_safe(&p, K0, K0, 12 + 1); + uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits); + uasm_i_or(&p, K0, K0, K1); + uasm_il_bnez(&p, &r, K0, label_leave); + /* No need for uasm_i_nop */ + } #ifdef CONFIG_64BIT - build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ + build_get_pmde64(&p, &l, &r, K0, K1); /* get pmd in K1 */ #else - build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ + build_get_pgde32(&p, K0, K1); /* get pgd in K1 */ #endif #ifdef CONFIG_HUGETLB_PAGE - build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); + build_is_huge_pte(&p, &r, K0, K1, label_tlb_huge_update); #endif - build_get_ptep(&p, K0, K1); - build_update_entries(&p, K0, K1); - build_tlb_write_entry(&p, &l, &r, tlb_random); - uasm_l_leave(&l, p); - uasm_i_eret(&p); /* return from trap */ - + build_get_ptep(&p, K0, K1); + build_update_entries(&p, K0, K1); + build_tlb_write_entry(&p, &l, &r, tlb_random); + uasm_l_leave(&l, p); + uasm_i_eret(&p); /* return from trap */ + } #ifdef CONFIG_HUGETLB_PAGE uasm_l_tlb_huge_update(&l, p); - UASM_i_LW(&p, K0, 0, K1); - build_huge_update_entries(&p, K0, K1); - build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random); + build_huge_update_entries(&p, htlb_info.huge_pte, K1); + build_huge_tlb_write_entry(&p, &l, &r, K0, tlb_random, + htlb_info.restore_scratch); #endif #ifdef CONFIG_64BIT - build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, refill); + build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, vmalloc_mode); #endif /* @@ -1014,6 +1327,55 @@ static void __cpuinit build_r4000_tlb_refill_handler(void) u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned; u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned; u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned; +#ifdef CONFIG_MIPS_PGD_C0_CONTEXT +u32 tlbmiss_handler_setup_pgd[16] __cacheline_aligned; + +static void __cpuinit build_r4000_setup_pgd(void) +{ + const int a0 = 4; + const int a1 = 5; + u32 *p = tlbmiss_handler_setup_pgd; + struct uasm_label *l = labels; + struct uasm_reloc *r = relocs; + + memset(tlbmiss_handler_setup_pgd, 0, sizeof(tlbmiss_handler_setup_pgd)); + memset(labels, 0, sizeof(labels)); + memset(relocs, 0, sizeof(relocs)); + + pgd_reg = allocate_kscratch(); + + if (pgd_reg == -1) { + /* PGD << 11 in c0_Context */ + /* + * If it is a ckseg0 address, convert to a physical + * address. Shifting right by 29 and adding 4 will + * result in zero for these addresses. + * + */ + UASM_i_SRA(&p, a1, a0, 29); + UASM_i_ADDIU(&p, a1, a1, 4); + uasm_il_bnez(&p, &r, a1, label_tlbl_goaround1); + uasm_i_nop(&p); + uasm_i_dinsm(&p, a0, 0, 29, 64 - 29); + uasm_l_tlbl_goaround1(&l, p); + UASM_i_SLL(&p, a0, a0, 11); + uasm_i_jr(&p, 31); + UASM_i_MTC0(&p, a0, C0_CONTEXT); + } else { + /* PGD in c0_KScratch */ + uasm_i_jr(&p, 31); + UASM_i_MTC0(&p, a0, 31, pgd_reg); + } + if (p - tlbmiss_handler_setup_pgd > ARRAY_SIZE(tlbmiss_handler_setup_pgd)) + panic("tlbmiss_handler_setup_pgd space exceeded"); + uasm_resolve_relocs(relocs, labels); + pr_debug("Wrote tlbmiss_handler_setup_pgd (%u instructions).\n", + (unsigned int)(p - tlbmiss_handler_setup_pgd)); + + dump_handler(tlbmiss_handler_setup_pgd, + ARRAY_SIZE(tlbmiss_handler_setup_pgd)); +} +#endif static void __cpuinit iPTE_LW(u32 **p, unsigned int pte, unsigned int ptr) @@ -1100,14 +1462,20 @@ build_pte_present(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, enum label_id lid) { if (kernel_uses_smartmips_rixi) { - uasm_i_andi(p, pte, pte, _PAGE_PRESENT); - uasm_il_beqz(p, r, pte, lid); + if (use_bbit_insns()) { + uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid); + uasm_i_nop(p); + } else { + uasm_i_andi(p, pte, pte, _PAGE_PRESENT); + uasm_il_beqz(p, r, pte, lid); + iPTE_LW(p, pte, ptr); + } } else { uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ); uasm_il_bnez(p, r, pte, lid); + iPTE_LW(p, pte, ptr); } - iPTE_LW(p, pte, ptr); } /* Make PTE valid, store result in PTR. */ @@ -1128,10 +1496,17 @@ static void __cpuinit build_pte_writable(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, enum label_id lid) { - uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); - uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); - uasm_il_bnez(p, r, pte, lid); - iPTE_LW(p, pte, ptr); + if (use_bbit_insns()) { + uasm_il_bbit0(p, r, pte, ilog2(_PAGE_PRESENT), lid); + uasm_i_nop(p); + uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid); + uasm_i_nop(p); + } else { + uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); + uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_WRITE); + uasm_il_bnez(p, r, pte, lid); + iPTE_LW(p, pte, ptr); + } } /* Make PTE writable, update software status bits as well, then store @@ -1155,12 +1530,19 @@ static void __cpuinit build_pte_modifiable(u32 **p, struct uasm_reloc **r, unsigned int pte, unsigned int ptr, enum label_id lid) { - uasm_i_andi(p, pte, pte, _PAGE_WRITE); - uasm_il_beqz(p, r, pte, lid); - iPTE_LW(p, pte, ptr); + if (use_bbit_insns()) { + uasm_il_bbit0(p, r, pte, ilog2(_PAGE_WRITE), lid); + uasm_i_nop(p); + } else { + uasm_i_andi(p, pte, pte, _PAGE_WRITE); + uasm_il_beqz(p, r, pte, lid); + iPTE_LW(p, pte, ptr); + } } #ifndef CONFIG_MIPS_PGD_C0_CONTEXT + + /* * R3000 style TLB load/store/modify handlers. */ @@ -1402,14 +1784,23 @@ static void __cpuinit build_r4000_tlb_load_handler(void) * If the page is not _PAGE_VALID, RI or XI could not * have triggered it. Skip the expensive test.. */ - uasm_i_andi(&p, K0, K0, _PAGE_VALID); - uasm_il_beqz(&p, &r, K0, label_tlbl_goaround1); + if (use_bbit_insns()) { + uasm_il_bbit0(&p, &r, K0, ilog2(_PAGE_VALID), + label_tlbl_goaround1); + } else { + uasm_i_andi(&p, K0, K0, _PAGE_VALID); + uasm_il_beqz(&p, &r, K0, label_tlbl_goaround1); + } uasm_i_nop(&p); uasm_i_tlbr(&p); /* Examine entrylo 0 or 1 based on ptr. */ - uasm_i_andi(&p, K0, K1, sizeof(pte_t)); - uasm_i_beqz(&p, K0, 8); + if (use_bbit_insns()) { + uasm_i_bbit0(&p, K1, ilog2(sizeof(pte_t)), 8); + } else { + uasm_i_andi(&p, K0, K1, sizeof(pte_t)); + uasm_i_beqz(&p, K0, 8); + } UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/ UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */ @@ -1417,12 +1808,18 @@ static void __cpuinit build_r4000_tlb_load_handler(void) * If the entryLo (now in K0) is valid (bit 1), RI or * XI must have triggered it. */ - uasm_i_andi(&p, K0, K0, 2); - uasm_il_bnez(&p, &r, K0, label_nopage_tlbl); - - uasm_l_tlbl_goaround1(&l, p); - /* Reload the PTE value */ - iPTE_LW(&p, K0, K1); + if (use_bbit_insns()) { + uasm_il_bbit1(&p, &r, K0, 1, label_nopage_tlbl); + /* Reload the PTE value */ + iPTE_LW(&p, K0, K1); + uasm_l_tlbl_goaround1(&l, p); + } else { + uasm_i_andi(&p, K0, K0, 2); + uasm_il_bnez(&p, &r, K0, label_nopage_tlbl); + uasm_l_tlbl_goaround1(&l, p); + /* Reload the PTE value */ + iPTE_LW(&p, K0, K1); + } } build_make_valid(&p, &r, K0, K1); build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1); @@ -1442,23 +1839,35 @@ static void __cpuinit build_r4000_tlb_load_handler(void) * If the page is not _PAGE_VALID, RI or XI could not * have triggered it. Skip the expensive test.. */ - uasm_i_andi(&p, K0, K0, _PAGE_VALID); - uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2); + if (use_bbit_insns()) { + uasm_il_bbit0(&p, &r, K0, ilog2(_PAGE_VALID), + label_tlbl_goaround2); + } else { + uasm_i_andi(&p, K0, K0, _PAGE_VALID); + uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2); + } uasm_i_nop(&p); uasm_i_tlbr(&p); /* Examine entrylo 0 or 1 based on ptr. */ - uasm_i_andi(&p, K0, K1, sizeof(pte_t)); - uasm_i_beqz(&p, K0, 8); - + if (use_bbit_insns()) { + uasm_i_bbit0(&p, K1, ilog2(sizeof(pte_t)), 8); + } else { + uasm_i_andi(&p, K0, K1, sizeof(pte_t)); + uasm_i_beqz(&p, K0, 8); + } UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/ UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */ /* * If the entryLo (now in K0) is valid (bit 1), RI or * XI must have triggered it. */ - uasm_i_andi(&p, K0, K0, 2); - uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2); + if (use_bbit_insns()) { + uasm_il_bbit0(&p, &r, K0, 1, label_tlbl_goaround2); + } else { + uasm_i_andi(&p, K0, K0, 2); + uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2); + } /* Reload the PTE value */ iPTE_LW(&p, K0, K1); @@ -1466,7 +1875,7 @@ static void __cpuinit build_r4000_tlb_load_handler(void) * We clobbered C0_PAGEMASK, restore it. On the other branch * it is restored in build_huge_tlb_write_entry. */ - build_restore_pagemask(&p, &r, K0, label_nopage_tlbl); + build_restore_pagemask(&p, &r, K0, label_nopage_tlbl, 0); uasm_l_tlbl_goaround2(&l, p); } @@ -1623,13 +2032,16 @@ void __cpuinit build_tlb_refill_handler(void) break; default: - build_r4000_tlb_refill_handler(); if (!run_once) { +#ifdef CONFIG_MIPS_PGD_C0_CONTEXT + build_r4000_setup_pgd(); +#endif build_r4000_tlb_load_handler(); build_r4000_tlb_store_handler(); build_r4000_tlb_modify_handler(); run_once++; } + build_r4000_tlb_refill_handler(); } } @@ -1641,4 +2053,8 @@ void __cpuinit flush_tlb_handlers(void) (unsigned long)handle_tlbs + sizeof(handle_tlbs)); local_flush_icache_range((unsigned long)handle_tlbm, (unsigned long)handle_tlbm + sizeof(handle_tlbm)); +#ifdef CONFIG_MIPS_PGD_C0_CONTEXT + local_flush_icache_range((unsigned long)tlbmiss_handler_setup_pgd, + (unsigned long)tlbmiss_handler_setup_pgd + sizeof(handle_tlbm)); +#endif } |