summaryrefslogtreecommitdiffstats
path: root/arch/mips/mm/tlbex.c
diff options
context:
space:
mode:
Diffstat (limited to 'arch/mips/mm/tlbex.c')
-rw-r--r--arch/mips/mm/tlbex.c330
1 files changed, 258 insertions, 72 deletions
diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c
index badcf5e8d69..86f004dc835 100644
--- a/arch/mips/mm/tlbex.c
+++ b/arch/mips/mm/tlbex.c
@@ -29,8 +29,17 @@
#include <asm/mmu_context.h>
#include <asm/war.h>
+#include <asm/uasm.h>
+
+/*
+ * TLB load/store/modify handlers.
+ *
+ * Only the fastpath gets synthesized at runtime, the slowpath for
+ * do_page_fault remains normal asm.
+ */
+extern void tlb_do_page_fault_0(void);
+extern void tlb_do_page_fault_1(void);
-#include "uasm.h"
static inline int r45k_bvahwbug(void)
{
@@ -77,11 +86,14 @@ enum label_id {
label_vmalloc_done,
label_tlbw_hazard,
label_split,
+ label_tlbl_goaround1,
+ label_tlbl_goaround2,
label_nopage_tlbl,
label_nopage_tlbs,
label_nopage_tlbm,
label_smp_pgtable_change,
label_r3000_write_probe_fail,
+ label_large_segbits_fault,
#ifdef CONFIG_HUGETLB_PAGE
label_tlb_huge_update,
#endif
@@ -93,11 +105,14 @@ UASM_L_LA(_vmalloc)
UASM_L_LA(_vmalloc_done)
UASM_L_LA(_tlbw_hazard)
UASM_L_LA(_split)
+UASM_L_LA(_tlbl_goaround1)
+UASM_L_LA(_tlbl_goaround2)
UASM_L_LA(_nopage_tlbl)
UASM_L_LA(_nopage_tlbs)
UASM_L_LA(_nopage_tlbm)
UASM_L_LA(_smp_pgtable_change)
UASM_L_LA(_r3000_write_probe_fail)
+UASM_L_LA(_large_segbits_fault)
#ifdef CONFIG_HUGETLB_PAGE
UASM_L_LA(_tlb_huge_update)
#endif
@@ -154,6 +169,10 @@ static u32 tlb_handler[128] __cpuinitdata;
static struct uasm_label labels[128] __cpuinitdata;
static struct uasm_reloc relocs[128] __cpuinitdata;
+#ifdef CONFIG_64BIT
+static int check_for_high_segbits __cpuinitdata;
+#endif
+
#ifndef CONFIG_MIPS_PGD_C0_CONTEXT
/*
* CONFIG_MIPS_PGD_C0_CONTEXT implies 64 bit and lack of pgd_current,
@@ -397,36 +416,60 @@ static void __cpuinit build_tlb_write_entry(u32 **p, struct uasm_label **l,
}
}
-#ifdef CONFIG_HUGETLB_PAGE
-static __cpuinit void build_huge_tlb_write_entry(u32 **p,
- struct uasm_label **l,
- struct uasm_reloc **r,
- unsigned int tmp,
- enum tlb_write_entry wmode)
+static __cpuinit __maybe_unused void build_convert_pte_to_entrylo(u32 **p,
+ unsigned int reg)
{
- /* Set huge page tlb entry size */
- uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
- uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff);
- uasm_i_mtc0(p, tmp, C0_PAGEMASK);
+ if (kernel_uses_smartmips_rixi) {
+ UASM_i_SRL(p, reg, reg, ilog2(_PAGE_NO_EXEC));
+ UASM_i_ROTR(p, reg, reg, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+ } else {
+#ifdef CONFIG_64BIT_PHYS_ADDR
+ uasm_i_dsrl_safe(p, reg, reg, ilog2(_PAGE_GLOBAL));
+#else
+ UASM_i_SRL(p, reg, reg, ilog2(_PAGE_GLOBAL));
+#endif
+ }
+}
- build_tlb_write_entry(p, l, r, wmode);
+#ifdef CONFIG_HUGETLB_PAGE
+static __cpuinit void build_restore_pagemask(u32 **p,
+ struct uasm_reloc **r,
+ unsigned int tmp,
+ enum label_id lid)
+{
/* Reset default page size */
if (PM_DEFAULT_MASK >> 16) {
uasm_i_lui(p, tmp, PM_DEFAULT_MASK >> 16);
uasm_i_ori(p, tmp, tmp, PM_DEFAULT_MASK & 0xffff);
- uasm_il_b(p, r, label_leave);
+ uasm_il_b(p, r, lid);
uasm_i_mtc0(p, tmp, C0_PAGEMASK);
} else if (PM_DEFAULT_MASK) {
uasm_i_ori(p, tmp, 0, PM_DEFAULT_MASK);
- uasm_il_b(p, r, label_leave);
+ uasm_il_b(p, r, lid);
uasm_i_mtc0(p, tmp, C0_PAGEMASK);
} else {
- uasm_il_b(p, r, label_leave);
+ uasm_il_b(p, r, lid);
uasm_i_mtc0(p, 0, C0_PAGEMASK);
}
}
+static __cpuinit void build_huge_tlb_write_entry(u32 **p,
+ struct uasm_label **l,
+ struct uasm_reloc **r,
+ unsigned int tmp,
+ enum tlb_write_entry wmode)
+{
+ /* Set huge page tlb entry size */
+ uasm_i_lui(p, tmp, PM_HUGE_MASK >> 16);
+ uasm_i_ori(p, tmp, tmp, PM_HUGE_MASK & 0xffff);
+ uasm_i_mtc0(p, tmp, C0_PAGEMASK);
+
+ build_tlb_write_entry(p, l, r, wmode);
+
+ build_restore_pagemask(p, r, tmp, label_leave);
+}
+
/*
* Check if Huge PTE is present, if so then jump to LABEL.
*/
@@ -460,15 +503,15 @@ static __cpuinit void build_huge_update_entries(u32 **p,
if (!small_sequence)
uasm_i_lui(p, tmp, HPAGE_SIZE >> (7 + 16));
- UASM_i_SRL(p, pte, pte, 6); /* convert to entrylo */
- uasm_i_mtc0(p, pte, C0_ENTRYLO0); /* load it */
+ build_convert_pte_to_entrylo(p, pte);
+ UASM_i_MTC0(p, pte, C0_ENTRYLO0); /* load it */
/* convert to entrylo1 */
if (small_sequence)
UASM_i_ADDIU(p, pte, pte, HPAGE_SIZE >> 7);
else
UASM_i_ADDU(p, pte, pte, tmp);
- uasm_i_mtc0(p, pte, C0_ENTRYLO1); /* load it */
+ UASM_i_MTC0(p, pte, C0_ENTRYLO1); /* load it */
}
static __cpuinit void build_huge_handler_tail(u32 **p,
@@ -505,7 +548,24 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
* The vmalloc handling is not in the hotpath.
*/
uasm_i_dmfc0(p, tmp, C0_BADVADDR);
- uasm_il_bltz(p, r, tmp, label_vmalloc);
+
+ if (check_for_high_segbits) {
+ /*
+ * The kernel currently implicitely assumes that the
+ * MIPS SEGBITS parameter for the processor is
+ * (PGDIR_SHIFT+PGDIR_BITS) or less, and will never
+ * allocate virtual addresses outside the maximum
+ * range for SEGBITS = (PGDIR_SHIFT+PGDIR_BITS). But
+ * that doesn't prevent user code from accessing the
+ * higher xuseg addresses. Here, we make sure that
+ * everything but the lower xuseg addresses goes down
+ * the module_alloc/vmalloc path.
+ */
+ uasm_i_dsrl_safe(p, ptr, tmp, PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
+ uasm_il_bnez(p, r, ptr, label_vmalloc);
+ } else {
+ uasm_il_bltz(p, r, tmp, label_vmalloc);
+ }
/* No uasm_i_nop needed here, since the next insn doesn't touch TMP. */
#ifdef CONFIG_MIPS_PGD_C0_CONTEXT
@@ -522,14 +582,14 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
* SMTC uses TCBind value as "CPU" index
*/
uasm_i_mfc0(p, ptr, C0_TCBIND);
- uasm_i_dsrl(p, ptr, ptr, 19);
+ uasm_i_dsrl_safe(p, ptr, ptr, 19);
# else
/*
* 64 bit SMP running in XKPHYS has smp_processor_id() << 3
* stored in CONTEXT.
*/
uasm_i_dmfc0(p, ptr, C0_CONTEXT);
- uasm_i_dsrl(p, ptr, ptr, 23);
+ uasm_i_dsrl_safe(p, ptr, ptr, 23);
# endif
UASM_i_LA_mostly(p, tmp, pgdc);
uasm_i_daddu(p, ptr, ptr, tmp);
@@ -542,42 +602,78 @@ build_get_pmde64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
uasm_l_vmalloc_done(l, *p);
- if (PGDIR_SHIFT - 3 < 32) /* get pgd offset in bytes */
- uasm_i_dsrl(p, tmp, tmp, PGDIR_SHIFT-3);
- else
- uasm_i_dsrl32(p, tmp, tmp, PGDIR_SHIFT - 3 - 32);
+ /* get pgd offset in bytes */
+ uasm_i_dsrl_safe(p, tmp, tmp, PGDIR_SHIFT - 3);
uasm_i_andi(p, tmp, tmp, (PTRS_PER_PGD - 1)<<3);
uasm_i_daddu(p, ptr, ptr, tmp); /* add in pgd offset */
+#ifndef __PAGETABLE_PMD_FOLDED
uasm_i_dmfc0(p, tmp, C0_BADVADDR); /* get faulting address */
uasm_i_ld(p, ptr, 0, ptr); /* get pmd pointer */
- uasm_i_dsrl(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
+ uasm_i_dsrl_safe(p, tmp, tmp, PMD_SHIFT-3); /* get pmd offset in bytes */
uasm_i_andi(p, tmp, tmp, (PTRS_PER_PMD - 1)<<3);
uasm_i_daddu(p, ptr, ptr, tmp); /* add in pmd offset */
+#endif
}
+enum vmalloc64_mode {not_refill, refill};
/*
* BVADDR is the faulting address, PTR is scratch.
* PTR will hold the pgd for vmalloc.
*/
static void __cpuinit
build_get_pgd_vmalloc64(u32 **p, struct uasm_label **l, struct uasm_reloc **r,
- unsigned int bvaddr, unsigned int ptr)
+ unsigned int bvaddr, unsigned int ptr,
+ enum vmalloc64_mode mode)
{
long swpd = (long)swapper_pg_dir;
+ int single_insn_swpd;
+ int did_vmalloc_branch = 0;
+
+ single_insn_swpd = uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd);
uasm_l_vmalloc(l, *p);
- if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) {
- uasm_il_b(p, r, label_vmalloc_done);
- uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
- } else {
- UASM_i_LA_mostly(p, ptr, swpd);
- uasm_il_b(p, r, label_vmalloc_done);
- if (uasm_in_compat_space_p(swpd))
- uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
- else
- uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
+ if (mode == refill && check_for_high_segbits) {
+ if (single_insn_swpd) {
+ uasm_il_bltz(p, r, bvaddr, label_vmalloc_done);
+ uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
+ did_vmalloc_branch = 1;
+ /* fall through */
+ } else {
+ uasm_il_bgez(p, r, bvaddr, label_large_segbits_fault);
+ }
+ }
+ if (!did_vmalloc_branch) {
+ if (uasm_in_compat_space_p(swpd) && !uasm_rel_lo(swpd)) {
+ uasm_il_b(p, r, label_vmalloc_done);
+ uasm_i_lui(p, ptr, uasm_rel_hi(swpd));
+ } else {
+ UASM_i_LA_mostly(p, ptr, swpd);
+ uasm_il_b(p, r, label_vmalloc_done);
+ if (uasm_in_compat_space_p(swpd))
+ uasm_i_addiu(p, ptr, ptr, uasm_rel_lo(swpd));
+ else
+ uasm_i_daddiu(p, ptr, ptr, uasm_rel_lo(swpd));
+ }
+ }
+ if (mode == refill && check_for_high_segbits) {
+ uasm_l_large_segbits_fault(l, *p);
+ /*
+ * We get here if we are an xsseg address, or if we are
+ * an xuseg address above (PGDIR_SHIFT+PGDIR_BITS) boundary.
+ *
+ * Ignoring xsseg (assume disabled so would generate
+ * (address errors?), the only remaining possibility
+ * is the upper xuseg addresses. On processors with
+ * TLB_SEGBITS <= PGDIR_SHIFT+PGDIR_BITS, these
+ * addresses would have taken an address error. We try
+ * to mimic that here by taking a load/istream page
+ * fault.
+ */
+ UASM_i_LA(p, ptr, (unsigned long)tlb_do_page_fault_0);
+ uasm_i_jr(p, ptr);
+ uasm_i_nop(p);
}
}
@@ -684,35 +780,53 @@ static void __cpuinit build_update_entries(u32 **p, unsigned int tmp,
if (cpu_has_64bits) {
uasm_i_ld(p, tmp, 0, ptep); /* get even pte */
uasm_i_ld(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
- uasm_i_dsrl(p, tmp, tmp, 6); /* convert to entrylo0 */
- uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
- uasm_i_dsrl(p, ptep, ptep, 6); /* convert to entrylo1 */
- uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
+ if (kernel_uses_smartmips_rixi) {
+ UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC));
+ UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC));
+ UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+ UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
+ UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+ } else {
+ uasm_i_dsrl_safe(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */
+ UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
+ uasm_i_dsrl_safe(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */
+ }
+ UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
} else {
int pte_off_even = sizeof(pte_t) / 2;
int pte_off_odd = pte_off_even + sizeof(pte_t);
/* The pte entries are pre-shifted */
uasm_i_lw(p, tmp, pte_off_even, ptep); /* get even pte */
- uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
+ UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
uasm_i_lw(p, ptep, pte_off_odd, ptep); /* get odd pte */
- uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
+ UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
}
#else
UASM_i_LW(p, tmp, 0, ptep); /* get even pte */
UASM_i_LW(p, ptep, sizeof(pte_t), ptep); /* get odd pte */
if (r45k_bvahwbug())
build_tlb_probe_entry(p);
- UASM_i_SRL(p, tmp, tmp, 6); /* convert to entrylo0 */
- if (r4k_250MHZhwbug())
- uasm_i_mtc0(p, 0, C0_ENTRYLO0);
- uasm_i_mtc0(p, tmp, C0_ENTRYLO0); /* load it */
- UASM_i_SRL(p, ptep, ptep, 6); /* convert to entrylo1 */
- if (r45k_bvahwbug())
- uasm_i_mfc0(p, tmp, C0_INDEX);
+ if (kernel_uses_smartmips_rixi) {
+ UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_NO_EXEC));
+ UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_NO_EXEC));
+ UASM_i_ROTR(p, tmp, tmp, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+ if (r4k_250MHZhwbug())
+ UASM_i_MTC0(p, 0, C0_ENTRYLO0);
+ UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
+ UASM_i_ROTR(p, ptep, ptep, ilog2(_PAGE_GLOBAL) - ilog2(_PAGE_NO_EXEC));
+ } else {
+ UASM_i_SRL(p, tmp, tmp, ilog2(_PAGE_GLOBAL)); /* convert to entrylo0 */
+ if (r4k_250MHZhwbug())
+ UASM_i_MTC0(p, 0, C0_ENTRYLO0);
+ UASM_i_MTC0(p, tmp, C0_ENTRYLO0); /* load it */
+ UASM_i_SRL(p, ptep, ptep, ilog2(_PAGE_GLOBAL)); /* convert to entrylo1 */
+ if (r45k_bvahwbug())
+ uasm_i_mfc0(p, tmp, C0_INDEX);
+ }
if (r4k_250MHZhwbug())
- uasm_i_mtc0(p, 0, C0_ENTRYLO1);
- uasm_i_mtc0(p, ptep, C0_ENTRYLO1); /* load it */
+ UASM_i_MTC0(p, 0, C0_ENTRYLO1);
+ UASM_i_MTC0(p, ptep, C0_ENTRYLO1); /* load it */
#endif
}
@@ -741,10 +855,15 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
* create the plain linear handler
*/
if (bcm1250_m3_war()) {
- UASM_i_MFC0(&p, K0, C0_BADVADDR);
- UASM_i_MFC0(&p, K1, C0_ENTRYHI);
+ unsigned int segbits = 44;
+
+ uasm_i_dmfc0(&p, K0, C0_BADVADDR);
+ uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
uasm_i_xor(&p, K0, K0, K1);
- UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
+ uasm_i_dsrl_safe(&p, K1, K0, 62);
+ uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
+ uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
+ uasm_i_or(&p, K0, K0, K1);
uasm_il_bnez(&p, &r, K0, label_leave);
/* No need for uasm_i_nop */
}
@@ -773,7 +892,7 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
#endif
#ifdef CONFIG_64BIT
- build_get_pgd_vmalloc64(&p, &l, &r, K0, K1);
+ build_get_pgd_vmalloc64(&p, &l, &r, K0, K1, refill);
#endif
/*
@@ -883,15 +1002,6 @@ static void __cpuinit build_r4000_tlb_refill_handler(void)
}
/*
- * TLB load/store/modify handlers.
- *
- * Only the fastpath gets synthesized at runtime, the slowpath for
- * do_page_fault remains normal asm.
- */
-extern void tlb_do_page_fault_0(void);
-extern void tlb_do_page_fault_1(void);
-
-/*
* 128 instructions for the fastpath handler is generous and should
* never be exceeded.
*/
@@ -985,9 +1095,14 @@ static void __cpuinit
build_pte_present(u32 **p, struct uasm_reloc **r,
unsigned int pte, unsigned int ptr, enum label_id lid)
{
- uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
- uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
- uasm_il_bnez(p, r, pte, lid);
+ if (kernel_uses_smartmips_rixi) {
+ uasm_i_andi(p, pte, pte, _PAGE_PRESENT);
+ uasm_il_beqz(p, r, pte, lid);
+ } else {
+ uasm_i_andi(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
+ uasm_i_xori(p, pte, pte, _PAGE_PRESENT | _PAGE_READ);
+ uasm_il_bnez(p, r, pte, lid);
+ }
iPTE_LW(p, pte, ptr);
}
@@ -1245,7 +1360,7 @@ build_r4000_tlbchange_handler_tail(u32 **p, struct uasm_label **l,
uasm_i_eret(p); /* return from trap */
#ifdef CONFIG_64BIT
- build_get_pgd_vmalloc64(p, l, r, tmp, ptr);
+ build_get_pgd_vmalloc64(p, l, r, tmp, ptr, not_refill);
#endif
}
@@ -1260,10 +1375,15 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
memset(relocs, 0, sizeof(relocs));
if (bcm1250_m3_war()) {
- UASM_i_MFC0(&p, K0, C0_BADVADDR);
- UASM_i_MFC0(&p, K1, C0_ENTRYHI);
+ unsigned int segbits = 44;
+
+ uasm_i_dmfc0(&p, K0, C0_BADVADDR);
+ uasm_i_dmfc0(&p, K1, C0_ENTRYHI);
uasm_i_xor(&p, K0, K0, K1);
- UASM_i_SRL(&p, K0, K0, PAGE_SHIFT + 1);
+ uasm_i_dsrl_safe(&p, K1, K0, 62);
+ uasm_i_dsrl_safe(&p, K0, K0, 12 + 1);
+ uasm_i_dsll_safe(&p, K0, K0, 64 + 12 + 1 - segbits);
+ uasm_i_or(&p, K0, K0, K1);
uasm_il_bnez(&p, &r, K0, label_leave);
/* No need for uasm_i_nop */
}
@@ -1272,6 +1392,34 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
build_pte_present(&p, &r, K0, K1, label_nopage_tlbl);
if (m4kc_tlbp_war())
build_tlb_probe_entry(&p);
+
+ if (kernel_uses_smartmips_rixi) {
+ /*
+ * If the page is not _PAGE_VALID, RI or XI could not
+ * have triggered it. Skip the expensive test..
+ */
+ uasm_i_andi(&p, K0, K0, _PAGE_VALID);
+ uasm_il_beqz(&p, &r, K0, label_tlbl_goaround1);
+ uasm_i_nop(&p);
+
+ uasm_i_tlbr(&p);
+ /* Examine entrylo 0 or 1 based on ptr. */
+ uasm_i_andi(&p, K0, K1, sizeof(pte_t));
+ uasm_i_beqz(&p, K0, 8);
+
+ UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/
+ UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */
+ /*
+ * If the entryLo (now in K0) is valid (bit 1), RI or
+ * XI must have triggered it.
+ */
+ uasm_i_andi(&p, K0, K0, 2);
+ uasm_il_bnez(&p, &r, K0, label_nopage_tlbl);
+
+ uasm_l_tlbl_goaround1(&l, p);
+ /* Reload the PTE value */
+ iPTE_LW(&p, K0, K1);
+ }
build_make_valid(&p, &r, K0, K1);
build_r4000_tlbchange_handler_tail(&p, &l, &r, K0, K1);
@@ -1284,6 +1432,40 @@ static void __cpuinit build_r4000_tlb_load_handler(void)
iPTE_LW(&p, K0, K1);
build_pte_present(&p, &r, K0, K1, label_nopage_tlbl);
build_tlb_probe_entry(&p);
+
+ if (kernel_uses_smartmips_rixi) {
+ /*
+ * If the page is not _PAGE_VALID, RI or XI could not
+ * have triggered it. Skip the expensive test..
+ */
+ uasm_i_andi(&p, K0, K0, _PAGE_VALID);
+ uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2);
+ uasm_i_nop(&p);
+
+ uasm_i_tlbr(&p);
+ /* Examine entrylo 0 or 1 based on ptr. */
+ uasm_i_andi(&p, K0, K1, sizeof(pte_t));
+ uasm_i_beqz(&p, K0, 8);
+
+ UASM_i_MFC0(&p, K0, C0_ENTRYLO0); /* load it in the delay slot*/
+ UASM_i_MFC0(&p, K0, C0_ENTRYLO1); /* load it if ptr is odd */
+ /*
+ * If the entryLo (now in K0) is valid (bit 1), RI or
+ * XI must have triggered it.
+ */
+ uasm_i_andi(&p, K0, K0, 2);
+ uasm_il_beqz(&p, &r, K0, label_tlbl_goaround2);
+ /* Reload the PTE value */
+ iPTE_LW(&p, K0, K1);
+
+ /*
+ * We clobbered C0_PAGEMASK, restore it. On the other branch
+ * it is restored in build_huge_tlb_write_entry.
+ */
+ build_restore_pagemask(&p, &r, K0, label_nopage_tlbl);
+
+ uasm_l_tlbl_goaround2(&l, p);
+ }
uasm_i_ori(&p, K0, K0, (_PAGE_ACCESSED | _PAGE_VALID));
build_huge_handler_tail(&p, &r, &l, K0, K1);
#endif
@@ -1402,6 +1584,10 @@ void __cpuinit build_tlb_refill_handler(void)
*/
static int run_once = 0;
+#ifdef CONFIG_64BIT
+ check_for_high_segbits = current_cpu_data.vmbits > (PGDIR_SHIFT + PGD_ORDER + PAGE_SHIFT - 3);
+#endif
+
switch (current_cpu_type()) {
case CPU_R2000:
case CPU_R3000:
generated by cgit v1.2.3-70-g09d2 (git 2.46.0) at 2024-12-01 01:07:15 +0000