diff options
| -rw-r--r-- | arch/tile/include/asm/irqflags.h | 34 | ||||
| -rw-r--r-- | arch/tile/kernel/entry.S | 3 | ||||
| -rw-r--r-- | arch/tile/kernel/intvec_64.S | 78 | ||||
| -rw-r--r-- | arch/tile/mm/init.c | 4 | ||||
| -rw-r--r-- | arch/tile/mm/migrate.h | 6 | ||||
| -rw-r--r-- | arch/tile/mm/migrate_32.S | 36 | ||||
| -rw-r--r-- | arch/tile/mm/migrate_64.S | 34 |
7 files changed, 96 insertions, 99 deletions
diff --git a/arch/tile/include/asm/irqflags.h b/arch/tile/include/asm/irqflags.h index 5db0ce54284..b4e96fef2cf 100644 --- a/arch/tile/include/asm/irqflags.h +++ b/arch/tile/include/asm/irqflags.h @@ -28,10 +28,10 @@ */ #if CHIP_HAS_AUX_PERF_COUNTERS() #define LINUX_MASKABLE_INTERRUPTS_HI \ - (~(INT_MASK_HI(INT_PERF_COUNT) | INT_MASK_HI(INT_AUX_PERF_COUNT))) + (~(INT_MASK_HI(INT_PERF_COUNT) | INT_MASK_HI(INT_AUX_PERF_COUNT))) #else #define LINUX_MASKABLE_INTERRUPTS_HI \ - (~(INT_MASK_HI(INT_PERF_COUNT))) + (~(INT_MASK_HI(INT_PERF_COUNT))) #endif #else @@ -90,6 +90,14 @@ __insn_mtspr(SPR_INTERRUPT_MASK_RESET_K_0, (unsigned long)(__m)); \ __insn_mtspr(SPR_INTERRUPT_MASK_RESET_K_1, (unsigned long)(__m>>32)); \ } while (0) +#define interrupt_mask_save_mask() \ + (__insn_mfspr(SPR_INTERRUPT_MASK_SET_K_0) | \ + (((unsigned long long)__insn_mfspr(SPR_INTERRUPT_MASK_SET_K_1))<<32)) +#define interrupt_mask_restore_mask(mask) do { \ + unsigned long long __m = (mask); \ + __insn_mtspr(SPR_INTERRUPT_MASK_K_0, (unsigned long)(__m)); \ + __insn_mtspr(SPR_INTERRUPT_MASK_K_1, (unsigned long)(__m>>32)); \ +} while (0) #else #define interrupt_mask_set(n) \ __insn_mtspr(SPR_INTERRUPT_MASK_SET_K, (1UL << (n))) @@ -101,6 +109,10 @@ __insn_mtspr(SPR_INTERRUPT_MASK_SET_K, (mask)) #define interrupt_mask_reset_mask(mask) \ __insn_mtspr(SPR_INTERRUPT_MASK_RESET_K, (mask)) +#define interrupt_mask_save_mask() \ + __insn_mfspr(SPR_INTERRUPT_MASK_K) +#define interrupt_mask_restore_mask(mask) \ + __insn_mtspr(SPR_INTERRUPT_MASK_K, (mask)) #endif /* @@ -122,7 +134,7 @@ DECLARE_PER_CPU(unsigned long long, interrupts_enabled_mask); /* Disable all interrupts, including NMIs. */ #define arch_local_irq_disable_all() \ - interrupt_mask_set_mask(-1UL) + interrupt_mask_set_mask(-1ULL) /* Re-enable all maskable interrupts. */ #define arch_local_irq_enable() \ @@ -179,7 +191,7 @@ DECLARE_PER_CPU(unsigned long long, interrupts_enabled_mask); #ifdef __tilegx__ #if INT_MEM_ERROR != 0 -# error Fix IRQ_DISABLED() macro +# error Fix IRQS_DISABLED() macro #endif /* Return 0 or 1 to indicate whether interrupts are currently disabled. */ @@ -207,9 +219,10 @@ DECLARE_PER_CPU(unsigned long long, interrupts_enabled_mask); mtspr SPR_INTERRUPT_MASK_SET_K, tmp /* Enable interrupts. */ -#define IRQ_ENABLE(tmp0, tmp1) \ +#define IRQ_ENABLE_LOAD(tmp0, tmp1) \ GET_INTERRUPTS_ENABLED_MASK_PTR(tmp0); \ - ld tmp0, tmp0; \ + ld tmp0, tmp0 +#define IRQ_ENABLE_APPLY(tmp0, tmp1) \ mtspr SPR_INTERRUPT_MASK_RESET_K, tmp0 #else /* !__tilegx__ */ @@ -253,17 +266,22 @@ DECLARE_PER_CPU(unsigned long long, interrupts_enabled_mask); mtspr SPR_INTERRUPT_MASK_SET_K_1, tmp /* Enable interrupts. */ -#define IRQ_ENABLE(tmp0, tmp1) \ +#define IRQ_ENABLE_LOAD(tmp0, tmp1) \ GET_INTERRUPTS_ENABLED_MASK_PTR(tmp0); \ { \ lw tmp0, tmp0; \ addi tmp1, tmp0, 4 \ }; \ - lw tmp1, tmp1; \ + lw tmp1, tmp1 +#define IRQ_ENABLE_APPLY(tmp0, tmp1) \ mtspr SPR_INTERRUPT_MASK_RESET_K_0, tmp0; \ mtspr SPR_INTERRUPT_MASK_RESET_K_1, tmp1 #endif +#define IRQ_ENABLE(tmp0, tmp1) \ + IRQ_ENABLE_LOAD(tmp0, tmp1); \ + IRQ_ENABLE_APPLY(tmp0, tmp1) + /* * Do the CPU's IRQ-state tracing from assembly code. We call a * C function, but almost everywhere we do, we don't mind clobbering diff --git a/arch/tile/kernel/entry.S b/arch/tile/kernel/entry.S index ec91568df88..133c4b56a99 100644 --- a/arch/tile/kernel/entry.S +++ b/arch/tile/kernel/entry.S @@ -100,8 +100,9 @@ STD_ENTRY(smp_nap) */ STD_ENTRY(_cpu_idle) movei r1, 1 + IRQ_ENABLE_LOAD(r2, r3) mtspr INTERRUPT_CRITICAL_SECTION, r1 - IRQ_ENABLE(r2, r3) /* unmask, but still with ICS set */ + IRQ_ENABLE_APPLY(r2, r3) /* unmask, but still with ICS set */ mtspr INTERRUPT_CRITICAL_SECTION, zero .global _cpu_idle_nap _cpu_idle_nap: diff --git a/arch/tile/kernel/intvec_64.S b/arch/tile/kernel/intvec_64.S index 30ae76e50c4..0ae8723ea57 100644 --- a/arch/tile/kernel/intvec_64.S +++ b/arch/tile/kernel/intvec_64.S @@ -220,7 +220,9 @@ intvec_\vecname: * This routine saves just the first four registers, plus the * stack context so we can do proper backtracing right away, * and defers to handle_interrupt to save the rest. - * The backtracer needs pc, ex1, lr, sp, r52, and faultnum. + * The backtracer needs pc, ex1, lr, sp, r52, and faultnum, + * and needs sp set to its final location at the bottom of + * the stack frame. */ addli r0, r0, PTREGS_OFFSET_LR - (PTREGS_SIZE + KSTK_PTREGS_GAP) wh64 r0 /* cache line 7 */ @@ -450,23 +452,6 @@ intvec_\vecname: push_reg r5, r52 st r52, r4 - /* Load tp with our per-cpu offset. */ -#ifdef CONFIG_SMP - { - mfspr r20, SPR_SYSTEM_SAVE_K_0 - moveli r21, hw2_last(__per_cpu_offset) - } - { - shl16insli r21, r21, hw1(__per_cpu_offset) - bfextu r20, r20, 0, LOG2_THREAD_SIZE-1 - } - shl16insli r21, r21, hw0(__per_cpu_offset) - shl3add r20, r20, r21 - ld tp, r20 -#else - move tp, zero -#endif - /* * If we will be returning to the kernel, we will need to * reset the interrupt masks to the state they had before. @@ -489,6 +474,44 @@ intvec_\vecname: .endif st r21, r32 + /* + * we've captured enough state to the stack (including in + * particular our EX_CONTEXT state) that we can now release + * the interrupt critical section and replace it with our + * standard "interrupts disabled" mask value. This allows + * synchronous interrupts (and profile interrupts) to punch + * through from this point onwards. + * + * It's important that no code before this point touch memory + * other than our own stack (to keep the invariant that this + * is all that gets touched under ICS), and that no code after + * this point reference any interrupt-specific SPR, in particular + * the EX_CONTEXT_K_ values. + */ + .ifc \function,handle_nmi + IRQ_DISABLE_ALL(r20) + .else + IRQ_DISABLE(r20, r21) + .endif + mtspr INTERRUPT_CRITICAL_SECTION, zero + + /* Load tp with our per-cpu offset. */ +#ifdef CONFIG_SMP + { + mfspr r20, SPR_SYSTEM_SAVE_K_0 + moveli r21, hw2_last(__per_cpu_offset) + } + { + shl16insli r21, r21, hw1(__per_cpu_offset) + bfextu r20, r20, 0, LOG2_THREAD_SIZE-1 + } + shl16insli r21, r21, hw0(__per_cpu_offset) + shl3add r20, r20, r21 + ld tp, r20 +#else + move tp, zero +#endif + #ifdef __COLLECT_LINKER_FEEDBACK__ /* * Notify the feedback routines that we were in the @@ -513,21 +536,6 @@ intvec_\vecname: #endif /* - * we've captured enough state to the stack (including in - * particular our EX_CONTEXT state) that we can now release - * the interrupt critical section and replace it with our - * standard "interrupts disabled" mask value. This allows - * synchronous interrupts (and profile interrupts) to punch - * through from this point onwards. - */ - .ifc \function,handle_nmi - IRQ_DISABLE_ALL(r20) - .else - IRQ_DISABLE(r20, r21) - .endif - mtspr INTERRUPT_CRITICAL_SECTION, zero - - /* * Prepare the first 256 stack bytes to be rapidly accessible * without having to fetch the background data. */ @@ -736,9 +744,10 @@ STD_ENTRY(interrupt_return) beqzt r30, .Lrestore_regs j 3f 2: TRACE_IRQS_ON + IRQ_ENABLE_LOAD(r20, r21) movei r0, 1 mtspr INTERRUPT_CRITICAL_SECTION, r0 - IRQ_ENABLE(r20, r21) + IRQ_ENABLE_APPLY(r20, r21) beqzt r30, .Lrestore_regs 3: @@ -755,7 +764,6 @@ STD_ENTRY(interrupt_return) * that will save some cycles if this turns out to be a syscall. */ .Lrestore_regs: - FEEDBACK_REENTER(interrupt_return) /* called from elsewhere */ /* * Rotate so we have one high bit and one low bit to test. diff --git a/arch/tile/mm/init.c b/arch/tile/mm/init.c index 6a9d20ddc34..1e4633520b3 100644 --- a/arch/tile/mm/init.c +++ b/arch/tile/mm/init.c @@ -444,6 +444,7 @@ static pgd_t pgtables[PTRS_PER_PGD] */ static void __init kernel_physical_mapping_init(pgd_t *pgd_base) { + unsigned long long irqmask; unsigned long address, pfn; pmd_t *pmd; pte_t *pte; @@ -633,10 +634,13 @@ static void __init kernel_physical_mapping_init(pgd_t *pgd_base) * - install pgtables[] as the real page table * - flush the TLB so the new page table takes effect */ + irqmask = interrupt_mask_save_mask(); + interrupt_mask_set_mask(-1ULL); rc = flush_and_install_context(__pa(pgtables), init_pgprot((unsigned long)pgtables), __get_cpu_var(current_asid), cpumask_bits(my_cpu_mask)); + interrupt_mask_restore_mask(irqmask); BUG_ON(rc != 0); /* Copy the page table back to the normal swapper_pg_dir. */ diff --git a/arch/tile/mm/migrate.h b/arch/tile/mm/migrate.h index cd45a0837fa..91683d97917 100644 --- a/arch/tile/mm/migrate.h +++ b/arch/tile/mm/migrate.h @@ -24,6 +24,9 @@ /* * This function is used as a helper when setting up the initial * page table (swapper_pg_dir). + * + * You must mask ALL interrupts prior to invoking this code, since + * you can't legally touch the stack during the cache flush. */ extern int flush_and_install_context(HV_PhysAddr page_table, HV_PTE access, HV_ASID asid, @@ -39,6 +42,9 @@ extern int flush_and_install_context(HV_PhysAddr page_table, HV_PTE access, * * Note that any non-NULL pointers must not point to the page that * is handled by the stack_pte itself. + * + * You must mask ALL interrupts prior to invoking this code, since + * you can't legally touch the stack during the cache flush. */ extern int homecache_migrate_stack_and_flush(pte_t stack_pte, unsigned long va, size_t length, pte_t *stack_ptep, diff --git a/arch/tile/mm/migrate_32.S b/arch/tile/mm/migrate_32.S index ac01a7cdf77..5305814bf18 100644 --- a/arch/tile/mm/migrate_32.S +++ b/arch/tile/mm/migrate_32.S @@ -40,8 +40,7 @@ #define FRAME_R32 16 #define FRAME_R33 20 #define FRAME_R34 24 -#define FRAME_R35 28 -#define FRAME_SIZE 32 +#define FRAME_SIZE 28 @@ -66,12 +65,11 @@ #define r_my_cpumask r5 /* Locals (callee-save); must not be more than FRAME_xxx above. */ -#define r_save_ics r30 -#define r_context_lo r31 -#define r_context_hi r32 -#define r_access_lo r33 -#define r_access_hi r34 -#define r_asid r35 +#define r_context_lo r30 +#define r_context_hi r31 +#define r_access_lo r32 +#define r_access_hi r33 +#define r_asid r34 STD_ENTRY(flush_and_install_context) /* @@ -104,11 +102,7 @@ STD_ENTRY(flush_and_install_context) sw r_tmp, r33 addi r_tmp, sp, FRAME_R34 } - { - sw r_tmp, r34 - addi r_tmp, sp, FRAME_R35 - } - sw r_tmp, r35 + sw r_tmp, r34 /* Move some arguments to callee-save registers. */ { @@ -121,13 +115,6 @@ STD_ENTRY(flush_and_install_context) } move r_asid, r_asid_in - /* Disable interrupts, since we can't use our stack. */ - { - mfspr r_save_ics, INTERRUPT_CRITICAL_SECTION - movei r_tmp, 1 - } - mtspr INTERRUPT_CRITICAL_SECTION, r_tmp - /* First, flush our L2 cache. */ { move r0, zero /* cache_pa */ @@ -163,7 +150,7 @@ STD_ENTRY(flush_and_install_context) } { move r4, r_asid - movei r5, HV_CTX_DIRECTIO + moveli r5, HV_CTX_DIRECTIO | CTX_PAGE_FLAG } jal hv_install_context bnz r0, .Ldone @@ -175,9 +162,6 @@ STD_ENTRY(flush_and_install_context) } .Ldone: - /* Reset interrupts back how they were before. */ - mtspr INTERRUPT_CRITICAL_SECTION, r_save_ics - /* Restore the callee-saved registers and return. */ addli lr, sp, FRAME_SIZE { @@ -202,10 +186,6 @@ STD_ENTRY(flush_and_install_context) } { lw r34, r_tmp - addli r_tmp, sp, FRAME_R35 - } - { - lw r35, r_tmp addi sp, sp, FRAME_SIZE } jrp lr diff --git a/arch/tile/mm/migrate_64.S b/arch/tile/mm/migrate_64.S index e76fea688be..1d15b10833d 100644 --- a/arch/tile/mm/migrate_64.S +++ b/arch/tile/mm/migrate_64.S @@ -38,8 +38,7 @@ #define FRAME_R30 16 #define FRAME_R31 24 #define FRAME_R32 32 -#define FRAME_R33 40 -#define FRAME_SIZE 48 +#define FRAME_SIZE 40 @@ -60,10 +59,9 @@ #define r_my_cpumask r3 /* Locals (callee-save); must not be more than FRAME_xxx above. */ -#define r_save_ics r30 -#define r_context r31 -#define r_access r32 -#define r_asid r33 +#define r_context r30 +#define r_access r31 +#define r_asid r32 /* * Caller-save locals and frame constants are the same as @@ -93,11 +91,7 @@ STD_ENTRY(flush_and_install_context) st r_tmp, r31 addi r_tmp, sp, FRAME_R32 } - { - st r_tmp, r32 - addi r_tmp, sp, FRAME_R33 - } - st r_tmp, r33 + st r_tmp, r32 /* Move some arguments to callee-save registers. */ { @@ -106,13 +100,6 @@ STD_ENTRY(flush_and_install_context) } move r_asid, r_asid_in - /* Disable interrupts, since we can't use our stack. */ - { - mfspr r_save_ics, INTERRUPT_CRITICAL_SECTION - movei r_tmp, 1 - } - mtspr INTERRUPT_CRITICAL_SECTION, r_tmp - /* First, flush our L2 cache. */ { move r0, zero /* cache_pa */ @@ -147,7 +134,7 @@ STD_ENTRY(flush_and_install_context) } { move r2, r_asid - movei r3, HV_CTX_DIRECTIO + moveli r3, HV_CTX_DIRECTIO | CTX_PAGE_FLAG } jal hv_install_context bnez r0, 1f @@ -158,10 +145,7 @@ STD_ENTRY(flush_and_install_context) jal hv_flush_all } -1: /* Reset interrupts back how they were before. */ - mtspr INTERRUPT_CRITICAL_SECTION, r_save_ics - - /* Restore the callee-saved registers and return. */ +1: /* Restore the callee-saved registers and return. */ addli lr, sp, FRAME_SIZE { ld lr, lr @@ -177,10 +161,6 @@ STD_ENTRY(flush_and_install_context) } { ld r32, r_tmp - addli r_tmp, sp, FRAME_R33 - } - { - ld r33, r_tmp addi sp, sp, FRAME_SIZE } jrp lr |