MIPS: Octeon: Make interrupt controller work with threaded handlers.

For CIUv1 controllers, we were relying on all calls to the irq_chip
functions to be done from the CPU that received the irq, and that they
would all be done from interrupt contest.  These assumptions do not
hold for threaded handlers.

We make all the masking actually mask the irq source, and use real
raw_spin_locks instead of manually twiddling the Status[IE] bit.

Signed-off-by: David Daney <david.daney@cavium.com>

1 files changed, 137 insertions, 143 deletions

diff --git a/arch/mips/cavium-octeon/octeon-irq.c b/arch/mips/cavium-octeon/octeon-irq.c
index aba085b2c0d..fadec885916 100644
--- a/arch/mips/cavium-octeon/octeon-irq.c
+++ b/arch/mips/cavium-octeon/octeon-irq.c
@@ -18,11 +18,9 @@
 #include <asm/octeon/octeon.h>
 #include <asm/octeon/cvmx-ciu2-defs.h>
 
-static DEFINE_RAW_SPINLOCK(octeon_irq_ciu0_lock);
-static DEFINE_RAW_SPINLOCK(octeon_irq_ciu1_lock);
-
 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu0_en_mirror);
 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu1_en_mirror);
+static DEFINE_PER_CPU(raw_spinlock_t, octeon_irq_ciu_spinlock);
 
 static __read_mostly u8 octeon_irq_ciu_to_irq[8][64];
 
@@ -234,22 +232,31 @@ static void octeon_irq_ciu_enable(struct irq_data *data)
 	unsigned long *pen;
 	unsigned long flags;
 	union octeon_ciu_chip_data cd;
+	raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
 
 	cd.p = irq_data_get_irq_chip_data(data);
 
+	raw_spin_lock_irqsave(lock, flags);
 	if (cd.s.line == 0) {
-		raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
 		pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
-		set_bit(cd.s.bit, pen);
+		__set_bit(cd.s.bit, pen);
+		/*
+		 * Must be visible to octeon_irq_ip{2,3}_ciu() before
+		 * enabling the irq.
+		 */
+		wmb();
 		cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
-		raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
 	} else {
-		raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
 		pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
-		set_bit(cd.s.bit, pen);
+		__set_bit(cd.s.bit, pen);
+		/*
+		 * Must be visible to octeon_irq_ip{2,3}_ciu() before
+		 * enabling the irq.
+		 */
+		wmb();
 		cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
-		raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
 	}
+	raw_spin_unlock_irqrestore(lock, flags);
 }
 
 static void octeon_irq_ciu_enable_local(struct irq_data *data)
@@ -257,22 +264,31 @@ static void octeon_irq_ciu_enable_local(struct irq_data *data)
 	unsigned long *pen;
 	unsigned long flags;
 	union octeon_ciu_chip_data cd;
+	raw_spinlock_t *lock = &__get_cpu_var(octeon_irq_ciu_spinlock);
 
 	cd.p = irq_data_get_irq_chip_data(data);
 
+	raw_spin_lock_irqsave(lock, flags);
 	if (cd.s.line == 0) {
-		raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
 		pen = &__get_cpu_var(octeon_irq_ciu0_en_mirror);
-		set_bit(cd.s.bit, pen);
+		__set_bit(cd.s.bit, pen);
+		/*
+		 * Must be visible to octeon_irq_ip{2,3}_ciu() before
+		 * enabling the irq.
+		 */
+		wmb();
 		cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen);
-		raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
 	} else {
-		raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
 		pen = &__get_cpu_var(octeon_irq_ciu1_en_mirror);
-		set_bit(cd.s.bit, pen);
+		__set_bit(cd.s.bit, pen);
+		/*
+		 * Must be visible to octeon_irq_ip{2,3}_ciu() before
+		 * enabling the irq.
+		 */
+		wmb();
 		cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen);
-		raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
 	}
+	raw_spin_unlock_irqrestore(lock, flags);
 }
 
 static void octeon_irq_ciu_disable_local(struct irq_data *data)
@@ -280,22 +296,31 @@ static void octeon_irq_ciu_disable_local(struct irq_data *data)
 	unsigned long *pen;
 	unsigned long flags;
 	union octeon_ciu_chip_data cd;
+	raw_spinlock_t *lock = &__get_cpu_var(octeon_irq_ciu_spinlock);
 
 	cd.p = irq_data_get_irq_chip_data(data);
 
+	raw_spin_lock_irqsave(lock, flags);
 	if (cd.s.line == 0) {
-		raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
 		pen = &__get_cpu_var(octeon_irq_ciu0_en_mirror);
-		clear_bit(cd.s.bit, pen);
+		__clear_bit(cd.s.bit, pen);
+		/*
+		 * Must be visible to octeon_irq_ip{2,3}_ciu() before
+		 * enabling the irq.
+		 */
+		wmb();
 		cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen);
-		raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
 	} else {
-		raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
 		pen = &__get_cpu_var(octeon_irq_ciu1_en_mirror);
-		clear_bit(cd.s.bit, pen);
+		__clear_bit(cd.s.bit, pen);
+		/*
+		 * Must be visible to octeon_irq_ip{2,3}_ciu() before
+		 * enabling the irq.
+		 */
+		wmb();
 		cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen);
-		raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
 	}
+	raw_spin_unlock_irqrestore(lock, flags);
 }
 
 static void octeon_irq_ciu_disable_all(struct irq_data *data)
@@ -304,29 +329,30 @@ static void octeon_irq_ciu_disable_all(struct irq_data *data)
 	unsigned long *pen;
 	int cpu;
 	union octeon_ciu_chip_data cd;
-
-	wmb(); /* Make sure flag changes arrive before register updates. */
+	raw_spinlock_t *lock;
 
 	cd.p = irq_data_get_irq_chip_data(data);
 
-	if (cd.s.line == 0) {
-		raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
-		for_each_online_cpu(cpu) {
-			int coreid = octeon_coreid_for_cpu(cpu);
+	for_each_online_cpu(cpu) {
+		int coreid = octeon_coreid_for_cpu(cpu);
+		lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
+		if (cd.s.line == 0)
 			pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
-			clear_bit(cd.s.bit, pen);
-			cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
-		}
-		raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
-	} else {
-		raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
-		for_each_online_cpu(cpu) {
-			int coreid = octeon_coreid_for_cpu(cpu);
+		else
 			pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
-			clear_bit(cd.s.bit, pen);
+
+		raw_spin_lock_irqsave(lock, flags);
+		__clear_bit(cd.s.bit, pen);
+		/*
+		 * Must be visible to octeon_irq_ip{2,3}_ciu() before
+		 * enabling the irq.
+		 */
+		wmb();
+		if (cd.s.line == 0)
+			cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
+		else
 			cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
-		}
-		raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
+		raw_spin_unlock_irqrestore(lock, flags);
 	}
 }
 
@@ -336,27 +362,30 @@ static void octeon_irq_ciu_enable_all(struct irq_data *data)
 	unsigned long *pen;
 	int cpu;
 	union octeon_ciu_chip_data cd;
+	raw_spinlock_t *lock;
 
 	cd.p = irq_data_get_irq_chip_data(data);
 
-	if (cd.s.line == 0) {
-		raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
-		for_each_online_cpu(cpu) {
-			int coreid = octeon_coreid_for_cpu(cpu);
+	for_each_online_cpu(cpu) {
+		int coreid = octeon_coreid_for_cpu(cpu);
+		lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
+		if (cd.s.line == 0)
 			pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
-			set_bit(cd.s.bit, pen);
-			cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
-		}
-		raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
-	} else {
-		raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
-		for_each_online_cpu(cpu) {
-			int coreid = octeon_coreid_for_cpu(cpu);
+		else
 			pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
-			set_bit(cd.s.bit, pen);
+
+		raw_spin_lock_irqsave(lock, flags);
+		__set_bit(cd.s.bit, pen);
+		/*
+		 * Must be visible to octeon_irq_ip{2,3}_ciu() before
+		 * enabling the irq.
+		 */
+		wmb();
+		if (cd.s.line == 0)
+			cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
+		else
 			cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
-		}
-		raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
+		raw_spin_unlock_irqrestore(lock, flags);
 	}
 }
 
@@ -459,8 +488,6 @@ static void octeon_irq_ciu_disable_all_v2(struct irq_data *data)
 	u64 mask;
 	union octeon_ciu_chip_data cd;
 
-	wmb(); /* Make sure flag changes arrive before register updates. */
-
 	cd.p = irq_data_get_irq_chip_data(data);
 	mask = 1ull << (cd.s.bit);
 
@@ -618,6 +645,8 @@ static int octeon_irq_ciu_set_affinity(struct irq_data *data,
 	bool enable_one = !irqd_irq_disabled(data) && !irqd_irq_masked(data);
 	unsigned long flags;
 	union octeon_ciu_chip_data cd;
+	unsigned long *pen;
+	raw_spinlock_t *lock;
 
 	cd.p = irq_data_get_irq_chip_data(data);
 
@@ -632,36 +661,36 @@ static int octeon_irq_ciu_set_affinity(struct irq_data *data,
 	if (!enable_one)
 		return 0;
 
-	if (cd.s.line == 0) {
-		raw_spin_lock_irqsave(&octeon_irq_ciu0_lock, flags);
-		for_each_online_cpu(cpu) {
-			int coreid = octeon_coreid_for_cpu(cpu);
-			unsigned long *pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
 
-			if (cpumask_test_cpu(cpu, dest) && enable_one) {
-				enable_one = false;
-				set_bit(cd.s.bit, pen);
-			} else {
-				clear_bit(cd.s.bit, pen);
-			}
-			cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
+	for_each_online_cpu(cpu) {
+		int coreid = octeon_coreid_for_cpu(cpu);
+
+		lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
+		raw_spin_lock_irqsave(lock, flags);
+
+		if (cd.s.line == 0)
+			pen = &per_cpu(octeon_irq_ciu0_en_mirror, cpu);
+		else
+			pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
+
+		if (cpumask_test_cpu(cpu, dest) && enable_one) {
+			enable_one = 0;
+			__set_bit(cd.s.bit, pen);
+		} else {
+			__clear_bit(cd.s.bit, pen);
 		}
-		raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock, flags);
-	} else {
-		raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
-		for_each_online_cpu(cpu) {
-			int coreid = octeon_coreid_for_cpu(cpu);
-			unsigned long *pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
+		/*
+		 * Must be visible to octeon_irq_ip{2,3}_ciu() before
+		 * enabling the irq.
+		 */
+		wmb();
 
-			if (cpumask_test_cpu(cpu, dest) && enable_one) {
-				enable_one = false;
-				set_bit(cd.s.bit, pen);
-			} else {
-				clear_bit(cd.s.bit, pen);
-			}
+		if (cd.s.line == 0)
+			cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), *pen);
+		else
 			cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
-		}
-		raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
+
+		raw_spin_unlock_irqrestore(lock, flags);
 	}
 	return 0;
 }
@@ -717,14 +746,6 @@ static int octeon_irq_ciu_set_affinity_v2(struct irq_data *data,
 #endif
 
 /*
- * The v1 CIU code already masks things, so supply a dummy version to
- * the core chip code.
- */
-static void octeon_irq_dummy_mask(struct irq_data *data)
-{
-}
-
-/*
  * Newer octeon chips have support for lockless CIU operation.
  */
 static struct irq_chip octeon_irq_chip_ciu_v2 = {
@@ -745,7 +766,8 @@ static struct irq_chip octeon_irq_chip_ciu = {
 	.irq_enable = octeon_irq_ciu_enable,
 	.irq_disable = octeon_irq_ciu_disable_all,
 	.irq_ack = octeon_irq_ciu_ack,
-	.irq_mask = octeon_irq_dummy_mask,
+	.irq_mask = octeon_irq_ciu_disable_local,
+	.irq_unmask = octeon_irq_ciu_enable,
 #ifdef CONFIG_SMP
 	.irq_set_affinity = octeon_irq_ciu_set_affinity,
 	.irq_cpu_offline = octeon_irq_cpu_offline_ciu,
@@ -769,6 +791,8 @@ static struct irq_chip octeon_irq_chip_ciu_mbox = {
 	.name = "CIU-M",
 	.irq_enable = octeon_irq_ciu_enable_all,
 	.irq_disable = octeon_irq_ciu_disable_all,
+	.irq_ack = octeon_irq_ciu_disable_local,
+	.irq_eoi = octeon_irq_ciu_enable_local,
 
 	.irq_cpu_online = octeon_irq_ciu_enable_local,
 	.irq_cpu_offline = octeon_irq_ciu_disable_local,
@@ -793,7 +817,8 @@ static struct irq_chip octeon_irq_chip_ciu_gpio = {
 	.name = "CIU-GPIO",
 	.irq_enable = octeon_irq_ciu_enable_gpio,
 	.irq_disable = octeon_irq_ciu_disable_gpio,
-	.irq_mask = octeon_irq_dummy_mask,
+	.irq_mask = octeon_irq_ciu_disable_local,
+	.irq_unmask = octeon_irq_ciu_enable,
 	.irq_ack = octeon_irq_ciu_gpio_ack,
 	.irq_set_type = octeon_irq_ciu_gpio_set_type,
 #ifdef CONFIG_SMP
@@ -812,12 +837,18 @@ static void octeon_irq_ciu_wd_enable(struct irq_data *data)
 	unsigned long *pen;
 	int coreid = data->irq - OCTEON_IRQ_WDOG0;	/* Bit 0-63 of EN1 */
 	int cpu = octeon_cpu_for_coreid(coreid);
+	raw_spinlock_t *lock = &per_cpu(octeon_irq_ciu_spinlock, cpu);
 
-	raw_spin_lock_irqsave(&octeon_irq_ciu1_lock, flags);
+	raw_spin_lock_irqsave(lock, flags);
 	pen = &per_cpu(octeon_irq_ciu1_en_mirror, cpu);
-	set_bit(coreid, pen);
+	__set_bit(coreid, pen);
+	/*
+	 * Must be visible to octeon_irq_ip{2,3}_ciu() before enabling
+	 * the irq.
+	 */
+	wmb();
 	cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), *pen);
-	raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock, flags);
+	raw_spin_unlock_irqrestore(lock, flags);
 }
 
 /*
@@ -846,7 +877,8 @@ static struct irq_chip octeon_irq_chip_ciu_wd = {
 	.name = "CIU-W",
 	.irq_enable = octeon_irq_ciu_wd_enable,
 	.irq_disable = octeon_irq_ciu_disable_all,
-	.irq_mask = octeon_irq_dummy_mask,
+	.irq_mask = octeon_irq_ciu_disable_local,
+	.irq_unmask = octeon_irq_ciu_enable_local,
 };
 
 static bool octeon_irq_ciu_is_edge(unsigned int line, unsigned int bit)
@@ -1027,27 +1059,7 @@ static struct irq_domain_ops octeon_irq_domain_gpio_ops = {
 	.xlate = octeon_irq_gpio_xlat,
 };
 
-static void octeon_irq_ip2_v1(void)
-{
-	const unsigned long core_id = cvmx_get_core_num();
-	u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2));
-
-	ciu_sum &= __get_cpu_var(octeon_irq_ciu0_en_mirror);
-	clear_c0_status(STATUSF_IP2);
-	if (likely(ciu_sum)) {
-		int bit = fls64(ciu_sum) - 1;
-		int irq = octeon_irq_ciu_to_irq[0][bit];
-		if (likely(irq))
-			do_IRQ(irq);
-		else
-			spurious_interrupt();
-	} else {
-		spurious_interrupt();
-	}
-	set_c0_status(STATUSF_IP2);
-}
-
-static void octeon_irq_ip2_v2(void)
+static void octeon_irq_ip2_ciu(void)
 {
 	const unsigned long core_id = cvmx_get_core_num();
 	u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id * 2));
@@ -1064,26 +1076,8 @@ static void octeon_irq_ip2_v2(void)
 		spurious_interrupt();
 	}
 }
-static void octeon_irq_ip3_v1(void)
-{
-	u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1);
 
-	ciu_sum &= __get_cpu_var(octeon_irq_ciu1_en_mirror);
-	clear_c0_status(STATUSF_IP3);
-	if (likely(ciu_sum)) {
-		int bit = fls64(ciu_sum) - 1;
-		int irq = octeon_irq_ciu_to_irq[1][bit];
-		if (likely(irq))
-			do_IRQ(irq);
-		else
-			spurious_interrupt();
-	} else {
-		spurious_interrupt();
-	}
-	set_c0_status(STATUSF_IP3);
-}
-
-static void octeon_irq_ip3_v2(void)
+static void octeon_irq_ip3_ciu(void)
 {
 	u64 ciu_sum = cvmx_read_csr(CVMX_CIU_INT_SUM1);
 
@@ -1134,6 +1128,12 @@ static void __cpuinit octeon_irq_percpu_enable(void)
 static void __cpuinit octeon_irq_init_ciu_percpu(void)
 {
 	int coreid = cvmx_get_core_num();
+
+
+	__get_cpu_var(octeon_irq_ciu0_en_mirror) = 0;
+	__get_cpu_var(octeon_irq_ciu1_en_mirror) = 0;
+	wmb();
+	raw_spin_lock_init(&__get_cpu_var(octeon_irq_ciu_spinlock));
 	/*
 	 * Disable All CIU Interrupts. The ones we need will be
 	 * enabled later.  Read the SUM register so we know the write
@@ -1170,10 +1170,6 @@ static void octeon_irq_init_ciu2_percpu(void)
 
 static void __cpuinit octeon_irq_setup_secondary_ciu(void)
 {
-
-	__get_cpu_var(octeon_irq_ciu0_en_mirror) = 0;
-	__get_cpu_var(octeon_irq_ciu1_en_mirror) = 0;
-
 	octeon_irq_init_ciu_percpu();
 	octeon_irq_percpu_enable();
 
@@ -1208,19 +1204,17 @@ static void __init octeon_irq_init_ciu(void)
 	octeon_irq_init_ciu_percpu();
 	octeon_irq_setup_secondary = octeon_irq_setup_secondary_ciu;
 
+	octeon_irq_ip2 = octeon_irq_ip2_ciu;
+	octeon_irq_ip3 = octeon_irq_ip3_ciu;
 	if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X) ||
 	    OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X) ||
 	    OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X) ||
 	    OCTEON_IS_MODEL(OCTEON_CN6XXX)) {
-		octeon_irq_ip2 = octeon_irq_ip2_v2;
-		octeon_irq_ip3 = octeon_irq_ip3_v2;
 		chip = &octeon_irq_chip_ciu_v2;
 		chip_mbox = &octeon_irq_chip_ciu_mbox_v2;
 		chip_wd = &octeon_irq_chip_ciu_wd_v2;
 		octeon_irq_gpio_chip = &octeon_irq_chip_ciu_gpio_v2;
 	} else {
-		octeon_irq_ip2 = octeon_irq_ip2_v1;
-		octeon_irq_ip3 = octeon_irq_ip3_v1;
 		chip = &octeon_irq_chip_ciu;
 		chip_mbox = &octeon_irq_chip_ciu_mbox;
 		chip_wd = &octeon_irq_chip_ciu_wd;