From 58411d1c012dca53ec9107bd98acb63f648e2435 Mon Sep 17 00:00:00 2001 From: John Gregor Date: Wed, 16 Apr 2008 21:09:24 -0700 Subject: IB/ipath: Head of Line blocking vs forward progress of user apps There's a conflict between our need to quiesce PSM-based applications to avoid HoL blocking when the IB link goes down and the apps' desire to remain running so that their quiescence timout mechanism can keep running. The compromise is to STOP the processes for a fixed period of time and then alternate between CONT and STOP until the link is again active. If there are poor interactions with subnet manager configuration at a given site, the interval can be adjusted via a module paramter. Signed-off-by: John Gregor Signed-off-by: Roland Dreier --- drivers/infiniband/hw/ipath/ipath_debug.h | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/infiniband/hw/ipath/ipath_debug.h') diff --git a/drivers/infiniband/hw/ipath/ipath_debug.h b/drivers/infiniband/hw/ipath/ipath_debug.h index d6f69532d83..7170bd20cfb 100644 --- a/drivers/infiniband/hw/ipath/ipath_debug.h +++ b/drivers/infiniband/hw/ipath/ipath_debug.h @@ -66,6 +66,7 @@ #define __IPATH_IPATHERR 0x40000 /* Ethernet (IPATH) errors */ #define __IPATH_IPATHPD 0x80000 /* Ethernet (IPATH) packet dump */ #define __IPATH_IPATHTABLE 0x100000 /* Ethernet (IPATH) table dump */ +#define __IPATH_LINKVERBDBG 0x200000 /* very verbose linkchange debug */ #else /* _IPATH_DEBUGGING */ -- cgit v1.2.3-70-g09d2 From 4330e4dad780467d930b394b5119c0218a1e2dbe Mon Sep 17 00:00:00 2001 From: Michael Albaugh Date: Wed, 16 Apr 2008 21:09:26 -0700 Subject: IB/ipath: Prevent link-recovery code from negating admin disable The link can be put in LINKDOWN_DISABLE state either locally or via a MAD. However, the link-recovery code will take it out of that state as a side-effect of attempts to clear SerDes/XGXS issues. We add a flag to indicate "link is down on purpose, leave it alone." Signed-off-by: Michael Albaugh Signed-off-by: Roland Dreier --- drivers/infiniband/hw/ipath/ipath_debug.h | 1 + drivers/infiniband/hw/ipath/ipath_driver.c | 70 ++++++++++++++++----------- drivers/infiniband/hw/ipath/ipath_init_chip.c | 5 +- drivers/infiniband/hw/ipath/ipath_intr.c | 2 + drivers/infiniband/hw/ipath/ipath_kernel.h | 2 + drivers/infiniband/hw/ipath/ipath_mad.c | 4 ++ 6 files changed, 55 insertions(+), 29 deletions(-) (limited to 'drivers/infiniband/hw/ipath/ipath_debug.h') diff --git a/drivers/infiniband/hw/ipath/ipath_debug.h b/drivers/infiniband/hw/ipath/ipath_debug.h index 7170bd20cfb..65926cd3575 100644 --- a/drivers/infiniband/hw/ipath/ipath_debug.h +++ b/drivers/infiniband/hw/ipath/ipath_debug.h @@ -90,6 +90,7 @@ #define __IPATH_IPATHERR 0x0 /* Ethernet (IPATH) errors on */ #define __IPATH_IPATHPD 0x0 /* Ethernet (IPATH) packet dump on */ #define __IPATH_IPATHTABLE 0x0 /* Ethernet (IPATH) packet dump on */ +#define __IPATH_LINKVERBDBG 0x0 /* very verbose linkchange debug */ #endif /* _IPATH_DEBUGGING */ diff --git a/drivers/infiniband/hw/ipath/ipath_driver.c b/drivers/infiniband/hw/ipath/ipath_driver.c index 130f4db7b8c..1299171c1e5 100644 --- a/drivers/infiniband/hw/ipath/ipath_driver.c +++ b/drivers/infiniband/hw/ipath/ipath_driver.c @@ -1664,30 +1664,48 @@ void ipath_cancel_sends(struct ipath_devdata *dd, int restore_sendctrl) ipath_read_kreg64(dd, dd->ipath_kregs->kr_scratch); } - -static void ipath_set_ib_lstate(struct ipath_devdata *dd, int which) +static void ipath_set_ib_lstate(struct ipath_devdata *dd, int linkcmd, + int linitcmd) { + u64 mod_wd; static const char *what[4] = { [0] = "NOP", [INFINIPATH_IBCC_LINKCMD_DOWN] = "DOWN", [INFINIPATH_IBCC_LINKCMD_ARMED] = "ARMED", [INFINIPATH_IBCC_LINKCMD_ACTIVE] = "ACTIVE" }; - int linkcmd = (which >> INFINIPATH_IBCC_LINKCMD_SHIFT) & - INFINIPATH_IBCC_LINKCMD_MASK; - ipath_cdbg(VERBOSE, "Trying to move unit %u to %s, current ltstate " - "is %s\n", dd->ipath_unit, - what[linkcmd], - ipath_ibcstatus_str[ipath_ib_linktrstate(dd, + if (linitcmd == INFINIPATH_IBCC_LINKINITCMD_DISABLE) { + /* + * If we are told to disable, note that so link-recovery + * code does not attempt to bring us back up. + */ + preempt_disable(); + dd->ipath_flags |= IPATH_IB_LINK_DISABLED; + preempt_enable(); + } else if (linitcmd) { + /* + * Any other linkinitcmd will lead to LINKDOWN and then + * to INIT (if all is well), so clear flag to let + * link-recovery code attempt to bring us back up. + */ + preempt_disable(); + dd->ipath_flags &= ~IPATH_IB_LINK_DISABLED; + preempt_enable(); + } + + mod_wd = (linkcmd << dd->ibcc_lc_shift) | + (linitcmd << INFINIPATH_IBCC_LINKINITCMD_SHIFT); + ipath_cdbg(VERBOSE, + "Moving unit %u to %s (initcmd=0x%x), current ltstate is %s\n", + dd->ipath_unit, what[linkcmd], linitcmd, + ipath_ibcstatus_str[ipath_ib_linktrstate(dd, ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus))]); - /* flush all queued sends when going to DOWN to be sure that - * they don't block MAD packets */ - if (linkcmd == INFINIPATH_IBCC_LINKCMD_DOWN) - ipath_cancel_sends(dd, 1); ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl, - dd->ipath_ibcctrl | which); + dd->ipath_ibcctrl | mod_wd); + /* read from chip so write is flushed */ + (void) ipath_read_kreg64(dd, dd->ipath_kregs->kr_ibcstatus); } int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate) @@ -1697,30 +1715,28 @@ int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate) switch (newstate) { case IPATH_IB_LINKDOWN_ONLY: - ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN << - INFINIPATH_IBCC_LINKCMD_SHIFT); + ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN, 0); /* don't wait */ ret = 0; goto bail; case IPATH_IB_LINKDOWN: - ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_POLL << - INFINIPATH_IBCC_LINKINITCMD_SHIFT); + ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN, + INFINIPATH_IBCC_LINKINITCMD_POLL); /* don't wait */ ret = 0; goto bail; case IPATH_IB_LINKDOWN_SLEEP: - ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_SLEEP << - INFINIPATH_IBCC_LINKINITCMD_SHIFT); + ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN, + INFINIPATH_IBCC_LINKINITCMD_SLEEP); /* don't wait */ ret = 0; goto bail; case IPATH_IB_LINKDOWN_DISABLE: - ipath_set_ib_lstate(dd, - INFINIPATH_IBCC_LINKINITCMD_DISABLE << - INFINIPATH_IBCC_LINKINITCMD_SHIFT); + ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_DOWN, + INFINIPATH_IBCC_LINKINITCMD_DISABLE); /* don't wait */ ret = 0; goto bail; @@ -1735,8 +1751,8 @@ int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate) ret = -EINVAL; goto bail; } - ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ARMED << - INFINIPATH_IBCC_LINKCMD_SHIFT); + ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ARMED, 0); + /* * Since the port can transition to ACTIVE by receiving * a non VL 15 packet, wait for either state. @@ -1753,8 +1769,7 @@ int ipath_set_linkstate(struct ipath_devdata *dd, u8 newstate) ret = -EINVAL; goto bail; } - ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ACTIVE << - INFINIPATH_IBCC_LINKCMD_SHIFT); + ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKCMD_ACTIVE, 0); lstate = IPATH_LINKACTIVE; break; @@ -2026,8 +2041,7 @@ void ipath_shutdown_device(struct ipath_devdata *dd) */ udelay(5); - ipath_set_ib_lstate(dd, INFINIPATH_IBCC_LINKINITCMD_DISABLE << - INFINIPATH_IBCC_LINKINITCMD_SHIFT); + ipath_set_ib_lstate(dd, 0, INFINIPATH_IBCC_LINKINITCMD_DISABLE); ipath_cancel_sends(dd, 0); signal_ib_event(dd, IB_EVENT_PORT_ERR); diff --git a/drivers/infiniband/hw/ipath/ipath_init_chip.c b/drivers/infiniband/hw/ipath/ipath_init_chip.c index 524fdf7d8c8..46c70656d53 100644 --- a/drivers/infiniband/hw/ipath/ipath_init_chip.c +++ b/drivers/infiniband/hw/ipath/ipath_init_chip.c @@ -180,10 +180,13 @@ static int bringup_link(struct ipath_devdata *dd) /* * Want to start out with both LINKCMD and LINKINITCMD in NOP * (0 and 0). Don't put linkinitcmd in ipath_ibcctrl, want that - * to stay a NOP + * to stay a NOP. Flag that we are disabled, for the (unlikely) + * case that some recovery path is trying to bring the link up + * before we are ready. */ ibc |= INFINIPATH_IBCC_LINKINITCMD_DISABLE << INFINIPATH_IBCC_LINKINITCMD_SHIFT; + dd->ipath_flags |= IPATH_IB_LINK_DISABLED; ipath_cdbg(VERBOSE, "Writing 0x%llx to ibcctrl\n", (unsigned long long) ibc); ipath_write_kreg(dd, dd->ipath_kregs->kr_ibcctrl, ibc); diff --git a/drivers/infiniband/hw/ipath/ipath_intr.c b/drivers/infiniband/hw/ipath/ipath_intr.c index ed82ecbb02d..5608e3268a6 100644 --- a/drivers/infiniband/hw/ipath/ipath_intr.c +++ b/drivers/infiniband/hw/ipath/ipath_intr.c @@ -309,6 +309,8 @@ static void handle_e_ibstatuschanged(struct ipath_devdata *dd, lastlstate == INFINIPATH_IBCS_L_STATE_DOWN) { /* transitioned to UP */ if (dd->ipath_f_ib_updown(dd, 1, ibcs)) { + /* link came up, so we must no longer be disabled */ + dd->ipath_flags &= ~IPATH_IB_LINK_DISABLED; ipath_cdbg(LINKVERB, "LinkUp handled, skipped\n"); goto skip_ibchange; /* chip-code handled */ } diff --git a/drivers/infiniband/hw/ipath/ipath_kernel.h b/drivers/infiniband/hw/ipath/ipath_kernel.h index b8b81cb745b..2510359b1c8 100644 --- a/drivers/infiniband/hw/ipath/ipath_kernel.h +++ b/drivers/infiniband/hw/ipath/ipath_kernel.h @@ -848,6 +848,8 @@ void ipath_hol_event(unsigned long); /* Suppress heartbeat, even if turning off loopback */ #define IPATH_NO_HRTBT 0x1000000 #define IPATH_HAS_MULT_IB_SPEED 0x8000000 + /* Linkdown-disable intentionally, Do not attempt to bring up */ +#define IPATH_IB_LINK_DISABLED 0x40000000 #define IPATH_IB_FORCE_NOTIFY 0x80000000 /* force notify on next ib change */ /* Bits in GPIO for the added interrupts */ diff --git a/drivers/infiniband/hw/ipath/ipath_mad.c b/drivers/infiniband/hw/ipath/ipath_mad.c index 7516a260898..30b2f44406b 100644 --- a/drivers/infiniband/hw/ipath/ipath_mad.c +++ b/drivers/infiniband/hw/ipath/ipath_mad.c @@ -590,6 +590,10 @@ static int recv_subn_set_portinfo(struct ib_smp *smp, else goto err; ipath_set_linkstate(dd, lstate); + if (lstate == IPATH_IB_LINKDOWN_DISABLE) { + ret = IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_CONSUMED; + goto done; + } ipath_wait_linkstate(dd, IPATH_LINKINIT | IPATH_LINKARMED | IPATH_LINKACTIVE, 1000); break; -- cgit v1.2.3-70-g09d2