From 289750d1f1fd4a715baa2a2c6dd0cec2b8317fd7 Mon Sep 17 00:00:00 2001
From: Robin Holt <holt@sgi.com>
Date: Tue, 15 Dec 2009 16:47:55 -0800
Subject: X86: uv: implement a gru_read_gpa kernel function

The BIOS has decided to store a pointer to the partition reserved page in
a scratch MMR.  The GRU is only able to read an MMR using a vload
instruction.  The gru_read_gpa() function will implemented.

Signed-off-by: Robin Holt <holt@sgi.com>
Signed-off-by: Jack Steiner <steiner@sgi.com>
Cc: Ingo Molnar <mingo@elte.hu>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
---
 drivers/misc/sgi-gru/grukservices.c | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

(limited to 'drivers/misc/sgi-gru/grukservices.c')

diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c
index 766e21e1557..d2b149facda 100644
--- a/drivers/misc/sgi-gru/grukservices.c
+++ b/drivers/misc/sgi-gru/grukservices.c
@@ -857,6 +857,29 @@ EXPORT_SYMBOL_GPL(gru_get_next_message);
 
 /* ---------------------- GRU DATA COPY FUNCTIONS ---------------------------*/
 
+/*
+ * Load a DW from a global GPA. The GPA can be a memory or MMR address.
+ */
+int gru_read_gpa(unsigned long *value, unsigned long gpa)
+{
+	void *cb;
+	void *dsr;
+	int ret, iaa;
+
+	STAT(read_gpa);
+	if (gru_get_cpu_resources(GRU_NUM_KERNEL_DSR_BYTES, &cb, &dsr))
+		return MQE_BUG_NO_RESOURCES;
+	iaa = gpa >> 62;
+	gru_vload_phys(cb, gpa, gru_get_tri(dsr), iaa, IMA);
+	ret = gru_wait(cb);
+	if (ret == CBS_IDLE)
+		*value = *(unsigned long *)dsr;
+	gru_free_cpu_resources(cb, dsr);
+	return ret;
+}
+EXPORT_SYMBOL_GPL(gru_read_gpa);
+
+
 /*
  * Copy a block of data using the GRU resources
  */
-- 
cgit v1.2.3-70-g09d2


From 091f1a10614db89a108cda980425799730d73d8a Mon Sep 17 00:00:00 2001
From: Jack Steiner <steiner@sgi.com>
Date: Tue, 15 Dec 2009 16:48:02 -0800
Subject: gru: add comments raised in previous code reviews

Add comments from previous code reviews.  The comments help explain some
of the more esoteric aspects of the driver.

Move a free() to the other side of an unlock.

Signed-off-by: Jack Steiner <steiner@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
---
 drivers/misc/sgi-gru/grufault.c     | 5 +++++
 drivers/misc/sgi-gru/grukservices.c | 4 +++-
 2 files changed, 8 insertions(+), 1 deletion(-)

(limited to 'drivers/misc/sgi-gru/grukservices.c')

diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c
index 679e0177828..2605edfbaeb 100644
--- a/drivers/misc/sgi-gru/grufault.c
+++ b/drivers/misc/sgi-gru/grufault.c
@@ -733,6 +733,11 @@ long gru_get_gseg_statistics(unsigned long arg)
 	if (copy_from_user(&req, (void __user *)arg, sizeof(req)))
 		return -EFAULT;
 
+	/*
+	 * The library creates arrays of contexts for threaded programs.
+	 * If no gts exists in the array, the context has never been used & all
+	 * statistics are implicitly 0.
+	 */
 	gts = gru_find_lock_gts(req.gseg);
 	if (gts) {
 		memcpy(&req.stats, &gts->ustats, sizeof(gts->ustats));
diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c
index d2b149facda..306855cc80f 100644
--- a/drivers/misc/sgi-gru/grukservices.c
+++ b/drivers/misc/sgi-gru/grukservices.c
@@ -200,13 +200,15 @@ static int gru_free_kernel_contexts(void)
 		bs = gru_base[bid];
 		if (!bs)
 			continue;
+
+		/* Ignore busy contexts. Don't want to block here.  */
 		if (down_write_trylock(&bs->bs_kgts_sema)) {
 			kgts = bs->bs_kgts;
 			if (kgts && kgts->ts_gru)
 				gru_unload_context(kgts, 0);
-			kfree(kgts);
 			bs->bs_kgts = NULL;
 			up_write(&bs->bs_kgts_sema);
+			kfree(kgts);
 		} else {
 			ret++;
 		}
-- 
cgit v1.2.3-70-g09d2


From 55484c45dbeca2eec7642932ec3f60f8a2d4bdbf Mon Sep 17 00:00:00 2001
From: Jack Steiner <steiner@sgi.com>
Date: Tue, 15 Dec 2009 16:48:05 -0800
Subject: gru: allow users to specify gru chiplet 2

Add support to the GRU driver to allow users to specify the blade &
chiplet for allocation of GRU contexts.  Add new statistics for context
loading/unloading/retargeting.  Also deleted a few GRU stats that were no
longer being unused.

Signed-off-by: Jack Steiner <steiner@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
---
 drivers/misc/sgi-gru/grufault.c     |  13 +---
 drivers/misc/sgi-gru/grufile.c      |  15 +++--
 drivers/misc/sgi-gru/grukservices.c |   8 ++-
 drivers/misc/sgi-gru/grumain.c      | 128 ++++++++++++++++++++++++------------
 drivers/misc/sgi-gru/gruprocfs.c    |   9 +--
 drivers/misc/sgi-gru/grutables.h    |  16 ++---
 6 files changed, 109 insertions(+), 80 deletions(-)

(limited to 'drivers/misc/sgi-gru/grukservices.c')

diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c
index a69d119921f..321ad809f34 100644
--- a/drivers/misc/sgi-gru/grufault.c
+++ b/drivers/misc/sgi-gru/grufault.c
@@ -546,17 +546,7 @@ int gru_handle_user_call_os(unsigned long cb)
 	if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE)
 		goto exit;
 
-	/*
-	 * If force_unload is set, the UPM TLB fault is phony. The task
-	 * has migrated to another node and the GSEG must be moved. Just
-	 * unload the context. The task will page fault and assign a new
-	 * context.
-	 */
-	if (gts->ts_tgid_owner == current->tgid && gts->ts_blade >= 0 &&
-				gts->ts_blade != uv_numa_blade_id()) {
-		STAT(call_os_offnode_reference);
-		gts->ts_force_unload = 1;
-	}
+	gru_check_context_placement(gts);
 
 	/*
 	 * CCH may contain stale data if ts_force_cch_reload is set.
@@ -771,6 +761,7 @@ int gru_set_context_option(unsigned long arg)
 		} else {
 			gts->ts_user_blade_id = req.val1;
 			gts->ts_user_chiplet_id = req.val0;
+			gru_check_context_placement(gts);
 		}
 		break;
 	case sco_gseg_owner:
diff --git a/drivers/misc/sgi-gru/grufile.c b/drivers/misc/sgi-gru/grufile.c
index 2e574fd5d3f..0a6d2a5a01f 100644
--- a/drivers/misc/sgi-gru/grufile.c
+++ b/drivers/misc/sgi-gru/grufile.c
@@ -232,23 +232,24 @@ static long gru_file_unlocked_ioctl(struct file *file, unsigned int req,
  * system.
  */
 static void gru_init_chiplet(struct gru_state *gru, unsigned long paddr,
-			     void *vaddr, int nid, int bid, int grunum)
+			     void *vaddr, int blade_id, int chiplet_id)
 {
 	spin_lock_init(&gru->gs_lock);
 	spin_lock_init(&gru->gs_asid_lock);
 	gru->gs_gru_base_paddr = paddr;
 	gru->gs_gru_base_vaddr = vaddr;
-	gru->gs_gid = bid * GRU_CHIPLETS_PER_BLADE + grunum;
-	gru->gs_blade = gru_base[bid];
-	gru->gs_blade_id = bid;
+	gru->gs_gid = blade_id * GRU_CHIPLETS_PER_BLADE + chiplet_id;
+	gru->gs_blade = gru_base[blade_id];
+	gru->gs_blade_id = blade_id;
+	gru->gs_chiplet_id = chiplet_id;
 	gru->gs_cbr_map = (GRU_CBR_AU == 64) ? ~0 : (1UL << GRU_CBR_AU) - 1;
 	gru->gs_dsr_map = (1UL << GRU_DSR_AU) - 1;
 	gru->gs_asid_limit = MAX_ASID;
 	gru_tgh_flush_init(gru);
 	if (gru->gs_gid >= gru_max_gids)
 		gru_max_gids = gru->gs_gid + 1;
-	gru_dbg(grudev, "bid %d, nid %d, gid %d, vaddr %p (0x%lx)\n",
-		bid, nid, gru->gs_gid, gru->gs_gru_base_vaddr,
+	gru_dbg(grudev, "bid %d, gid %d, vaddr %p (0x%lx)\n",
+		blade_id, gru->gs_gid, gru->gs_gru_base_vaddr,
 		gru->gs_gru_base_paddr);
 }
 
@@ -283,7 +284,7 @@ static int gru_init_tables(unsigned long gru_base_paddr, void *gru_base_vaddr)
 				chip++, gru++) {
 			paddr = gru_chiplet_paddr(gru_base_paddr, pnode, chip);
 			vaddr = gru_chiplet_vaddr(gru_base_vaddr, pnode, chip);
-			gru_init_chiplet(gru, paddr, vaddr, nid, bid, chip);
+			gru_init_chiplet(gru, paddr, vaddr, bid, chip);
 			n = hweight64(gru->gs_cbr_map) * GRU_CBR_AU_SIZE;
 			cbrs = max(cbrs, n);
 			n = hweight64(gru->gs_dsr_map) * GRU_DSR_AU_BYTES;
diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c
index 306855cc80f..e0d4b53d1fc 100644
--- a/drivers/misc/sgi-gru/grukservices.c
+++ b/drivers/misc/sgi-gru/grukservices.c
@@ -160,8 +160,10 @@ static void gru_load_kernel_context(struct gru_blade_state *bs, int blade_id)
 	up_read(&bs->bs_kgts_sema);
 	down_write(&bs->bs_kgts_sema);
 
-	if (!bs->bs_kgts)
+	if (!bs->bs_kgts) {
 		bs->bs_kgts = gru_alloc_gts(NULL, 0, 0, 0, 0);
+		bs->bs_kgts->ts_user_blade_id = blade_id;
+	}
 	kgts = bs->bs_kgts;
 
 	if (!kgts->ts_gru) {
@@ -172,9 +174,9 @@ static void gru_load_kernel_context(struct gru_blade_state *bs, int blade_id)
 		kgts->ts_dsr_au_count = GRU_DS_BYTES_TO_AU(
 			GRU_NUM_KERNEL_DSR_BYTES * ncpus +
 				bs->bs_async_dsr_bytes);
-		while (!gru_assign_gru_context(kgts, blade_id)) {
+		while (!gru_assign_gru_context(kgts)) {
 			msleep(1);
-			gru_steal_context(kgts, blade_id);
+			gru_steal_context(kgts);
 		}
 		gru_load_context(kgts);
 		gru = bs->bs_kgts->ts_gru;
diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c
index f449c2dbc1e..54ad7544a9a 100644
--- a/drivers/misc/sgi-gru/grumain.c
+++ b/drivers/misc/sgi-gru/grumain.c
@@ -684,6 +684,40 @@ static int gru_retarget_intr(struct gru_thread_state *gts)
 	return gru_update_cch(gts, 0);
 }
 
+/*
+ * Unload the gru context if it is not assigned to the correct blade or
+ * chiplet. Misassignment can occur if the process migrates to a different
+ * blade or if the user changes the selected blade/chiplet.
+ * 	Return 0 if  context correct placed, otherwise 1
+ */
+void gru_check_context_placement(struct gru_thread_state *gts)
+{
+	struct gru_state *gru;
+	int blade_id, chiplet_id;
+
+	/*
+	 * If the current task is the context owner, verify that the
+	 * context is correctly placed. This test is skipped for non-owner
+	 * references. Pthread apps use non-owner references to the CBRs.
+	 */
+	gru = gts->ts_gru;
+	if (!gru || gts->ts_tgid_owner != current->tgid)
+		return;
+
+	blade_id = gts->ts_user_blade_id;
+	if (blade_id < 0)
+		blade_id = uv_numa_blade_id();
+
+	chiplet_id = gts->ts_user_chiplet_id;
+	if (gru->gs_blade_id != blade_id ||
+	    (chiplet_id >= 0 && chiplet_id != gru->gs_chiplet_id)) {
+		STAT(check_context_unload);
+		gru_unload_context(gts, 1);
+	} else if (gru_retarget_intr(gts)) {
+		STAT(check_context_retarget_intr);
+	}
+}
+
 
 /*
  * Insufficient GRU resources available on the local blade. Steal a context from
@@ -714,13 +748,17 @@ static void gts_stolen(struct gru_thread_state *gts,
 	}
 }
 
-void gru_steal_context(struct gru_thread_state *gts, int blade_id)
+void gru_steal_context(struct gru_thread_state *gts)
 {
 	struct gru_blade_state *blade;
 	struct gru_state *gru, *gru0;
 	struct gru_thread_state *ngts = NULL;
 	int ctxnum, ctxnum0, flag = 0, cbr, dsr;
+	int blade_id = gts->ts_user_blade_id;
+	int chiplet_id = gts->ts_user_chiplet_id;
 
+	if (blade_id < 0)
+		blade_id = uv_numa_blade_id();
 	cbr = gts->ts_cbr_au_count;
 	dsr = gts->ts_dsr_au_count;
 
@@ -731,35 +769,39 @@ void gru_steal_context(struct gru_thread_state *gts, int blade_id)
 	gru = blade->bs_lru_gru;
 	if (ctxnum == 0)
 		gru = next_gru(blade, gru);
+	blade->bs_lru_gru = gru;
+	blade->bs_lru_ctxnum = ctxnum;
 	ctxnum0 = ctxnum;
 	gru0 = gru;
 	while (1) {
-		if (check_gru_resources(gru, cbr, dsr, GRU_NUM_CCH))
-			break;
-		spin_lock(&gru->gs_lock);
-		for (; ctxnum < GRU_NUM_CCH; ctxnum++) {
-			if (flag && gru == gru0 && ctxnum == ctxnum0)
+		if (chiplet_id < 0 || chiplet_id == gru->gs_chiplet_id) {
+			if (check_gru_resources(gru, cbr, dsr, GRU_NUM_CCH))
 				break;
-			ngts = gru->gs_gts[ctxnum];
-			/*
-			 * We are grabbing locks out of order, so trylock is
-			 * needed. GTSs are usually not locked, so the odds of
-			 * success are high. If trylock fails, try to steal a
-			 * different GSEG.
-			 */
-			if (ngts && is_gts_stealable(ngts, blade))
+			spin_lock(&gru->gs_lock);
+			for (; ctxnum < GRU_NUM_CCH; ctxnum++) {
+				if (flag && gru == gru0 && ctxnum == ctxnum0)
+					break;
+				ngts = gru->gs_gts[ctxnum];
+				/*
+			 	* We are grabbing locks out of order, so trylock is
+			 	* needed. GTSs are usually not locked, so the odds of
+			 	* success are high. If trylock fails, try to steal a
+			 	* different GSEG.
+			 	*/
+				if (ngts && is_gts_stealable(ngts, blade))
+					break;
+				ngts = NULL;
+			}
+			spin_unlock(&gru->gs_lock);
+			if (ngts || (flag && gru == gru0 && ctxnum == ctxnum0))
 				break;
-			ngts = NULL;
-			flag = 1;
 		}
-		spin_unlock(&gru->gs_lock);
-		if (ngts || (flag && gru == gru0 && ctxnum == ctxnum0))
+		if (flag && gru == gru0)
 			break;
+		flag = 1;
 		ctxnum = 0;
 		gru = next_gru(blade, gru);
 	}
-	blade->bs_lru_gru = gru;
-	blade->bs_lru_ctxnum = ctxnum;
 	spin_unlock(&blade->bs_lock);
 
 	if (ngts) {
@@ -777,20 +819,36 @@ void gru_steal_context(struct gru_thread_state *gts, int blade_id)
 		hweight64(gru->gs_dsr_map));
 }
 
+/*
+ * Assign a gru context.
+ */
+static int gru_assign_context_number(struct gru_state *gru)
+{
+	int ctxnum;
+
+	ctxnum = find_first_zero_bit(&gru->gs_context_map, GRU_NUM_CCH);
+	__set_bit(ctxnum, &gru->gs_context_map);
+	return ctxnum;
+}
+
 /*
  * Scan the GRUs on the local blade & assign a GRU context.
  */
-struct gru_state *gru_assign_gru_context(struct gru_thread_state *gts,
-						int blade)
+struct gru_state *gru_assign_gru_context(struct gru_thread_state *gts)
 {
 	struct gru_state *gru, *grux;
 	int i, max_active_contexts;
+	int blade_id = gts->ts_user_blade_id;
+	int chiplet_id = gts->ts_user_chiplet_id;
 
-
+	if (blade_id < 0)
+		blade_id = uv_numa_blade_id();
 again:
 	gru = NULL;
 	max_active_contexts = GRU_NUM_CCH;
-	for_each_gru_on_blade(grux, blade, i) {
+	for_each_gru_on_blade(grux, blade_id, i) {
+		if (chiplet_id >= 0 && chiplet_id != grux->gs_chiplet_id)
+			continue;
 		if (check_gru_resources(grux, gts->ts_cbr_au_count,
 					gts->ts_dsr_au_count,
 					max_active_contexts)) {
@@ -811,12 +869,9 @@ again:
 		reserve_gru_resources(gru, gts);
 		gts->ts_gru = gru;
 		gts->ts_blade = gru->gs_blade_id;
-		gts->ts_ctxnum =
-		    find_first_zero_bit(&gru->gs_context_map, GRU_NUM_CCH);
-		BUG_ON(gts->ts_ctxnum == GRU_NUM_CCH);
+		gts->ts_ctxnum = gru_assign_context_number(gru);
 		atomic_inc(&gts->ts_refcnt);
 		gru->gs_gts[gts->ts_ctxnum] = gts;
-		__set_bit(gts->ts_ctxnum, &gru->gs_context_map);
 		spin_unlock(&gru->gs_lock);
 
 		STAT(assign_context);
@@ -844,7 +899,6 @@ int gru_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 {
 	struct gru_thread_state *gts;
 	unsigned long paddr, vaddr;
-	int blade_id;
 
 	vaddr = (unsigned long)vmf->virtual_address;
 	gru_dbg(grudev, "vma %p, vaddr 0x%lx (0x%lx)\n",
@@ -859,28 +913,18 @@ int gru_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
 again:
 	mutex_lock(&gts->ts_ctxlock);
 	preempt_disable();
-	blade_id = uv_numa_blade_id();
 
-	if (gts->ts_gru) {
-		if (gts->ts_gru->gs_blade_id != blade_id) {
-			STAT(migrated_nopfn_unload);
-			gru_unload_context(gts, 1);
-		} else {
-			if (gru_retarget_intr(gts))
-				STAT(migrated_nopfn_retarget);
-		}
-	}
+	gru_check_context_placement(gts);
 
 	if (!gts->ts_gru) {
 		STAT(load_user_context);
-		if (!gru_assign_gru_context(gts, blade_id)) {
+		if (!gru_assign_gru_context(gts)) {
 			preempt_enable();
 			mutex_unlock(&gts->ts_ctxlock);
 			set_current_state(TASK_INTERRUPTIBLE);
 			schedule_timeout(GRU_ASSIGN_DELAY);  /* true hack ZZZ */
-			blade_id = uv_numa_blade_id();
 			if (gts->ts_steal_jiffies + GRU_STEAL_DELAY < jiffies)
-				gru_steal_context(gts, blade_id);
+				gru_steal_context(gts);
 			goto again;
 		}
 		gru_load_context(gts);
diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c
index cf6b3e3034d..a8a2e760cca 100644
--- a/drivers/misc/sgi-gru/gruprocfs.c
+++ b/drivers/misc/sgi-gru/gruprocfs.c
@@ -67,19 +67,14 @@ static int statistics_show(struct seq_file *s, void *p)
 	printstat(s, intr);
 	printstat(s, intr_mm_lock_failed);
 	printstat(s, call_os);
-	printstat(s, call_os_offnode_reference);
 	printstat(s, call_os_check_for_bug);
 	printstat(s, call_os_wait_queue);
 	printstat(s, user_flush_tlb);
 	printstat(s, user_unload_context);
 	printstat(s, user_exception);
 	printstat(s, set_context_option);
-	printstat(s, migrate_check);
-	printstat(s, migrated_retarget);
-	printstat(s, migrated_unload);
-	printstat(s, migrated_unload_delay);
-	printstat(s, migrated_nopfn_retarget);
-	printstat(s, migrated_nopfn_unload);
+	printstat(s, check_context_retarget_intr);
+	printstat(s, check_context_unload);
 	printstat(s, tlb_dropin);
 	printstat(s, tlb_dropin_fail_no_asid);
 	printstat(s, tlb_dropin_fail_upm);
diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h
index 27131fb2253..8a61a8599ba 100644
--- a/drivers/misc/sgi-gru/grutables.h
+++ b/drivers/misc/sgi-gru/grutables.h
@@ -192,19 +192,14 @@ struct gru_stats_s {
 	atomic_long_t intr;
 	atomic_long_t intr_mm_lock_failed;
 	atomic_long_t call_os;
-	atomic_long_t call_os_offnode_reference;
 	atomic_long_t call_os_check_for_bug;
 	atomic_long_t call_os_wait_queue;
 	atomic_long_t user_flush_tlb;
 	atomic_long_t user_unload_context;
 	atomic_long_t user_exception;
 	atomic_long_t set_context_option;
-	atomic_long_t migrate_check;
-	atomic_long_t migrated_retarget;
-	atomic_long_t migrated_unload;
-	atomic_long_t migrated_unload_delay;
-	atomic_long_t migrated_nopfn_retarget;
-	atomic_long_t migrated_nopfn_unload;
+	atomic_long_t check_context_retarget_intr;
+	atomic_long_t check_context_unload;
 	atomic_long_t tlb_dropin;
 	atomic_long_t tlb_dropin_fail_no_asid;
 	atomic_long_t tlb_dropin_fail_upm;
@@ -425,6 +420,7 @@ struct gru_state {
 							   gru segments (64) */
 	unsigned short		gs_gid;			/* unique GRU number */
 	unsigned short		gs_blade_id;		/* blade of GRU */
+	unsigned char		gs_chiplet_id;		/* blade chiplet of GRU */
 	unsigned char		gs_tgh_local_shift;	/* used to pick TGH for
 							   local flush */
 	unsigned char		gs_tgh_first_remote;	/* starting TGH# for
@@ -636,10 +632,9 @@ extern struct gru_thread_state *gru_find_thread_state(struct vm_area_struct
 				*vma, int tsid);
 extern struct gru_thread_state *gru_alloc_thread_state(struct vm_area_struct
 				*vma, int tsid);
-extern struct gru_state *gru_assign_gru_context(struct gru_thread_state *gts,
-		int blade);
+extern struct gru_state *gru_assign_gru_context(struct gru_thread_state *gts);
 extern void gru_load_context(struct gru_thread_state *gts);
-extern void gru_steal_context(struct gru_thread_state *gts, int blade_id);
+extern void gru_steal_context(struct gru_thread_state *gts);
 extern void gru_unload_context(struct gru_thread_state *gts, int savestate);
 extern int gru_update_cch(struct gru_thread_state *gts, int force_unload);
 extern void gts_drop(struct gru_thread_state *gts);
@@ -654,6 +649,7 @@ extern int gru_user_flush_tlb(unsigned long arg);
 extern int gru_user_unload_context(unsigned long arg);
 extern int gru_get_exception_detail(unsigned long arg);
 extern int gru_set_context_option(unsigned long address);
+extern void gru_check_context_placement(struct gru_thread_state *gts);
 extern int gru_cpu_fault_map_id(void);
 extern struct vm_area_struct *gru_find_vma(unsigned long vaddr);
 extern void gru_flush_all_tlb(struct gru_state *gru);
-- 
cgit v1.2.3-70-g09d2


From 33f3648342dc40f8bd6383a5a1a91c22e06f6b77 Mon Sep 17 00:00:00 2001
From: Jack Steiner <steiner@sgi.com>
Date: Tue, 15 Dec 2009 16:48:09 -0800
Subject: gru: add test for gru_copy_gpa

Improve existing driver self-tests.  Add a new debugging test to the SGI
GRU driver for verifying the global GRU copy function.

Signed-off-by: Jack Steiner <steiner@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
---
 drivers/misc/sgi-gru/grukservices.c | 48 ++++++++++++++++++++++++++++++-------
 1 file changed, 39 insertions(+), 9 deletions(-)

(limited to 'drivers/misc/sgi-gru/grukservices.c')

diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c
index e0d4b53d1fc..24ec109e61c 100644
--- a/drivers/misc/sgi-gru/grukservices.c
+++ b/drivers/misc/sgi-gru/grukservices.c
@@ -1004,6 +1004,7 @@ static int quicktest2(unsigned long arg)
 	int ret = 0;
 	unsigned long *buf;
 	void *cb0, *cb;
+	struct gru_control_block_status *gen;
 	int i, k, istatus, bytes;
 
 	bytes = numcb * 4 * 8;
@@ -1023,20 +1024,30 @@ static int quicktest2(unsigned long arg)
 				XTYPE_DW, 4, 1, IMA_INTERRUPT);
 
 	ret = 0;
-	for (k = 0; k < numcb; k++) {
+	k = numcb;
+	do {
 		gru_wait_async_cbr(han);
 		for (i = 0; i < numcb; i++) {
 			cb = cb0 + i * GRU_HANDLE_STRIDE;
 			istatus = gru_check_status(cb);
-			if (istatus == CBS_ACTIVE)
-				continue;
-			if (istatus == CBS_EXCEPTION)
-				ret = -EFAULT;
-			else if (buf[i] || buf[i + 1] || buf[i + 2] ||
-					buf[i + 3])
-				ret = -EIO;
+			if (istatus != CBS_ACTIVE && istatus != CBS_CALL_OS)
+				break;
 		}
-	}
+		if (i == numcb)
+			continue;
+		if (istatus != CBS_IDLE) {
+			printk(KERN_DEBUG "GRU:%d quicktest2: cb %d, exception\n", smp_processor_id(), i);
+			ret = -EFAULT;
+		} else if (buf[4 * i] || buf[4 * i + 1] || buf[4 * i + 2] ||
+				buf[4 * i + 3]) {
+			printk(KERN_DEBUG "GRU:%d quicktest2:cb %d,  buf 0x%lx, 0x%lx, 0x%lx, 0x%lx\n",
+			       smp_processor_id(), i, buf[4 * i], buf[4 * i + 1], buf[4 * i + 2], buf[4 * i + 3]);
+			ret = -EIO;
+		}
+		k--;
+		gen = cb;
+		gen->istatus = CBS_CALL_OS; /* don't handle this CBR again */
+	} while (k);
 	BUG_ON(cmp.done);
 
 	gru_unlock_async_resource(han);
@@ -1046,6 +1057,22 @@ done:
 	return ret;
 }
 
+#define BUFSIZE 200
+static int quicktest3(unsigned long arg)
+{
+	char buf1[BUFSIZE], buf2[BUFSIZE];
+	int ret = 0;
+
+	memset(buf2, 0, sizeof(buf2));
+	memset(buf1, get_cycles() & 255, sizeof(buf1));
+	gru_copy_gpa(uv_gpa(buf2), uv_gpa(buf1), BUFSIZE);
+	if (memcmp(buf1, buf2, BUFSIZE)) {
+		printk(KERN_DEBUG "GRU quicktest3 error\n");
+		ret = -EIO;
+	}
+	return ret;
+}
+
 /*
  * Debugging only. User hook for various kernel tests
  * of driver & gru.
@@ -1064,6 +1091,9 @@ int gru_ktest(unsigned long arg)
 	case 2:
 		ret = quicktest2(arg);
 		break;
+	case 3:
+		ret = quicktest3(arg);
+		break;
 	case 99:
 		ret = gru_free_kernel_contexts();
 		break;
-- 
cgit v1.2.3-70-g09d2


From 67bf04a5c2574e9495f660f418f6df776821d578 Mon Sep 17 00:00:00 2001
From: Jack Steiner <steiner@sgi.com>
Date: Tue, 15 Dec 2009 16:48:11 -0800
Subject: gru: fix prefetch and speculation bugs

Fix several bugs related to prefetch, ordering & speculation:

	- GRU cch_allocate() instruction causes cacheable memory
	  to be created. Add a barriers to prevent speculation
	  from prefetching data before it exists.
	- Add memory barriers before cache-flush instructions to ensure
	  that previously stored data is included in the line flushed to memory.

Signed-off-by: Jack Steiner <steiner@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
---
 drivers/misc/sgi-gru/gru_instructions.h |  1 +
 drivers/misc/sgi-gru/grufault.c         |  2 ++
 drivers/misc/sgi-gru/gruhandles.c       | 22 ++++++++++++++++++++--
 drivers/misc/sgi-gru/grukservices.c     | 10 ++++++----
 drivers/misc/sgi-gru/grumain.c          |  9 +++++++++
 5 files changed, 38 insertions(+), 6 deletions(-)

(limited to 'drivers/misc/sgi-gru/grukservices.c')

diff --git a/drivers/misc/sgi-gru/gru_instructions.h b/drivers/misc/sgi-gru/gru_instructions.h
index e033b6ce4a3..32f358d9f42 100644
--- a/drivers/misc/sgi-gru/gru_instructions.h
+++ b/drivers/misc/sgi-gru/gru_instructions.h
@@ -325,6 +325,7 @@ static inline void gru_flush_cache(void *p)
 static inline void gru_start_instruction(struct gru_instruction *ins, int op32)
 {
 	gru_ordered_store_int(ins, op32);
+	mb();
 	gru_flush_cache(ins);
 }
 
diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c
index 7466234450f..d3cacd696b3 100644
--- a/drivers/misc/sgi-gru/grufault.c
+++ b/drivers/misc/sgi-gru/grufault.c
@@ -333,6 +333,7 @@ static int gru_try_dropin(struct gru_thread_state *gts,
 	 */
 	if (tfh->status != TFHSTATUS_EXCEPTION) {
 		gru_flush_cache(tfh);
+		sync_core();
 		if (tfh->status != TFHSTATUS_EXCEPTION)
 			goto failnoexception;
 		STAT(tfh_stale_on_fault);
@@ -599,6 +600,7 @@ int gru_get_exception_detail(unsigned long arg)
 		cbrnum = thread_cbr_number(gts, ucbnum);
 		cbe = get_cbe_by_index(gts->ts_gru, cbrnum);
 		gru_flush_cache(cbe);	/* CBE not coherent */
+		sync_core();		/* make sure we are have current data */
 		excdet.opc = cbe->opccpy;
 		excdet.exopc = cbe->exopccpy;
 		excdet.ecause = cbe->ecause;
diff --git a/drivers/misc/sgi-gru/gruhandles.c b/drivers/misc/sgi-gru/gruhandles.c
index 806419a6b44..f1117a7637a 100644
--- a/drivers/misc/sgi-gru/gruhandles.c
+++ b/drivers/misc/sgi-gru/gruhandles.c
@@ -91,9 +91,18 @@ static int wait_instruction_complete(void *h, enum mcs_op opc)
 
 int cch_allocate(struct gru_context_configuration_handle *cch)
 {
+	int ret;
+
 	cch->opc = CCHOP_ALLOCATE;
 	start_instruction(cch);
-	return wait_instruction_complete(cch, cchop_allocate);
+	ret = wait_instruction_complete(cch, cchop_allocate);
+
+	/*
+	 * Stop speculation into the GSEG being mapped by the previous ALLOCATE.
+	 * The GSEG memory does not exist until the ALLOCATE completes.
+	 */
+	sync_core();
+	return ret;
 }
 
 int cch_start(struct gru_context_configuration_handle *cch)
@@ -112,9 +121,18 @@ int cch_interrupt(struct gru_context_configuration_handle *cch)
 
 int cch_deallocate(struct gru_context_configuration_handle *cch)
 {
+	int ret;
+
 	cch->opc = CCHOP_DEALLOCATE;
 	start_instruction(cch);
-	return wait_instruction_complete(cch, cchop_deallocate);
+	ret = wait_instruction_complete(cch, cchop_deallocate);
+
+	/*
+	 * Stop speculation into the GSEG being unmapped by the previous
+	 * DEALLOCATE.
+	 */
+	sync_core();
+	return ret;
 }
 
 int cch_interrupt_sync(struct gru_context_configuration_handle
diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c
index 24ec109e61c..8c81aca0463 100644
--- a/drivers/misc/sgi-gru/grukservices.c
+++ b/drivers/misc/sgi-gru/grukservices.c
@@ -395,6 +395,7 @@ int gru_get_cb_exception_detail(void *cb,
 	cbrnum = thread_cbr_number(bs->bs_kgts, get_cb_number(cb));
 	cbe = get_cbe(GRUBASE(cb), cbrnum);
 	gru_flush_cache(cbe);	/* CBE not coherent */
+	sync_core();
 	excdet->opc = cbe->opccpy;
 	excdet->exopc = cbe->exopccpy;
 	excdet->ecause = cbe->ecause;
@@ -461,9 +462,10 @@ int gru_check_status_proc(void *cb)
 	int ret;
 
 	ret = gen->istatus;
-	if (ret != CBS_EXCEPTION)
-		return ret;
-	return gru_retry_exception(cb);
+	if (ret == CBS_EXCEPTION)
+		ret = gru_retry_exception(cb);
+	rmb();
+	return ret;
 
 }
 
@@ -475,7 +477,7 @@ int gru_wait_proc(void *cb)
 	ret = gru_wait_idle_or_exception(gen);
 	if (ret == CBS_EXCEPTION)
 		ret = gru_retry_exception(cb);
-
+	rmb();
 	return ret;
 }
 
diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c
index 94402887188..a383271d391 100644
--- a/drivers/misc/sgi-gru/grumain.c
+++ b/drivers/misc/sgi-gru/grumain.c
@@ -499,6 +499,9 @@ static void gru_load_context_data(void *save, void *grubase, int ctxnum,
 			memset(cbe + i * GRU_HANDLE_STRIDE, 0,
 						GRU_CACHE_LINE_BYTES);
 		}
+		/* Flush CBE to hide race in context restart */
+		mb();
+		gru_flush_cache(cbe + i * GRU_HANDLE_STRIDE);
 		cb += GRU_HANDLE_STRIDE;
 	}
 
@@ -519,6 +522,12 @@ static void gru_unload_context_data(void *save, void *grubase, int ctxnum,
 	cb = gseg + GRU_CB_BASE;
 	cbe = grubase + GRU_CBE_BASE;
 	length = hweight64(dsrmap) * GRU_DSR_AU_BYTES;
+
+	/* CBEs may not be coherent. Flush them from cache */
+	for_each_cbr_in_allocation_map(i, &cbrmap, scr)
+		gru_flush_cache(cbe + i * GRU_HANDLE_STRIDE);
+	mb();		/* Let the CL flush complete */
+
 	gru_prefetch_context(gseg, cb, cbe, cbrmap, length);
 
 	for_each_cbr_in_allocation_map(i, &cbrmap, scr) {
-- 
cgit v1.2.3-70-g09d2


From 563447d7eb04c9b382f90a132be126a21a635647 Mon Sep 17 00:00:00 2001
From: Jack Steiner <steiner@sgi.com>
Date: Tue, 15 Dec 2009 16:48:12 -0800
Subject: gru: add additional GRU statistics

Add additional GRU statistics & debug messages.

Signed-off-by: Jack Steiner <steiner@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
---
 drivers/misc/sgi-gru/grufault.c     | 18 +++++++++++-------
 drivers/misc/sgi-gru/grufile.c      |  2 +-
 drivers/misc/sgi-gru/gruhandles.c   | 11 ++++++++---
 drivers/misc/sgi-gru/grukservices.c | 35 ++++++++++++++++++++++++-----------
 drivers/misc/sgi-gru/grumain.c      | 10 +++++++---
 drivers/misc/sgi-gru/gruprocfs.c    | 25 +++++++++++++------------
 drivers/misc/sgi-gru/grutables.h    | 18 +++++++++---------
 drivers/misc/sgi-gru/grutlbpurge.c  |  6 ++++--
 8 files changed, 77 insertions(+), 48 deletions(-)

(limited to 'drivers/misc/sgi-gru/grukservices.c')

diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c
index a78aa798d50..7d757e9c42f 100644
--- a/drivers/misc/sgi-gru/grufault.c
+++ b/drivers/misc/sgi-gru/grufault.c
@@ -303,7 +303,7 @@ static int gru_try_dropin(struct gru_thread_state *gts,
 			  struct gru_tlb_fault_handle *tfh,
 			  struct gru_instruction_bits *cbk)
 {
-	int pageshift = 0, asid, write, ret, atomic = !cbk;
+	int pageshift = 0, asid, write, ret, atomic = !cbk, indexway;
 	unsigned long gpa = 0, vaddr = 0;
 
 	/*
@@ -333,6 +333,7 @@ static int gru_try_dropin(struct gru_thread_state *gts,
 	write = (tfh->cause & TFHCAUSE_TLB_MOD) != 0;
 	vaddr = tfh->missvaddr;
 	asid = tfh->missasid;
+	indexway = tfh->indexway;
 	if (asid == 0)
 		goto failnoasid;
 
@@ -361,11 +362,12 @@ static int gru_try_dropin(struct gru_thread_state *gts,
 	gru_cb_set_istatus_active(cbk);
 	tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write,
 			  GRU_PAGESIZE(pageshift));
-	STAT(tlb_dropin);
 	gru_dbg(grudev,
-		"%s: tfh 0x%p, vaddr 0x%lx, asid 0x%x, ps %d, gpa 0x%lx\n",
-		ret ? "non-atomic" : "atomic", tfh, vaddr, asid,
-		pageshift, gpa);
+		"%s: gid %d, gts 0x%p, tfh 0x%p, vaddr 0x%lx, asid 0x%x, indexway 0x%x,"
+		" rw %d, ps %d, gpa 0x%lx\n",
+		atomic ? "atomic" : "non-atomic", gts->ts_gru->gs_gid, gts, tfh, vaddr, asid,
+		indexway, write, pageshift, gpa);
+	STAT(tlb_dropin);
 	return 0;
 
 failnoasid:
@@ -460,12 +462,14 @@ static irqreturn_t gru_intr(int chiplet, int blade)
 		dmap.fault_bits[0], dmap.fault_bits[1]);
 
 	for_each_cbr_in_tfm(cbrnum, dmap.fault_bits) {
+		STAT(intr_cbr);
 		complete(gru->gs_blade->bs_async_wq);
 		gru_dbg(grudev, "gid %d, cbr_done %d, done %d\n",
 			gru->gs_gid, cbrnum, gru->gs_blade->bs_async_wq->done);
 	}
 
 	for_each_cbr_in_tfm(cbrnum, imap.fault_bits) {
+		STAT(intr_tfh);
 		tfh = get_tfh_by_index(gru, cbrnum);
 		prefetchw(tfh);	/* Helps on hdw, required for emulator */
 
@@ -551,7 +555,6 @@ int gru_handle_user_call_os(unsigned long cb)
 	int ucbnum, cbrnum, ret = -EINVAL;
 
 	STAT(call_os);
-	gru_dbg(grudev, "address 0x%lx\n", cb);
 
 	/* sanity check the cb pointer */
 	ucbnum = get_cb_number((void *)cb);
@@ -561,6 +564,7 @@ int gru_handle_user_call_os(unsigned long cb)
 	gts = gru_find_lock_gts(cb);
 	if (!gts)
 		return -EINVAL;
+	gru_dbg(grudev, "address 0x%lx, gid %d, gts 0x%p\n", cb, gts->ts_gru ? gts->ts_gru->gs_gid : -1, gts);
 
 	if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE)
 		goto exit;
@@ -603,11 +607,11 @@ int gru_get_exception_detail(unsigned long arg)
 	if (copy_from_user(&excdet, (void __user *)arg, sizeof(excdet)))
 		return -EFAULT;
 
-	gru_dbg(grudev, "address 0x%lx\n", excdet.cb);
 	gts = gru_find_lock_gts(excdet.cb);
 	if (!gts)
 		return -EINVAL;
 
+	gru_dbg(grudev, "address 0x%lx, gid %d, gts 0x%p\n", excdet.cb, gts->ts_gru ? gts->ts_gru->gs_gid : -1, gts);
 	ucbnum = get_cb_number((void *)excdet.cb);
 	if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) {
 		ret = -EINVAL;
diff --git a/drivers/misc/sgi-gru/grufile.c b/drivers/misc/sgi-gru/grufile.c
index 22b8b2733a2..9d41208a6c9 100644
--- a/drivers/misc/sgi-gru/grufile.c
+++ b/drivers/misc/sgi-gru/grufile.c
@@ -192,7 +192,7 @@ static long gru_file_unlocked_ioctl(struct file *file, unsigned int req,
 {
 	int err = -EBADRQC;
 
-	gru_dbg(grudev, "file %p\n", file);
+	gru_dbg(grudev, "file %p, req 0x%x, 0x%lx\n", file, req, arg);
 
 	switch (req) {
 	case GRU_CREATE_CONTEXT:
diff --git a/drivers/misc/sgi-gru/gruhandles.c b/drivers/misc/sgi-gru/gruhandles.c
index f1117a7637a..4bc9ee811fb 100644
--- a/drivers/misc/sgi-gru/gruhandles.c
+++ b/drivers/misc/sgi-gru/gruhandles.c
@@ -27,9 +27,11 @@
 #ifdef CONFIG_IA64
 #include <asm/processor.h>
 #define GRU_OPERATION_TIMEOUT	(((cycles_t) local_cpu_data->itc_freq)*10)
+#define CLKS2NSEC(c)		((c) *1000000000 / local_cpu_data->itc_freq)
 #else
 #include <asm/tsc.h>
 #define GRU_OPERATION_TIMEOUT	((cycles_t) tsc_khz*10*1000)
+#define CLKS2NSEC(c)		((c) * 1000000 / tsc_khz)
 #endif
 
 /* Extract the status field from a kernel handle */
@@ -39,10 +41,13 @@ struct mcs_op_statistic mcs_op_statistics[mcsop_last];
 
 static void update_mcs_stats(enum mcs_op op, unsigned long clks)
 {
+	unsigned long nsec;
+
+	nsec = CLKS2NSEC(clks);
 	atomic_long_inc(&mcs_op_statistics[op].count);
-	atomic_long_add(clks, &mcs_op_statistics[op].total);
-	if (mcs_op_statistics[op].max < clks)
-		mcs_op_statistics[op].max = clks;
+	atomic_long_add(nsec, &mcs_op_statistics[op].total);
+	if (mcs_op_statistics[op].max < nsec)
+		mcs_op_statistics[op].max = nsec;
 }
 
 static void start_instruction(void *h)
diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c
index 8c81aca0463..4da6f56833d 100644
--- a/drivers/misc/sgi-gru/grukservices.c
+++ b/drivers/misc/sgi-gru/grukservices.c
@@ -414,8 +414,8 @@ char *gru_get_cb_exception_detail_str(int ret, void *cb,
 	if (ret > 0 && gen->istatus == CBS_EXCEPTION) {
 		gru_get_cb_exception_detail(cb, &excdet);
 		snprintf(buf, size,
-			"GRU exception: cb %p, opc %d, exopc %d, ecause 0x%x,"
-			"excdet0 0x%lx, excdet1 0x%x",
+			"GRU:%d exception: cb %p, opc %d, exopc %d, ecause 0x%x,"
+			"excdet0 0x%lx, excdet1 0x%x", smp_processor_id(),
 			gen, excdet.opc, excdet.exopc, excdet.ecause,
 			excdet.exceptdet0, excdet.exceptdet1);
 	} else {
@@ -604,6 +604,8 @@ static int send_noop_message(void *cb, struct gru_message_queue_desc *mqd,
 				ret = MQE_UNEXPECTED_CB_ERR;
 			break;
 		case CBSS_PAGE_OVERFLOW:
+			STAT(mesq_noop_page_overflow);
+			/* fallthru */
 		default:
 			BUG();
 		}
@@ -745,6 +747,9 @@ static int send_message_failure(void *cb, struct gru_message_queue_desc *mqd,
 		STAT(mesq_send_put_nacked);
 		ret = send_message_put_nacked(cb, mqd, mesg, lines);
 		break;
+	case CBSS_PAGE_OVERFLOW:
+		STAT(mesq_page_overflow);
+		/* fallthru */
 	default:
 		BUG();
 	}
@@ -837,7 +842,6 @@ void *gru_get_next_message(struct gru_message_queue_desc *mqd)
 	int present = mhdr->present;
 
 	/* skip NOOP messages */
-	STAT(mesq_receive);
 	while (present == MQS_NOOP) {
 		gru_free_message(mqd, mhdr);
 		mhdr = mq->next;
@@ -857,6 +861,7 @@ void *gru_get_next_message(struct gru_message_queue_desc *mqd)
 	if (mhdr->lines == 2)
 		restore_present2(mhdr, mhdr->present2);
 
+	STAT(mesq_receive);
 	return mhdr;
 }
 EXPORT_SYMBOL_GPL(gru_get_next_message);
@@ -927,24 +932,24 @@ static int quicktest0(unsigned long arg)
 
 	gru_vload(cb, uv_gpa(&word0), gru_get_tri(dsr), XTYPE_DW, 1, 1, IMA);
 	if (gru_wait(cb) != CBS_IDLE) {
-		printk(KERN_DEBUG "GRU quicktest0: CBR failure 1\n");
+		printk(KERN_DEBUG "GRU:%d quicktest0: CBR failure 1\n", smp_processor_id());
 		goto done;
 	}
 
 	if (*p != MAGIC) {
-		printk(KERN_DEBUG "GRU: quicktest0 bad magic 0x%lx\n", *p);
+		printk(KERN_DEBUG "GRU:%d quicktest0 bad magic 0x%lx\n", smp_processor_id(), *p);
 		goto done;
 	}
 	gru_vstore(cb, uv_gpa(&word1), gru_get_tri(dsr), XTYPE_DW, 1, 1, IMA);
 	if (gru_wait(cb) != CBS_IDLE) {
-		printk(KERN_DEBUG "GRU quicktest0: CBR failure 2\n");
+		printk(KERN_DEBUG "GRU:%d quicktest0: CBR failure 2\n", smp_processor_id());
 		goto done;
 	}
 
 	if (word0 != word1 || word1 != MAGIC) {
 		printk(KERN_DEBUG
-		       "GRU quicktest0 err: found 0x%lx, expected 0x%lx\n",
-		     word1, MAGIC);
+		       "GRU:%d quicktest0 err: found 0x%lx, expected 0x%lx\n",
+		     smp_processor_id(), word1, MAGIC);
 		goto done;
 	}
 	ret = 0;
@@ -981,8 +986,11 @@ static int quicktest1(unsigned long arg)
 		if (ret)
 			break;
 	}
-	if (ret != MQE_QUEUE_FULL || i != 4)
+	if (ret != MQE_QUEUE_FULL || i != 4) {
+		printk(KERN_DEBUG "GRU:%d quicktest1: unexpect status %d, i %d\n",
+		       smp_processor_id(), ret, i);
 		goto done;
+	}
 
 	for (i = 0; i < 6; i++) {
 		m = gru_get_next_message(&mqd);
@@ -990,7 +998,12 @@ static int quicktest1(unsigned long arg)
 			break;
 		gru_free_message(&mqd, m);
 	}
-	ret = (i == 4) ? 0 : -EIO;
+	if (i != 4) {
+		printk(KERN_DEBUG "GRU:%d quicktest2: bad message, i %d, m %p, m8 %d\n",
+			smp_processor_id(), i, m, m ? m[8] : -1);
+		goto done;
+	}
+	ret = 0;
 
 done:
 	kfree(p);
@@ -1069,7 +1082,7 @@ static int quicktest3(unsigned long arg)
 	memset(buf1, get_cycles() & 255, sizeof(buf1));
 	gru_copy_gpa(uv_gpa(buf2), uv_gpa(buf1), BUFSIZE);
 	if (memcmp(buf1, buf2, BUFSIZE)) {
-		printk(KERN_DEBUG "GRU quicktest3 error\n");
+		printk(KERN_DEBUG "GRU:%d quicktest3 error\n", smp_processor_id());
 		ret = -EIO;
 	}
 	return ret;
diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c
index 120c70c5a28..ebabbdcbb97 100644
--- a/drivers/misc/sgi-gru/grumain.c
+++ b/drivers/misc/sgi-gru/grumain.c
@@ -370,6 +370,7 @@ struct gru_vma_data *gru_alloc_vma_data(struct vm_area_struct *vma, int tsid)
 	if (!vdata)
 		return NULL;
 
+	STAT(vdata_alloc);
 	INIT_LIST_HEAD(&vdata->vd_head);
 	spin_lock_init(&vdata->vd_lock);
 	gru_dbg(grudev, "alloc vdata %p\n", vdata);
@@ -552,7 +553,8 @@ void gru_unload_context(struct gru_thread_state *gts, int savestate)
 		zap_vma_ptes(gts->ts_vma, UGRUADDR(gts), GRU_GSEG_PAGESIZE);
 	cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
 
-	gru_dbg(grudev, "gts %p\n", gts);
+	gru_dbg(grudev, "gts %p, cbrmap 0x%lx, dsrmap 0x%lx\n",
+		gts, gts->ts_cbr_map, gts->ts_dsr_map);
 	lock_cch_handle(cch);
 	if (cch_interrupt_sync(cch))
 		BUG();
@@ -583,9 +585,7 @@ void gru_load_context(struct gru_thread_state *gts)
 	struct gru_context_configuration_handle *cch;
 	int i, err, asid, ctxnum = gts->ts_ctxnum;
 
-	gru_dbg(grudev, "gts %p\n", gts);
 	cch = get_cch(gru->gs_gru_base_vaddr, ctxnum);
-
 	lock_cch_handle(cch);
 	cch->tfm_fault_bit_enable =
 	    (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL
@@ -635,6 +635,10 @@ void gru_load_context(struct gru_thread_state *gts)
 	if (cch_start(cch))
 		BUG();
 	unlock_cch_handle(cch);
+
+	gru_dbg(grudev, "gid %d, gts %p, cbrmap 0x%lx, dsrmap 0x%lx, tie %d, tis %d\n",
+		gts->ts_gru->gs_gid, gts, gts->ts_cbr_map, gts->ts_dsr_map,
+		(gts->ts_user_options == GRU_OPT_MISS_FMM_INTR), gts->ts_tlb_int_select);
 }
 
 /*
diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c
index 4f6fc11a263..0a57ab29cd3 100644
--- a/drivers/misc/sgi-gru/gruprocfs.c
+++ b/drivers/misc/sgi-gru/gruprocfs.c
@@ -36,8 +36,7 @@ static void printstat_val(struct seq_file *s, atomic_long_t *v, char *id)
 {
 	unsigned long val = atomic_long_read(v);
 
-	if (val)
-		seq_printf(s, "%16lu %s\n", val, id);
+	seq_printf(s, "%16lu %s\n", val, id);
 }
 
 static int statistics_show(struct seq_file *s, void *p)
@@ -46,7 +45,8 @@ static int statistics_show(struct seq_file *s, void *p)
 	printstat(s, vdata_free);
 	printstat(s, gts_alloc);
 	printstat(s, gts_free);
-	printstat(s, vdata_double_alloc);
+	printstat(s, gms_alloc);
+	printstat(s, gms_free);
 	printstat(s, gts_double_allocate);
 	printstat(s, assign_context);
 	printstat(s, assign_context_failed);
@@ -59,15 +59,15 @@ static int statistics_show(struct seq_file *s, void *p)
 	printstat(s, steal_kernel_context);
 	printstat(s, steal_context_failed);
 	printstat(s, nopfn);
-	printstat(s, break_cow);
 	printstat(s, asid_new);
 	printstat(s, asid_next);
 	printstat(s, asid_wrap);
 	printstat(s, asid_reuse);
 	printstat(s, intr);
+	printstat(s, intr_cbr);
+	printstat(s, intr_tfh);
 	printstat(s, intr_mm_lock_failed);
 	printstat(s, call_os);
-	printstat(s, call_os_check_for_bug);
 	printstat(s, call_os_wait_queue);
 	printstat(s, user_flush_tlb);
 	printstat(s, user_unload_context);
@@ -83,11 +83,9 @@ static int statistics_show(struct seq_file *s, void *p)
 	printstat(s, tlb_dropin_fail_idle);
 	printstat(s, tlb_dropin_fail_fmm);
 	printstat(s, tlb_dropin_fail_no_exception);
-	printstat(s, tlb_dropin_fail_no_exception_war);
 	printstat(s, tfh_stale_on_fault);
 	printstat(s, mmu_invalidate_range);
 	printstat(s, mmu_invalidate_page);
-	printstat(s, mmu_clear_flush_young);
 	printstat(s, flush_tlb);
 	printstat(s, flush_tlb_gru);
 	printstat(s, flush_tlb_gru_tgh);
@@ -104,7 +102,6 @@ static int statistics_show(struct seq_file *s, void *p)
 	printstat(s, mesq_send_qlimit_reached);
 	printstat(s, mesq_send_amo_nacked);
 	printstat(s, mesq_send_put_nacked);
-	printstat(s, mesq_qf_not_full);
 	printstat(s, mesq_qf_locked);
 	printstat(s, mesq_qf_noop_not_full);
 	printstat(s, mesq_qf_switch_head_failed);
@@ -114,6 +111,7 @@ static int statistics_show(struct seq_file *s, void *p)
 	printstat(s, mesq_noop_qlimit_reached);
 	printstat(s, mesq_noop_amo_nacked);
 	printstat(s, mesq_noop_put_nacked);
+	printstat(s, mesq_noop_page_overflow);
 	return 0;
 }
 
@@ -131,6 +129,7 @@ static int mcs_statistics_show(struct seq_file *s, void *p)
 	static char *id[] = {"cch_allocate", "cch_start", "cch_interrupt",
 		"cch_interrupt_sync", "cch_deallocate", "tgh_invalidate"};
 
+	seq_printf(s, "%-20s%12s%12s%12s\n", "#id", "count", "aver-clks", "max-clks");
 	for (op = 0; op < mcsop_last; op++) {
 		count = atomic_long_read(&mcs_op_statistics[op].count);
 		total = atomic_long_read(&mcs_op_statistics[op].total);
@@ -150,6 +149,7 @@ static ssize_t mcs_statistics_write(struct file *file,
 
 static int options_show(struct seq_file *s, void *p)
 {
+	seq_printf(s, "#bitmask: 1=trace, 2=statistics\n");
 	seq_printf(s, "0x%lx\n", gru_options);
 	return 0;
 }
@@ -179,16 +179,17 @@ static int cch_seq_show(struct seq_file *file, void *data)
 	const char *mode[] = { "??", "UPM", "INTR", "OS_POLL" };
 
 	if (gid == 0)
-		seq_printf(file, "#%5s%5s%6s%9s%6s%8s%8s\n", "gid", "bid",
-			   "ctx#", "pid", "cbrs", "dsbytes", "mode");
+		seq_printf(file, "#%5s%5s%6s%7s%9s%6s%8s%8s\n", "gid", "bid",
+			   "ctx#", "asid", "pid", "cbrs", "dsbytes", "mode");
 	if (gru)
 		for (i = 0; i < GRU_NUM_CCH; i++) {
 			ts = gru->gs_gts[i];
 			if (!ts)
 				continue;
-			seq_printf(file, " %5d%5d%6d%9d%6d%8d%8s\n",
+			seq_printf(file, " %5d%5d%6d%7d%9d%6d%8d%8s\n",
 				   gru->gs_gid, gru->gs_blade_id, i,
-				   ts->ts_tgid_owner,
+				   is_kernel_context(ts) ? 0 : ts->ts_gms->ms_asids[gid].mt_asid,
+				   is_kernel_context(ts) ? 0 : ts->ts_tgid_owner,
 				   ts->ts_cbr_au_count * GRU_CBR_AU_SIZE,
 				   ts->ts_cbr_au_count * GRU_DSR_AU_BYTES,
 				   mode[ts->ts_user_options &
diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h
index d83e36715e6..76fe2987fc9 100644
--- a/drivers/misc/sgi-gru/grutables.h
+++ b/drivers/misc/sgi-gru/grutables.h
@@ -171,7 +171,8 @@ struct gru_stats_s {
 	atomic_long_t vdata_free;
 	atomic_long_t gts_alloc;
 	atomic_long_t gts_free;
-	atomic_long_t vdata_double_alloc;
+	atomic_long_t gms_alloc;
+	atomic_long_t gms_free;
 	atomic_long_t gts_double_allocate;
 	atomic_long_t assign_context;
 	atomic_long_t assign_context_failed;
@@ -184,15 +185,15 @@ struct gru_stats_s {
 	atomic_long_t steal_kernel_context;
 	atomic_long_t steal_context_failed;
 	atomic_long_t nopfn;
-	atomic_long_t break_cow;
 	atomic_long_t asid_new;
 	atomic_long_t asid_next;
 	atomic_long_t asid_wrap;
 	atomic_long_t asid_reuse;
 	atomic_long_t intr;
+	atomic_long_t intr_cbr;
+	atomic_long_t intr_tfh;
 	atomic_long_t intr_mm_lock_failed;
 	atomic_long_t call_os;
-	atomic_long_t call_os_check_for_bug;
 	atomic_long_t call_os_wait_queue;
 	atomic_long_t user_flush_tlb;
 	atomic_long_t user_unload_context;
@@ -208,11 +209,9 @@ struct gru_stats_s {
 	atomic_long_t tlb_dropin_fail_idle;
 	atomic_long_t tlb_dropin_fail_fmm;
 	atomic_long_t tlb_dropin_fail_no_exception;
-	atomic_long_t tlb_dropin_fail_no_exception_war;
 	atomic_long_t tfh_stale_on_fault;
 	atomic_long_t mmu_invalidate_range;
 	atomic_long_t mmu_invalidate_page;
-	atomic_long_t mmu_clear_flush_young;
 	atomic_long_t flush_tlb;
 	atomic_long_t flush_tlb_gru;
 	atomic_long_t flush_tlb_gru_tgh;
@@ -231,7 +230,7 @@ struct gru_stats_s {
 	atomic_long_t mesq_send_qlimit_reached;
 	atomic_long_t mesq_send_amo_nacked;
 	atomic_long_t mesq_send_put_nacked;
-	atomic_long_t mesq_qf_not_full;
+	atomic_long_t mesq_page_overflow;
 	atomic_long_t mesq_qf_locked;
 	atomic_long_t mesq_qf_noop_not_full;
 	atomic_long_t mesq_qf_switch_head_failed;
@@ -241,6 +240,7 @@ struct gru_stats_s {
 	atomic_long_t mesq_noop_qlimit_reached;
 	atomic_long_t mesq_noop_amo_nacked;
 	atomic_long_t mesq_noop_put_nacked;
+	atomic_long_t mesq_noop_page_overflow;
 
 };
 
@@ -255,8 +255,8 @@ struct mcs_op_statistic {
 
 extern struct mcs_op_statistic mcs_op_statistics[mcsop_last];
 
-#define OPT_DPRINT	1
-#define OPT_STATS	2
+#define OPT_DPRINT		1
+#define OPT_STATS		2
 
 
 #define IRQ_GRU			110	/* Starting IRQ number for interrupts */
@@ -279,7 +279,7 @@ extern struct mcs_op_statistic mcs_op_statistics[mcsop_last];
 #define gru_dbg(dev, fmt, x...)						\
 	do {								\
 		if (gru_options & OPT_DPRINT)				\
-			dev_dbg(dev, "%s: " fmt, __func__, x);		\
+			printk(KERN_DEBUG "GRU:%d %s: " fmt, smp_processor_id(), __func__, x);\
 	} while (0)
 #else
 #define gru_dbg(x...)
diff --git a/drivers/misc/sgi-gru/grutlbpurge.c b/drivers/misc/sgi-gru/grutlbpurge.c
index 421c548e51f..240a6d36166 100644
--- a/drivers/misc/sgi-gru/grutlbpurge.c
+++ b/drivers/misc/sgi-gru/grutlbpurge.c
@@ -184,8 +184,8 @@ void gru_flush_tlb_range(struct gru_mm_struct *gms, unsigned long start,
 			STAT(flush_tlb_gru_tgh);
 			asid = GRUASID(asid, start);
 			gru_dbg(grudev,
-	"  FLUSH gruid %d, asid 0x%x, num %ld, cbmap 0x%x\n",
-				gid, asid, num, asids->mt_ctxbitmap);
+	"  FLUSH gruid %d, asid 0x%x, vaddr 0x%lx, vamask 0x%x, num %ld, cbmap 0x%x\n",
+			      gid, asid, start, grupagesize, num, asids->mt_ctxbitmap);
 			tgh = get_lock_tgh_handle(gru);
 			tgh_invalidate(tgh, start, ~0, asid, grupagesize, 0,
 				       num - 1, asids->mt_ctxbitmap);
@@ -308,6 +308,7 @@ struct gru_mm_struct *gru_register_mmu_notifier(void)
 	} else {
 		gms = kzalloc(sizeof(*gms), GFP_KERNEL);
 		if (gms) {
+			STAT(gms_alloc);
 			spin_lock_init(&gms->ms_asid_lock);
 			gms->ms_notifier.ops = &gru_mmuops;
 			atomic_set(&gms->ms_refcnt, 1);
@@ -333,6 +334,7 @@ void gru_drop_mmu_notifier(struct gru_mm_struct *gms)
 		if (!gms->ms_released)
 			mmu_notifier_unregister(&gms->ms_notifier, current->mm);
 		kfree(gms);
+		STAT(gms_free);
 	}
 }
 
-- 
cgit v1.2.3-70-g09d2


From c550222f64265c9384c45d7f691445386025a82b Mon Sep 17 00:00:00 2001
From: Jack Steiner <steiner@sgi.com>
Date: Tue, 15 Dec 2009 16:48:13 -0800
Subject: gru: preload tlb for bcopy instructions

Add anticipatory TLB dropins for GRU TLB misses that occur on BCOPY
instructions that copy large amounts of data.

Signed-off-by: Jack Steiner <steiner@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
---
 drivers/misc/sgi-gru/grufault.c     | 89 +++++++++++++++++++++++++++++++++++--
 drivers/misc/sgi-gru/grufile.c      |  1 +
 drivers/misc/sgi-gru/gruhandles.c   | 11 +++--
 drivers/misc/sgi-gru/gruhandles.h   | 20 ++++++++-
 drivers/misc/sgi-gru/grukservices.c |  2 +-
 drivers/misc/sgi-gru/grulib.h       |  1 +
 drivers/misc/sgi-gru/grumain.c      |  8 +++-
 drivers/misc/sgi-gru/gruprocfs.c    |  4 +-
 drivers/misc/sgi-gru/grutables.h    |  9 +++-
 9 files changed, 129 insertions(+), 16 deletions(-)

(limited to 'drivers/misc/sgi-gru/grukservices.c')

diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c
index 7d757e9c42f..a1b3a1d66af 100644
--- a/drivers/misc/sgi-gru/grufault.c
+++ b/drivers/misc/sgi-gru/grufault.c
@@ -289,6 +289,61 @@ upm:
 }
 
 
+/*
+ * Flush a CBE from cache. The CBE is clean in the cache. Dirty the
+ * CBE cacheline so that the line will be written back to home agent.
+ * Otherwise the line may be silently dropped. This has no impact
+ * except on performance.
+ */
+static void gru_flush_cache_cbe(struct gru_control_block_extended *cbe)
+{
+	if (unlikely(cbe)) {
+		cbe->cbrexecstatus = 0;         /* make CL dirty */
+		gru_flush_cache(cbe);
+	}
+}
+
+/*
+ * Preload the TLB with entries that may be required. Currently, preloading
+ * is implemented only for BCOPY. Preload  <tlb_preload_count> pages OR to
+ * the end of the bcopy tranfer, whichever is smaller.
+ */
+static void gru_preload_tlb(struct gru_state *gru,
+			struct gru_thread_state *gts, int atomic,
+			unsigned long fault_vaddr, int asid, int write,
+			unsigned char tlb_preload_count,
+			struct gru_tlb_fault_handle *tfh,
+			struct gru_control_block_extended *cbe)
+{
+	unsigned long vaddr = 0, gpa;
+	int ret, pageshift;
+
+	if (cbe->opccpy != OP_BCOPY)
+		return;
+
+	if (fault_vaddr == cbe->cbe_baddr0)
+		vaddr = fault_vaddr + GRU_CACHE_LINE_BYTES * cbe->cbe_src_cl - 1;
+	else if (fault_vaddr == cbe->cbe_baddr1)
+		vaddr = fault_vaddr + (1 << cbe->xtypecpy) * cbe->cbe_nelemcur - 1;
+
+	fault_vaddr &= PAGE_MASK;
+	vaddr &= PAGE_MASK;
+	vaddr = min(vaddr, fault_vaddr + tlb_preload_count * PAGE_SIZE);
+
+	while (vaddr > fault_vaddr) {
+		ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift);
+		if (ret || tfh_write_only(tfh, gpa, GAA_RAM, vaddr, asid, write,
+					  GRU_PAGESIZE(pageshift)))
+			return;
+		gru_dbg(grudev,
+			"%s: gid %d, gts 0x%p, tfh 0x%p, vaddr 0x%lx, asid 0x%x, rw %d, ps %d, gpa 0x%lx\n",
+			atomic ? "atomic" : "non-atomic", gru->gs_gid, gts, tfh,
+			vaddr, asid, write, pageshift, gpa);
+		vaddr -= PAGE_SIZE;
+		STAT(tlb_preload_page);
+	}
+}
+
 /*
  * Drop a TLB entry into the GRU. The fault is described by info in an TFH.
  *	Input:
@@ -303,6 +358,8 @@ static int gru_try_dropin(struct gru_thread_state *gts,
 			  struct gru_tlb_fault_handle *tfh,
 			  struct gru_instruction_bits *cbk)
 {
+	struct gru_control_block_extended *cbe = NULL;
+	unsigned char tlb_preload_count = gts->ts_tlb_preload_count;
 	int pageshift = 0, asid, write, ret, atomic = !cbk, indexway;
 	unsigned long gpa = 0, vaddr = 0;
 
@@ -313,6 +370,14 @@ static int gru_try_dropin(struct gru_thread_state *gts,
 	 * the dropin is ignored. This eliminates the need for additional locks.
 	 */
 
+	/*
+	 * Prefetch the CBE if doing TLB preloading
+	 */
+	if (unlikely(tlb_preload_count)) {
+		cbe = gru_tfh_to_cbe(tfh);
+		prefetchw(cbe);
+	}
+
 	/*
 	 * Error if TFH state is IDLE or FMM mode & the user issuing a UPM call.
 	 * Might be a hardware race OR a stupid user. Ignore FMM because FMM
@@ -359,6 +424,12 @@ static int gru_try_dropin(struct gru_thread_state *gts,
 			goto failupm;
 		}
 	}
+
+	if (unlikely(cbe) && pageshift == PAGE_SHIFT) {
+		gru_preload_tlb(gts->ts_gru, gts, atomic, vaddr, asid, write, tlb_preload_count, tfh, cbe);
+		gru_flush_cache_cbe(cbe);
+	}
+
 	gru_cb_set_istatus_active(cbk);
 	tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write,
 			  GRU_PAGESIZE(pageshift));
@@ -378,11 +449,13 @@ failnoasid:
 		tfh_user_polling_mode(tfh);
 	else
 		gru_flush_cache(tfh);
+	gru_flush_cache_cbe(cbe);
 	return -EAGAIN;
 
 failupm:
 	/* Atomic failure switch CBR to UPM */
 	tfh_user_polling_mode(tfh);
+	gru_flush_cache_cbe(cbe);
 	STAT(tlb_dropin_fail_upm);
 	gru_dbg(grudev, "FAILED upm tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr);
 	return 1;
@@ -390,6 +463,7 @@ failupm:
 failfmm:
 	/* FMM state on UPM call */
 	gru_flush_cache(tfh);
+	gru_flush_cache_cbe(cbe);
 	STAT(tlb_dropin_fail_fmm);
 	gru_dbg(grudev, "FAILED fmm tfh: 0x%p, state %d\n", tfh, tfh->state);
 	return 0;
@@ -397,6 +471,7 @@ failfmm:
 failnoexception:
 	/* TFH status did not show exception pending */
 	gru_flush_cache(tfh);
+	gru_flush_cache_cbe(cbe);
 	if (cbk)
 		gru_flush_cache(cbk);
 	STAT(tlb_dropin_fail_no_exception);
@@ -407,6 +482,7 @@ failnoexception:
 failidle:
 	/* TFH state was idle  - no miss pending */
 	gru_flush_cache(tfh);
+	gru_flush_cache_cbe(cbe);
 	if (cbk)
 		gru_flush_cache(cbk);
 	STAT(tlb_dropin_fail_idle);
@@ -416,6 +492,7 @@ failidle:
 failinval:
 	/* All errors (atomic & non-atomic) switch CBR to EXCEPTION state */
 	tfh_exception(tfh);
+	gru_flush_cache_cbe(cbe);
 	STAT(tlb_dropin_fail_invalid);
 	gru_dbg(grudev, "FAILED inval tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr);
 	return -EFAULT;
@@ -426,6 +503,7 @@ failactive:
 		tfh_user_polling_mode(tfh);
 	else
 		gru_flush_cache(tfh);
+	gru_flush_cache_cbe(cbe);
 	STAT(tlb_dropin_fail_range_active);
 	gru_dbg(grudev, "FAILED range active: tfh 0x%p, vaddr 0x%lx\n",
 		tfh, vaddr);
@@ -627,7 +705,7 @@ int gru_get_exception_detail(unsigned long arg)
 		excdet.exceptdet1 = cbe->idef3upd;
 		excdet.cbrstate = cbe->cbrstate;
 		excdet.cbrexecstatus = cbe->cbrexecstatus;
-		gru_flush_cache(cbe);
+		gru_flush_cache_cbe(cbe);
 		ret = 0;
 	} else {
 		ret = -EAGAIN;
@@ -770,9 +848,12 @@ int gru_set_context_option(unsigned long arg)
 		return -EFAULT;
 	gru_dbg(grudev, "op %d, gseg 0x%lx, value1 0x%lx\n", req.op, req.gseg, req.val1);
 
-	gts = gru_alloc_locked_gts(req.gseg);
-	if (IS_ERR(gts))
-		return PTR_ERR(gts);
+	gts = gru_find_lock_gts(req.gseg);
+	if (!gts) {
+		gts = gru_alloc_locked_gts(req.gseg);
+		if (IS_ERR(gts))
+			return PTR_ERR(gts);
+	}
 
 	switch (req.op) {
 	case sco_blade_chiplet:
diff --git a/drivers/misc/sgi-gru/grufile.c b/drivers/misc/sgi-gru/grufile.c
index 9d41208a6c9..cb3b4d22847 100644
--- a/drivers/misc/sgi-gru/grufile.c
+++ b/drivers/misc/sgi-gru/grufile.c
@@ -152,6 +152,7 @@ static int gru_create_new_context(unsigned long arg)
 		vdata->vd_dsr_au_count =
 		    GRU_DS_BYTES_TO_AU(req.data_segment_bytes);
 		vdata->vd_cbr_au_count = GRU_CB_COUNT_TO_AU(req.control_blocks);
+		vdata->vd_tlb_preload_count = req.tlb_preload_count;
 		ret = 0;
 	}
 	up_write(&current->mm->mmap_sem);
diff --git a/drivers/misc/sgi-gru/gruhandles.c b/drivers/misc/sgi-gru/gruhandles.c
index 66d67d9bc9b..2f30badc6ff 100644
--- a/drivers/misc/sgi-gru/gruhandles.c
+++ b/drivers/misc/sgi-gru/gruhandles.c
@@ -165,17 +165,20 @@ int tgh_invalidate(struct gru_tlb_global_handle *tgh,
 	return wait_instruction_complete(tgh, tghop_invalidate);
 }
 
-void tfh_write_only(struct gru_tlb_fault_handle *tfh,
-				  unsigned long pfn, unsigned long vaddr,
-				  int asid, int dirty, int pagesize)
+int tfh_write_only(struct gru_tlb_fault_handle *tfh,
+				  unsigned long paddr, int gaa,
+				  unsigned long vaddr, int asid, int dirty,
+				  int pagesize)
 {
 	tfh->fillasid = asid;
 	tfh->fillvaddr = vaddr;
-	tfh->pfn = pfn;
+	tfh->pfn = paddr >> GRU_PADDR_SHIFT;
+	tfh->gaa = gaa;
 	tfh->dirty = dirty;
 	tfh->pagesize = pagesize;
 	tfh->opc = TFHOP_WRITE_ONLY;
 	start_instruction(tfh);
+	return wait_instruction_complete(tfh, tfhop_write_only);
 }
 
 void tfh_write_restart(struct gru_tlb_fault_handle *tfh,
diff --git a/drivers/misc/sgi-gru/gruhandles.h b/drivers/misc/sgi-gru/gruhandles.h
index 47b762f89e0..ea584ebf65b 100644
--- a/drivers/misc/sgi-gru/gruhandles.h
+++ b/drivers/misc/sgi-gru/gruhandles.h
@@ -164,6 +164,16 @@ static inline void *gru_chiplet_vaddr(void *vaddr, int pnode, int chiplet)
 	return vaddr + GRU_SIZE * (2 * pnode  + chiplet);
 }
 
+static inline struct gru_control_block_extended *gru_tfh_to_cbe(
+					struct gru_tlb_fault_handle *tfh)
+{
+	unsigned long cbe;
+
+	cbe = (unsigned long)tfh - GRU_TFH_BASE + GRU_CBE_BASE;
+	return (struct gru_control_block_extended*)cbe;
+}
+
+
 
 
 /*
@@ -446,6 +456,12 @@ struct gru_control_block_extended {
 	unsigned int cbrexecstatus:8;
 };
 
+/* CBE fields for active BCOPY instructions */
+#define cbe_baddr0	idef1upd
+#define cbe_baddr1	idef3upd
+#define cbe_src_cl	idef6cpy
+#define cbe_nelemcur	idef5upd
+
 enum gru_cbr_state {
 	CBRSTATE_INACTIVE,
 	CBRSTATE_IDLE,
@@ -493,8 +509,8 @@ int cch_interrupt_sync(struct gru_context_configuration_handle *cch);
 int tgh_invalidate(struct gru_tlb_global_handle *tgh, unsigned long vaddr,
 	unsigned long vaddrmask, int asid, int pagesize, int global, int n,
 	unsigned short ctxbitmap);
-void tfh_write_only(struct gru_tlb_fault_handle *tfh, unsigned long pfn,
-	unsigned long vaddr, int asid, int dirty, int pagesize);
+int tfh_write_only(struct gru_tlb_fault_handle *tfh, unsigned long paddr,
+	int gaa, unsigned long vaddr, int asid, int dirty, int pagesize);
 void tfh_write_restart(struct gru_tlb_fault_handle *tfh, unsigned long paddr,
 	int gaa, unsigned long vaddr, int asid, int dirty, int pagesize);
 void tfh_restart(struct gru_tlb_fault_handle *tfh);
diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c
index 4da6f56833d..d9ff0289a1c 100644
--- a/drivers/misc/sgi-gru/grukservices.c
+++ b/drivers/misc/sgi-gru/grukservices.c
@@ -161,7 +161,7 @@ static void gru_load_kernel_context(struct gru_blade_state *bs, int blade_id)
 	down_write(&bs->bs_kgts_sema);
 
 	if (!bs->bs_kgts) {
-		bs->bs_kgts = gru_alloc_gts(NULL, 0, 0, 0, 0);
+		bs->bs_kgts = gru_alloc_gts(NULL, 0, 0, 0, 0, 0);
 		bs->bs_kgts->ts_user_blade_id = blade_id;
 	}
 	kgts = bs->bs_kgts;
diff --git a/drivers/misc/sgi-gru/grulib.h b/drivers/misc/sgi-gru/grulib.h
index e033b36df7e..c6928af7393 100644
--- a/drivers/misc/sgi-gru/grulib.h
+++ b/drivers/misc/sgi-gru/grulib.h
@@ -86,6 +86,7 @@ struct gru_create_context_req {
 	unsigned int		control_blocks;
 	unsigned int		maximum_thread_count;
 	unsigned int		options;
+	unsigned char		tlb_preload_count;
 };
 
 /*
diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c
index ebabbdcbb97..ade0925eab0 100644
--- a/drivers/misc/sgi-gru/grumain.c
+++ b/drivers/misc/sgi-gru/grumain.c
@@ -316,7 +316,8 @@ static struct gru_thread_state *gru_find_current_gts_nolock(struct gru_vma_data
  * Allocate a thread state structure.
  */
 struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma,
-		int cbr_au_count, int dsr_au_count, int options, int tsid)
+		int cbr_au_count, int dsr_au_count,
+		unsigned char tlb_preload_count, int options, int tsid)
 {
 	struct gru_thread_state *gts;
 	struct gru_mm_struct *gms;
@@ -334,6 +335,7 @@ struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma,
 	mutex_init(&gts->ts_ctxlock);
 	gts->ts_cbr_au_count = cbr_au_count;
 	gts->ts_dsr_au_count = dsr_au_count;
+	gts->ts_tlb_preload_count = tlb_preload_count;
 	gts->ts_user_options = options;
 	gts->ts_user_blade_id = -1;
 	gts->ts_user_chiplet_id = -1;
@@ -403,7 +405,9 @@ struct gru_thread_state *gru_alloc_thread_state(struct vm_area_struct *vma,
 	struct gru_vma_data *vdata = vma->vm_private_data;
 	struct gru_thread_state *gts, *ngts;
 
-	gts = gru_alloc_gts(vma, vdata->vd_cbr_au_count, vdata->vd_dsr_au_count,
+	gts = gru_alloc_gts(vma, vdata->vd_cbr_au_count,
+			    vdata->vd_dsr_au_count,
+			    vdata->vd_tlb_preload_count,
 			    vdata->vd_user_options, tsid);
 	if (IS_ERR(gts))
 		return gts;
diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c
index 0a57ab29cd3..54a5a1c35ad 100644
--- a/drivers/misc/sgi-gru/gruprocfs.c
+++ b/drivers/misc/sgi-gru/gruprocfs.c
@@ -76,6 +76,7 @@ static int statistics_show(struct seq_file *s, void *p)
 	printstat(s, check_context_retarget_intr);
 	printstat(s, check_context_unload);
 	printstat(s, tlb_dropin);
+	printstat(s, tlb_preload_page);
 	printstat(s, tlb_dropin_fail_no_asid);
 	printstat(s, tlb_dropin_fail_upm);
 	printstat(s, tlb_dropin_fail_invalid);
@@ -127,7 +128,8 @@ static int mcs_statistics_show(struct seq_file *s, void *p)
 	int op;
 	unsigned long total, count, max;
 	static char *id[] = {"cch_allocate", "cch_start", "cch_interrupt",
-		"cch_interrupt_sync", "cch_deallocate", "tgh_invalidate"};
+		"cch_interrupt_sync", "cch_deallocate", "tfh_write_only",
+		"tfh_write_restart", "tgh_invalidate"};
 
 	seq_printf(s, "%-20s%12s%12s%12s\n", "#id", "count", "aver-clks", "max-clks");
 	for (op = 0; op < mcsop_last; op++) {
diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h
index 76fe2987fc9..adaf691d59f 100644
--- a/drivers/misc/sgi-gru/grutables.h
+++ b/drivers/misc/sgi-gru/grutables.h
@@ -202,6 +202,7 @@ struct gru_stats_s {
 	atomic_long_t check_context_retarget_intr;
 	atomic_long_t check_context_unload;
 	atomic_long_t tlb_dropin;
+	atomic_long_t tlb_preload_page;
 	atomic_long_t tlb_dropin_fail_no_asid;
 	atomic_long_t tlb_dropin_fail_upm;
 	atomic_long_t tlb_dropin_fail_invalid;
@@ -245,7 +246,8 @@ struct gru_stats_s {
 };
 
 enum mcs_op {cchop_allocate, cchop_start, cchop_interrupt, cchop_interrupt_sync,
-	cchop_deallocate, tghop_invalidate, mcsop_last};
+	cchop_deallocate, tfhop_write_only, tfhop_write_restart,
+	tghop_invalidate, mcsop_last};
 
 struct mcs_op_statistic {
 	atomic_long_t	count;
@@ -335,6 +337,7 @@ struct gru_vma_data {
 	long			vd_user_options;/* misc user option flags */
 	int			vd_cbr_au_count;
 	int			vd_dsr_au_count;
+	unsigned char		vd_tlb_preload_count;
 };
 
 /*
@@ -350,6 +353,7 @@ struct gru_thread_state {
 	struct gru_state	*ts_gru;	/* GRU where the context is
 						   loaded */
 	struct gru_mm_struct	*ts_gms;	/* asid & ioproc struct */
+	unsigned char		ts_tlb_preload_count; /* TLB preload pages */
 	unsigned long		ts_cbr_map;	/* map of allocated CBRs */
 	unsigned long		ts_dsr_map;	/* map of allocated DATA
 						   resources */
@@ -661,7 +665,8 @@ extern int gru_proc_init(void);
 extern void gru_proc_exit(void);
 
 extern struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma,
-		int cbr_au_count, int dsr_au_count, int options, int tsid);
+		int cbr_au_count, int dsr_au_count,
+		unsigned char tlb_preload_count, int options, int tsid);
 extern unsigned long gru_reserve_cb_resources(struct gru_state *gru,
 		int cbr_au_count, char *cbmap);
 extern unsigned long gru_reserve_ds_resources(struct gru_state *gru,
-- 
cgit v1.2.3-70-g09d2


From 1848a710287b33a92c3c5b2f4c3ae201a879eb7e Mon Sep 17 00:00:00 2001
From: Jack Steiner <steiner@sgi.com>
Date: Tue, 15 Dec 2009 16:48:14 -0800
Subject: gru: fix bug in exception handling

Fix a GRU driver bug converting a CBR address to the context that contains
the CBR.  The conversion is rarely done so performance does not matter.

Signed-off-by: Jack Steiner <steiner@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
---
 drivers/misc/sgi-gru/grukservices.c | 30 ++++++++++++++++++++++--------
 1 file changed, 22 insertions(+), 8 deletions(-)

(limited to 'drivers/misc/sgi-gru/grukservices.c')

diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c
index d9ff0289a1c..c13342dda5f 100644
--- a/drivers/misc/sgi-gru/grukservices.c
+++ b/drivers/misc/sgi-gru/grukservices.c
@@ -97,9 +97,6 @@
 #define ASYNC_HAN_TO_BID(h)	((h) - 1)
 #define ASYNC_BID_TO_HAN(b)	((b) + 1)
 #define ASYNC_HAN_TO_BS(h)	gru_base[ASYNC_HAN_TO_BID(h)]
-#define KCB_TO_GID(cb)		((cb - gru_start_vaddr) /		\
-					(GRU_SIZE * GRU_CHIPLETS_PER_BLADE))
-#define KCB_TO_BS(cb)		gru_base[KCB_TO_GID(cb)]
 
 #define GRU_NUM_KERNEL_CBR	1
 #define GRU_NUM_KERNEL_DSR_BYTES 256
@@ -388,11 +385,28 @@ int gru_get_cb_exception_detail(void *cb,
 		struct control_block_extended_exc_detail *excdet)
 {
 	struct gru_control_block_extended *cbe;
-	struct gru_blade_state *bs;
-	int cbrnum;
-
-	bs = KCB_TO_BS(cb);
-	cbrnum = thread_cbr_number(bs->bs_kgts, get_cb_number(cb));
+	struct gru_thread_state *kgts = NULL;
+	unsigned long off;
+	int cbrnum, bid;
+
+	/*
+	 * Locate kgts for cb. This algorithm is SLOW but
+	 * this function is rarely called (ie., almost never).
+	 * Performance does not matter.
+	 */
+	for_each_possible_blade(bid) {
+		if (!gru_base[bid])
+			break;
+		kgts = gru_base[bid]->bs_kgts;
+		if (!kgts || !kgts->ts_gru)
+			continue;
+		off = cb - kgts->ts_gru->gs_gru_base_vaddr;
+		if (off < GRU_SIZE)
+			break;
+		kgts = NULL;
+	}
+	BUG_ON(!kgts);
+	cbrnum = thread_cbr_number(kgts, get_cb_number(cb));
 	cbe = get_cbe(GRUBASE(cb), cbrnum);
 	gru_flush_cache(cbe);	/* CBE not coherent */
 	sync_core();
-- 
cgit v1.2.3-70-g09d2


From 0cd2b0813aac660f5f7a6574083157a70c152dd5 Mon Sep 17 00:00:00 2001
From: Jack Steiner <steiner@sgi.com>
Date: Tue, 15 Dec 2009 16:48:17 -0800
Subject: gru: fix bug in allocation of kernel contexts

Fix a bug in the assignment of GRU contexts used for kernel functions.  If
a sleep occurs on the wait for a semaphore, the thread could switch cpus
and allocate resources on the wrong blade.

Signed-off-by: Jack Steiner <steiner@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
---
 drivers/misc/sgi-gru/grukservices.c | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

(limited to 'drivers/misc/sgi-gru/grukservices.c')

diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c
index c13342dda5f..bfbf8fdbcd3 100644
--- a/drivers/misc/sgi-gru/grukservices.c
+++ b/drivers/misc/sgi-gru/grukservices.c
@@ -221,13 +221,21 @@ static int gru_free_kernel_contexts(void)
 static struct gru_blade_state *gru_lock_kernel_context(int blade_id)
 {
 	struct gru_blade_state *bs;
+	int bid;
 
 	STAT(lock_kernel_context);
-	bs = gru_base[blade_id];
+again:
+	bid = blade_id < 0 ? uv_numa_blade_id() : blade_id;
+	bs = gru_base[bid];
 
+	/* Handle the case where migration occured while waiting for the sema */
 	down_read(&bs->bs_kgts_sema);
+	if (blade_id < 0 && bid != uv_numa_blade_id()) {
+		up_read(&bs->bs_kgts_sema);
+		goto again;
+	}
 	if (!bs->bs_kgts || !bs->bs_kgts->ts_gru)
-		gru_load_kernel_context(bs, blade_id);
+		gru_load_kernel_context(bs, bid);
 	return bs;
 
 }
@@ -256,7 +264,7 @@ static int gru_get_cpu_resources(int dsr_bytes, void **cb, void **dsr)
 
 	BUG_ON(dsr_bytes > GRU_NUM_KERNEL_DSR_BYTES);
 	preempt_disable();
-	bs = gru_lock_kernel_context(uv_numa_blade_id());
+	bs = gru_lock_kernel_context(-1);
 	lcpu = uv_blade_processor_id();
 	*cb = bs->kernel_cb + lcpu * GRU_HANDLE_STRIDE;
 	*dsr = bs->kernel_dsr + lcpu * GRU_NUM_KERNEL_DSR_BYTES;
-- 
cgit v1.2.3-70-g09d2


From 76148df19cbd5437dc5358408a58c7cc6366ecf4 Mon Sep 17 00:00:00 2001
From: Jack Steiner <steiner@sgi.com>
Date: Tue, 15 Dec 2009 16:48:21 -0800
Subject: gru: send cross partition interrupts using the gru

GRU Message queue instructions are used to deliver messages to other SSIs
within the numalink domain.  In most cases, a single GRU mesq instruction
will deliver both the message AND an interrupt to notify the other SSI
that a messsage is present.  In some cases, however, the interrupt must be
sent explicitly.

To improve resilency, the GRU driver should send these explicit interrupts
using the GRU to write the remote chipset register.  Current code sends
the interrupt using a cpu instruction to write the chipset register.

Signed-off-by: Jack Steiner <steiner@sgi.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
---
 drivers/misc/sgi-gru/gru_instructions.h | 12 ++++++++++
 drivers/misc/sgi-gru/grukservices.c     | 39 ++++++++++++++++++++-------------
 2 files changed, 36 insertions(+), 15 deletions(-)

(limited to 'drivers/misc/sgi-gru/grukservices.c')

diff --git a/drivers/misc/sgi-gru/gru_instructions.h b/drivers/misc/sgi-gru/gru_instructions.h
index 848c7a8f02b..d95587cc794 100644
--- a/drivers/misc/sgi-gru/gru_instructions.h
+++ b/drivers/misc/sgi-gru/gru_instructions.h
@@ -367,6 +367,18 @@ static inline void gru_vload_phys(void *cb, unsigned long gpa,
 					(unsigned long)tri0, CB_IMA(hints)));
 }
 
+static inline void gru_vstore_phys(void *cb, unsigned long gpa,
+		unsigned int tri0, int iaa, unsigned long hints)
+{
+	struct gru_instruction *ins = (struct gru_instruction *)cb;
+
+	ins->baddr0 = (long)gpa | ((unsigned long)iaa << 62);
+	ins->nelem = 1;
+	ins->op1_stride = 1;
+	gru_start_instruction(ins, __opdword(OP_VSTORE, 0, XTYPE_DW, iaa, 0,
+					(unsigned long)tri0, CB_IMA(hints)));
+}
+
 static inline void gru_vload(void *cb, unsigned long mem_addr,
 		unsigned int tri0, unsigned char xtype, unsigned long nelem,
 		unsigned long stride, unsigned long hints)
diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c
index bfbf8fdbcd3..34749ee88df 100644
--- a/drivers/misc/sgi-gru/grukservices.c
+++ b/drivers/misc/sgi-gru/grukservices.c
@@ -31,6 +31,7 @@
 #include <linux/interrupt.h>
 #include <linux/uaccess.h>
 #include <linux/delay.h>
+#include <asm/io_apic.h>
 #include "gru.h"
 #include "grulib.h"
 #include "grutables.h"
@@ -566,7 +567,7 @@ int gru_create_message_queue(struct gru_message_queue_desc *mqd,
 	mqd->mq = mq;
 	mqd->mq_gpa = uv_gpa(mq);
 	mqd->qlines = qlines;
-	mqd->interrupt_pnode = UV_NASID_TO_PNODE(nasid);
+	mqd->interrupt_pnode = nasid >> 1;
 	mqd->interrupt_vector = vector;
 	mqd->interrupt_apicid = apicid;
 	return 0;
@@ -702,18 +703,6 @@ cberr:
 	return MQE_UNEXPECTED_CB_ERR;
 }
 
-/*
- * Send a cross-partition interrupt to the SSI that contains the target
- * message queue. Normally, the interrupt is automatically delivered by hardware
- * but some error conditions require explicit delivery.
- */
-static void send_message_queue_interrupt(struct gru_message_queue_desc *mqd)
-{
-	if (mqd->interrupt_vector)
-		uv_hub_send_ipi(mqd->interrupt_pnode, mqd->interrupt_apicid,
-				mqd->interrupt_vector);
-}
-
 /*
  * Handle a PUT failure. Note: if message was a 2-line message, one of the
  * lines might have successfully have been written. Before sending the
@@ -723,7 +712,8 @@ static void send_message_queue_interrupt(struct gru_message_queue_desc *mqd)
 static int send_message_put_nacked(void *cb, struct gru_message_queue_desc *mqd,
 			void *mesg, int lines)
 {
-	unsigned long m;
+	unsigned long m, *val = mesg, gpa, save;
+	int ret;
 
 	m = mqd->mq_gpa + (gru_get_amo_value_head(cb) << 6);
 	if (lines == 2) {
@@ -734,7 +724,26 @@ static int send_message_put_nacked(void *cb, struct gru_message_queue_desc *mqd,
 	gru_vstore(cb, m, gru_get_tri(mesg), XTYPE_CL, lines, 1, IMA);
 	if (gru_wait(cb) != CBS_IDLE)
 		return MQE_UNEXPECTED_CB_ERR;
-	send_message_queue_interrupt(mqd);
+
+	if (!mqd->interrupt_vector)
+		return MQE_OK;
+
+	/*
+	 * Send a cross-partition interrupt to the SSI that contains the target
+	 * message queue. Normally, the interrupt is automatically delivered by
+	 * hardware but some error conditions require explicit delivery.
+	 * Use the GRU to deliver the interrupt. Otherwise partition failures
+	 * could cause unrecovered errors.
+	 */
+	gpa = uv_global_gru_mmr_address(mqd->interrupt_pnode, UVH_IPI_INT);
+	save = *val;
+	*val = uv_hub_ipi_value(mqd->interrupt_apicid, mqd->interrupt_vector,
+				dest_Fixed);
+	gru_vstore_phys(cb, gpa, gru_get_tri(mesg), IAA_REGISTER, IMA);
+	ret = gru_wait(cb);
+	*val = save;
+	if (ret != CBS_IDLE)
+		return MQE_UNEXPECTED_CB_ERR;
 	return MQE_OK;
 }
 
-- 
cgit v1.2.3-70-g09d2