From 518e5cd4aae476042bdee511e0e00c8670c0df42 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Tue, 15 Dec 2009 16:48:04 -0800 Subject: gru: allow users to specify gru chiplet 1 Add table & user request infrastructure that is needed to allow users to specify the blade and chiplet for allocation of GRU contexts. Use of this information is in a subsequent patch. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grumain.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'drivers/misc/sgi-gru/grumain.c') diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c index 3bc643dad60..f449c2dbc1e 100644 --- a/drivers/misc/sgi-gru/grumain.c +++ b/drivers/misc/sgi-gru/grumain.c @@ -328,6 +328,8 @@ struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma, gts->ts_cbr_au_count = cbr_au_count; gts->ts_dsr_au_count = dsr_au_count; gts->ts_user_options = options; + gts->ts_user_blade_id = -1; + gts->ts_user_chiplet_id = -1; gts->ts_tsid = tsid; gts->ts_ctxnum = NULLCTX; gts->ts_tlb_int_select = -1; -- cgit v1.2.3-70-g09d2 From 55484c45dbeca2eec7642932ec3f60f8a2d4bdbf Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Tue, 15 Dec 2009 16:48:05 -0800 Subject: gru: allow users to specify gru chiplet 2 Add support to the GRU driver to allow users to specify the blade & chiplet for allocation of GRU contexts. Add new statistics for context loading/unloading/retargeting. Also deleted a few GRU stats that were no longer being unused. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 13 +--- drivers/misc/sgi-gru/grufile.c | 15 +++-- drivers/misc/sgi-gru/grukservices.c | 8 ++- drivers/misc/sgi-gru/grumain.c | 128 ++++++++++++++++++++++++------------ drivers/misc/sgi-gru/gruprocfs.c | 9 +-- drivers/misc/sgi-gru/grutables.h | 16 ++--- 6 files changed, 109 insertions(+), 80 deletions(-) (limited to 'drivers/misc/sgi-gru/grumain.c') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index a69d119921f..321ad809f34 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -546,17 +546,7 @@ int gru_handle_user_call_os(unsigned long cb) if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) goto exit; - /* - * If force_unload is set, the UPM TLB fault is phony. The task - * has migrated to another node and the GSEG must be moved. Just - * unload the context. The task will page fault and assign a new - * context. - */ - if (gts->ts_tgid_owner == current->tgid && gts->ts_blade >= 0 && - gts->ts_blade != uv_numa_blade_id()) { - STAT(call_os_offnode_reference); - gts->ts_force_unload = 1; - } + gru_check_context_placement(gts); /* * CCH may contain stale data if ts_force_cch_reload is set. @@ -771,6 +761,7 @@ int gru_set_context_option(unsigned long arg) } else { gts->ts_user_blade_id = req.val1; gts->ts_user_chiplet_id = req.val0; + gru_check_context_placement(gts); } break; case sco_gseg_owner: diff --git a/drivers/misc/sgi-gru/grufile.c b/drivers/misc/sgi-gru/grufile.c index 2e574fd5d3f..0a6d2a5a01f 100644 --- a/drivers/misc/sgi-gru/grufile.c +++ b/drivers/misc/sgi-gru/grufile.c @@ -232,23 +232,24 @@ static long gru_file_unlocked_ioctl(struct file *file, unsigned int req, * system. */ static void gru_init_chiplet(struct gru_state *gru, unsigned long paddr, - void *vaddr, int nid, int bid, int grunum) + void *vaddr, int blade_id, int chiplet_id) { spin_lock_init(&gru->gs_lock); spin_lock_init(&gru->gs_asid_lock); gru->gs_gru_base_paddr = paddr; gru->gs_gru_base_vaddr = vaddr; - gru->gs_gid = bid * GRU_CHIPLETS_PER_BLADE + grunum; - gru->gs_blade = gru_base[bid]; - gru->gs_blade_id = bid; + gru->gs_gid = blade_id * GRU_CHIPLETS_PER_BLADE + chiplet_id; + gru->gs_blade = gru_base[blade_id]; + gru->gs_blade_id = blade_id; + gru->gs_chiplet_id = chiplet_id; gru->gs_cbr_map = (GRU_CBR_AU == 64) ? ~0 : (1UL << GRU_CBR_AU) - 1; gru->gs_dsr_map = (1UL << GRU_DSR_AU) - 1; gru->gs_asid_limit = MAX_ASID; gru_tgh_flush_init(gru); if (gru->gs_gid >= gru_max_gids) gru_max_gids = gru->gs_gid + 1; - gru_dbg(grudev, "bid %d, nid %d, gid %d, vaddr %p (0x%lx)\n", - bid, nid, gru->gs_gid, gru->gs_gru_base_vaddr, + gru_dbg(grudev, "bid %d, gid %d, vaddr %p (0x%lx)\n", + blade_id, gru->gs_gid, gru->gs_gru_base_vaddr, gru->gs_gru_base_paddr); } @@ -283,7 +284,7 @@ static int gru_init_tables(unsigned long gru_base_paddr, void *gru_base_vaddr) chip++, gru++) { paddr = gru_chiplet_paddr(gru_base_paddr, pnode, chip); vaddr = gru_chiplet_vaddr(gru_base_vaddr, pnode, chip); - gru_init_chiplet(gru, paddr, vaddr, nid, bid, chip); + gru_init_chiplet(gru, paddr, vaddr, bid, chip); n = hweight64(gru->gs_cbr_map) * GRU_CBR_AU_SIZE; cbrs = max(cbrs, n); n = hweight64(gru->gs_dsr_map) * GRU_DSR_AU_BYTES; diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c index 306855cc80f..e0d4b53d1fc 100644 --- a/drivers/misc/sgi-gru/grukservices.c +++ b/drivers/misc/sgi-gru/grukservices.c @@ -160,8 +160,10 @@ static void gru_load_kernel_context(struct gru_blade_state *bs, int blade_id) up_read(&bs->bs_kgts_sema); down_write(&bs->bs_kgts_sema); - if (!bs->bs_kgts) + if (!bs->bs_kgts) { bs->bs_kgts = gru_alloc_gts(NULL, 0, 0, 0, 0); + bs->bs_kgts->ts_user_blade_id = blade_id; + } kgts = bs->bs_kgts; if (!kgts->ts_gru) { @@ -172,9 +174,9 @@ static void gru_load_kernel_context(struct gru_blade_state *bs, int blade_id) kgts->ts_dsr_au_count = GRU_DS_BYTES_TO_AU( GRU_NUM_KERNEL_DSR_BYTES * ncpus + bs->bs_async_dsr_bytes); - while (!gru_assign_gru_context(kgts, blade_id)) { + while (!gru_assign_gru_context(kgts)) { msleep(1); - gru_steal_context(kgts, blade_id); + gru_steal_context(kgts); } gru_load_context(kgts); gru = bs->bs_kgts->ts_gru; diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c index f449c2dbc1e..54ad7544a9a 100644 --- a/drivers/misc/sgi-gru/grumain.c +++ b/drivers/misc/sgi-gru/grumain.c @@ -684,6 +684,40 @@ static int gru_retarget_intr(struct gru_thread_state *gts) return gru_update_cch(gts, 0); } +/* + * Unload the gru context if it is not assigned to the correct blade or + * chiplet. Misassignment can occur if the process migrates to a different + * blade or if the user changes the selected blade/chiplet. + * Return 0 if context correct placed, otherwise 1 + */ +void gru_check_context_placement(struct gru_thread_state *gts) +{ + struct gru_state *gru; + int blade_id, chiplet_id; + + /* + * If the current task is the context owner, verify that the + * context is correctly placed. This test is skipped for non-owner + * references. Pthread apps use non-owner references to the CBRs. + */ + gru = gts->ts_gru; + if (!gru || gts->ts_tgid_owner != current->tgid) + return; + + blade_id = gts->ts_user_blade_id; + if (blade_id < 0) + blade_id = uv_numa_blade_id(); + + chiplet_id = gts->ts_user_chiplet_id; + if (gru->gs_blade_id != blade_id || + (chiplet_id >= 0 && chiplet_id != gru->gs_chiplet_id)) { + STAT(check_context_unload); + gru_unload_context(gts, 1); + } else if (gru_retarget_intr(gts)) { + STAT(check_context_retarget_intr); + } +} + /* * Insufficient GRU resources available on the local blade. Steal a context from @@ -714,13 +748,17 @@ static void gts_stolen(struct gru_thread_state *gts, } } -void gru_steal_context(struct gru_thread_state *gts, int blade_id) +void gru_steal_context(struct gru_thread_state *gts) { struct gru_blade_state *blade; struct gru_state *gru, *gru0; struct gru_thread_state *ngts = NULL; int ctxnum, ctxnum0, flag = 0, cbr, dsr; + int blade_id = gts->ts_user_blade_id; + int chiplet_id = gts->ts_user_chiplet_id; + if (blade_id < 0) + blade_id = uv_numa_blade_id(); cbr = gts->ts_cbr_au_count; dsr = gts->ts_dsr_au_count; @@ -731,35 +769,39 @@ void gru_steal_context(struct gru_thread_state *gts, int blade_id) gru = blade->bs_lru_gru; if (ctxnum == 0) gru = next_gru(blade, gru); + blade->bs_lru_gru = gru; + blade->bs_lru_ctxnum = ctxnum; ctxnum0 = ctxnum; gru0 = gru; while (1) { - if (check_gru_resources(gru, cbr, dsr, GRU_NUM_CCH)) - break; - spin_lock(&gru->gs_lock); - for (; ctxnum < GRU_NUM_CCH; ctxnum++) { - if (flag && gru == gru0 && ctxnum == ctxnum0) + if (chiplet_id < 0 || chiplet_id == gru->gs_chiplet_id) { + if (check_gru_resources(gru, cbr, dsr, GRU_NUM_CCH)) break; - ngts = gru->gs_gts[ctxnum]; - /* - * We are grabbing locks out of order, so trylock is - * needed. GTSs are usually not locked, so the odds of - * success are high. If trylock fails, try to steal a - * different GSEG. - */ - if (ngts && is_gts_stealable(ngts, blade)) + spin_lock(&gru->gs_lock); + for (; ctxnum < GRU_NUM_CCH; ctxnum++) { + if (flag && gru == gru0 && ctxnum == ctxnum0) + break; + ngts = gru->gs_gts[ctxnum]; + /* + * We are grabbing locks out of order, so trylock is + * needed. GTSs are usually not locked, so the odds of + * success are high. If trylock fails, try to steal a + * different GSEG. + */ + if (ngts && is_gts_stealable(ngts, blade)) + break; + ngts = NULL; + } + spin_unlock(&gru->gs_lock); + if (ngts || (flag && gru == gru0 && ctxnum == ctxnum0)) break; - ngts = NULL; - flag = 1; } - spin_unlock(&gru->gs_lock); - if (ngts || (flag && gru == gru0 && ctxnum == ctxnum0)) + if (flag && gru == gru0) break; + flag = 1; ctxnum = 0; gru = next_gru(blade, gru); } - blade->bs_lru_gru = gru; - blade->bs_lru_ctxnum = ctxnum; spin_unlock(&blade->bs_lock); if (ngts) { @@ -777,20 +819,36 @@ void gru_steal_context(struct gru_thread_state *gts, int blade_id) hweight64(gru->gs_dsr_map)); } +/* + * Assign a gru context. + */ +static int gru_assign_context_number(struct gru_state *gru) +{ + int ctxnum; + + ctxnum = find_first_zero_bit(&gru->gs_context_map, GRU_NUM_CCH); + __set_bit(ctxnum, &gru->gs_context_map); + return ctxnum; +} + /* * Scan the GRUs on the local blade & assign a GRU context. */ -struct gru_state *gru_assign_gru_context(struct gru_thread_state *gts, - int blade) +struct gru_state *gru_assign_gru_context(struct gru_thread_state *gts) { struct gru_state *gru, *grux; int i, max_active_contexts; + int blade_id = gts->ts_user_blade_id; + int chiplet_id = gts->ts_user_chiplet_id; - + if (blade_id < 0) + blade_id = uv_numa_blade_id(); again: gru = NULL; max_active_contexts = GRU_NUM_CCH; - for_each_gru_on_blade(grux, blade, i) { + for_each_gru_on_blade(grux, blade_id, i) { + if (chiplet_id >= 0 && chiplet_id != grux->gs_chiplet_id) + continue; if (check_gru_resources(grux, gts->ts_cbr_au_count, gts->ts_dsr_au_count, max_active_contexts)) { @@ -811,12 +869,9 @@ again: reserve_gru_resources(gru, gts); gts->ts_gru = gru; gts->ts_blade = gru->gs_blade_id; - gts->ts_ctxnum = - find_first_zero_bit(&gru->gs_context_map, GRU_NUM_CCH); - BUG_ON(gts->ts_ctxnum == GRU_NUM_CCH); + gts->ts_ctxnum = gru_assign_context_number(gru); atomic_inc(>s->ts_refcnt); gru->gs_gts[gts->ts_ctxnum] = gts; - __set_bit(gts->ts_ctxnum, &gru->gs_context_map); spin_unlock(&gru->gs_lock); STAT(assign_context); @@ -844,7 +899,6 @@ int gru_fault(struct vm_area_struct *vma, struct vm_fault *vmf) { struct gru_thread_state *gts; unsigned long paddr, vaddr; - int blade_id; vaddr = (unsigned long)vmf->virtual_address; gru_dbg(grudev, "vma %p, vaddr 0x%lx (0x%lx)\n", @@ -859,28 +913,18 @@ int gru_fault(struct vm_area_struct *vma, struct vm_fault *vmf) again: mutex_lock(>s->ts_ctxlock); preempt_disable(); - blade_id = uv_numa_blade_id(); - if (gts->ts_gru) { - if (gts->ts_gru->gs_blade_id != blade_id) { - STAT(migrated_nopfn_unload); - gru_unload_context(gts, 1); - } else { - if (gru_retarget_intr(gts)) - STAT(migrated_nopfn_retarget); - } - } + gru_check_context_placement(gts); if (!gts->ts_gru) { STAT(load_user_context); - if (!gru_assign_gru_context(gts, blade_id)) { + if (!gru_assign_gru_context(gts)) { preempt_enable(); mutex_unlock(>s->ts_ctxlock); set_current_state(TASK_INTERRUPTIBLE); schedule_timeout(GRU_ASSIGN_DELAY); /* true hack ZZZ */ - blade_id = uv_numa_blade_id(); if (gts->ts_steal_jiffies + GRU_STEAL_DELAY < jiffies) - gru_steal_context(gts, blade_id); + gru_steal_context(gts); goto again; } gru_load_context(gts); diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c index cf6b3e3034d..a8a2e760cca 100644 --- a/drivers/misc/sgi-gru/gruprocfs.c +++ b/drivers/misc/sgi-gru/gruprocfs.c @@ -67,19 +67,14 @@ static int statistics_show(struct seq_file *s, void *p) printstat(s, intr); printstat(s, intr_mm_lock_failed); printstat(s, call_os); - printstat(s, call_os_offnode_reference); printstat(s, call_os_check_for_bug); printstat(s, call_os_wait_queue); printstat(s, user_flush_tlb); printstat(s, user_unload_context); printstat(s, user_exception); printstat(s, set_context_option); - printstat(s, migrate_check); - printstat(s, migrated_retarget); - printstat(s, migrated_unload); - printstat(s, migrated_unload_delay); - printstat(s, migrated_nopfn_retarget); - printstat(s, migrated_nopfn_unload); + printstat(s, check_context_retarget_intr); + printstat(s, check_context_unload); printstat(s, tlb_dropin); printstat(s, tlb_dropin_fail_no_asid); printstat(s, tlb_dropin_fail_upm); diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h index 27131fb2253..8a61a8599ba 100644 --- a/drivers/misc/sgi-gru/grutables.h +++ b/drivers/misc/sgi-gru/grutables.h @@ -192,19 +192,14 @@ struct gru_stats_s { atomic_long_t intr; atomic_long_t intr_mm_lock_failed; atomic_long_t call_os; - atomic_long_t call_os_offnode_reference; atomic_long_t call_os_check_for_bug; atomic_long_t call_os_wait_queue; atomic_long_t user_flush_tlb; atomic_long_t user_unload_context; atomic_long_t user_exception; atomic_long_t set_context_option; - atomic_long_t migrate_check; - atomic_long_t migrated_retarget; - atomic_long_t migrated_unload; - atomic_long_t migrated_unload_delay; - atomic_long_t migrated_nopfn_retarget; - atomic_long_t migrated_nopfn_unload; + atomic_long_t check_context_retarget_intr; + atomic_long_t check_context_unload; atomic_long_t tlb_dropin; atomic_long_t tlb_dropin_fail_no_asid; atomic_long_t tlb_dropin_fail_upm; @@ -425,6 +420,7 @@ struct gru_state { gru segments (64) */ unsigned short gs_gid; /* unique GRU number */ unsigned short gs_blade_id; /* blade of GRU */ + unsigned char gs_chiplet_id; /* blade chiplet of GRU */ unsigned char gs_tgh_local_shift; /* used to pick TGH for local flush */ unsigned char gs_tgh_first_remote; /* starting TGH# for @@ -636,10 +632,9 @@ extern struct gru_thread_state *gru_find_thread_state(struct vm_area_struct *vma, int tsid); extern struct gru_thread_state *gru_alloc_thread_state(struct vm_area_struct *vma, int tsid); -extern struct gru_state *gru_assign_gru_context(struct gru_thread_state *gts, - int blade); +extern struct gru_state *gru_assign_gru_context(struct gru_thread_state *gts); extern void gru_load_context(struct gru_thread_state *gts); -extern void gru_steal_context(struct gru_thread_state *gts, int blade_id); +extern void gru_steal_context(struct gru_thread_state *gts); extern void gru_unload_context(struct gru_thread_state *gts, int savestate); extern int gru_update_cch(struct gru_thread_state *gts, int force_unload); extern void gts_drop(struct gru_thread_state *gts); @@ -654,6 +649,7 @@ extern int gru_user_flush_tlb(unsigned long arg); extern int gru_user_unload_context(unsigned long arg); extern int gru_get_exception_detail(unsigned long arg); extern int gru_set_context_option(unsigned long address); +extern void gru_check_context_placement(struct gru_thread_state *gts); extern int gru_cpu_fault_map_id(void); extern struct vm_area_struct *gru_find_vma(unsigned long vaddr); extern void gru_flush_all_tlb(struct gru_state *gru); -- cgit v1.2.3-70-g09d2 From 99f7c229b32bdf7424fbeb1d0d1b3883e14e97d0 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Tue, 15 Dec 2009 16:48:06 -0800 Subject: gru: allow users to specify gru chiplet 3 This patch builds on the infrastructure introduced in the patches that allow user specification of GRU blades & chiplets for context allocation. This patch simplifies the algorithms for migrating GRU contexts between blades. No new functionality is introduced. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 8 +++----- drivers/misc/sgi-gru/grumain.c | 30 +++++++++--------------------- drivers/misc/sgi-gru/grutables.h | 4 +--- 3 files changed, 13 insertions(+), 29 deletions(-) (limited to 'drivers/misc/sgi-gru/grumain.c') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 321ad809f34..9470303a9cb 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -362,7 +362,7 @@ static int gru_try_dropin(struct gru_thread_state *gts, if (!(gts->ts_sizeavail & GRU_SIZEAVAIL(pageshift))) { gts->ts_sizeavail |= GRU_SIZEAVAIL(pageshift); - if (atomic || !gru_update_cch(gts, 0)) { + if (atomic || !gru_update_cch(gts)) { gts->ts_force_cch_reload = 1; goto failupm; } @@ -553,14 +553,12 @@ int gru_handle_user_call_os(unsigned long cb) */ if (gts->ts_gru && gts->ts_force_cch_reload) { gts->ts_force_cch_reload = 0; - gru_update_cch(gts, 0); + gru_update_cch(gts); } ret = -EAGAIN; cbrnum = thread_cbr_number(gts, ucbnum); - if (gts->ts_force_unload) { - gru_unload_context(gts, 1); - } else if (gts->ts_gru) { + if (gts->ts_gru) { tfh = get_tfh_by_index(gts->ts_gru, cbrnum); cbk = get_gseg_base_address_cb(gts->ts_gru->gs_gru_base_vaddr, gts->ts_ctxnum, ucbnum); diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c index 54ad7544a9a..9ec54bde447 100644 --- a/drivers/misc/sgi-gru/grumain.c +++ b/drivers/misc/sgi-gru/grumain.c @@ -551,7 +551,6 @@ void gru_unload_context(struct gru_thread_state *gts, int savestate) if (cch_deallocate(cch)) BUG(); - gts->ts_force_unload = 0; /* ts_force_unload locked by CCH lock */ unlock_cch_handle(cch); gru_free_gru_context(gts); @@ -624,11 +623,8 @@ void gru_load_context(struct gru_thread_state *gts) * Update fields in an active CCH: * - retarget interrupts on local blade * - update sizeavail mask - * - force a delayed context unload by clearing the CCH asids. This - * forces TLB misses for new GRU instructions. The context is unloaded - * when the next TLB miss occurs. */ -int gru_update_cch(struct gru_thread_state *gts, int force_unload) +int gru_update_cch(struct gru_thread_state *gts) { struct gru_context_configuration_handle *cch; struct gru_state *gru = gts->ts_gru; @@ -642,21 +638,13 @@ int gru_update_cch(struct gru_thread_state *gts, int force_unload) goto exit; if (cch_interrupt(cch)) BUG(); - if (!force_unload) { - for (i = 0; i < 8; i++) - cch->sizeavail[i] = gts->ts_sizeavail; - gts->ts_tlb_int_select = gru_cpu_fault_map_id(); - cch->tlb_int_select = gru_cpu_fault_map_id(); - cch->tfm_fault_bit_enable = - (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL - || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR); - } else { - for (i = 0; i < 8; i++) - cch->asid[i] = 0; - cch->tfm_fault_bit_enable = 0; - cch->tlb_int_enable = 0; - gts->ts_force_unload = 1; - } + for (i = 0; i < 8; i++) + cch->sizeavail[i] = gts->ts_sizeavail; + gts->ts_tlb_int_select = gru_cpu_fault_map_id(); + cch->tlb_int_select = gru_cpu_fault_map_id(); + cch->tfm_fault_bit_enable = + (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL + || gts->ts_user_options == GRU_OPT_MISS_FMM_INTR); if (cch_start(cch)) BUG(); ret = 1; @@ -681,7 +669,7 @@ static int gru_retarget_intr(struct gru_thread_state *gts) gru_dbg(grudev, "retarget from %d to %d\n", gts->ts_tlb_int_select, gru_cpu_fault_map_id()); - return gru_update_cch(gts, 0); + return gru_update_cch(gts); } /* diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h index 8a61a8599ba..676da782199 100644 --- a/drivers/misc/sgi-gru/grutables.h +++ b/drivers/misc/sgi-gru/grutables.h @@ -382,8 +382,6 @@ struct gru_thread_state { char ts_blade; /* If >= 0, migrate context if ref from diferent blade */ char ts_force_cch_reload; - char ts_force_unload;/* force context to be unloaded - after migration */ char ts_cbr_idx[GRU_CBR_AU];/* CBR numbers of each allocated CB */ int ts_data_valid; /* Indicates if ts_gdata has @@ -636,7 +634,7 @@ extern struct gru_state *gru_assign_gru_context(struct gru_thread_state *gts); extern void gru_load_context(struct gru_thread_state *gts); extern void gru_steal_context(struct gru_thread_state *gts); extern void gru_unload_context(struct gru_thread_state *gts, int savestate); -extern int gru_update_cch(struct gru_thread_state *gts, int force_unload); +extern int gru_update_cch(struct gru_thread_state *gts); extern void gts_drop(struct gru_thread_state *gts); extern void gru_tgh_flush_init(struct gru_state *gru); extern int gru_kservices_init(void); -- cgit v1.2.3-70-g09d2 From 7f2251b1bcdd3d2971b2fde3008b270ea11b8780 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Tue, 15 Dec 2009 16:48:08 -0800 Subject: gru: handle failures to mmu_notifier_register Under some conditions, mmu_notifier_register() will fail to register a mmu_notifier. Fix the GRU driver to correctly handle these failures. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 6 +++--- drivers/misc/sgi-gru/grumain.c | 18 +++++++++++------- drivers/misc/sgi-gru/grutlbpurge.c | 8 +++++++- 3 files changed, 21 insertions(+), 11 deletions(-) (limited to 'drivers/misc/sgi-gru/grumain.c') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 9470303a9cb..ada7df7e096 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -96,7 +96,7 @@ static struct gru_thread_state *gru_alloc_locked_gts(unsigned long vaddr) vma = gru_find_vma(vaddr); if (vma) gts = gru_alloc_thread_state(vma, TSID(vaddr, vma)); - if (gts) { + if (!IS_ERR(gts)) { mutex_lock(>s->ts_ctxlock); downgrade_write(&mm->mmap_sem); } else { @@ -747,8 +747,8 @@ int gru_set_context_option(unsigned long arg) gru_dbg(grudev, "op %d, gseg 0x%lx, value1 0x%lx\n", req.op, req.gseg, req.val1); gts = gru_alloc_locked_gts(req.gseg); - if (!gts) - return -EINVAL; + if (IS_ERR(gts)) + return PTR_ERR(gts); switch (req.op) { case sco_blade_chiplet: diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c index 9ec54bde447..94402887188 100644 --- a/drivers/misc/sgi-gru/grumain.c +++ b/drivers/misc/sgi-gru/grumain.c @@ -27,6 +27,7 @@ #include #include #include +#include #include #include "gru.h" #include "grutables.h" @@ -286,7 +287,8 @@ static void gru_unload_mm_tracker(struct gru_state *gru, void gts_drop(struct gru_thread_state *gts) { if (gts && atomic_dec_return(>s->ts_refcnt) == 0) { - gru_drop_mmu_notifier(gts->ts_gms); + if (gts->ts_gms) + gru_drop_mmu_notifier(gts->ts_gms); kfree(gts); STAT(gts_free); } @@ -313,13 +315,14 @@ struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma, int cbr_au_count, int dsr_au_count, int options, int tsid) { struct gru_thread_state *gts; + struct gru_mm_struct *gms; int bytes; bytes = DSR_BYTES(dsr_au_count) + CBR_BYTES(cbr_au_count); bytes += sizeof(struct gru_thread_state); gts = kmalloc(bytes, GFP_KERNEL); if (!gts) - return NULL; + return ERR_PTR(-ENOMEM); STAT(gts_alloc); memset(gts, 0, sizeof(struct gru_thread_state)); /* zero out header */ @@ -338,9 +341,10 @@ struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma, if (vma) { gts->ts_mm = current->mm; gts->ts_vma = vma; - gts->ts_gms = gru_register_mmu_notifier(); - if (!gts->ts_gms) + gms = gru_register_mmu_notifier(); + if (IS_ERR(gms)) goto err; + gts->ts_gms = gms; } gru_dbg(grudev, "alloc gts %p\n", gts); @@ -348,7 +352,7 @@ struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma, err: gts_drop(gts); - return NULL; + return ERR_CAST(gms); } /* @@ -396,8 +400,8 @@ struct gru_thread_state *gru_alloc_thread_state(struct vm_area_struct *vma, gts = gru_alloc_gts(vma, vdata->vd_cbr_au_count, vdata->vd_dsr_au_count, vdata->vd_user_options, tsid); - if (!gts) - return NULL; + if (IS_ERR(gts)) + return gts; spin_lock(&vdata->vd_lock); ngts = gru_find_current_gts_nolock(vdata, tsid); diff --git a/drivers/misc/sgi-gru/grutlbpurge.c b/drivers/misc/sgi-gru/grutlbpurge.c index 1d125091f5e..421c548e51f 100644 --- a/drivers/misc/sgi-gru/grutlbpurge.c +++ b/drivers/misc/sgi-gru/grutlbpurge.c @@ -299,6 +299,7 @@ struct gru_mm_struct *gru_register_mmu_notifier(void) { struct gru_mm_struct *gms; struct mmu_notifier *mn; + int err; mn = mmu_find_ops(current->mm, &gru_mmuops); if (mn) { @@ -311,12 +312,17 @@ struct gru_mm_struct *gru_register_mmu_notifier(void) gms->ms_notifier.ops = &gru_mmuops; atomic_set(&gms->ms_refcnt, 1); init_waitqueue_head(&gms->ms_wait_queue); - __mmu_notifier_register(&gms->ms_notifier, current->mm); + err = __mmu_notifier_register(&gms->ms_notifier, current->mm); + if (err) + goto error; } } gru_dbg(grudev, "gms %p, refcnt %d\n", gms, atomic_read(&gms->ms_refcnt)); return gms; +error: + kfree(gms); + return ERR_PTR(err); } void gru_drop_mmu_notifier(struct gru_mm_struct *gms) -- cgit v1.2.3-70-g09d2 From 67bf04a5c2574e9495f660f418f6df776821d578 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Tue, 15 Dec 2009 16:48:11 -0800 Subject: gru: fix prefetch and speculation bugs Fix several bugs related to prefetch, ordering & speculation: - GRU cch_allocate() instruction causes cacheable memory to be created. Add a barriers to prevent speculation from prefetching data before it exists. - Add memory barriers before cache-flush instructions to ensure that previously stored data is included in the line flushed to memory. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/gru_instructions.h | 1 + drivers/misc/sgi-gru/grufault.c | 2 ++ drivers/misc/sgi-gru/gruhandles.c | 22 ++++++++++++++++++++-- drivers/misc/sgi-gru/grukservices.c | 10 ++++++---- drivers/misc/sgi-gru/grumain.c | 9 +++++++++ 5 files changed, 38 insertions(+), 6 deletions(-) (limited to 'drivers/misc/sgi-gru/grumain.c') diff --git a/drivers/misc/sgi-gru/gru_instructions.h b/drivers/misc/sgi-gru/gru_instructions.h index e033b6ce4a3..32f358d9f42 100644 --- a/drivers/misc/sgi-gru/gru_instructions.h +++ b/drivers/misc/sgi-gru/gru_instructions.h @@ -325,6 +325,7 @@ static inline void gru_flush_cache(void *p) static inline void gru_start_instruction(struct gru_instruction *ins, int op32) { gru_ordered_store_int(ins, op32); + mb(); gru_flush_cache(ins); } diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 7466234450f..d3cacd696b3 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -333,6 +333,7 @@ static int gru_try_dropin(struct gru_thread_state *gts, */ if (tfh->status != TFHSTATUS_EXCEPTION) { gru_flush_cache(tfh); + sync_core(); if (tfh->status != TFHSTATUS_EXCEPTION) goto failnoexception; STAT(tfh_stale_on_fault); @@ -599,6 +600,7 @@ int gru_get_exception_detail(unsigned long arg) cbrnum = thread_cbr_number(gts, ucbnum); cbe = get_cbe_by_index(gts->ts_gru, cbrnum); gru_flush_cache(cbe); /* CBE not coherent */ + sync_core(); /* make sure we are have current data */ excdet.opc = cbe->opccpy; excdet.exopc = cbe->exopccpy; excdet.ecause = cbe->ecause; diff --git a/drivers/misc/sgi-gru/gruhandles.c b/drivers/misc/sgi-gru/gruhandles.c index 806419a6b44..f1117a7637a 100644 --- a/drivers/misc/sgi-gru/gruhandles.c +++ b/drivers/misc/sgi-gru/gruhandles.c @@ -91,9 +91,18 @@ static int wait_instruction_complete(void *h, enum mcs_op opc) int cch_allocate(struct gru_context_configuration_handle *cch) { + int ret; + cch->opc = CCHOP_ALLOCATE; start_instruction(cch); - return wait_instruction_complete(cch, cchop_allocate); + ret = wait_instruction_complete(cch, cchop_allocate); + + /* + * Stop speculation into the GSEG being mapped by the previous ALLOCATE. + * The GSEG memory does not exist until the ALLOCATE completes. + */ + sync_core(); + return ret; } int cch_start(struct gru_context_configuration_handle *cch) @@ -112,9 +121,18 @@ int cch_interrupt(struct gru_context_configuration_handle *cch) int cch_deallocate(struct gru_context_configuration_handle *cch) { + int ret; + cch->opc = CCHOP_DEALLOCATE; start_instruction(cch); - return wait_instruction_complete(cch, cchop_deallocate); + ret = wait_instruction_complete(cch, cchop_deallocate); + + /* + * Stop speculation into the GSEG being unmapped by the previous + * DEALLOCATE. + */ + sync_core(); + return ret; } int cch_interrupt_sync(struct gru_context_configuration_handle diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c index 24ec109e61c..8c81aca0463 100644 --- a/drivers/misc/sgi-gru/grukservices.c +++ b/drivers/misc/sgi-gru/grukservices.c @@ -395,6 +395,7 @@ int gru_get_cb_exception_detail(void *cb, cbrnum = thread_cbr_number(bs->bs_kgts, get_cb_number(cb)); cbe = get_cbe(GRUBASE(cb), cbrnum); gru_flush_cache(cbe); /* CBE not coherent */ + sync_core(); excdet->opc = cbe->opccpy; excdet->exopc = cbe->exopccpy; excdet->ecause = cbe->ecause; @@ -461,9 +462,10 @@ int gru_check_status_proc(void *cb) int ret; ret = gen->istatus; - if (ret != CBS_EXCEPTION) - return ret; - return gru_retry_exception(cb); + if (ret == CBS_EXCEPTION) + ret = gru_retry_exception(cb); + rmb(); + return ret; } @@ -475,7 +477,7 @@ int gru_wait_proc(void *cb) ret = gru_wait_idle_or_exception(gen); if (ret == CBS_EXCEPTION) ret = gru_retry_exception(cb); - + rmb(); return ret; } diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c index 94402887188..a383271d391 100644 --- a/drivers/misc/sgi-gru/grumain.c +++ b/drivers/misc/sgi-gru/grumain.c @@ -499,6 +499,9 @@ static void gru_load_context_data(void *save, void *grubase, int ctxnum, memset(cbe + i * GRU_HANDLE_STRIDE, 0, GRU_CACHE_LINE_BYTES); } + /* Flush CBE to hide race in context restart */ + mb(); + gru_flush_cache(cbe + i * GRU_HANDLE_STRIDE); cb += GRU_HANDLE_STRIDE; } @@ -519,6 +522,12 @@ static void gru_unload_context_data(void *save, void *grubase, int ctxnum, cb = gseg + GRU_CB_BASE; cbe = grubase + GRU_CBE_BASE; length = hweight64(dsrmap) * GRU_DSR_AU_BYTES; + + /* CBEs may not be coherent. Flush them from cache */ + for_each_cbr_in_allocation_map(i, &cbrmap, scr) + gru_flush_cache(cbe + i * GRU_HANDLE_STRIDE); + mb(); /* Let the CL flush complete */ + gru_prefetch_context(gseg, cb, cbe, cbrmap, length); for_each_cbr_in_allocation_map(i, &cbrmap, scr) { -- cgit v1.2.3-70-g09d2 From 4107e1d38a23028c2a3bc23dd948265dbe6becba Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Tue, 15 Dec 2009 16:48:11 -0800 Subject: gru: update irq infrastructure Update the GRU irq allocate/free functions to use the latest upstream infrastructure. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 49 +++++--- drivers/misc/sgi-gru/grufile.c | 258 +++++++++++++++++++++++++++++++-------- drivers/misc/sgi-gru/grumain.c | 9 +- drivers/misc/sgi-gru/grutables.h | 14 ++- 4 files changed, 260 insertions(+), 70 deletions(-) (limited to 'drivers/misc/sgi-gru/grumain.c') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index d3cacd696b3..a78aa798d50 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -133,19 +133,6 @@ static void gru_cb_set_istatus_active(struct gru_instruction_bits *cbk) } } -/* - * Convert a interrupt IRQ to a pointer to the GRU GTS that caused the - * interrupt. Interrupts are always sent to a cpu on the blade that contains the - * GRU (except for headless blades which are not currently supported). A blade - * has N grus; a block of N consecutive IRQs is assigned to the GRUs. The IRQ - * number uniquely identifies the GRU chiplet on the local blade that caused the - * interrupt. Always called in interrupt context. - */ -static inline struct gru_state *irq_to_gru(int irq) -{ - return &gru_base[uv_numa_blade_id()]->bs_grus[irq - IRQ_GRU]; -} - /* * Read & clear a TFM * @@ -449,7 +436,7 @@ failactive: * Note that this is the interrupt handler that is registered with linux * interrupt handlers. */ -irqreturn_t gru_intr(int irq, void *dev_id) +static irqreturn_t gru_intr(int chiplet, int blade) { struct gru_state *gru; struct gru_tlb_fault_map imap, dmap; @@ -459,13 +446,18 @@ irqreturn_t gru_intr(int irq, void *dev_id) STAT(intr); - gru = irq_to_gru(irq); + gru = &gru_base[blade]->bs_grus[chiplet]; if (!gru) { - dev_err(grudev, "GRU: invalid interrupt: cpu %d, irq %d\n", - raw_smp_processor_id(), irq); + dev_err(grudev, "GRU: invalid interrupt: cpu %d, chiplet %d\n", + raw_smp_processor_id(), chiplet); return IRQ_NONE; } get_clear_fault_map(gru, &imap, &dmap); + gru_dbg(grudev, + "cpu %d, chiplet %d, gid %d, imap %016lx %016lx, dmap %016lx %016lx\n", + smp_processor_id(), chiplet, gru->gs_gid, + imap.fault_bits[0], imap.fault_bits[1], + dmap.fault_bits[0], dmap.fault_bits[1]); for_each_cbr_in_tfm(cbrnum, dmap.fault_bits) { complete(gru->gs_blade->bs_async_wq); @@ -503,6 +495,29 @@ irqreturn_t gru_intr(int irq, void *dev_id) return IRQ_HANDLED; } +irqreturn_t gru0_intr(int irq, void *dev_id) +{ + return gru_intr(0, uv_numa_blade_id()); +} + +irqreturn_t gru1_intr(int irq, void *dev_id) +{ + return gru_intr(1, uv_numa_blade_id()); +} + +irqreturn_t gru_intr_mblade(int irq, void *dev_id) +{ + int blade; + + for_each_possible_blade(blade) { + if (uv_blade_nr_possible_cpus(blade)) + continue; + gru_intr(0, blade); + gru_intr(1, blade); + } + return IRQ_HANDLED; +} + static int gru_user_dropin(struct gru_thread_state *gts, struct gru_tlb_fault_handle *tfh, diff --git a/drivers/misc/sgi-gru/grufile.c b/drivers/misc/sgi-gru/grufile.c index 0a6d2a5a01f..22b8b2733a2 100644 --- a/drivers/misc/sgi-gru/grufile.c +++ b/drivers/misc/sgi-gru/grufile.c @@ -35,6 +35,9 @@ #include #include #include +#ifdef CONFIG_X86_64 +#include +#endif #include #include "gru.h" #include "grulib.h" @@ -130,7 +133,6 @@ static int gru_create_new_context(unsigned long arg) struct gru_vma_data *vdata; int ret = -EINVAL; - if (copy_from_user(&req, (void __user *)arg, sizeof(req))) return -EFAULT; @@ -302,34 +304,210 @@ fail: return -ENOMEM; } -#ifdef CONFIG_IA64 +static void gru_free_tables(void) +{ + int bid; + int order = get_order(sizeof(struct gru_state) * + GRU_CHIPLETS_PER_BLADE); -static int get_base_irq(void) + for (bid = 0; bid < GRU_MAX_BLADES; bid++) + free_pages((unsigned long)gru_base[bid], order); +} + +static unsigned long gru_chiplet_cpu_to_mmr(int chiplet, int cpu, int *corep) { - return IRQ_GRU; + unsigned long mmr = 0; + int core; + + /* + * We target the cores of a blade and not the hyperthreads themselves. + * There is a max of 8 cores per socket and 2 sockets per blade, + * making for a max total of 16 cores (i.e., 16 CPUs without + * hyperthreading and 32 CPUs with hyperthreading). + */ + core = uv_cpu_core_number(cpu) + UV_MAX_INT_CORES * uv_cpu_socket_number(cpu); + if (core >= GRU_NUM_TFM || uv_cpu_ht_number(cpu)) + return 0; + + if (chiplet == 0) { + mmr = UVH_GR0_TLB_INT0_CONFIG + + core * (UVH_GR0_TLB_INT1_CONFIG - UVH_GR0_TLB_INT0_CONFIG); + } else if (chiplet == 1) { + mmr = UVH_GR1_TLB_INT0_CONFIG + + core * (UVH_GR1_TLB_INT1_CONFIG - UVH_GR1_TLB_INT0_CONFIG); + } else { + BUG(); + } + + *corep = core; + return mmr; } -#elif defined CONFIG_X86_64 +#ifdef CONFIG_IA64 + +static int gru_irq_count[GRU_CHIPLETS_PER_BLADE]; -static void noop(unsigned int irq) +static void gru_noop(unsigned int irq) { } -static struct irq_chip gru_chip = { - .name = "gru", - .mask = noop, - .unmask = noop, - .ack = noop, +static struct irq_chip gru_chip[GRU_CHIPLETS_PER_BLADE] = { + [0 ... GRU_CHIPLETS_PER_BLADE - 1] { + .mask = gru_noop, + .unmask = gru_noop, + .ack = gru_noop + } }; -static int get_base_irq(void) +static int gru_chiplet_setup_tlb_irq(int chiplet, char *irq_name, + irq_handler_t irq_handler, int cpu, int blade) +{ + unsigned long mmr; + int irq = IRQ_GRU + chiplet; + int ret, core; + + mmr = gru_chiplet_cpu_to_mmr(chiplet, cpu, &core); + if (mmr == 0) + return 0; + + if (gru_irq_count[chiplet] == 0) { + gru_chip[chiplet].name = irq_name; + ret = set_irq_chip(irq, &gru_chip[chiplet]); + if (ret) { + printk(KERN_ERR "%s: set_irq_chip failed, errno=%d\n", + GRU_DRIVER_ID_STR, -ret); + return ret; + } + + ret = request_irq(irq, irq_handler, 0, irq_name, NULL); + if (ret) { + printk(KERN_ERR "%s: request_irq failed, errno=%d\n", + GRU_DRIVER_ID_STR, -ret); + return ret; + } + } + gru_irq_count[chiplet]++; + + return 0; +} + +static void gru_chiplet_teardown_tlb_irq(int chiplet, int cpu, int blade) +{ + unsigned long mmr; + int core, irq = IRQ_GRU + chiplet; + + if (gru_irq_count[chiplet] == 0) + return; + + mmr = gru_chiplet_cpu_to_mmr(chiplet, cpu, &core); + if (mmr == 0) + return; + + if (--gru_irq_count[chiplet] == 0) + free_irq(irq, NULL); +} + +#elif defined CONFIG_X86_64 + +static int gru_chiplet_setup_tlb_irq(int chiplet, char *irq_name, + irq_handler_t irq_handler, int cpu, int blade) +{ + unsigned long mmr; + int irq, core; + int ret; + + mmr = gru_chiplet_cpu_to_mmr(chiplet, cpu, &core); + if (mmr == 0) + return 0; + + irq = uv_setup_irq(irq_name, cpu, blade, mmr, UV_AFFINITY_CPU); + if (irq < 0) { + printk(KERN_ERR "%s: uv_setup_irq failed, errno=%d\n", + GRU_DRIVER_ID_STR, -irq); + return irq; + } + + ret = request_irq(irq, irq_handler, 0, irq_name, NULL); + if (ret) { + uv_teardown_irq(irq); + printk(KERN_ERR "%s: request_irq failed, errno=%d\n", + GRU_DRIVER_ID_STR, -ret); + return ret; + } + gru_base[blade]->bs_grus[chiplet].gs_irq[core] = irq; + return 0; +} + +static void gru_chiplet_teardown_tlb_irq(int chiplet, int cpu, int blade) { - set_irq_chip(IRQ_GRU, &gru_chip); - set_irq_chip(IRQ_GRU + 1, &gru_chip); - return IRQ_GRU; + int irq, core; + unsigned long mmr; + + mmr = gru_chiplet_cpu_to_mmr(chiplet, cpu, &core); + if (mmr) { + irq = gru_base[blade]->bs_grus[chiplet].gs_irq[core]; + if (irq) { + free_irq(irq, NULL); + uv_teardown_irq(irq); + } + } } + #endif +static void gru_teardown_tlb_irqs(void) +{ + int blade; + int cpu; + + for_each_online_cpu(cpu) { + blade = uv_cpu_to_blade_id(cpu); + gru_chiplet_teardown_tlb_irq(0, cpu, blade); + gru_chiplet_teardown_tlb_irq(1, cpu, blade); + } + for_each_possible_blade(blade) { + if (uv_blade_nr_possible_cpus(blade)) + continue; + gru_chiplet_teardown_tlb_irq(0, 0, blade); + gru_chiplet_teardown_tlb_irq(1, 0, blade); + } +} + +static int gru_setup_tlb_irqs(void) +{ + int blade; + int cpu; + int ret; + + for_each_online_cpu(cpu) { + blade = uv_cpu_to_blade_id(cpu); + ret = gru_chiplet_setup_tlb_irq(0, "GRU0_TLB", gru0_intr, cpu, blade); + if (ret != 0) + goto exit1; + + ret = gru_chiplet_setup_tlb_irq(1, "GRU1_TLB", gru1_intr, cpu, blade); + if (ret != 0) + goto exit1; + } + for_each_possible_blade(blade) { + if (uv_blade_nr_possible_cpus(blade)) + continue; + ret = gru_chiplet_setup_tlb_irq(0, "GRU0_TLB", gru_intr_mblade, 0, blade); + if (ret != 0) + goto exit1; + + ret = gru_chiplet_setup_tlb_irq(1, "GRU1_TLB", gru_intr_mblade, 0, blade); + if (ret != 0) + goto exit1; + } + + return 0; + +exit1: + gru_teardown_tlb_irqs(); + return ret; +} + /* * gru_init * @@ -337,8 +515,7 @@ static int get_base_irq(void) */ static int __init gru_init(void) { - int ret, irq, chip; - char id[10]; + int ret; if (!is_uv_system()) return 0; @@ -353,41 +530,29 @@ static int __init gru_init(void) gru_end_paddr = gru_start_paddr + GRU_MAX_BLADES * GRU_SIZE; printk(KERN_INFO "GRU space: 0x%lx - 0x%lx\n", gru_start_paddr, gru_end_paddr); - irq = get_base_irq(); - for (chip = 0; chip < GRU_CHIPLETS_PER_BLADE; chip++) { - ret = request_irq(irq + chip, gru_intr, 0, id, NULL); - /* TODO: fix irq handling on x86. For now ignore failure because - * interrupts are not required & not yet fully supported */ - if (ret) { - printk(KERN_WARNING - "!!!WARNING: GRU ignoring request failure!!!\n"); - ret = 0; - } - if (ret) { - printk(KERN_ERR "%s: request_irq failed\n", - GRU_DRIVER_ID_STR); - goto exit1; - } - } - ret = misc_register(&gru_miscdev); if (ret) { printk(KERN_ERR "%s: misc_register failed\n", GRU_DRIVER_ID_STR); - goto exit1; + goto exit0; } ret = gru_proc_init(); if (ret) { printk(KERN_ERR "%s: proc init failed\n", GRU_DRIVER_ID_STR); - goto exit2; + goto exit1; } ret = gru_init_tables(gru_start_paddr, gru_start_vaddr); if (ret) { printk(KERN_ERR "%s: init tables failed\n", GRU_DRIVER_ID_STR); - goto exit3; + goto exit2; } + + ret = gru_setup_tlb_irqs(); + if (ret != 0) + goto exit3; + gru_kservices_init(); printk(KERN_INFO "%s: v%s\n", GRU_DRIVER_ID_STR, @@ -395,31 +560,24 @@ static int __init gru_init(void) return 0; exit3: - gru_proc_exit(); + gru_free_tables(); exit2: - misc_deregister(&gru_miscdev); + gru_proc_exit(); exit1: - for (--chip; chip >= 0; chip--) - free_irq(irq + chip, NULL); + misc_deregister(&gru_miscdev); +exit0: return ret; } static void __exit gru_exit(void) { - int i, bid; - int order = get_order(sizeof(struct gru_state) * - GRU_CHIPLETS_PER_BLADE); - if (!is_uv_system()) return; - for (i = 0; i < GRU_CHIPLETS_PER_BLADE; i++) - free_irq(IRQ_GRU + i, NULL); + gru_teardown_tlb_irqs(); gru_kservices_exit(); - for (bid = 0; bid < GRU_MAX_BLADES; bid++) - free_pages((unsigned long)gru_base[bid], order); - + gru_free_tables(); misc_deregister(&gru_miscdev); gru_proc_exit(); } diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c index a383271d391..120c70c5a28 100644 --- a/drivers/misc/sgi-gru/grumain.c +++ b/drivers/misc/sgi-gru/grumain.c @@ -49,12 +49,16 @@ struct device *grudev = &gru_device; /* * Select a gru fault map to be used by the current cpu. Note that * multiple cpus may be using the same map. - * ZZZ should "shift" be used?? Depends on HT cpu numbering * ZZZ should be inline but did not work on emulator */ int gru_cpu_fault_map_id(void) { - return uv_blade_processor_id() % GRU_NUM_TFM; + int cpu = smp_processor_id(); + int id, core; + + core = uv_cpu_core_number(cpu); + id = core + UV_MAX_INT_CORES * uv_cpu_socket_number(cpu); + return id; } /*--------- ASID Management ------------------------------------------- @@ -605,6 +609,7 @@ void gru_load_context(struct gru_thread_state *gts) cch->unmap_enable = 1; cch->tfm_done_bit_enable = 1; cch->cb_int_enable = 1; + cch->tlb_int_select = 0; /* For now, ints go to cpu 0 */ } else { cch->unmap_enable = 0; cch->tfm_done_bit_enable = 0; diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h index f0c7308a36b..d83e36715e6 100644 --- a/drivers/misc/sgi-gru/grutables.h +++ b/drivers/misc/sgi-gru/grutables.h @@ -444,6 +444,7 @@ struct gru_state { in use */ struct gru_thread_state *gs_gts[GRU_NUM_CCH]; /* GTS currently using the context */ + int gs_irq[GRU_NUM_TFM]; /* Interrupt irqs */ }; /* @@ -610,6 +611,15 @@ static inline int is_kernel_context(struct gru_thread_state *gts) return !gts->ts_mm; } +/* + * The following are for Nehelem-EX. A more general scheme is needed for + * future processors. + */ +#define UV_MAX_INT_CORES 8 +#define uv_cpu_socket_number(p) ((cpu_physical_id(p) >> 5) & 1) +#define uv_cpu_ht_number(p) (cpu_physical_id(p) & 1) +#define uv_cpu_core_number(p) (((cpu_physical_id(p) >> 2) & 4) | \ + ((cpu_physical_id(p) >> 1) & 3)) /*----------------------------------------------------------------------------- * Function prototypes & externs */ @@ -633,9 +643,11 @@ extern void gts_drop(struct gru_thread_state *gts); extern void gru_tgh_flush_init(struct gru_state *gru); extern int gru_kservices_init(void); extern void gru_kservices_exit(void); +extern irqreturn_t gru0_intr(int irq, void *dev_id); +extern irqreturn_t gru1_intr(int irq, void *dev_id); +extern irqreturn_t gru_intr_mblade(int irq, void *dev_id); extern int gru_dump_chiplet_request(unsigned long arg); extern long gru_get_gseg_statistics(unsigned long arg); -extern irqreturn_t gru_intr(int irq, void *dev_id); extern int gru_handle_user_call_os(unsigned long address); extern int gru_user_flush_tlb(unsigned long arg); extern int gru_user_unload_context(unsigned long arg); -- cgit v1.2.3-70-g09d2 From 563447d7eb04c9b382f90a132be126a21a635647 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Tue, 15 Dec 2009 16:48:12 -0800 Subject: gru: add additional GRU statistics Add additional GRU statistics & debug messages. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 18 +++++++++++------- drivers/misc/sgi-gru/grufile.c | 2 +- drivers/misc/sgi-gru/gruhandles.c | 11 ++++++++--- drivers/misc/sgi-gru/grukservices.c | 35 ++++++++++++++++++++++++----------- drivers/misc/sgi-gru/grumain.c | 10 +++++++--- drivers/misc/sgi-gru/gruprocfs.c | 25 +++++++++++++------------ drivers/misc/sgi-gru/grutables.h | 18 +++++++++--------- drivers/misc/sgi-gru/grutlbpurge.c | 6 ++++-- 8 files changed, 77 insertions(+), 48 deletions(-) (limited to 'drivers/misc/sgi-gru/grumain.c') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index a78aa798d50..7d757e9c42f 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -303,7 +303,7 @@ static int gru_try_dropin(struct gru_thread_state *gts, struct gru_tlb_fault_handle *tfh, struct gru_instruction_bits *cbk) { - int pageshift = 0, asid, write, ret, atomic = !cbk; + int pageshift = 0, asid, write, ret, atomic = !cbk, indexway; unsigned long gpa = 0, vaddr = 0; /* @@ -333,6 +333,7 @@ static int gru_try_dropin(struct gru_thread_state *gts, write = (tfh->cause & TFHCAUSE_TLB_MOD) != 0; vaddr = tfh->missvaddr; asid = tfh->missasid; + indexway = tfh->indexway; if (asid == 0) goto failnoasid; @@ -361,11 +362,12 @@ static int gru_try_dropin(struct gru_thread_state *gts, gru_cb_set_istatus_active(cbk); tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write, GRU_PAGESIZE(pageshift)); - STAT(tlb_dropin); gru_dbg(grudev, - "%s: tfh 0x%p, vaddr 0x%lx, asid 0x%x, ps %d, gpa 0x%lx\n", - ret ? "non-atomic" : "atomic", tfh, vaddr, asid, - pageshift, gpa); + "%s: gid %d, gts 0x%p, tfh 0x%p, vaddr 0x%lx, asid 0x%x, indexway 0x%x," + " rw %d, ps %d, gpa 0x%lx\n", + atomic ? "atomic" : "non-atomic", gts->ts_gru->gs_gid, gts, tfh, vaddr, asid, + indexway, write, pageshift, gpa); + STAT(tlb_dropin); return 0; failnoasid: @@ -460,12 +462,14 @@ static irqreturn_t gru_intr(int chiplet, int blade) dmap.fault_bits[0], dmap.fault_bits[1]); for_each_cbr_in_tfm(cbrnum, dmap.fault_bits) { + STAT(intr_cbr); complete(gru->gs_blade->bs_async_wq); gru_dbg(grudev, "gid %d, cbr_done %d, done %d\n", gru->gs_gid, cbrnum, gru->gs_blade->bs_async_wq->done); } for_each_cbr_in_tfm(cbrnum, imap.fault_bits) { + STAT(intr_tfh); tfh = get_tfh_by_index(gru, cbrnum); prefetchw(tfh); /* Helps on hdw, required for emulator */ @@ -551,7 +555,6 @@ int gru_handle_user_call_os(unsigned long cb) int ucbnum, cbrnum, ret = -EINVAL; STAT(call_os); - gru_dbg(grudev, "address 0x%lx\n", cb); /* sanity check the cb pointer */ ucbnum = get_cb_number((void *)cb); @@ -561,6 +564,7 @@ int gru_handle_user_call_os(unsigned long cb) gts = gru_find_lock_gts(cb); if (!gts) return -EINVAL; + gru_dbg(grudev, "address 0x%lx, gid %d, gts 0x%p\n", cb, gts->ts_gru ? gts->ts_gru->gs_gid : -1, gts); if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) goto exit; @@ -603,11 +607,11 @@ int gru_get_exception_detail(unsigned long arg) if (copy_from_user(&excdet, (void __user *)arg, sizeof(excdet))) return -EFAULT; - gru_dbg(grudev, "address 0x%lx\n", excdet.cb); gts = gru_find_lock_gts(excdet.cb); if (!gts) return -EINVAL; + gru_dbg(grudev, "address 0x%lx, gid %d, gts 0x%p\n", excdet.cb, gts->ts_gru ? gts->ts_gru->gs_gid : -1, gts); ucbnum = get_cb_number((void *)excdet.cb); if (ucbnum >= gts->ts_cbr_au_count * GRU_CBR_AU_SIZE) { ret = -EINVAL; diff --git a/drivers/misc/sgi-gru/grufile.c b/drivers/misc/sgi-gru/grufile.c index 22b8b2733a2..9d41208a6c9 100644 --- a/drivers/misc/sgi-gru/grufile.c +++ b/drivers/misc/sgi-gru/grufile.c @@ -192,7 +192,7 @@ static long gru_file_unlocked_ioctl(struct file *file, unsigned int req, { int err = -EBADRQC; - gru_dbg(grudev, "file %p\n", file); + gru_dbg(grudev, "file %p, req 0x%x, 0x%lx\n", file, req, arg); switch (req) { case GRU_CREATE_CONTEXT: diff --git a/drivers/misc/sgi-gru/gruhandles.c b/drivers/misc/sgi-gru/gruhandles.c index f1117a7637a..4bc9ee811fb 100644 --- a/drivers/misc/sgi-gru/gruhandles.c +++ b/drivers/misc/sgi-gru/gruhandles.c @@ -27,9 +27,11 @@ #ifdef CONFIG_IA64 #include #define GRU_OPERATION_TIMEOUT (((cycles_t) local_cpu_data->itc_freq)*10) +#define CLKS2NSEC(c) ((c) *1000000000 / local_cpu_data->itc_freq) #else #include #define GRU_OPERATION_TIMEOUT ((cycles_t) tsc_khz*10*1000) +#define CLKS2NSEC(c) ((c) * 1000000 / tsc_khz) #endif /* Extract the status field from a kernel handle */ @@ -39,10 +41,13 @@ struct mcs_op_statistic mcs_op_statistics[mcsop_last]; static void update_mcs_stats(enum mcs_op op, unsigned long clks) { + unsigned long nsec; + + nsec = CLKS2NSEC(clks); atomic_long_inc(&mcs_op_statistics[op].count); - atomic_long_add(clks, &mcs_op_statistics[op].total); - if (mcs_op_statistics[op].max < clks) - mcs_op_statistics[op].max = clks; + atomic_long_add(nsec, &mcs_op_statistics[op].total); + if (mcs_op_statistics[op].max < nsec) + mcs_op_statistics[op].max = nsec; } static void start_instruction(void *h) diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c index 8c81aca0463..4da6f56833d 100644 --- a/drivers/misc/sgi-gru/grukservices.c +++ b/drivers/misc/sgi-gru/grukservices.c @@ -414,8 +414,8 @@ char *gru_get_cb_exception_detail_str(int ret, void *cb, if (ret > 0 && gen->istatus == CBS_EXCEPTION) { gru_get_cb_exception_detail(cb, &excdet); snprintf(buf, size, - "GRU exception: cb %p, opc %d, exopc %d, ecause 0x%x," - "excdet0 0x%lx, excdet1 0x%x", + "GRU:%d exception: cb %p, opc %d, exopc %d, ecause 0x%x," + "excdet0 0x%lx, excdet1 0x%x", smp_processor_id(), gen, excdet.opc, excdet.exopc, excdet.ecause, excdet.exceptdet0, excdet.exceptdet1); } else { @@ -604,6 +604,8 @@ static int send_noop_message(void *cb, struct gru_message_queue_desc *mqd, ret = MQE_UNEXPECTED_CB_ERR; break; case CBSS_PAGE_OVERFLOW: + STAT(mesq_noop_page_overflow); + /* fallthru */ default: BUG(); } @@ -745,6 +747,9 @@ static int send_message_failure(void *cb, struct gru_message_queue_desc *mqd, STAT(mesq_send_put_nacked); ret = send_message_put_nacked(cb, mqd, mesg, lines); break; + case CBSS_PAGE_OVERFLOW: + STAT(mesq_page_overflow); + /* fallthru */ default: BUG(); } @@ -837,7 +842,6 @@ void *gru_get_next_message(struct gru_message_queue_desc *mqd) int present = mhdr->present; /* skip NOOP messages */ - STAT(mesq_receive); while (present == MQS_NOOP) { gru_free_message(mqd, mhdr); mhdr = mq->next; @@ -857,6 +861,7 @@ void *gru_get_next_message(struct gru_message_queue_desc *mqd) if (mhdr->lines == 2) restore_present2(mhdr, mhdr->present2); + STAT(mesq_receive); return mhdr; } EXPORT_SYMBOL_GPL(gru_get_next_message); @@ -927,24 +932,24 @@ static int quicktest0(unsigned long arg) gru_vload(cb, uv_gpa(&word0), gru_get_tri(dsr), XTYPE_DW, 1, 1, IMA); if (gru_wait(cb) != CBS_IDLE) { - printk(KERN_DEBUG "GRU quicktest0: CBR failure 1\n"); + printk(KERN_DEBUG "GRU:%d quicktest0: CBR failure 1\n", smp_processor_id()); goto done; } if (*p != MAGIC) { - printk(KERN_DEBUG "GRU: quicktest0 bad magic 0x%lx\n", *p); + printk(KERN_DEBUG "GRU:%d quicktest0 bad magic 0x%lx\n", smp_processor_id(), *p); goto done; } gru_vstore(cb, uv_gpa(&word1), gru_get_tri(dsr), XTYPE_DW, 1, 1, IMA); if (gru_wait(cb) != CBS_IDLE) { - printk(KERN_DEBUG "GRU quicktest0: CBR failure 2\n"); + printk(KERN_DEBUG "GRU:%d quicktest0: CBR failure 2\n", smp_processor_id()); goto done; } if (word0 != word1 || word1 != MAGIC) { printk(KERN_DEBUG - "GRU quicktest0 err: found 0x%lx, expected 0x%lx\n", - word1, MAGIC); + "GRU:%d quicktest0 err: found 0x%lx, expected 0x%lx\n", + smp_processor_id(), word1, MAGIC); goto done; } ret = 0; @@ -981,8 +986,11 @@ static int quicktest1(unsigned long arg) if (ret) break; } - if (ret != MQE_QUEUE_FULL || i != 4) + if (ret != MQE_QUEUE_FULL || i != 4) { + printk(KERN_DEBUG "GRU:%d quicktest1: unexpect status %d, i %d\n", + smp_processor_id(), ret, i); goto done; + } for (i = 0; i < 6; i++) { m = gru_get_next_message(&mqd); @@ -990,7 +998,12 @@ static int quicktest1(unsigned long arg) break; gru_free_message(&mqd, m); } - ret = (i == 4) ? 0 : -EIO; + if (i != 4) { + printk(KERN_DEBUG "GRU:%d quicktest2: bad message, i %d, m %p, m8 %d\n", + smp_processor_id(), i, m, m ? m[8] : -1); + goto done; + } + ret = 0; done: kfree(p); @@ -1069,7 +1082,7 @@ static int quicktest3(unsigned long arg) memset(buf1, get_cycles() & 255, sizeof(buf1)); gru_copy_gpa(uv_gpa(buf2), uv_gpa(buf1), BUFSIZE); if (memcmp(buf1, buf2, BUFSIZE)) { - printk(KERN_DEBUG "GRU quicktest3 error\n"); + printk(KERN_DEBUG "GRU:%d quicktest3 error\n", smp_processor_id()); ret = -EIO; } return ret; diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c index 120c70c5a28..ebabbdcbb97 100644 --- a/drivers/misc/sgi-gru/grumain.c +++ b/drivers/misc/sgi-gru/grumain.c @@ -370,6 +370,7 @@ struct gru_vma_data *gru_alloc_vma_data(struct vm_area_struct *vma, int tsid) if (!vdata) return NULL; + STAT(vdata_alloc); INIT_LIST_HEAD(&vdata->vd_head); spin_lock_init(&vdata->vd_lock); gru_dbg(grudev, "alloc vdata %p\n", vdata); @@ -552,7 +553,8 @@ void gru_unload_context(struct gru_thread_state *gts, int savestate) zap_vma_ptes(gts->ts_vma, UGRUADDR(gts), GRU_GSEG_PAGESIZE); cch = get_cch(gru->gs_gru_base_vaddr, ctxnum); - gru_dbg(grudev, "gts %p\n", gts); + gru_dbg(grudev, "gts %p, cbrmap 0x%lx, dsrmap 0x%lx\n", + gts, gts->ts_cbr_map, gts->ts_dsr_map); lock_cch_handle(cch); if (cch_interrupt_sync(cch)) BUG(); @@ -583,9 +585,7 @@ void gru_load_context(struct gru_thread_state *gts) struct gru_context_configuration_handle *cch; int i, err, asid, ctxnum = gts->ts_ctxnum; - gru_dbg(grudev, "gts %p\n", gts); cch = get_cch(gru->gs_gru_base_vaddr, ctxnum); - lock_cch_handle(cch); cch->tfm_fault_bit_enable = (gts->ts_user_options == GRU_OPT_MISS_FMM_POLL @@ -635,6 +635,10 @@ void gru_load_context(struct gru_thread_state *gts) if (cch_start(cch)) BUG(); unlock_cch_handle(cch); + + gru_dbg(grudev, "gid %d, gts %p, cbrmap 0x%lx, dsrmap 0x%lx, tie %d, tis %d\n", + gts->ts_gru->gs_gid, gts, gts->ts_cbr_map, gts->ts_dsr_map, + (gts->ts_user_options == GRU_OPT_MISS_FMM_INTR), gts->ts_tlb_int_select); } /* diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c index 4f6fc11a263..0a57ab29cd3 100644 --- a/drivers/misc/sgi-gru/gruprocfs.c +++ b/drivers/misc/sgi-gru/gruprocfs.c @@ -36,8 +36,7 @@ static void printstat_val(struct seq_file *s, atomic_long_t *v, char *id) { unsigned long val = atomic_long_read(v); - if (val) - seq_printf(s, "%16lu %s\n", val, id); + seq_printf(s, "%16lu %s\n", val, id); } static int statistics_show(struct seq_file *s, void *p) @@ -46,7 +45,8 @@ static int statistics_show(struct seq_file *s, void *p) printstat(s, vdata_free); printstat(s, gts_alloc); printstat(s, gts_free); - printstat(s, vdata_double_alloc); + printstat(s, gms_alloc); + printstat(s, gms_free); printstat(s, gts_double_allocate); printstat(s, assign_context); printstat(s, assign_context_failed); @@ -59,15 +59,15 @@ static int statistics_show(struct seq_file *s, void *p) printstat(s, steal_kernel_context); printstat(s, steal_context_failed); printstat(s, nopfn); - printstat(s, break_cow); printstat(s, asid_new); printstat(s, asid_next); printstat(s, asid_wrap); printstat(s, asid_reuse); printstat(s, intr); + printstat(s, intr_cbr); + printstat(s, intr_tfh); printstat(s, intr_mm_lock_failed); printstat(s, call_os); - printstat(s, call_os_check_for_bug); printstat(s, call_os_wait_queue); printstat(s, user_flush_tlb); printstat(s, user_unload_context); @@ -83,11 +83,9 @@ static int statistics_show(struct seq_file *s, void *p) printstat(s, tlb_dropin_fail_idle); printstat(s, tlb_dropin_fail_fmm); printstat(s, tlb_dropin_fail_no_exception); - printstat(s, tlb_dropin_fail_no_exception_war); printstat(s, tfh_stale_on_fault); printstat(s, mmu_invalidate_range); printstat(s, mmu_invalidate_page); - printstat(s, mmu_clear_flush_young); printstat(s, flush_tlb); printstat(s, flush_tlb_gru); printstat(s, flush_tlb_gru_tgh); @@ -104,7 +102,6 @@ static int statistics_show(struct seq_file *s, void *p) printstat(s, mesq_send_qlimit_reached); printstat(s, mesq_send_amo_nacked); printstat(s, mesq_send_put_nacked); - printstat(s, mesq_qf_not_full); printstat(s, mesq_qf_locked); printstat(s, mesq_qf_noop_not_full); printstat(s, mesq_qf_switch_head_failed); @@ -114,6 +111,7 @@ static int statistics_show(struct seq_file *s, void *p) printstat(s, mesq_noop_qlimit_reached); printstat(s, mesq_noop_amo_nacked); printstat(s, mesq_noop_put_nacked); + printstat(s, mesq_noop_page_overflow); return 0; } @@ -131,6 +129,7 @@ static int mcs_statistics_show(struct seq_file *s, void *p) static char *id[] = {"cch_allocate", "cch_start", "cch_interrupt", "cch_interrupt_sync", "cch_deallocate", "tgh_invalidate"}; + seq_printf(s, "%-20s%12s%12s%12s\n", "#id", "count", "aver-clks", "max-clks"); for (op = 0; op < mcsop_last; op++) { count = atomic_long_read(&mcs_op_statistics[op].count); total = atomic_long_read(&mcs_op_statistics[op].total); @@ -150,6 +149,7 @@ static ssize_t mcs_statistics_write(struct file *file, static int options_show(struct seq_file *s, void *p) { + seq_printf(s, "#bitmask: 1=trace, 2=statistics\n"); seq_printf(s, "0x%lx\n", gru_options); return 0; } @@ -179,16 +179,17 @@ static int cch_seq_show(struct seq_file *file, void *data) const char *mode[] = { "??", "UPM", "INTR", "OS_POLL" }; if (gid == 0) - seq_printf(file, "#%5s%5s%6s%9s%6s%8s%8s\n", "gid", "bid", - "ctx#", "pid", "cbrs", "dsbytes", "mode"); + seq_printf(file, "#%5s%5s%6s%7s%9s%6s%8s%8s\n", "gid", "bid", + "ctx#", "asid", "pid", "cbrs", "dsbytes", "mode"); if (gru) for (i = 0; i < GRU_NUM_CCH; i++) { ts = gru->gs_gts[i]; if (!ts) continue; - seq_printf(file, " %5d%5d%6d%9d%6d%8d%8s\n", + seq_printf(file, " %5d%5d%6d%7d%9d%6d%8d%8s\n", gru->gs_gid, gru->gs_blade_id, i, - ts->ts_tgid_owner, + is_kernel_context(ts) ? 0 : ts->ts_gms->ms_asids[gid].mt_asid, + is_kernel_context(ts) ? 0 : ts->ts_tgid_owner, ts->ts_cbr_au_count * GRU_CBR_AU_SIZE, ts->ts_cbr_au_count * GRU_DSR_AU_BYTES, mode[ts->ts_user_options & diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h index d83e36715e6..76fe2987fc9 100644 --- a/drivers/misc/sgi-gru/grutables.h +++ b/drivers/misc/sgi-gru/grutables.h @@ -171,7 +171,8 @@ struct gru_stats_s { atomic_long_t vdata_free; atomic_long_t gts_alloc; atomic_long_t gts_free; - atomic_long_t vdata_double_alloc; + atomic_long_t gms_alloc; + atomic_long_t gms_free; atomic_long_t gts_double_allocate; atomic_long_t assign_context; atomic_long_t assign_context_failed; @@ -184,15 +185,15 @@ struct gru_stats_s { atomic_long_t steal_kernel_context; atomic_long_t steal_context_failed; atomic_long_t nopfn; - atomic_long_t break_cow; atomic_long_t asid_new; atomic_long_t asid_next; atomic_long_t asid_wrap; atomic_long_t asid_reuse; atomic_long_t intr; + atomic_long_t intr_cbr; + atomic_long_t intr_tfh; atomic_long_t intr_mm_lock_failed; atomic_long_t call_os; - atomic_long_t call_os_check_for_bug; atomic_long_t call_os_wait_queue; atomic_long_t user_flush_tlb; atomic_long_t user_unload_context; @@ -208,11 +209,9 @@ struct gru_stats_s { atomic_long_t tlb_dropin_fail_idle; atomic_long_t tlb_dropin_fail_fmm; atomic_long_t tlb_dropin_fail_no_exception; - atomic_long_t tlb_dropin_fail_no_exception_war; atomic_long_t tfh_stale_on_fault; atomic_long_t mmu_invalidate_range; atomic_long_t mmu_invalidate_page; - atomic_long_t mmu_clear_flush_young; atomic_long_t flush_tlb; atomic_long_t flush_tlb_gru; atomic_long_t flush_tlb_gru_tgh; @@ -231,7 +230,7 @@ struct gru_stats_s { atomic_long_t mesq_send_qlimit_reached; atomic_long_t mesq_send_amo_nacked; atomic_long_t mesq_send_put_nacked; - atomic_long_t mesq_qf_not_full; + atomic_long_t mesq_page_overflow; atomic_long_t mesq_qf_locked; atomic_long_t mesq_qf_noop_not_full; atomic_long_t mesq_qf_switch_head_failed; @@ -241,6 +240,7 @@ struct gru_stats_s { atomic_long_t mesq_noop_qlimit_reached; atomic_long_t mesq_noop_amo_nacked; atomic_long_t mesq_noop_put_nacked; + atomic_long_t mesq_noop_page_overflow; }; @@ -255,8 +255,8 @@ struct mcs_op_statistic { extern struct mcs_op_statistic mcs_op_statistics[mcsop_last]; -#define OPT_DPRINT 1 -#define OPT_STATS 2 +#define OPT_DPRINT 1 +#define OPT_STATS 2 #define IRQ_GRU 110 /* Starting IRQ number for interrupts */ @@ -279,7 +279,7 @@ extern struct mcs_op_statistic mcs_op_statistics[mcsop_last]; #define gru_dbg(dev, fmt, x...) \ do { \ if (gru_options & OPT_DPRINT) \ - dev_dbg(dev, "%s: " fmt, __func__, x); \ + printk(KERN_DEBUG "GRU:%d %s: " fmt, smp_processor_id(), __func__, x);\ } while (0) #else #define gru_dbg(x...) diff --git a/drivers/misc/sgi-gru/grutlbpurge.c b/drivers/misc/sgi-gru/grutlbpurge.c index 421c548e51f..240a6d36166 100644 --- a/drivers/misc/sgi-gru/grutlbpurge.c +++ b/drivers/misc/sgi-gru/grutlbpurge.c @@ -184,8 +184,8 @@ void gru_flush_tlb_range(struct gru_mm_struct *gms, unsigned long start, STAT(flush_tlb_gru_tgh); asid = GRUASID(asid, start); gru_dbg(grudev, - " FLUSH gruid %d, asid 0x%x, num %ld, cbmap 0x%x\n", - gid, asid, num, asids->mt_ctxbitmap); + " FLUSH gruid %d, asid 0x%x, vaddr 0x%lx, vamask 0x%x, num %ld, cbmap 0x%x\n", + gid, asid, start, grupagesize, num, asids->mt_ctxbitmap); tgh = get_lock_tgh_handle(gru); tgh_invalidate(tgh, start, ~0, asid, grupagesize, 0, num - 1, asids->mt_ctxbitmap); @@ -308,6 +308,7 @@ struct gru_mm_struct *gru_register_mmu_notifier(void) } else { gms = kzalloc(sizeof(*gms), GFP_KERNEL); if (gms) { + STAT(gms_alloc); spin_lock_init(&gms->ms_asid_lock); gms->ms_notifier.ops = &gru_mmuops; atomic_set(&gms->ms_refcnt, 1); @@ -333,6 +334,7 @@ void gru_drop_mmu_notifier(struct gru_mm_struct *gms) if (!gms->ms_released) mmu_notifier_unregister(&gms->ms_notifier, current->mm); kfree(gms); + STAT(gms_free); } } -- cgit v1.2.3-70-g09d2 From c550222f64265c9384c45d7f691445386025a82b Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Tue, 15 Dec 2009 16:48:13 -0800 Subject: gru: preload tlb for bcopy instructions Add anticipatory TLB dropins for GRU TLB misses that occur on BCOPY instructions that copy large amounts of data. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grufault.c | 89 +++++++++++++++++++++++++++++++++++-- drivers/misc/sgi-gru/grufile.c | 1 + drivers/misc/sgi-gru/gruhandles.c | 11 +++-- drivers/misc/sgi-gru/gruhandles.h | 20 ++++++++- drivers/misc/sgi-gru/grukservices.c | 2 +- drivers/misc/sgi-gru/grulib.h | 1 + drivers/misc/sgi-gru/grumain.c | 8 +++- drivers/misc/sgi-gru/gruprocfs.c | 4 +- drivers/misc/sgi-gru/grutables.h | 9 +++- 9 files changed, 129 insertions(+), 16 deletions(-) (limited to 'drivers/misc/sgi-gru/grumain.c') diff --git a/drivers/misc/sgi-gru/grufault.c b/drivers/misc/sgi-gru/grufault.c index 7d757e9c42f..a1b3a1d66af 100644 --- a/drivers/misc/sgi-gru/grufault.c +++ b/drivers/misc/sgi-gru/grufault.c @@ -289,6 +289,61 @@ upm: } +/* + * Flush a CBE from cache. The CBE is clean in the cache. Dirty the + * CBE cacheline so that the line will be written back to home agent. + * Otherwise the line may be silently dropped. This has no impact + * except on performance. + */ +static void gru_flush_cache_cbe(struct gru_control_block_extended *cbe) +{ + if (unlikely(cbe)) { + cbe->cbrexecstatus = 0; /* make CL dirty */ + gru_flush_cache(cbe); + } +} + +/* + * Preload the TLB with entries that may be required. Currently, preloading + * is implemented only for BCOPY. Preload pages OR to + * the end of the bcopy tranfer, whichever is smaller. + */ +static void gru_preload_tlb(struct gru_state *gru, + struct gru_thread_state *gts, int atomic, + unsigned long fault_vaddr, int asid, int write, + unsigned char tlb_preload_count, + struct gru_tlb_fault_handle *tfh, + struct gru_control_block_extended *cbe) +{ + unsigned long vaddr = 0, gpa; + int ret, pageshift; + + if (cbe->opccpy != OP_BCOPY) + return; + + if (fault_vaddr == cbe->cbe_baddr0) + vaddr = fault_vaddr + GRU_CACHE_LINE_BYTES * cbe->cbe_src_cl - 1; + else if (fault_vaddr == cbe->cbe_baddr1) + vaddr = fault_vaddr + (1 << cbe->xtypecpy) * cbe->cbe_nelemcur - 1; + + fault_vaddr &= PAGE_MASK; + vaddr &= PAGE_MASK; + vaddr = min(vaddr, fault_vaddr + tlb_preload_count * PAGE_SIZE); + + while (vaddr > fault_vaddr) { + ret = gru_vtop(gts, vaddr, write, atomic, &gpa, &pageshift); + if (ret || tfh_write_only(tfh, gpa, GAA_RAM, vaddr, asid, write, + GRU_PAGESIZE(pageshift))) + return; + gru_dbg(grudev, + "%s: gid %d, gts 0x%p, tfh 0x%p, vaddr 0x%lx, asid 0x%x, rw %d, ps %d, gpa 0x%lx\n", + atomic ? "atomic" : "non-atomic", gru->gs_gid, gts, tfh, + vaddr, asid, write, pageshift, gpa); + vaddr -= PAGE_SIZE; + STAT(tlb_preload_page); + } +} + /* * Drop a TLB entry into the GRU. The fault is described by info in an TFH. * Input: @@ -303,6 +358,8 @@ static int gru_try_dropin(struct gru_thread_state *gts, struct gru_tlb_fault_handle *tfh, struct gru_instruction_bits *cbk) { + struct gru_control_block_extended *cbe = NULL; + unsigned char tlb_preload_count = gts->ts_tlb_preload_count; int pageshift = 0, asid, write, ret, atomic = !cbk, indexway; unsigned long gpa = 0, vaddr = 0; @@ -313,6 +370,14 @@ static int gru_try_dropin(struct gru_thread_state *gts, * the dropin is ignored. This eliminates the need for additional locks. */ + /* + * Prefetch the CBE if doing TLB preloading + */ + if (unlikely(tlb_preload_count)) { + cbe = gru_tfh_to_cbe(tfh); + prefetchw(cbe); + } + /* * Error if TFH state is IDLE or FMM mode & the user issuing a UPM call. * Might be a hardware race OR a stupid user. Ignore FMM because FMM @@ -359,6 +424,12 @@ static int gru_try_dropin(struct gru_thread_state *gts, goto failupm; } } + + if (unlikely(cbe) && pageshift == PAGE_SHIFT) { + gru_preload_tlb(gts->ts_gru, gts, atomic, vaddr, asid, write, tlb_preload_count, tfh, cbe); + gru_flush_cache_cbe(cbe); + } + gru_cb_set_istatus_active(cbk); tfh_write_restart(tfh, gpa, GAA_RAM, vaddr, asid, write, GRU_PAGESIZE(pageshift)); @@ -378,11 +449,13 @@ failnoasid: tfh_user_polling_mode(tfh); else gru_flush_cache(tfh); + gru_flush_cache_cbe(cbe); return -EAGAIN; failupm: /* Atomic failure switch CBR to UPM */ tfh_user_polling_mode(tfh); + gru_flush_cache_cbe(cbe); STAT(tlb_dropin_fail_upm); gru_dbg(grudev, "FAILED upm tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr); return 1; @@ -390,6 +463,7 @@ failupm: failfmm: /* FMM state on UPM call */ gru_flush_cache(tfh); + gru_flush_cache_cbe(cbe); STAT(tlb_dropin_fail_fmm); gru_dbg(grudev, "FAILED fmm tfh: 0x%p, state %d\n", tfh, tfh->state); return 0; @@ -397,6 +471,7 @@ failfmm: failnoexception: /* TFH status did not show exception pending */ gru_flush_cache(tfh); + gru_flush_cache_cbe(cbe); if (cbk) gru_flush_cache(cbk); STAT(tlb_dropin_fail_no_exception); @@ -407,6 +482,7 @@ failnoexception: failidle: /* TFH state was idle - no miss pending */ gru_flush_cache(tfh); + gru_flush_cache_cbe(cbe); if (cbk) gru_flush_cache(cbk); STAT(tlb_dropin_fail_idle); @@ -416,6 +492,7 @@ failidle: failinval: /* All errors (atomic & non-atomic) switch CBR to EXCEPTION state */ tfh_exception(tfh); + gru_flush_cache_cbe(cbe); STAT(tlb_dropin_fail_invalid); gru_dbg(grudev, "FAILED inval tfh: 0x%p, vaddr 0x%lx\n", tfh, vaddr); return -EFAULT; @@ -426,6 +503,7 @@ failactive: tfh_user_polling_mode(tfh); else gru_flush_cache(tfh); + gru_flush_cache_cbe(cbe); STAT(tlb_dropin_fail_range_active); gru_dbg(grudev, "FAILED range active: tfh 0x%p, vaddr 0x%lx\n", tfh, vaddr); @@ -627,7 +705,7 @@ int gru_get_exception_detail(unsigned long arg) excdet.exceptdet1 = cbe->idef3upd; excdet.cbrstate = cbe->cbrstate; excdet.cbrexecstatus = cbe->cbrexecstatus; - gru_flush_cache(cbe); + gru_flush_cache_cbe(cbe); ret = 0; } else { ret = -EAGAIN; @@ -770,9 +848,12 @@ int gru_set_context_option(unsigned long arg) return -EFAULT; gru_dbg(grudev, "op %d, gseg 0x%lx, value1 0x%lx\n", req.op, req.gseg, req.val1); - gts = gru_alloc_locked_gts(req.gseg); - if (IS_ERR(gts)) - return PTR_ERR(gts); + gts = gru_find_lock_gts(req.gseg); + if (!gts) { + gts = gru_alloc_locked_gts(req.gseg); + if (IS_ERR(gts)) + return PTR_ERR(gts); + } switch (req.op) { case sco_blade_chiplet: diff --git a/drivers/misc/sgi-gru/grufile.c b/drivers/misc/sgi-gru/grufile.c index 9d41208a6c9..cb3b4d22847 100644 --- a/drivers/misc/sgi-gru/grufile.c +++ b/drivers/misc/sgi-gru/grufile.c @@ -152,6 +152,7 @@ static int gru_create_new_context(unsigned long arg) vdata->vd_dsr_au_count = GRU_DS_BYTES_TO_AU(req.data_segment_bytes); vdata->vd_cbr_au_count = GRU_CB_COUNT_TO_AU(req.control_blocks); + vdata->vd_tlb_preload_count = req.tlb_preload_count; ret = 0; } up_write(¤t->mm->mmap_sem); diff --git a/drivers/misc/sgi-gru/gruhandles.c b/drivers/misc/sgi-gru/gruhandles.c index 66d67d9bc9b..2f30badc6ff 100644 --- a/drivers/misc/sgi-gru/gruhandles.c +++ b/drivers/misc/sgi-gru/gruhandles.c @@ -165,17 +165,20 @@ int tgh_invalidate(struct gru_tlb_global_handle *tgh, return wait_instruction_complete(tgh, tghop_invalidate); } -void tfh_write_only(struct gru_tlb_fault_handle *tfh, - unsigned long pfn, unsigned long vaddr, - int asid, int dirty, int pagesize) +int tfh_write_only(struct gru_tlb_fault_handle *tfh, + unsigned long paddr, int gaa, + unsigned long vaddr, int asid, int dirty, + int pagesize) { tfh->fillasid = asid; tfh->fillvaddr = vaddr; - tfh->pfn = pfn; + tfh->pfn = paddr >> GRU_PADDR_SHIFT; + tfh->gaa = gaa; tfh->dirty = dirty; tfh->pagesize = pagesize; tfh->opc = TFHOP_WRITE_ONLY; start_instruction(tfh); + return wait_instruction_complete(tfh, tfhop_write_only); } void tfh_write_restart(struct gru_tlb_fault_handle *tfh, diff --git a/drivers/misc/sgi-gru/gruhandles.h b/drivers/misc/sgi-gru/gruhandles.h index 47b762f89e0..ea584ebf65b 100644 --- a/drivers/misc/sgi-gru/gruhandles.h +++ b/drivers/misc/sgi-gru/gruhandles.h @@ -164,6 +164,16 @@ static inline void *gru_chiplet_vaddr(void *vaddr, int pnode, int chiplet) return vaddr + GRU_SIZE * (2 * pnode + chiplet); } +static inline struct gru_control_block_extended *gru_tfh_to_cbe( + struct gru_tlb_fault_handle *tfh) +{ + unsigned long cbe; + + cbe = (unsigned long)tfh - GRU_TFH_BASE + GRU_CBE_BASE; + return (struct gru_control_block_extended*)cbe; +} + + /* @@ -446,6 +456,12 @@ struct gru_control_block_extended { unsigned int cbrexecstatus:8; }; +/* CBE fields for active BCOPY instructions */ +#define cbe_baddr0 idef1upd +#define cbe_baddr1 idef3upd +#define cbe_src_cl idef6cpy +#define cbe_nelemcur idef5upd + enum gru_cbr_state { CBRSTATE_INACTIVE, CBRSTATE_IDLE, @@ -493,8 +509,8 @@ int cch_interrupt_sync(struct gru_context_configuration_handle *cch); int tgh_invalidate(struct gru_tlb_global_handle *tgh, unsigned long vaddr, unsigned long vaddrmask, int asid, int pagesize, int global, int n, unsigned short ctxbitmap); -void tfh_write_only(struct gru_tlb_fault_handle *tfh, unsigned long pfn, - unsigned long vaddr, int asid, int dirty, int pagesize); +int tfh_write_only(struct gru_tlb_fault_handle *tfh, unsigned long paddr, + int gaa, unsigned long vaddr, int asid, int dirty, int pagesize); void tfh_write_restart(struct gru_tlb_fault_handle *tfh, unsigned long paddr, int gaa, unsigned long vaddr, int asid, int dirty, int pagesize); void tfh_restart(struct gru_tlb_fault_handle *tfh); diff --git a/drivers/misc/sgi-gru/grukservices.c b/drivers/misc/sgi-gru/grukservices.c index 4da6f56833d..d9ff0289a1c 100644 --- a/drivers/misc/sgi-gru/grukservices.c +++ b/drivers/misc/sgi-gru/grukservices.c @@ -161,7 +161,7 @@ static void gru_load_kernel_context(struct gru_blade_state *bs, int blade_id) down_write(&bs->bs_kgts_sema); if (!bs->bs_kgts) { - bs->bs_kgts = gru_alloc_gts(NULL, 0, 0, 0, 0); + bs->bs_kgts = gru_alloc_gts(NULL, 0, 0, 0, 0, 0); bs->bs_kgts->ts_user_blade_id = blade_id; } kgts = bs->bs_kgts; diff --git a/drivers/misc/sgi-gru/grulib.h b/drivers/misc/sgi-gru/grulib.h index e033b36df7e..c6928af7393 100644 --- a/drivers/misc/sgi-gru/grulib.h +++ b/drivers/misc/sgi-gru/grulib.h @@ -86,6 +86,7 @@ struct gru_create_context_req { unsigned int control_blocks; unsigned int maximum_thread_count; unsigned int options; + unsigned char tlb_preload_count; }; /* diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c index ebabbdcbb97..ade0925eab0 100644 --- a/drivers/misc/sgi-gru/grumain.c +++ b/drivers/misc/sgi-gru/grumain.c @@ -316,7 +316,8 @@ static struct gru_thread_state *gru_find_current_gts_nolock(struct gru_vma_data * Allocate a thread state structure. */ struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma, - int cbr_au_count, int dsr_au_count, int options, int tsid) + int cbr_au_count, int dsr_au_count, + unsigned char tlb_preload_count, int options, int tsid) { struct gru_thread_state *gts; struct gru_mm_struct *gms; @@ -334,6 +335,7 @@ struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma, mutex_init(>s->ts_ctxlock); gts->ts_cbr_au_count = cbr_au_count; gts->ts_dsr_au_count = dsr_au_count; + gts->ts_tlb_preload_count = tlb_preload_count; gts->ts_user_options = options; gts->ts_user_blade_id = -1; gts->ts_user_chiplet_id = -1; @@ -403,7 +405,9 @@ struct gru_thread_state *gru_alloc_thread_state(struct vm_area_struct *vma, struct gru_vma_data *vdata = vma->vm_private_data; struct gru_thread_state *gts, *ngts; - gts = gru_alloc_gts(vma, vdata->vd_cbr_au_count, vdata->vd_dsr_au_count, + gts = gru_alloc_gts(vma, vdata->vd_cbr_au_count, + vdata->vd_dsr_au_count, + vdata->vd_tlb_preload_count, vdata->vd_user_options, tsid); if (IS_ERR(gts)) return gts; diff --git a/drivers/misc/sgi-gru/gruprocfs.c b/drivers/misc/sgi-gru/gruprocfs.c index 0a57ab29cd3..54a5a1c35ad 100644 --- a/drivers/misc/sgi-gru/gruprocfs.c +++ b/drivers/misc/sgi-gru/gruprocfs.c @@ -76,6 +76,7 @@ static int statistics_show(struct seq_file *s, void *p) printstat(s, check_context_retarget_intr); printstat(s, check_context_unload); printstat(s, tlb_dropin); + printstat(s, tlb_preload_page); printstat(s, tlb_dropin_fail_no_asid); printstat(s, tlb_dropin_fail_upm); printstat(s, tlb_dropin_fail_invalid); @@ -127,7 +128,8 @@ static int mcs_statistics_show(struct seq_file *s, void *p) int op; unsigned long total, count, max; static char *id[] = {"cch_allocate", "cch_start", "cch_interrupt", - "cch_interrupt_sync", "cch_deallocate", "tgh_invalidate"}; + "cch_interrupt_sync", "cch_deallocate", "tfh_write_only", + "tfh_write_restart", "tgh_invalidate"}; seq_printf(s, "%-20s%12s%12s%12s\n", "#id", "count", "aver-clks", "max-clks"); for (op = 0; op < mcsop_last; op++) { diff --git a/drivers/misc/sgi-gru/grutables.h b/drivers/misc/sgi-gru/grutables.h index 76fe2987fc9..adaf691d59f 100644 --- a/drivers/misc/sgi-gru/grutables.h +++ b/drivers/misc/sgi-gru/grutables.h @@ -202,6 +202,7 @@ struct gru_stats_s { atomic_long_t check_context_retarget_intr; atomic_long_t check_context_unload; atomic_long_t tlb_dropin; + atomic_long_t tlb_preload_page; atomic_long_t tlb_dropin_fail_no_asid; atomic_long_t tlb_dropin_fail_upm; atomic_long_t tlb_dropin_fail_invalid; @@ -245,7 +246,8 @@ struct gru_stats_s { }; enum mcs_op {cchop_allocate, cchop_start, cchop_interrupt, cchop_interrupt_sync, - cchop_deallocate, tghop_invalidate, mcsop_last}; + cchop_deallocate, tfhop_write_only, tfhop_write_restart, + tghop_invalidate, mcsop_last}; struct mcs_op_statistic { atomic_long_t count; @@ -335,6 +337,7 @@ struct gru_vma_data { long vd_user_options;/* misc user option flags */ int vd_cbr_au_count; int vd_dsr_au_count; + unsigned char vd_tlb_preload_count; }; /* @@ -350,6 +353,7 @@ struct gru_thread_state { struct gru_state *ts_gru; /* GRU where the context is loaded */ struct gru_mm_struct *ts_gms; /* asid & ioproc struct */ + unsigned char ts_tlb_preload_count; /* TLB preload pages */ unsigned long ts_cbr_map; /* map of allocated CBRs */ unsigned long ts_dsr_map; /* map of allocated DATA resources */ @@ -661,7 +665,8 @@ extern int gru_proc_init(void); extern void gru_proc_exit(void); extern struct gru_thread_state *gru_alloc_gts(struct vm_area_struct *vma, - int cbr_au_count, int dsr_au_count, int options, int tsid); + int cbr_au_count, int dsr_au_count, + unsigned char tlb_preload_count, int options, int tsid); extern unsigned long gru_reserve_cb_resources(struct gru_state *gru, int cbr_au_count, char *cbmap); extern unsigned long gru_reserve_ds_resources(struct gru_state *gru, -- cgit v1.2.3-70-g09d2 From 41314790883c8ad2dfb2cb295eb8177069627044 Mon Sep 17 00:00:00 2001 From: Jack Steiner Date: Tue, 15 Dec 2009 16:48:15 -0800 Subject: gru: check for correct GRU chiplet assignment Simplify the code that checks for correct assignment of GRU contexts to users. Signed-off-by: Jack Steiner Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- drivers/misc/sgi-gru/grumain.c | 44 ++++++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 14 deletions(-) (limited to 'drivers/misc/sgi-gru/grumain.c') diff --git a/drivers/misc/sgi-gru/grumain.c b/drivers/misc/sgi-gru/grumain.c index ade0925eab0..f8538bbd0bf 100644 --- a/drivers/misc/sgi-gru/grumain.c +++ b/drivers/misc/sgi-gru/grumain.c @@ -53,12 +53,16 @@ struct device *grudev = &gru_device; */ int gru_cpu_fault_map_id(void) { +#ifdef CONFIG_IA64 + return uv_blade_processor_id() % GRU_NUM_TFM; +#else int cpu = smp_processor_id(); int id, core; core = uv_cpu_core_number(cpu); id = core + UV_MAX_INT_CORES * uv_cpu_socket_number(cpu); return id; +#endif } /*--------- ASID Management ------------------------------------------- @@ -698,16 +702,35 @@ static int gru_retarget_intr(struct gru_thread_state *gts) return gru_update_cch(gts); } +/* + * Check if a GRU context is allowed to use a specific chiplet. By default + * a context is assigned to any blade-local chiplet. However, users can + * override this. + * Returns 1 if assignment allowed, 0 otherwise + */ +static int gru_check_chiplet_assignment(struct gru_state *gru, + struct gru_thread_state *gts) +{ + int blade_id; + int chiplet_id; + + blade_id = gts->ts_user_blade_id; + if (blade_id < 0) + blade_id = uv_numa_blade_id(); + + chiplet_id = gts->ts_user_chiplet_id; + return gru->gs_blade_id == blade_id && + (chiplet_id < 0 || chiplet_id == gru->gs_chiplet_id); +} + /* * Unload the gru context if it is not assigned to the correct blade or * chiplet. Misassignment can occur if the process migrates to a different * blade or if the user changes the selected blade/chiplet. - * Return 0 if context correct placed, otherwise 1 */ void gru_check_context_placement(struct gru_thread_state *gts) { struct gru_state *gru; - int blade_id, chiplet_id; /* * If the current task is the context owner, verify that the @@ -718,13 +741,7 @@ void gru_check_context_placement(struct gru_thread_state *gts) if (!gru || gts->ts_tgid_owner != current->tgid) return; - blade_id = gts->ts_user_blade_id; - if (blade_id < 0) - blade_id = uv_numa_blade_id(); - - chiplet_id = gts->ts_user_chiplet_id; - if (gru->gs_blade_id != blade_id || - (chiplet_id >= 0 && chiplet_id != gru->gs_chiplet_id)) { + if (!gru_check_chiplet_assignment(gru, gts)) { STAT(check_context_unload); gru_unload_context(gts, 1); } else if (gru_retarget_intr(gts)) { @@ -768,9 +785,9 @@ void gru_steal_context(struct gru_thread_state *gts) struct gru_state *gru, *gru0; struct gru_thread_state *ngts = NULL; int ctxnum, ctxnum0, flag = 0, cbr, dsr; - int blade_id = gts->ts_user_blade_id; - int chiplet_id = gts->ts_user_chiplet_id; + int blade_id; + blade_id = gts->ts_user_blade_id; if (blade_id < 0) blade_id = uv_numa_blade_id(); cbr = gts->ts_cbr_au_count; @@ -788,7 +805,7 @@ void gru_steal_context(struct gru_thread_state *gts) ctxnum0 = ctxnum; gru0 = gru; while (1) { - if (chiplet_id < 0 || chiplet_id == gru->gs_chiplet_id) { + if (gru_check_chiplet_assignment(gru, gts)) { if (check_gru_resources(gru, cbr, dsr, GRU_NUM_CCH)) break; spin_lock(&gru->gs_lock); @@ -853,7 +870,6 @@ struct gru_state *gru_assign_gru_context(struct gru_thread_state *gts) struct gru_state *gru, *grux; int i, max_active_contexts; int blade_id = gts->ts_user_blade_id; - int chiplet_id = gts->ts_user_chiplet_id; if (blade_id < 0) blade_id = uv_numa_blade_id(); @@ -861,7 +877,7 @@ again: gru = NULL; max_active_contexts = GRU_NUM_CCH; for_each_gru_on_blade(grux, blade_id, i) { - if (chiplet_id >= 0 && chiplet_id != grux->gs_chiplet_id) + if (!gru_check_chiplet_assignment(grux, gts)) continue; if (check_gru_resources(grux, gts->ts_cbr_au_count, gts->ts_dsr_au_count, -- cgit v1.2.3-70-g09d2