From 1238819a41b6e38e1560afe8f33bbc815671b4f7 Mon Sep 17 00:00:00 2001 From: Sebastian Siewior Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Make file-internal functions & variables static There are a few symbols used only in one file within spufs; this change makes them static where suitable. Signed-off-by: Sebastian Siewior Signed-off-by: Jeremy Kerr Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/file.c | 6 +++--- arch/powerpc/platforms/cell/spufs/run.c | 4 ++-- arch/powerpc/platforms/cell/spufs/syscalls.c | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c index 4100ddc52f0..a4a8770623d 100644 --- a/arch/powerpc/platforms/cell/spufs/file.c +++ b/arch/powerpc/platforms/cell/spufs/file.c @@ -199,9 +199,9 @@ static int spufs_mem_mmap(struct file *file, struct vm_area_struct *vma) } #ifdef CONFIG_SPU_FS_64K_LS -unsigned long spufs_get_unmapped_area(struct file *file, unsigned long addr, - unsigned long len, unsigned long pgoff, - unsigned long flags) +static unsigned long spufs_get_unmapped_area(struct file *file, + unsigned long addr, unsigned long len, unsigned long pgoff, + unsigned long flags) { struct spu_context *ctx = file->private_data; struct spu_state *csa = &ctx->csa; diff --git a/arch/powerpc/platforms/cell/spufs/run.c b/arch/powerpc/platforms/cell/spufs/run.c index 958f10e90fd..1ce5e22ea5f 100644 --- a/arch/powerpc/platforms/cell/spufs/run.c +++ b/arch/powerpc/platforms/cell/spufs/run.c @@ -205,7 +205,7 @@ static int spu_reacquire_runnable(struct spu_context *ctx, u32 *npc, * This means we can only do a very rough approximation of POSIX * signal semantics. */ -int spu_handle_restartsys(struct spu_context *ctx, long *spu_ret, +static int spu_handle_restartsys(struct spu_context *ctx, long *spu_ret, unsigned int *npc) { int ret; @@ -241,7 +241,7 @@ int spu_handle_restartsys(struct spu_context *ctx, long *spu_ret, return ret; } -int spu_process_callback(struct spu_context *ctx) +static int spu_process_callback(struct spu_context *ctx) { struct spu_syscall_block s; u32 ls_pointer, npc; diff --git a/arch/powerpc/platforms/cell/spufs/syscalls.c b/arch/powerpc/platforms/cell/spufs/syscalls.c index 43f0fb88abb..3d8c3281c76 100644 --- a/arch/powerpc/platforms/cell/spufs/syscalls.c +++ b/arch/powerpc/platforms/cell/spufs/syscalls.c @@ -76,8 +76,8 @@ asmlinkage long sys_spu_run(int fd, __u32 __user *unpc, __u32 __user *ustatus) } #endif -asmlinkage long do_spu_create(const char __user *pathname, unsigned int flags, - mode_t mode, struct file *neighbor) +static asmlinkage long do_spu_create(const char __user *pathname, + unsigned int flags, mode_t mode, struct file *neighbor) { char *tmp; int ret; -- cgit v1.2.3-70-g09d2 From 1e8b0f6d1b4dd5fafe732a24dfcb9ac9fd27cf08 Mon Sep 17 00:00:00 2001 From: Jeremy Kerr Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Remove asmlinkage from do_spu_create do_spu_create doesn't need the asmlinkage qualifier; remove it. Signed-off-by: Jeremy Kerr Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/syscalls.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/syscalls.c b/arch/powerpc/platforms/cell/spufs/syscalls.c index 3d8c3281c76..936e0a8af3a 100644 --- a/arch/powerpc/platforms/cell/spufs/syscalls.c +++ b/arch/powerpc/platforms/cell/spufs/syscalls.c @@ -76,8 +76,8 @@ asmlinkage long sys_spu_run(int fd, __u32 __user *unpc, __u32 __user *ustatus) } #endif -static asmlinkage long do_spu_create(const char __user *pathname, - unsigned int flags, mode_t mode, struct file *neighbor) +static long do_spu_create(const char __user *pathname, unsigned int flags, + mode_t mode, struct file *neighbor) { char *tmp; int ret; -- cgit v1.2.3-70-g09d2 From 6232a74f25f45a98d8cf64c5d4208f4795eb693d Mon Sep 17 00:00:00 2001 From: Jeremy Kerr Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Remove spu_harvest Based on an initial patch from Sebastian Siewior spu_harvest isn't used, remove it. Signed-off-by: Jeremy Kerr Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/switch.c | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/switch.c b/arch/powerpc/platforms/cell/spufs/switch.c index 27ffdae98e5..25066199150 100644 --- a/arch/powerpc/platforms/cell/spufs/switch.c +++ b/arch/powerpc/platforms/cell/spufs/switch.c @@ -2146,19 +2146,6 @@ int spu_restore(struct spu_state *new, struct spu *spu) } EXPORT_SYMBOL_GPL(spu_restore); -/** - * spu_harvest - SPU harvest (reset) operation - * @spu: pointer to SPU iomem structure. - * - * Perform SPU harvest (reset) operation. - */ -void spu_harvest(struct spu *spu) -{ - acquire_spu_lock(spu); - harvest(NULL, spu); - release_spu_lock(spu); -} - static void init_prob(struct spu_state *csa) { csa->spu_chnlcnt_RW[9] = 1; -- cgit v1.2.3-70-g09d2 From 8b0d3121a0b2cf91768ecef635e241b6abc3f1da Mon Sep 17 00:00:00 2001 From: Sebastian Siewior Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Make isolated loader properly aligned According to the comment in spufs_init_isolated_loader(), the isolated loader should be aligned on a 16 byte boundary. ARCH_{KMALLOC,SLAB}_MINALIGN is not defined so only 8 byte alignment is guaranteed. This enforces alignment via __get_free_pages. Signed-off-by: Sebastian Siewior Signed-off-by: Jeremy Kerr Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/inode.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/inode.c b/arch/powerpc/platforms/cell/spufs/inode.c index b3d0dd118dd..e210a4b259f 100644 --- a/arch/powerpc/platforms/cell/spufs/inode.c +++ b/arch/powerpc/platforms/cell/spufs/inode.c @@ -43,6 +43,7 @@ static struct kmem_cache *spufs_inode_cache; char *isolated_loader; +static int isolated_loader_size; static struct inode * spufs_alloc_inode(struct super_block *sb) @@ -667,7 +668,8 @@ spufs_parse_options(char *options, struct inode *root) static void spufs_exit_isolated_loader(void) { - kfree(isolated_loader); + free_pages((unsigned long) isolated_loader, + get_order(isolated_loader_size)); } static void @@ -685,11 +687,12 @@ spufs_init_isolated_loader(void) if (!loader) return; - /* kmalloc should align on a 16 byte boundary..* */ - isolated_loader = kmalloc(size, GFP_KERNEL); + /* the loader must be align on a 16 byte boundary */ + isolated_loader = (char *)__get_free_pages(GFP_KERNEL, get_order(size)); if (!isolated_loader) return; + isolated_loader_size = size; memcpy(isolated_loader, loader, size); printk(KERN_INFO "spufs: SPU isolation mode enabled\n"); } -- cgit v1.2.3-70-g09d2 From 36ddbb1380f282b4280c57efdb646dd8647a789f Mon Sep 17 00:00:00 2001 From: Andre Detsch Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Fix race condition on gang->aff_ref_spu Affinity reference point location (gang->aff_ref_spu) is reset when the whole gang is descheduled. However, the last member of a gang can be descheduled while we are trying to schedule another member of the gang. This was leading to a race condition, and the code was using gang->aff_ref_spu in an unsafe manner. By holding the gang->aff_mutex a little bit longer, and increment gang->aff_sched_count (which controls when gang->aff_ref_spu should be reset) a little bit earlier, the problem is fixed. Signed-off-by: Andre Detsch Signed-off-by: Jeremy Kerr Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/sched.c | 49 ++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 17 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/sched.c b/arch/powerpc/platforms/cell/spufs/sched.c index c784edd40ea..17806e001e5 100644 --- a/arch/powerpc/platforms/cell/spufs/sched.c +++ b/arch/powerpc/platforms/cell/spufs/sched.c @@ -230,8 +230,6 @@ static void spu_bind_context(struct spu *spu, struct spu_context *ctx) if (ctx->flags & SPU_CREATE_NOSCHED) atomic_inc(&cbe_spu_info[spu->node].reserved_spus); - if (!list_empty(&ctx->aff_list)) - atomic_inc(&ctx->gang->aff_sched_count); ctx->stats.slb_flt_base = spu->stats.slb_flt; ctx->stats.class2_intr_base = spu->stats.class2_intr; @@ -392,7 +390,6 @@ static int has_affinity(struct spu_context *ctx) if (list_empty(&ctx->aff_list)) return 0; - mutex_lock(&gang->aff_mutex); if (!gang->aff_ref_spu) { if (!(gang->aff_flags & AFF_MERGED)) aff_merge_remaining_ctxs(gang); @@ -400,7 +397,6 @@ static int has_affinity(struct spu_context *ctx) aff_set_offsets(gang); aff_set_ref_point_location(gang); } - mutex_unlock(&gang->aff_mutex); return gang->aff_ref_spu != NULL; } @@ -418,9 +414,16 @@ static void spu_unbind_context(struct spu *spu, struct spu_context *ctx) if (spu->ctx->flags & SPU_CREATE_NOSCHED) atomic_dec(&cbe_spu_info[spu->node].reserved_spus); - if (!list_empty(&ctx->aff_list)) - if (atomic_dec_and_test(&ctx->gang->aff_sched_count)) - ctx->gang->aff_ref_spu = NULL; + + if (ctx->gang){ + mutex_lock(&ctx->gang->aff_mutex); + if (has_affinity(ctx)) { + if (atomic_dec_and_test(&ctx->gang->aff_sched_count)) + ctx->gang->aff_ref_spu = NULL; + } + mutex_unlock(&ctx->gang->aff_mutex); + } + spu_switch_notify(spu, NULL); spu_unmap_mappings(ctx); spu_save(&ctx->csa, spu); @@ -511,20 +514,32 @@ static void spu_prio_wait(struct spu_context *ctx) static struct spu *spu_get_idle(struct spu_context *ctx) { - struct spu *spu; + struct spu *spu, *aff_ref_spu; int node, n; - if (has_affinity(ctx)) { - node = ctx->gang->aff_ref_spu->node; + if (ctx->gang) { + mutex_lock(&ctx->gang->aff_mutex); + if (has_affinity(ctx)) { + aff_ref_spu = ctx->gang->aff_ref_spu; + atomic_inc(&ctx->gang->aff_sched_count); + mutex_unlock(&ctx->gang->aff_mutex); + node = aff_ref_spu->node; - mutex_lock(&cbe_spu_info[node].list_mutex); - spu = ctx_location(ctx->gang->aff_ref_spu, ctx->aff_offset, node); - if (spu && spu->alloc_state == SPU_FREE) - goto found; - mutex_unlock(&cbe_spu_info[node].list_mutex); - return NULL; - } + mutex_lock(&cbe_spu_info[node].list_mutex); + spu = ctx_location(aff_ref_spu, ctx->aff_offset, node); + if (spu && spu->alloc_state == SPU_FREE) + goto found; + mutex_unlock(&cbe_spu_info[node].list_mutex); + mutex_lock(&ctx->gang->aff_mutex); + if (atomic_dec_and_test(&ctx->gang->aff_sched_count)) + ctx->gang->aff_ref_spu = NULL; + mutex_unlock(&ctx->gang->aff_mutex); + + return NULL; + } + mutex_unlock(&ctx->gang->aff_mutex); + } node = cpu_to_node(raw_smp_processor_id()); for (n = 0; n < MAX_NUMNODES; n++, node++) { node = (node < MAX_NUMNODES) ? node : 0; -- cgit v1.2.3-70-g09d2 From 98f06978ffebbec16abdea58489f217229580859 Mon Sep 17 00:00:00 2001 From: Jeremy Kerr Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] cell: Unify spufs syscall path At present, a built-in spufs will not use the spufs_calls callbacks, but directly call sys_spu_create. This saves us an indirect branch, but means we have duplicated functions - one for CONFIG_SPU_FS=y and one for =m. This change unifies the spufs syscall path, and provides access to the spufs_calls structure through a get/put pair. At present, the only user of the spufs_calls structure is spu_syscalls.c, but this will facilitate adding the coredump calls later. Everyone likes numbers, right? Here's a before/after comparison with CONFIG_SPU_FS=y, doing spu_create(); close(); 64k times. Before: [jk@cell ~]$ time ./spu_create performing 65536 spu_create calls real 0m24.075s user 0m0.146s sys 0m23.925s After: [jk@cell ~]$ time ./spu_create performing 65536 spu_create calls real 0m24.777s user 0m0.141s sys 0m24.631s So, we're adding around 11us per syscall, at the benefit of having only one syscall path. Signed-off-by: Jeremy Kerr Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/Makefile | 4 +- arch/powerpc/platforms/cell/spu_syscalls.c | 108 +++++++++++++++++---------- arch/powerpc/platforms/cell/spufs/spufs.h | 1 + arch/powerpc/platforms/cell/spufs/syscalls.c | 42 ----------- include/asm-powerpc/spu.h | 14 +--- 5 files changed, 72 insertions(+), 97 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/Makefile b/arch/powerpc/platforms/cell/Makefile index f88a7c76f29..40f78e90895 100644 --- a/arch/powerpc/platforms/cell/Makefile +++ b/arch/powerpc/platforms/cell/Makefile @@ -13,15 +13,13 @@ obj-$(CONFIG_PPC_CELL_NATIVE) += smp.o endif # needed only when building loadable spufs.ko -spufs-modular-$(CONFIG_SPU_FS) += spu_syscalls.o spu-priv1-$(CONFIG_PPC_CELL_NATIVE) += spu_priv1_mmio.o spu-manage-$(CONFIG_PPC_CELLEB) += spu_manage.o spu-manage-$(CONFIG_PPC_CELL_NATIVE) += spu_manage.o obj-$(CONFIG_SPU_BASE) += spu_callbacks.o spu_base.o \ - spu_coredump.o \ - $(spufs-modular-m) \ + spu_coredump.o spu_syscalls.o \ $(spu-priv1-y) \ $(spu-manage-y) \ spufs/ diff --git a/arch/powerpc/platforms/cell/spu_syscalls.c b/arch/powerpc/platforms/cell/spu_syscalls.c index 027ac32cc63..c0238dd9ff2 100644 --- a/arch/powerpc/platforms/cell/spu_syscalls.c +++ b/arch/powerpc/platforms/cell/spu_syscalls.c @@ -22,42 +22,70 @@ #include #include #include +#include #include -struct spufs_calls spufs_calls = { - .owner = NULL, -}; +/* protected by rcu */ +static struct spufs_calls *spufs_calls; -/* These stub syscalls are needed to have the actual implementation - * within a loadable module. When spufs is built into the kernel, - * this file is not used and the syscalls directly enter the fs code */ +#ifdef CONFIG_SPU_FS_MODULE + +static inline struct spufs_calls *spufs_calls_get(void) +{ + struct spufs_calls *calls = NULL; + + rcu_read_lock(); + calls = rcu_dereference(spufs_calls); + if (calls && !try_module_get(calls->owner)) + calls = NULL; + rcu_read_unlock(); + + return calls; +} + +static inline void spufs_calls_put(struct spufs_calls *calls) +{ + BUG_ON(calls != spufs_calls); + + /* we don't need to rcu this, as we hold a reference to the module */ + module_put(spufs_calls->owner); +} + +#else /* !defined CONFIG_SPU_FS_MODULE */ + +static inline struct spufs_calls *spufs_calls_get(void) +{ + return spufs_calls; +} + +static inline void spufs_calls_put(struct spufs_calls *calls) { } + +#endif /* CONFIG_SPU_FS_MODULE */ asmlinkage long sys_spu_create(const char __user *name, unsigned int flags, mode_t mode, int neighbor_fd) { long ret; - struct module *owner = spufs_calls.owner; struct file *neighbor; int fput_needed; + struct spufs_calls *calls; - ret = -ENOSYS; - if (owner && try_module_get(owner)) { - if (flags & SPU_CREATE_AFFINITY_SPU) { - neighbor = fget_light(neighbor_fd, &fput_needed); - ret = -EBADF; - if (neighbor) { - ret = spufs_calls.create_thread(name, flags, - mode, neighbor); - fput_light(neighbor, fput_needed); - } - } - else { - ret = spufs_calls.create_thread(name, flags, - mode, NULL); + calls = spufs_calls_get(); + if (!calls) + return -ENOSYS; + + if (flags & SPU_CREATE_AFFINITY_SPU) { + ret = -EBADF; + neighbor = fget_light(neighbor_fd, &fput_needed); + if (neighbor) { + ret = calls->create_thread(name, flags, mode, neighbor); + fput_light(neighbor, fput_needed); } - module_put(owner); - } + } else + ret = calls->create_thread(name, flags, mode, NULL); + + spufs_calls_put(calls); return ret; } @@ -66,37 +94,37 @@ asmlinkage long sys_spu_run(int fd, __u32 __user *unpc, __u32 __user *ustatus) long ret; struct file *filp; int fput_needed; - struct module *owner = spufs_calls.owner; + struct spufs_calls *calls; - ret = -ENOSYS; - if (owner && try_module_get(owner)) { - ret = -EBADF; - filp = fget_light(fd, &fput_needed); - if (filp) { - ret = spufs_calls.spu_run(filp, unpc, ustatus); - fput_light(filp, fput_needed); - } - module_put(owner); + calls = spufs_calls_get(); + if (!calls) + return -ENOSYS; + + ret = -EBADF; + filp = fget_light(fd, &fput_needed); + if (filp) { + ret = calls->spu_run(filp, unpc, ustatus); + fput_light(filp, fput_needed); } + + spufs_calls_put(calls); return ret; } int register_spu_syscalls(struct spufs_calls *calls) { - if (spufs_calls.owner) + if (spufs_calls) return -EBUSY; - spufs_calls.create_thread = calls->create_thread; - spufs_calls.spu_run = calls->spu_run; - smp_mb(); - spufs_calls.owner = calls->owner; + rcu_assign_pointer(spufs_calls, calls); return 0; } EXPORT_SYMBOL_GPL(register_spu_syscalls); void unregister_spu_syscalls(struct spufs_calls *calls) { - BUG_ON(spufs_calls.owner != calls->owner); - spufs_calls.owner = NULL; + BUG_ON(spufs_calls->owner != calls->owner); + rcu_assign_pointer(spufs_calls, NULL); + synchronize_rcu(); } EXPORT_SYMBOL_GPL(unregister_spu_syscalls); diff --git a/arch/powerpc/platforms/cell/spufs/spufs.h b/arch/powerpc/platforms/cell/spufs/spufs.h index 2bfdeb8ea8b..3dbffebb3ce 100644 --- a/arch/powerpc/platforms/cell/spufs/spufs.h +++ b/arch/powerpc/platforms/cell/spufs/spufs.h @@ -200,6 +200,7 @@ extern struct tree_descr spufs_dir_contents[]; extern struct tree_descr spufs_dir_nosched_contents[]; /* system call implementation */ +extern struct spufs_calls spufs_calls; long spufs_run_spu(struct spu_context *ctx, u32 *npc, u32 *status); long spufs_create(struct nameidata *nd, unsigned int flags, mode_t mode, struct file *filp); diff --git a/arch/powerpc/platforms/cell/spufs/syscalls.c b/arch/powerpc/platforms/cell/spufs/syscalls.c index 936e0a8af3a..22b138dc335 100644 --- a/arch/powerpc/platforms/cell/spufs/syscalls.c +++ b/arch/powerpc/platforms/cell/spufs/syscalls.c @@ -58,24 +58,6 @@ out: return ret; } -#ifndef MODULE -asmlinkage long sys_spu_run(int fd, __u32 __user *unpc, __u32 __user *ustatus) -{ - int fput_needed; - struct file *filp; - long ret; - - ret = -EBADF; - filp = fget_light(fd, &fput_needed); - if (filp) { - ret = do_spu_run(filp, unpc, ustatus); - fput_light(filp, fput_needed); - } - - return ret; -} -#endif - static long do_spu_create(const char __user *pathname, unsigned int flags, mode_t mode, struct file *neighbor) { @@ -99,30 +81,6 @@ static long do_spu_create(const char __user *pathname, unsigned int flags, return ret; } -#ifndef MODULE -asmlinkage long sys_spu_create(const char __user *pathname, unsigned int flags, - mode_t mode, int neighbor_fd) -{ - int fput_needed; - struct file *neighbor; - long ret; - - if (flags & SPU_CREATE_AFFINITY_SPU) { - ret = -EBADF; - neighbor = fget_light(neighbor_fd, &fput_needed); - if (neighbor) { - ret = do_spu_create(pathname, flags, mode, neighbor); - fput_light(neighbor, fput_needed); - } - } - else { - ret = do_spu_create(pathname, flags, mode, NULL); - } - - return ret; -} -#endif - struct spufs_calls spufs_calls = { .create_thread = do_spu_create, .spu_run = do_spu_run, diff --git a/include/asm-powerpc/spu.h b/include/asm-powerpc/spu.h index 5bde3980bf4..383ecfdf7eb 100644 --- a/include/asm-powerpc/spu.h +++ b/include/asm-powerpc/spu.h @@ -238,14 +238,14 @@ extern long spu_sys_callback(struct spu_syscall_block *s); /* syscalls implemented in spufs */ struct file; -extern struct spufs_calls { +struct spufs_calls { asmlinkage long (*create_thread)(const char __user *name, unsigned int flags, mode_t mode, struct file *neighbor); asmlinkage long (*spu_run)(struct file *filp, __u32 __user *unpc, __u32 __user *ustatus); struct module *owner; -} spufs_calls; +}; /* coredump calls implemented in spufs */ struct spu_coredump_calls { @@ -274,18 +274,8 @@ struct spu_coredump_calls { #define SPU_CREATE_FLAG_ALL 0x003f /* mask of all valid flags */ -#ifdef CONFIG_SPU_FS_MODULE int register_spu_syscalls(struct spufs_calls *calls); void unregister_spu_syscalls(struct spufs_calls *calls); -#else -static inline int register_spu_syscalls(struct spufs_calls *calls) -{ - return 0; -} -static inline void unregister_spu_syscalls(struct spufs_calls *calls) -{ -} -#endif /* MODULE */ int register_arch_coredump_calls(struct spu_coredump_calls *calls); void unregister_arch_coredump_calls(struct spu_coredump_calls *calls); -- cgit v1.2.3-70-g09d2 From 05a059f3296c07a7455290dd8188b23ecb380fc7 Mon Sep 17 00:00:00 2001 From: Jeremy Kerr Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Fix restore_decr_wrapped() to match CBE Handbook Based on an original patch from Masato Noguchi . We're currently not restoring the SPE decrementer as specified by the CBE handbook. This change fixes our implementation to match, and makes the function read more like the docs. Signed-off-by: Jeremy Kerr Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/switch.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/switch.c b/arch/powerpc/platforms/cell/spufs/switch.c index 25066199150..de7e5ee451d 100644 --- a/arch/powerpc/platforms/cell/spufs/switch.c +++ b/arch/powerpc/platforms/cell/spufs/switch.c @@ -1559,15 +1559,15 @@ static inline void restore_decr_wrapped(struct spu_state *csa, struct spu *spu) * "wrapped" flag is set, OR in a '1' to * CSA.SPU_Event_Status[Tm]. */ - if (csa->lscsa->decr_status.slot[0] & SPU_DECR_STATUS_WRAPPED) { - csa->spu_chnldata_RW[0] |= 0x20; - } - if ((csa->lscsa->decr_status.slot[0] & SPU_DECR_STATUS_WRAPPED) && - (csa->spu_chnlcnt_RW[0] == 0 && - ((csa->spu_chnldata_RW[2] & 0x20) == 0x0) && - ((csa->spu_chnldata_RW[0] & 0x20) != 0x1))) { + if (!(csa->lscsa->decr_status.slot[0] & SPU_DECR_STATUS_WRAPPED)) + return; + + if ((csa->spu_chnlcnt_RW[0] == 0) && + (csa->spu_chnldata_RW[1] & 0x20) && + !(csa->spu_chnldata_RW[0] & 0x20)) csa->spu_chnlcnt_RW[0] = 1; - } + + csa->spu_chnldata_RW[0] |= 0x20; } static inline void restore_ch_part1(struct spu_state *csa, struct spu *spu) -- cgit v1.2.3-70-g09d2 From a595ed662c96dcd920415bea3135ff1af60e9a00 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Extract the file descriptor search logic in SPU coredump code Extract the logic for searching through the file descriptors for spu contexts into a separate routine, coredump_next_context(), so we can use it elsewhere in future. In the process we flatten the for loop, and move the NOSCHED test into coredump_next_context(). Signed-off-by: Michael Ellerman Signed-off-by: Jeremy Kerr Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/coredump.c | 58 ++++++++++++++++++---------- 1 file changed, 38 insertions(+), 20 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/coredump.c b/arch/powerpc/platforms/cell/spufs/coredump.c index 5e31799b1e3..99f8e0b0089 100644 --- a/arch/powerpc/platforms/cell/spufs/coredump.c +++ b/arch/powerpc/platforms/cell/spufs/coredump.c @@ -109,16 +109,11 @@ static int spufs_ctx_note_size(struct spufs_ctx_info *ctx_info) return total; } -static int spufs_add_one_context(struct file *file, int dfd) +static int spufs_add_one_context(struct spu_context *ctx, int dfd) { - struct spu_context *ctx; struct spufs_ctx_info *ctx_info; int size; - ctx = SPUFS_I(file->f_dentry->d_inode)->i_ctx; - if (ctx->flags & SPU_CREATE_NOSCHED) - return 0; - ctx_info = kzalloc(sizeof(*ctx_info), GFP_KERNEL); if (unlikely(!ctx_info)) return -ENOMEM; @@ -142,22 +137,45 @@ static int spufs_add_one_context(struct file *file, int dfd) * internal functionality to dump them without needing to actually * open the files. */ -static int spufs_arch_notes_size(void) +static struct spu_context *coredump_next_context(int *fd) { struct fdtable *fdt = files_fdtable(current->files); - int size = 0, fd; - - for (fd = 0; fd < fdt->max_fds; fd++) { - if (FD_ISSET(fd, fdt->open_fds)) { - struct file *file = fcheck(fd); - - if (file && file->f_op == &spufs_context_fops) { - int rval = spufs_add_one_context(file, fd); - if (rval < 0) - break; - size += rval; - } - } + struct file *file; + struct spu_context *ctx = NULL; + + for (; *fd < fdt->max_fds; (*fd)++) { + if (!FD_ISSET(*fd, fdt->open_fds)) + continue; + + file = fcheck(*fd); + + if (!file || file->f_op != &spufs_context_fops) + continue; + + ctx = SPUFS_I(file->f_dentry->d_inode)->i_ctx; + if (ctx->flags & SPU_CREATE_NOSCHED) + continue; + + /* start searching the next fd next time we're called */ + (*fd)++; + break; + } + + return ctx; +} + +static int spufs_arch_notes_size(void) +{ + struct spu_context *ctx; + int size = 0, rc, fd; + + fd = 0; + while ((ctx = coredump_next_context(&fd)) != NULL) { + rc = spufs_add_one_context(ctx, fd); + if (rc < 0) + break; + + size += rc; } return size; -- cgit v1.2.3-70-g09d2 From f9b7bbe7a803c6f10e7b3a354c5d97f632060320 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Remove ctx_info and ctx_info_list Remove the ctx_info struct entirely, and also the ctx_info_list. This fixes a race where two processes can clobber each other's ctx_info structs. Instead of using the list, we just repeat the search through the file descriptor table. Signed-off-by: Michael Ellerman Signed-off-by: Jeremy Kerr Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/coredump.c | 79 +++++++--------------------- 1 file changed, 19 insertions(+), 60 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/coredump.c b/arch/powerpc/platforms/cell/spufs/coredump.c index 99f8e0b0089..66636693c9d 100644 --- a/arch/powerpc/platforms/cell/spufs/coredump.c +++ b/arch/powerpc/platforms/cell/spufs/coredump.c @@ -31,15 +31,6 @@ #include "spufs.h" -struct spufs_ctx_info { - struct list_head list; - int dfd; - int memsize; /* in bytes */ - struct spu_context *ctx; -}; - -static LIST_HEAD(ctx_info_list); - static ssize_t do_coredump_read(int num, struct spu_context *ctx, void __user *buffer, size_t size, loff_t *off) { @@ -73,25 +64,17 @@ static int spufs_dump_seek(struct file *file, loff_t off) return 1; } -static void spufs_fill_memsize(struct spufs_ctx_info *ctx_info) +static u64 ctx_ls_size(struct spu_context *ctx) { - struct spu_context *ctx; - unsigned long long lslr; - - ctx = ctx_info->ctx; - lslr = ctx->csa.priv2.spu_lslr_RW; - ctx_info->memsize = lslr + 1; + return ctx->csa.priv2.spu_lslr_RW + 1; } -static int spufs_ctx_note_size(struct spufs_ctx_info *ctx_info) +static int spufs_ctx_note_size(struct spu_context *ctx, int dfd) { - int dfd, memsize, i, sz, total = 0; + int i, sz, total = 0; char *name; char fullname[80]; - dfd = ctx_info->dfd; - memsize = ctx_info->memsize; - for (i = 0; spufs_coredump_read[i].name; i++) { name = spufs_coredump_read[i].name; sz = spufs_coredump_read[i].size; @@ -101,7 +84,7 @@ static int spufs_ctx_note_size(struct spufs_ctx_info *ctx_info) total += sizeof(struct elf_note); total += roundup(strlen(fullname) + 1, 4); if (!strcmp(name, "mem")) - total += roundup(memsize, 4); + total += roundup(ctx_ls_size(ctx), 4); else total += roundup(sz, 4); } @@ -109,25 +92,6 @@ static int spufs_ctx_note_size(struct spufs_ctx_info *ctx_info) return total; } -static int spufs_add_one_context(struct spu_context *ctx, int dfd) -{ - struct spufs_ctx_info *ctx_info; - int size; - - ctx_info = kzalloc(sizeof(*ctx_info), GFP_KERNEL); - if (unlikely(!ctx_info)) - return -ENOMEM; - - ctx_info->dfd = dfd; - ctx_info->ctx = ctx; - - spufs_fill_memsize(ctx_info); - - size = spufs_ctx_note_size(ctx_info); - list_add(&ctx_info->list, &ctx_info_list); - return size; -} - /* * The additional architecture-specific notes for Cell are various * context files in the spu context. @@ -171,7 +135,7 @@ static int spufs_arch_notes_size(void) fd = 0; while ((ctx = coredump_next_context(&fd)) != NULL) { - rc = spufs_add_one_context(ctx, fd); + rc = spufs_ctx_note_size(ctx, fd); if (rc < 0) break; @@ -181,12 +145,11 @@ static int spufs_arch_notes_size(void) return size; } -static void spufs_arch_write_note(struct spufs_ctx_info *ctx_info, int i, - struct file *file) +static void spufs_arch_write_note(struct spu_context *ctx, int i, + struct file *file, int dfd) { - struct spu_context *ctx; loff_t pos = 0; - int sz, dfd, rc, total = 0; + int sz, rc, total = 0; const int bufsz = PAGE_SIZE; char *name; char fullname[80], *buf; @@ -196,18 +159,13 @@ static void spufs_arch_write_note(struct spufs_ctx_info *ctx_info, int i, if (!buf) return; - dfd = ctx_info->dfd; name = spufs_coredump_read[i].name; if (!strcmp(name, "mem")) - sz = ctx_info->memsize; + sz = ctx_ls_size(ctx); else sz = spufs_coredump_read[i].size; - ctx = ctx_info->ctx; - if (!ctx) - goto out; - sprintf(fullname, "SPU/%d/%s", dfd, name); en.n_namesz = strlen(fullname) + 1; en.n_descsz = sz; @@ -237,16 +195,17 @@ out: static void spufs_arch_write_notes(struct file *file) { - int j; - struct spufs_ctx_info *ctx_info, *next; + struct spu_context *ctx; + int fd, j; + + fd = 0; + while ((ctx = coredump_next_context(&fd)) != NULL) { + spu_acquire_saved(ctx); - list_for_each_entry_safe(ctx_info, next, &ctx_info_list, list) { - spu_acquire_saved(ctx_info->ctx); for (j = 0; j < spufs_coredump_num_notes; j++) - spufs_arch_write_note(ctx_info, j, file); - spu_release_saved(ctx_info->ctx); - list_del(&ctx_info->list); - kfree(ctx_info); + spufs_arch_write_note(ctx, j, file, fd); + + spu_release_saved(ctx); } } -- cgit v1.2.3-70-g09d2 From 9a5080f11d67972d7972d824f1b1827fafbce126 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Call spu_acquire_saved() before calculating the SPU note sizes It makes sense to stop the SPU processes as soon as possible. Also if we dont acquire_saved() I think there's a possibility that the value in csa.priv2.spu_lslr_RW won't be accurate. Signed-off-by: Michael Ellerman Signed-off-by: Jeremy Kerr Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/coredump.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/coredump.c b/arch/powerpc/platforms/cell/spufs/coredump.c index 66636693c9d..21283f68288 100644 --- a/arch/powerpc/platforms/cell/spufs/coredump.c +++ b/arch/powerpc/platforms/cell/spufs/coredump.c @@ -135,7 +135,9 @@ static int spufs_arch_notes_size(void) fd = 0; while ((ctx = coredump_next_context(&fd)) != NULL) { + spu_acquire_saved(ctx); rc = spufs_ctx_note_size(ctx, fd); + spu_release_saved(ctx); if (rc < 0) break; -- cgit v1.2.3-70-g09d2 From 4fca9c425009c01d41db6c6ebf0189843ee90f0b Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Use computed sizes/#defines rather than literals in SPU coredump code The spufs_coredump_reader array contains the size of the data that will be returned by the read routine. Currently these are specified as literals, and though some are obvious, sizeof(u32) == 4, others are not, 69 * 8 == ??? Instead, use sizeof() whatever type is returned by each routine, or in the case of spufs_mem_read() the #define LS_SIZE. Signed-off-by: Michael Ellerman Signed-off-by: Jeremy Kerr Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/file.c | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c index a4a8770623d..18ddde8ba19 100644 --- a/arch/powerpc/platforms/cell/spufs/file.c +++ b/arch/powerpc/platforms/cell/spufs/file.c @@ -2231,23 +2231,24 @@ struct tree_descr spufs_dir_nosched_contents[] = { }; struct spufs_coredump_reader spufs_coredump_read[] = { - { "regs", __spufs_regs_read, NULL, 128 * 16 }, - { "fpcr", __spufs_fpcr_read, NULL, 16 }, + { "regs", __spufs_regs_read, NULL, sizeof(struct spu_reg128[128])}, + { "fpcr", __spufs_fpcr_read, NULL, sizeof(struct spu_reg128) }, { "lslr", NULL, __spufs_lslr_get, 11 }, { "decr", NULL, __spufs_decr_get, 11 }, { "decr_status", NULL, __spufs_decr_status_get, 11 }, - { "mem", __spufs_mem_read, NULL, 256 * 1024, }, - { "signal1", __spufs_signal1_read, NULL, 4 }, + { "mem", __spufs_mem_read, NULL, LS_SIZE, }, + { "signal1", __spufs_signal1_read, NULL, sizeof(u32) }, { "signal1_type", NULL, __spufs_signal1_type_get, 2 }, - { "signal2", __spufs_signal2_read, NULL, 4 }, + { "signal2", __spufs_signal2_read, NULL, sizeof(u32) }, { "signal2_type", NULL, __spufs_signal2_type_get, 2 }, { "event_mask", NULL, __spufs_event_mask_get, 8 }, { "event_status", NULL, __spufs_event_status_get, 8 }, - { "mbox_info", __spufs_mbox_info_read, NULL, 4 }, - { "ibox_info", __spufs_ibox_info_read, NULL, 4 }, - { "wbox_info", __spufs_wbox_info_read, NULL, 16 }, - { "dma_info", __spufs_dma_info_read, NULL, 69 * 8 }, - { "proxydma_info", __spufs_proxydma_info_read, NULL, 35 * 8 }, + { "mbox_info", __spufs_mbox_info_read, NULL, sizeof(u32) }, + { "ibox_info", __spufs_ibox_info_read, NULL, sizeof(u32) }, + { "wbox_info", __spufs_wbox_info_read, NULL, 4 * sizeof(u32)}, + { "dma_info", __spufs_dma_info_read, NULL, sizeof(struct spu_dma_info)}, + { "proxydma_info", __spufs_proxydma_info_read, + NULL, sizeof(struct spu_proxydma_info)}, { "object-id", NULL, __spufs_object_id_get, 19 }, { }, }; -- cgit v1.2.3-70-g09d2 From d464fb441071a3d65bde2264c5f97f9ca47ce5c3 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Write some SPU coredump values as ASCII Unfortunately GDB expects some of the SPU coredump values to be identical in format to what is found in spufs. This means we need to dump some of the values as ASCII strings, not the actual values. Because we don't know what the values will be, we always print the values with the format "0x%.16lx", that way we know the result will be 19 bytes. do_coredump_read() doesn't take a __user buffer, so remove the annotation, and because we know that it's safe to just snprintf() directly to it. Signed-off-by: Michael Ellerman Signed-off-by: Jeremy Kerr Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/coredump.c | 8 +++++--- arch/powerpc/platforms/cell/spufs/file.c | 14 +++++++------- 2 files changed, 12 insertions(+), 10 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/coredump.c b/arch/powerpc/platforms/cell/spufs/coredump.c index 21283f68288..c65b7178e77 100644 --- a/arch/powerpc/platforms/cell/spufs/coredump.c +++ b/arch/powerpc/platforms/cell/spufs/coredump.c @@ -31,7 +31,7 @@ #include "spufs.h" -static ssize_t do_coredump_read(int num, struct spu_context *ctx, void __user *buffer, +static ssize_t do_coredump_read(int num, struct spu_context *ctx, void *buffer, size_t size, loff_t *off) { u64 data; @@ -41,8 +41,10 @@ static ssize_t do_coredump_read(int num, struct spu_context *ctx, void __user *b return spufs_coredump_read[num].read(ctx, buffer, size, off); data = spufs_coredump_read[num].get(ctx); - ret = copy_to_user(buffer, &data, 8); - return ret ? -EFAULT : 8; + ret = snprintf(buffer, size, "0x%.16lx", data); + if (ret >= size) + return size; + return ++ret; /* count trailing NULL */ } /* diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c index 18ddde8ba19..85edbecf506 100644 --- a/arch/powerpc/platforms/cell/spufs/file.c +++ b/arch/powerpc/platforms/cell/spufs/file.c @@ -2233,16 +2233,16 @@ struct tree_descr spufs_dir_nosched_contents[] = { struct spufs_coredump_reader spufs_coredump_read[] = { { "regs", __spufs_regs_read, NULL, sizeof(struct spu_reg128[128])}, { "fpcr", __spufs_fpcr_read, NULL, sizeof(struct spu_reg128) }, - { "lslr", NULL, __spufs_lslr_get, 11 }, - { "decr", NULL, __spufs_decr_get, 11 }, - { "decr_status", NULL, __spufs_decr_status_get, 11 }, + { "lslr", NULL, __spufs_lslr_get, 19 }, + { "decr", NULL, __spufs_decr_get, 19 }, + { "decr_status", NULL, __spufs_decr_status_get, 19 }, { "mem", __spufs_mem_read, NULL, LS_SIZE, }, { "signal1", __spufs_signal1_read, NULL, sizeof(u32) }, - { "signal1_type", NULL, __spufs_signal1_type_get, 2 }, + { "signal1_type", NULL, __spufs_signal1_type_get, 19 }, { "signal2", __spufs_signal2_read, NULL, sizeof(u32) }, - { "signal2_type", NULL, __spufs_signal2_type_get, 2 }, - { "event_mask", NULL, __spufs_event_mask_get, 8 }, - { "event_status", NULL, __spufs_event_status_get, 8 }, + { "signal2_type", NULL, __spufs_signal2_type_get, 19 }, + { "event_mask", NULL, __spufs_event_mask_get, 19 }, + { "event_status", NULL, __spufs_event_status_get, 19 }, { "mbox_info", __spufs_mbox_info_read, NULL, sizeof(u32) }, { "ibox_info", __spufs_ibox_info_read, NULL, sizeof(u32) }, { "wbox_info", __spufs_wbox_info_read, NULL, 4 * sizeof(u32)}, -- cgit v1.2.3-70-g09d2 From 59000b53c7ea07531018b6cf1f5fcd21e881867a Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Correctly calculate the size of the local-store to dump The routine to dump the local store, __spufs_mem_read(), does not take the spu_lslr_RW value into account - so we shouldn't check it when we're calculating the size either. Signed-off-by: Michael Ellerman Signed-off-by: Jeremy Kerr Acked-by: Arnd Bergmann Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/coredump.c | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/coredump.c b/arch/powerpc/platforms/cell/spufs/coredump.c index c65b7178e77..52d62193fd8 100644 --- a/arch/powerpc/platforms/cell/spufs/coredump.c +++ b/arch/powerpc/platforms/cell/spufs/coredump.c @@ -66,11 +66,6 @@ static int spufs_dump_seek(struct file *file, loff_t off) return 1; } -static u64 ctx_ls_size(struct spu_context *ctx) -{ - return ctx->csa.priv2.spu_lslr_RW + 1; -} - static int spufs_ctx_note_size(struct spu_context *ctx, int dfd) { int i, sz, total = 0; @@ -85,10 +80,7 @@ static int spufs_ctx_note_size(struct spu_context *ctx, int dfd) total += sizeof(struct elf_note); total += roundup(strlen(fullname) + 1, 4); - if (!strcmp(name, "mem")) - total += roundup(ctx_ls_size(ctx), 4); - else - total += roundup(sz, 4); + total += roundup(sz, 4); } return total; @@ -164,11 +156,7 @@ static void spufs_arch_write_note(struct spu_context *ctx, int i, return; name = spufs_coredump_read[i].name; - - if (!strcmp(name, "mem")) - sz = ctx_ls_size(ctx); - else - sz = spufs_coredump_read[i].size; + sz = spufs_coredump_read[i].size; sprintf(fullname, "SPU/%d/%s", dfd, name); en.n_namesz = strlen(fullname) + 1; -- cgit v1.2.3-70-g09d2 From 936d5bf1d7dc69c56bf79ad68819e597307a1884 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Get rid of spufs_coredump_num_notes, it's not needed if we NULL terminate The spufs_coredump_read array is NULL terminated, and we also store the size. We only need one or the other, and the other arrays in file.c are NULL terminated, so do that. Signed-off-by: Michael Ellerman Signed-off-by: Jeremy Kerr Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/coredump.c | 4 ++-- arch/powerpc/platforms/cell/spufs/file.c | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/coredump.c b/arch/powerpc/platforms/cell/spufs/coredump.c index 52d62193fd8..fc988fd1ffb 100644 --- a/arch/powerpc/platforms/cell/spufs/coredump.c +++ b/arch/powerpc/platforms/cell/spufs/coredump.c @@ -72,7 +72,7 @@ static int spufs_ctx_note_size(struct spu_context *ctx, int dfd) char *name; char fullname[80]; - for (i = 0; spufs_coredump_read[i].name; i++) { + for (i = 0; spufs_coredump_read[i].name != NULL; i++) { name = spufs_coredump_read[i].name; sz = spufs_coredump_read[i].size; @@ -194,7 +194,7 @@ static void spufs_arch_write_notes(struct file *file) while ((ctx = coredump_next_context(&fd)) != NULL) { spu_acquire_saved(ctx); - for (j = 0; j < spufs_coredump_num_notes; j++) + for (j = 0; spufs_coredump_read[j].name != NULL; j++) spufs_arch_write_note(ctx, j, file, fd); spu_release_saved(ctx); diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c index 85edbecf506..6095fb18dbb 100644 --- a/arch/powerpc/platforms/cell/spufs/file.c +++ b/arch/powerpc/platforms/cell/spufs/file.c @@ -2250,7 +2250,5 @@ struct spufs_coredump_reader spufs_coredump_read[] = { { "proxydma_info", __spufs_proxydma_info_read, NULL, sizeof(struct spu_proxydma_info)}, { "object-id", NULL, __spufs_object_id_get, 19 }, - { }, + { NULL }, }; -int spufs_coredump_num_notes = ARRAY_SIZE(spufs_coredump_read) - 1; - -- cgit v1.2.3-70-g09d2 From 74de08bc10dd4d67870cf5b6c5aaf6875cd869c5 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Internal __spufs_get_foo() routines should take a spu_context * The SPUFS attribute get routines take a void * because the generic attribute code doesn't know what sort of data it's passing around. However our internal __spufs_get_foo() routines can take a spu_context * directly, which saves plonking it in and out of a void * again. Signed-off-by: Michael Ellerman Signed-off-by: Jeremy Kerr Acked-by: Arnd Bergmann Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/file.c | 40 +++++++++++++------------------ arch/powerpc/platforms/cell/spufs/spufs.h | 2 +- 2 files changed, 17 insertions(+), 25 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c index 6095fb18dbb..4cd34e53aca 100644 --- a/arch/powerpc/platforms/cell/spufs/file.c +++ b/arch/powerpc/platforms/cell/spufs/file.c @@ -1085,9 +1085,8 @@ static void spufs_signal1_type_set(void *data, u64 val) spu_release(ctx); } -static u64 __spufs_signal1_type_get(void *data) +static u64 __spufs_signal1_type_get(struct spu_context *ctx) { - struct spu_context *ctx = data; return ctx->ops->signal1_type_get(ctx); } @@ -1097,7 +1096,7 @@ static u64 spufs_signal1_type_get(void *data) u64 ret; spu_acquire(ctx); - ret = __spufs_signal1_type_get(data); + ret = __spufs_signal1_type_get(ctx); spu_release(ctx); return ret; @@ -1114,9 +1113,8 @@ static void spufs_signal2_type_set(void *data, u64 val) spu_release(ctx); } -static u64 __spufs_signal2_type_get(void *data) +static u64 __spufs_signal2_type_get(struct spu_context *ctx) { - struct spu_context *ctx = data; return ctx->ops->signal2_type_get(ctx); } @@ -1126,7 +1124,7 @@ static u64 spufs_signal2_type_get(void *data) u64 ret; spu_acquire(ctx); - ret = __spufs_signal2_type_get(data); + ret = __spufs_signal2_type_get(ctx); spu_release(ctx); return ret; @@ -1629,9 +1627,8 @@ static void spufs_decr_set(void *data, u64 val) spu_release_saved(ctx); } -static u64 __spufs_decr_get(void *data) +static u64 __spufs_decr_get(struct spu_context *ctx) { - struct spu_context *ctx = data; struct spu_lscsa *lscsa = ctx->csa.lscsa; return lscsa->decr.slot[0]; } @@ -1641,7 +1638,7 @@ static u64 spufs_decr_get(void *data) struct spu_context *ctx = data; u64 ret; spu_acquire_saved(ctx); - ret = __spufs_decr_get(data); + ret = __spufs_decr_get(ctx); spu_release_saved(ctx); return ret; } @@ -1659,9 +1656,8 @@ static void spufs_decr_status_set(void *data, u64 val) spu_release_saved(ctx); } -static u64 __spufs_decr_status_get(void *data) +static u64 __spufs_decr_status_get(struct spu_context *ctx) { - struct spu_context *ctx = data; if (ctx->csa.priv2.mfc_control_RW & MFC_CNTL_DECREMENTER_RUNNING) return SPU_DECR_STATUS_RUNNING; else @@ -1673,7 +1669,7 @@ static u64 spufs_decr_status_get(void *data) struct spu_context *ctx = data; u64 ret; spu_acquire_saved(ctx); - ret = __spufs_decr_status_get(data); + ret = __spufs_decr_status_get(ctx); spu_release_saved(ctx); return ret; } @@ -1689,9 +1685,8 @@ static void spufs_event_mask_set(void *data, u64 val) spu_release_saved(ctx); } -static u64 __spufs_event_mask_get(void *data) +static u64 __spufs_event_mask_get(struct spu_context *ctx) { - struct spu_context *ctx = data; struct spu_lscsa *lscsa = ctx->csa.lscsa; return lscsa->event_mask.slot[0]; } @@ -1701,16 +1696,15 @@ static u64 spufs_event_mask_get(void *data) struct spu_context *ctx = data; u64 ret; spu_acquire_saved(ctx); - ret = __spufs_event_mask_get(data); + ret = __spufs_event_mask_get(ctx); spu_release_saved(ctx); return ret; } DEFINE_SIMPLE_ATTRIBUTE(spufs_event_mask_ops, spufs_event_mask_get, spufs_event_mask_set, "0x%llx\n") -static u64 __spufs_event_status_get(void *data) +static u64 __spufs_event_status_get(struct spu_context *ctx) { - struct spu_context *ctx = data; struct spu_state *state = &ctx->csa; u64 stat; stat = state->spu_chnlcnt_RW[0]; @@ -1725,7 +1719,7 @@ static u64 spufs_event_status_get(void *data) u64 ret = 0; spu_acquire_saved(ctx); - ret = __spufs_event_status_get(data); + ret = __spufs_event_status_get(ctx); spu_release_saved(ctx); return ret; } @@ -1770,16 +1764,15 @@ static u64 spufs_id_get(void *data) } DEFINE_SIMPLE_ATTRIBUTE(spufs_id_ops, spufs_id_get, NULL, "0x%llx\n") -static u64 __spufs_object_id_get(void *data) +static u64 __spufs_object_id_get(struct spu_context *ctx) { - struct spu_context *ctx = data; return ctx->object_id; } static u64 spufs_object_id_get(void *data) { /* FIXME: Should there really be no locking here? */ - return __spufs_object_id_get(data); + return __spufs_object_id_get((struct spu_context *)data); } static void spufs_object_id_set(void *data, u64 id) @@ -1791,9 +1784,8 @@ static void spufs_object_id_set(void *data, u64 id) DEFINE_SIMPLE_ATTRIBUTE(spufs_object_id_ops, spufs_object_id_get, spufs_object_id_set, "0x%llx\n"); -static u64 __spufs_lslr_get(void *data) +static u64 __spufs_lslr_get(struct spu_context *ctx) { - struct spu_context *ctx = data; return ctx->csa.priv2.spu_lslr_RW; } @@ -1803,7 +1795,7 @@ static u64 spufs_lslr_get(void *data) u64 ret; spu_acquire_saved(ctx); - ret = __spufs_lslr_get(data); + ret = __spufs_lslr_get(ctx); spu_release_saved(ctx); return ret; diff --git a/arch/powerpc/platforms/cell/spufs/spufs.h b/arch/powerpc/platforms/cell/spufs/spufs.h index 3dbffebb3ce..f869a4b488b 100644 --- a/arch/powerpc/platforms/cell/spufs/spufs.h +++ b/arch/powerpc/platforms/cell/spufs/spufs.h @@ -296,7 +296,7 @@ struct spufs_coredump_reader { char *name; ssize_t (*read)(struct spu_context *ctx, char __user *buffer, size_t size, loff_t *pos); - u64 (*get)(void *data); + u64 (*get)(struct spu_context *ctx); size_t size; }; extern struct spufs_coredump_reader spufs_coredump_read[]; -- cgit v1.2.3-70-g09d2 From 78810ff6723f20015373b1ba8dd981f24c62f680 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Add contents of npc file to SPU coredumps Signed-off-by: Michael Ellerman Signed-off-by: Jeremy Kerr Acked-by: Arnd Bergmann Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/file.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c index 4cd34e53aca..985c86bb16d 100644 --- a/arch/powerpc/platforms/cell/spufs/file.c +++ b/arch/powerpc/platforms/cell/spufs/file.c @@ -1606,12 +1606,17 @@ static void spufs_npc_set(void *data, u64 val) spu_release(ctx); } +static u64 __spufs_npc_get(struct spu_context *ctx) +{ + return ctx->ops->npc_read(ctx); +} + static u64 spufs_npc_get(void *data) { struct spu_context *ctx = data; u64 ret; spu_acquire(ctx); - ret = ctx->ops->npc_read(ctx); + ret = __spufs_npc_get(ctx); spu_release(ctx); return ret; } @@ -2242,5 +2247,6 @@ struct spufs_coredump_reader spufs_coredump_read[] = { { "proxydma_info", __spufs_proxydma_info_read, NULL, sizeof(struct spu_proxydma_info)}, { "object-id", NULL, __spufs_object_id_get, 19 }, + { "npc", NULL, __spufs_npc_get, 19 }, { NULL }, }; -- cgit v1.2.3-70-g09d2 From 48cad41f7ee7b8a9a8317a4abbdaf09bc68b4773 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Combine spufs_coredump_calls with spufs_calls Because spufs might be built as a module, we can't have other parts of the kernel calling directly into it, we need stub routines that check first if the module is loaded. Currently we have two structures which hold callbacks for these stubs, the syscalls are in spufs_calls and the coredump calls are in spufs_coredump_calls. In both cases the logic for registering/unregistering is essentially the same, so we can simplify things by combining the two. Signed-off-by: Michael Ellerman Signed-off-by: Jeremy Kerr Acked-by: Arnd Bergmann Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/Makefile | 2 +- arch/powerpc/platforms/cell/spu_coredump.c | 83 ---------------------------- arch/powerpc/platforms/cell/spu_syscalls.c | 30 ++++++++++ arch/powerpc/platforms/cell/spufs/coredump.c | 10 +--- arch/powerpc/platforms/cell/spufs/inode.c | 6 -- arch/powerpc/platforms/cell/spufs/spufs.h | 4 ++ arch/powerpc/platforms/cell/spufs/syscalls.c | 2 + include/asm-powerpc/spu.h | 12 +--- 8 files changed, 41 insertions(+), 108 deletions(-) delete mode 100644 arch/powerpc/platforms/cell/spu_coredump.c (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/Makefile b/arch/powerpc/platforms/cell/Makefile index 40f78e90895..61d12f18303 100644 --- a/arch/powerpc/platforms/cell/Makefile +++ b/arch/powerpc/platforms/cell/Makefile @@ -19,7 +19,7 @@ spu-manage-$(CONFIG_PPC_CELLEB) += spu_manage.o spu-manage-$(CONFIG_PPC_CELL_NATIVE) += spu_manage.o obj-$(CONFIG_SPU_BASE) += spu_callbacks.o spu_base.o \ - spu_coredump.o spu_syscalls.o \ + spu_syscalls.o \ $(spu-priv1-y) \ $(spu-manage-y) \ spufs/ diff --git a/arch/powerpc/platforms/cell/spu_coredump.c b/arch/powerpc/platforms/cell/spu_coredump.c deleted file mode 100644 index 656a8c52cd3..00000000000 --- a/arch/powerpc/platforms/cell/spu_coredump.c +++ /dev/null @@ -1,83 +0,0 @@ -/* - * SPU core dump code - * - * (C) Copyright 2006 IBM Corp. - * - * Author: Dwayne Grant McConnell - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2, or (at your option) - * any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. - */ - -#include -#include -#include - -#include - -static struct spu_coredump_calls *spu_coredump_calls; -static DEFINE_MUTEX(spu_coredump_mutex); - -int arch_notes_size(void) -{ - int ret; - - mutex_lock(&spu_coredump_mutex); - - if (spu_coredump_calls && try_module_get(spu_coredump_calls->owner)) { - ret = spu_coredump_calls->arch_notes_size(); - module_put(spu_coredump_calls->owner); - } else { - ret = 0; - } - - mutex_unlock(&spu_coredump_mutex); - - return ret; -} - -void arch_write_notes(struct file *file) -{ - mutex_lock(&spu_coredump_mutex); - if (spu_coredump_calls && try_module_get(spu_coredump_calls->owner)) { - spu_coredump_calls->arch_write_notes(file); - module_put(spu_coredump_calls->owner); - } - mutex_unlock(&spu_coredump_mutex); -} - -int register_arch_coredump_calls(struct spu_coredump_calls *calls) -{ - int ret = 0; - - - mutex_lock(&spu_coredump_mutex); - if (spu_coredump_calls) - ret = -EBUSY; - else - spu_coredump_calls = calls; - mutex_unlock(&spu_coredump_mutex); - return ret; -} -EXPORT_SYMBOL_GPL(register_arch_coredump_calls); - -void unregister_arch_coredump_calls(struct spu_coredump_calls *calls) -{ - BUG_ON(spu_coredump_calls != calls); - - mutex_lock(&spu_coredump_mutex); - spu_coredump_calls = NULL; - mutex_unlock(&spu_coredump_mutex); -} -EXPORT_SYMBOL_GPL(unregister_arch_coredump_calls); diff --git a/arch/powerpc/platforms/cell/spu_syscalls.c b/arch/powerpc/platforms/cell/spu_syscalls.c index c0238dd9ff2..05841cdef4e 100644 --- a/arch/powerpc/platforms/cell/spu_syscalls.c +++ b/arch/powerpc/platforms/cell/spu_syscalls.c @@ -2,6 +2,7 @@ * SPU file system -- system call stubs * * (C) Copyright IBM Deutschland Entwicklung GmbH 2005 + * (C) Copyright 2006-2007, IBM Corporation * * Author: Arnd Bergmann * @@ -111,6 +112,35 @@ asmlinkage long sys_spu_run(int fd, __u32 __user *unpc, __u32 __user *ustatus) return ret; } +int arch_notes_size(void) +{ + struct spufs_calls *calls; + int ret; + + calls = spufs_calls_get(); + if (!calls) + return 0; + + ret = calls->coredump_extra_notes_size(); + + spufs_calls_put(calls); + + return ret; +} + +void arch_write_notes(struct file *file) +{ + struct spufs_calls *calls; + + calls = spufs_calls_get(); + if (!calls) + return; + + calls->coredump_extra_notes_write(file); + + spufs_calls_put(calls); +} + int register_spu_syscalls(struct spufs_calls *calls) { if (spufs_calls) diff --git a/arch/powerpc/platforms/cell/spufs/coredump.c b/arch/powerpc/platforms/cell/spufs/coredump.c index fc988fd1ffb..6c20e44dba6 100644 --- a/arch/powerpc/platforms/cell/spufs/coredump.c +++ b/arch/powerpc/platforms/cell/spufs/coredump.c @@ -122,7 +122,7 @@ static struct spu_context *coredump_next_context(int *fd) return ctx; } -static int spufs_arch_notes_size(void) +int spufs_coredump_extra_notes_size(void) { struct spu_context *ctx; int size = 0, rc, fd; @@ -185,7 +185,7 @@ out: free_page((unsigned long)buf); } -static void spufs_arch_write_notes(struct file *file) +void spufs_coredump_extra_notes_write(struct file *file) { struct spu_context *ctx; int fd, j; @@ -200,9 +200,3 @@ static void spufs_arch_write_notes(struct file *file) spu_release_saved(ctx); } } - -struct spu_coredump_calls spufs_coredump_calls = { - .arch_notes_size = spufs_arch_notes_size, - .arch_write_notes = spufs_arch_write_notes, - .owner = THIS_MODULE, -}; diff --git a/arch/powerpc/platforms/cell/spufs/inode.c b/arch/powerpc/platforms/cell/spufs/inode.c index e210a4b259f..11098747d09 100644 --- a/arch/powerpc/platforms/cell/spufs/inode.c +++ b/arch/powerpc/platforms/cell/spufs/inode.c @@ -790,16 +790,11 @@ static int __init spufs_init(void) ret = register_spu_syscalls(&spufs_calls); if (ret) goto out_fs; - ret = register_arch_coredump_calls(&spufs_coredump_calls); - if (ret) - goto out_syscalls; spufs_init_isolated_loader(); return 0; -out_syscalls: - unregister_spu_syscalls(&spufs_calls); out_fs: unregister_filesystem(&spufs_type); out_sched: @@ -815,7 +810,6 @@ static void __exit spufs_exit(void) { spu_sched_exit(); spufs_exit_isolated_loader(); - unregister_arch_coredump_calls(&spufs_coredump_calls); unregister_spu_syscalls(&spufs_calls); unregister_filesystem(&spufs_type); kmem_cache_destroy(spufs_inode_cache); diff --git a/arch/powerpc/platforms/cell/spufs/spufs.h b/arch/powerpc/platforms/cell/spufs/spufs.h index f869a4b488b..c7b4e035de4 100644 --- a/arch/powerpc/platforms/cell/spufs/spufs.h +++ b/arch/powerpc/platforms/cell/spufs/spufs.h @@ -204,6 +204,10 @@ extern struct spufs_calls spufs_calls; long spufs_run_spu(struct spu_context *ctx, u32 *npc, u32 *status); long spufs_create(struct nameidata *nd, unsigned int flags, mode_t mode, struct file *filp); +/* ELF coredump callbacks for writing SPU ELF notes */ +extern int spufs_coredump_extra_notes_size(void); +extern void spufs_coredump_extra_notes_write(struct file *file); + extern const struct file_operations spufs_context_fops; /* gang management */ diff --git a/arch/powerpc/platforms/cell/spufs/syscalls.c b/arch/powerpc/platforms/cell/spufs/syscalls.c index 22b138dc335..2c34f717019 100644 --- a/arch/powerpc/platforms/cell/spufs/syscalls.c +++ b/arch/powerpc/platforms/cell/spufs/syscalls.c @@ -84,5 +84,7 @@ static long do_spu_create(const char __user *pathname, unsigned int flags, struct spufs_calls spufs_calls = { .create_thread = do_spu_create, .spu_run = do_spu_run, + .coredump_extra_notes_size = spufs_coredump_extra_notes_size, + .coredump_extra_notes_write = spufs_coredump_extra_notes_write, .owner = THIS_MODULE, }; diff --git a/include/asm-powerpc/spu.h b/include/asm-powerpc/spu.h index eb1159cdb8a..f1b10a18798 100644 --- a/include/asm-powerpc/spu.h +++ b/include/asm-powerpc/spu.h @@ -244,13 +244,8 @@ struct spufs_calls { struct file *neighbor); long (*spu_run)(struct file *filp, __u32 __user *unpc, __u32 __user *ustatus); - struct module *owner; -}; - -/* coredump calls implemented in spufs */ -struct spu_coredump_calls { - asmlinkage int (*arch_notes_size)(void); - asmlinkage void (*arch_write_notes)(struct file *file); + int (*coredump_extra_notes_size)(void); + void (*coredump_extra_notes_write)(struct file *file); struct module *owner; }; @@ -277,9 +272,6 @@ struct spu_coredump_calls { int register_spu_syscalls(struct spufs_calls *calls); void unregister_spu_syscalls(struct spufs_calls *calls); -int register_arch_coredump_calls(struct spu_coredump_calls *calls); -void unregister_arch_coredump_calls(struct spu_coredump_calls *calls); - int spu_add_sysdev_attr(struct sysdev_attribute *attr); void spu_remove_sysdev_attr(struct sysdev_attribute *attr); -- cgit v1.2.3-70-g09d2 From 7af1443a9d319132087e1e9a3984b94c6998835c Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Handle errors in SPU coredump code, and support coredump to a pipe Rework spufs_coredump_extra_notes_write() to check for and return errors. If we're coredumping to a pipe we can't trust file->f_pos, we need to maintain the foffset value passed to us. The cleanest way to do this is to have the low level write routine increment foffset when we've successfully written. Signed-off-by: Michael Ellerman Signed-off-by: Jeremy Kerr Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spu_syscalls.c | 8 +-- arch/powerpc/platforms/cell/spufs/coredump.c | 89 +++++++++++++++++++--------- arch/powerpc/platforms/cell/spufs/spufs.h | 2 +- include/asm-powerpc/spu.h | 2 +- 4 files changed, 67 insertions(+), 34 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spu_syscalls.c b/arch/powerpc/platforms/cell/spu_syscalls.c index b0117a7c610..a9438b719fe 100644 --- a/arch/powerpc/platforms/cell/spu_syscalls.c +++ b/arch/powerpc/platforms/cell/spu_syscalls.c @@ -132,19 +132,17 @@ int elf_coredump_extra_notes_size(void) int elf_coredump_extra_notes_write(struct file *file, loff_t *foffset) { struct spufs_calls *calls; + int ret; calls = spufs_calls_get(); if (!calls) return 0; - calls->coredump_extra_notes_write(file); + ret = calls->coredump_extra_notes_write(file, foffset); spufs_calls_put(calls); - /* Fudge foffset for now */ - *foffset = file->f_pos; - - return 0; + return ret; } int register_spu_syscalls(struct spufs_calls *calls) diff --git a/arch/powerpc/platforms/cell/spufs/coredump.c b/arch/powerpc/platforms/cell/spufs/coredump.c index 6c20e44dba6..6b8aef6d7e6 100644 --- a/arch/powerpc/platforms/cell/spufs/coredump.c +++ b/arch/powerpc/platforms/cell/spufs/coredump.c @@ -51,19 +51,34 @@ static ssize_t do_coredump_read(int num, struct spu_context *ctx, void *buffer, * These are the only things you should do on a core-file: use only these * functions to write out all the necessary info. */ -static int spufs_dump_write(struct file *file, const void *addr, int nr) +static int spufs_dump_write(struct file *file, const void *addr, int nr, loff_t *foffset) { - return file->f_op->write(file, addr, nr, &file->f_pos) == nr; + ssize_t written; + + written = file->f_op->write(file, addr, nr, &file->f_pos); + *foffset += written; + + if (written != nr) + return -EIO; + + return 0; } -static int spufs_dump_seek(struct file *file, loff_t off) +static int spufs_dump_align(struct file *file, char *buf, loff_t new_off, + loff_t *foffset) { - if (file->f_op->llseek) { - if (file->f_op->llseek(file, off, 0) != off) - return 0; - } else - file->f_pos = off; - return 1; + int rc, size; + + size = min((loff_t)PAGE_SIZE, new_off - *foffset); + memset(buf, 0, size); + + rc = 0; + while (rc == 0 && new_off > *foffset) { + size = min((loff_t)PAGE_SIZE, new_off - *foffset); + rc = spufs_dump_write(file, buf, size, foffset); + } + + return rc; } static int spufs_ctx_note_size(struct spu_context *ctx, int dfd) @@ -141,11 +156,11 @@ int spufs_coredump_extra_notes_size(void) return size; } -static void spufs_arch_write_note(struct spu_context *ctx, int i, - struct file *file, int dfd) +static int spufs_arch_write_note(struct spu_context *ctx, int i, + struct file *file, int dfd, loff_t *foffset) { loff_t pos = 0; - int sz, rc, total = 0; + int sz, rc, nread, total = 0; const int bufsz = PAGE_SIZE; char *name; char fullname[80], *buf; @@ -153,7 +168,7 @@ static void spufs_arch_write_note(struct spu_context *ctx, int i, buf = (void *)get_zeroed_page(GFP_KERNEL); if (!buf) - return; + return -ENOMEM; name = spufs_coredump_read[i].name; sz = spufs_coredump_read[i].size; @@ -163,40 +178,60 @@ static void spufs_arch_write_note(struct spu_context *ctx, int i, en.n_descsz = sz; en.n_type = NT_SPU; - if (!spufs_dump_write(file, &en, sizeof(en))) + rc = spufs_dump_write(file, &en, sizeof(en), foffset); + if (rc) goto out; - if (!spufs_dump_write(file, fullname, en.n_namesz)) + + rc = spufs_dump_write(file, fullname, en.n_namesz, foffset); + if (rc) goto out; - if (!spufs_dump_seek(file, roundup((unsigned long)file->f_pos, 4))) + + rc = spufs_dump_align(file, buf, roundup(*foffset, 4), foffset); + if (rc) goto out; do { - rc = do_coredump_read(i, ctx, buf, bufsz, &pos); - if (rc > 0) { - if (!spufs_dump_write(file, buf, rc)) + nread = do_coredump_read(i, ctx, buf, bufsz, &pos); + if (nread > 0) { + rc = spufs_dump_write(file, buf, nread, foffset); + if (rc) goto out; - total += rc; + total += nread; } - } while (rc == bufsz && total < sz); + } while (nread == bufsz && total < sz); + + if (nread < 0) { + rc = nread; + goto out; + } + + rc = spufs_dump_align(file, buf, roundup(*foffset - total + sz, 4), + foffset); - spufs_dump_seek(file, roundup((unsigned long)file->f_pos - - total + sz, 4)); out: free_page((unsigned long)buf); + return rc; } -void spufs_coredump_extra_notes_write(struct file *file) +int spufs_coredump_extra_notes_write(struct file *file, loff_t *foffset) { struct spu_context *ctx; - int fd, j; + int fd, j, rc; fd = 0; while ((ctx = coredump_next_context(&fd)) != NULL) { spu_acquire_saved(ctx); - for (j = 0; spufs_coredump_read[j].name != NULL; j++) - spufs_arch_write_note(ctx, j, file, fd); + for (j = 0; spufs_coredump_read[j].name != NULL; j++) { + rc = spufs_arch_write_note(ctx, j, file, fd, foffset); + if (rc) { + spu_release_saved(ctx); + return rc; + } + } spu_release_saved(ctx); } + + return 0; } diff --git a/arch/powerpc/platforms/cell/spufs/spufs.h b/arch/powerpc/platforms/cell/spufs/spufs.h index c7b4e035de4..ca47b991bda 100644 --- a/arch/powerpc/platforms/cell/spufs/spufs.h +++ b/arch/powerpc/platforms/cell/spufs/spufs.h @@ -206,7 +206,7 @@ long spufs_create(struct nameidata *nd, unsigned int flags, mode_t mode, struct file *filp); /* ELF coredump callbacks for writing SPU ELF notes */ extern int spufs_coredump_extra_notes_size(void); -extern void spufs_coredump_extra_notes_write(struct file *file); +extern int spufs_coredump_extra_notes_write(struct file *file, loff_t *foffset); extern const struct file_operations spufs_context_fops; diff --git a/include/asm-powerpc/spu.h b/include/asm-powerpc/spu.h index f1b10a18798..b1accce77bb 100644 --- a/include/asm-powerpc/spu.h +++ b/include/asm-powerpc/spu.h @@ -245,7 +245,7 @@ struct spufs_calls { long (*spu_run)(struct file *filp, __u32 __user *unpc, __u32 __user *ustatus); int (*coredump_extra_notes_size)(void); - void (*coredump_extra_notes_write)(struct file *file); + int (*coredump_extra_notes_write)(struct file *file, loff_t *foffset); struct module *owner; }; -- cgit v1.2.3-70-g09d2 From 9e25ae6d91e7fb058c8957c2a64dc3ca0377dd5b Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Respect RLIMIT_CORE in spu coredump code Currently the spu coredump code doesn't respect the ulimit, it should. Signed-off-by: Michael Ellerman Signed-off-by: Jeremy Kerr Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/coredump.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/coredump.c b/arch/powerpc/platforms/cell/spufs/coredump.c index 6b8aef6d7e6..80f62363e1c 100644 --- a/arch/powerpc/platforms/cell/spufs/coredump.c +++ b/arch/powerpc/platforms/cell/spufs/coredump.c @@ -53,8 +53,12 @@ static ssize_t do_coredump_read(int num, struct spu_context *ctx, void *buffer, */ static int spufs_dump_write(struct file *file, const void *addr, int nr, loff_t *foffset) { + unsigned long limit = current->signal->rlim[RLIMIT_CORE].rlim_cur; ssize_t written; + if (*foffset + nr > limit) + return -EIO; + written = file->f_op->write(file, addr, nr, &file->f_pos); *foffset += written; -- cgit v1.2.3-70-g09d2 From 104f0cc2dcf7ce0ca7da041177233747d6aa0136 Mon Sep 17 00:00:00 2001 From: Michael Ellerman Date: Wed, 19 Sep 2007 14:38:12 +1000 Subject: [POWERPC] spufs: Add DEFINE_SPUFS_ATTRIBUTE() This patch adds DEFINE_SPUFS_ATTRIBUTE(), a wrapper around DEFINE_SIMPLE_ATTRIBUTE which does the specified locking for the get routine for us. Unfortunately we need two get routines (a locked and unlocked version) to support the coredump code. This hides one of those (the locked version) inside the macro foo. Signed-off-by: Michael Ellerman Signed-off-by: Jeremy Kerr Signed-off-by: Paul Mackerras --- arch/powerpc/platforms/cell/spufs/file.c | 216 +++++++++++-------------------- 1 file changed, 76 insertions(+), 140 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/platforms/cell/spufs/file.c b/arch/powerpc/platforms/cell/spufs/file.c index 985c86bb16d..b93a0275a21 100644 --- a/arch/powerpc/platforms/cell/spufs/file.c +++ b/arch/powerpc/platforms/cell/spufs/file.c @@ -1076,6 +1076,36 @@ static const struct file_operations spufs_signal2_nosched_fops = { .mmap = spufs_signal2_mmap, }; +/* + * This is a wrapper around DEFINE_SIMPLE_ATTRIBUTE which does the + * work of acquiring (or not) the SPU context before calling through + * to the actual get routine. The set routine is called directly. + */ +#define SPU_ATTR_NOACQUIRE 0 +#define SPU_ATTR_ACQUIRE 1 +#define SPU_ATTR_ACQUIRE_SAVED 2 + +#define DEFINE_SPUFS_ATTRIBUTE(__name, __get, __set, __fmt, __acquire) \ +static u64 __##__get(void *data) \ +{ \ + struct spu_context *ctx = data; \ + u64 ret; \ + \ + if (__acquire == SPU_ATTR_ACQUIRE) { \ + spu_acquire(ctx); \ + ret = __get(ctx); \ + spu_release(ctx); \ + } else if (__acquire == SPU_ATTR_ACQUIRE_SAVED) { \ + spu_acquire_saved(ctx); \ + ret = __get(ctx); \ + spu_release_saved(ctx); \ + } else \ + ret = __get(ctx); \ + \ + return ret; \ +} \ +DEFINE_SIMPLE_ATTRIBUTE(__name, __##__get, __set, __fmt); + static void spufs_signal1_type_set(void *data, u64 val) { struct spu_context *ctx = data; @@ -1085,24 +1115,13 @@ static void spufs_signal1_type_set(void *data, u64 val) spu_release(ctx); } -static u64 __spufs_signal1_type_get(struct spu_context *ctx) +static u64 spufs_signal1_type_get(struct spu_context *ctx) { return ctx->ops->signal1_type_get(ctx); } +DEFINE_SPUFS_ATTRIBUTE(spufs_signal1_type, spufs_signal1_type_get, + spufs_signal1_type_set, "%llu", SPU_ATTR_ACQUIRE); -static u64 spufs_signal1_type_get(void *data) -{ - struct spu_context *ctx = data; - u64 ret; - - spu_acquire(ctx); - ret = __spufs_signal1_type_get(ctx); - spu_release(ctx); - - return ret; -} -DEFINE_SIMPLE_ATTRIBUTE(spufs_signal1_type, spufs_signal1_type_get, - spufs_signal1_type_set, "%llu"); static void spufs_signal2_type_set(void *data, u64 val) { @@ -1113,24 +1132,12 @@ static void spufs_signal2_type_set(void *data, u64 val) spu_release(ctx); } -static u64 __spufs_signal2_type_get(struct spu_context *ctx) +static u64 spufs_signal2_type_get(struct spu_context *ctx) { return ctx->ops->signal2_type_get(ctx); } - -static u64 spufs_signal2_type_get(void *data) -{ - struct spu_context *ctx = data; - u64 ret; - - spu_acquire(ctx); - ret = __spufs_signal2_type_get(ctx); - spu_release(ctx); - - return ret; -} -DEFINE_SIMPLE_ATTRIBUTE(spufs_signal2_type, spufs_signal2_type_get, - spufs_signal2_type_set, "%llu"); +DEFINE_SPUFS_ATTRIBUTE(spufs_signal2_type, spufs_signal2_type_get, + spufs_signal2_type_set, "%llu", SPU_ATTR_ACQUIRE); #if SPUFS_MMAP_4K static unsigned long spufs_mss_mmap_nopfn(struct vm_area_struct *vma, @@ -1606,22 +1613,12 @@ static void spufs_npc_set(void *data, u64 val) spu_release(ctx); } -static u64 __spufs_npc_get(struct spu_context *ctx) +static u64 spufs_npc_get(struct spu_context *ctx) { return ctx->ops->npc_read(ctx); } - -static u64 spufs_npc_get(void *data) -{ - struct spu_context *ctx = data; - u64 ret; - spu_acquire(ctx); - ret = __spufs_npc_get(ctx); - spu_release(ctx); - return ret; -} -DEFINE_SIMPLE_ATTRIBUTE(spufs_npc_ops, spufs_npc_get, spufs_npc_set, - "0x%llx\n") +DEFINE_SPUFS_ATTRIBUTE(spufs_npc_ops, spufs_npc_get, spufs_npc_set, + "0x%llx\n", SPU_ATTR_ACQUIRE); static void spufs_decr_set(void *data, u64 val) { @@ -1632,23 +1629,13 @@ static void spufs_decr_set(void *data, u64 val) spu_release_saved(ctx); } -static u64 __spufs_decr_get(struct spu_context *ctx) +static u64 spufs_decr_get(struct spu_context *ctx) { struct spu_lscsa *lscsa = ctx->csa.lscsa; return lscsa->decr.slot[0]; } - -static u64 spufs_decr_get(void *data) -{ - struct spu_context *ctx = data; - u64 ret; - spu_acquire_saved(ctx); - ret = __spufs_decr_get(ctx); - spu_release_saved(ctx); - return ret; -} -DEFINE_SIMPLE_ATTRIBUTE(spufs_decr_ops, spufs_decr_get, spufs_decr_set, - "0x%llx\n") +DEFINE_SPUFS_ATTRIBUTE(spufs_decr_ops, spufs_decr_get, spufs_decr_set, + "0x%llx\n", SPU_ATTR_ACQUIRE_SAVED); static void spufs_decr_status_set(void *data, u64 val) { @@ -1661,25 +1648,16 @@ static void spufs_decr_status_set(void *data, u64 val) spu_release_saved(ctx); } -static u64 __spufs_decr_status_get(struct spu_context *ctx) +static u64 spufs_decr_status_get(struct spu_context *ctx) { if (ctx->csa.priv2.mfc_control_RW & MFC_CNTL_DECREMENTER_RUNNING) return SPU_DECR_STATUS_RUNNING; else return 0; } - -static u64 spufs_decr_status_get(void *data) -{ - struct spu_context *ctx = data; - u64 ret; - spu_acquire_saved(ctx); - ret = __spufs_decr_status_get(ctx); - spu_release_saved(ctx); - return ret; -} -DEFINE_SIMPLE_ATTRIBUTE(spufs_decr_status_ops, spufs_decr_status_get, - spufs_decr_status_set, "0x%llx\n") +DEFINE_SPUFS_ATTRIBUTE(spufs_decr_status_ops, spufs_decr_status_get, + spufs_decr_status_set, "0x%llx\n", + SPU_ATTR_ACQUIRE_SAVED); static void spufs_event_mask_set(void *data, u64 val) { @@ -1690,25 +1668,17 @@ static void spufs_event_mask_set(void *data, u64 val) spu_release_saved(ctx); } -static u64 __spufs_event_mask_get(struct spu_context *ctx) +static u64 spufs_event_mask_get(struct spu_context *ctx) { struct spu_lscsa *lscsa = ctx->csa.lscsa; return lscsa->event_mask.slot[0]; } -static u64 spufs_event_mask_get(void *data) -{ - struct spu_context *ctx = data; - u64 ret; - spu_acquire_saved(ctx); - ret = __spufs_event_mask_get(ctx); - spu_release_saved(ctx); - return ret; -} -DEFINE_SIMPLE_ATTRIBUTE(spufs_event_mask_ops, spufs_event_mask_get, - spufs_event_mask_set, "0x%llx\n") +DEFINE_SPUFS_ATTRIBUTE(spufs_event_mask_ops, spufs_event_mask_get, + spufs_event_mask_set, "0x%llx\n", + SPU_ATTR_ACQUIRE_SAVED); -static u64 __spufs_event_status_get(struct spu_context *ctx) +static u64 spufs_event_status_get(struct spu_context *ctx) { struct spu_state *state = &ctx->csa; u64 stat; @@ -1717,19 +1687,8 @@ static u64 __spufs_event_status_get(struct spu_context *ctx) return state->spu_chnldata_RW[0]; return 0; } - -static u64 spufs_event_status_get(void *data) -{ - struct spu_context *ctx = data; - u64 ret = 0; - - spu_acquire_saved(ctx); - ret = __spufs_event_status_get(ctx); - spu_release_saved(ctx); - return ret; -} -DEFINE_SIMPLE_ATTRIBUTE(spufs_event_status_ops, spufs_event_status_get, - NULL, "0x%llx\n") +DEFINE_SPUFS_ATTRIBUTE(spufs_event_status_ops, spufs_event_status_get, + NULL, "0x%llx\n", SPU_ATTR_ACQUIRE_SAVED) static void spufs_srr0_set(void *data, u64 val) { @@ -1740,44 +1699,32 @@ static void spufs_srr0_set(void *data, u64 val) spu_release_saved(ctx); } -static u64 spufs_srr0_get(void *data) +static u64 spufs_srr0_get(struct spu_context *ctx) { - struct spu_context *ctx = data; struct spu_lscsa *lscsa = ctx->csa.lscsa; - u64 ret; - spu_acquire_saved(ctx); - ret = lscsa->srr0.slot[0]; - spu_release_saved(ctx); - return ret; + return lscsa->srr0.slot[0]; } -DEFINE_SIMPLE_ATTRIBUTE(spufs_srr0_ops, spufs_srr0_get, spufs_srr0_set, - "0x%llx\n") +DEFINE_SPUFS_ATTRIBUTE(spufs_srr0_ops, spufs_srr0_get, spufs_srr0_set, + "0x%llx\n", SPU_ATTR_ACQUIRE_SAVED) -static u64 spufs_id_get(void *data) +static u64 spufs_id_get(struct spu_context *ctx) { - struct spu_context *ctx = data; u64 num; - spu_acquire(ctx); if (ctx->state == SPU_STATE_RUNNABLE) num = ctx->spu->number; else num = (unsigned int)-1; - spu_release(ctx); return num; } -DEFINE_SIMPLE_ATTRIBUTE(spufs_id_ops, spufs_id_get, NULL, "0x%llx\n") +DEFINE_SPUFS_ATTRIBUTE(spufs_id_ops, spufs_id_get, NULL, "0x%llx\n", + SPU_ATTR_ACQUIRE) -static u64 __spufs_object_id_get(struct spu_context *ctx) -{ - return ctx->object_id; -} - -static u64 spufs_object_id_get(void *data) +static u64 spufs_object_id_get(struct spu_context *ctx) { /* FIXME: Should there really be no locking here? */ - return __spufs_object_id_get((struct spu_context *)data); + return ctx->object_id; } static void spufs_object_id_set(void *data, u64 id) @@ -1786,26 +1733,15 @@ static void spufs_object_id_set(void *data, u64 id) ctx->object_id = id; } -DEFINE_SIMPLE_ATTRIBUTE(spufs_object_id_ops, spufs_object_id_get, - spufs_object_id_set, "0x%llx\n"); +DEFINE_SPUFS_ATTRIBUTE(spufs_object_id_ops, spufs_object_id_get, + spufs_object_id_set, "0x%llx\n", SPU_ATTR_NOACQUIRE); -static u64 __spufs_lslr_get(struct spu_context *ctx) +static u64 spufs_lslr_get(struct spu_context *ctx) { return ctx->csa.priv2.spu_lslr_RW; } - -static u64 spufs_lslr_get(void *data) -{ - struct spu_context *ctx = data; - u64 ret; - - spu_acquire_saved(ctx); - ret = __spufs_lslr_get(ctx); - spu_release_saved(ctx); - - return ret; -} -DEFINE_SIMPLE_ATTRIBUTE(spufs_lslr_ops, spufs_lslr_get, NULL, "0x%llx\n") +DEFINE_SPUFS_ATTRIBUTE(spufs_lslr_ops, spufs_lslr_get, NULL, "0x%llx\n", + SPU_ATTR_ACQUIRE_SAVED); static int spufs_info_open(struct inode *inode, struct file *file) { @@ -2230,23 +2166,23 @@ struct tree_descr spufs_dir_nosched_contents[] = { struct spufs_coredump_reader spufs_coredump_read[] = { { "regs", __spufs_regs_read, NULL, sizeof(struct spu_reg128[128])}, { "fpcr", __spufs_fpcr_read, NULL, sizeof(struct spu_reg128) }, - { "lslr", NULL, __spufs_lslr_get, 19 }, - { "decr", NULL, __spufs_decr_get, 19 }, - { "decr_status", NULL, __spufs_decr_status_get, 19 }, + { "lslr", NULL, spufs_lslr_get, 19 }, + { "decr", NULL, spufs_decr_get, 19 }, + { "decr_status", NULL, spufs_decr_status_get, 19 }, { "mem", __spufs_mem_read, NULL, LS_SIZE, }, { "signal1", __spufs_signal1_read, NULL, sizeof(u32) }, - { "signal1_type", NULL, __spufs_signal1_type_get, 19 }, + { "signal1_type", NULL, spufs_signal1_type_get, 19 }, { "signal2", __spufs_signal2_read, NULL, sizeof(u32) }, - { "signal2_type", NULL, __spufs_signal2_type_get, 19 }, - { "event_mask", NULL, __spufs_event_mask_get, 19 }, - { "event_status", NULL, __spufs_event_status_get, 19 }, + { "signal2_type", NULL, spufs_signal2_type_get, 19 }, + { "event_mask", NULL, spufs_event_mask_get, 19 }, + { "event_status", NULL, spufs_event_status_get, 19 }, { "mbox_info", __spufs_mbox_info_read, NULL, sizeof(u32) }, { "ibox_info", __spufs_ibox_info_read, NULL, sizeof(u32) }, { "wbox_info", __spufs_wbox_info_read, NULL, 4 * sizeof(u32)}, { "dma_info", __spufs_dma_info_read, NULL, sizeof(struct spu_dma_info)}, { "proxydma_info", __spufs_proxydma_info_read, NULL, sizeof(struct spu_proxydma_info)}, - { "object-id", NULL, __spufs_object_id_get, 19 }, - { "npc", NULL, __spufs_npc_get, 19 }, + { "object-id", NULL, spufs_object_id_get, 19 }, + { "npc", NULL, spufs_npc_get, 19 }, { NULL }, }; -- cgit v1.2.3-70-g09d2 From 1189be6508d45183013ddb82b18f4934193de274 Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Thu, 11 Oct 2007 20:37:10 +1000 Subject: [POWERPC] Use 1TB segments This makes the kernel use 1TB segments for all kernel mappings and for user addresses of 1TB and above, on machines which support them (currently POWER5+, POWER6 and PA6T). We detect that the machine supports 1TB segments by looking at the ibm,processor-segment-sizes property in the device tree. We don't currently use 1TB segments for user addresses < 1T, since that would effectively prevent 32-bit processes from using huge pages unless we also had a way to revert to using 256MB segments. That would be possible but would involve extra complications (such as keeping track of which segment size was used when HPTEs were inserted) and is not addressed here. Parts of this patch were originally written by Ben Herrenschmidt. Signed-off-by: Paul Mackerras --- arch/powerpc/kernel/entry_64.S | 14 ++- arch/powerpc/kernel/head_64.S | 2 +- arch/powerpc/kernel/process.c | 9 +- arch/powerpc/mm/hash_low_64.S | 73 +++++++++++++-- arch/powerpc/mm/hash_native_64.c | 92 +++++++++---------- arch/powerpc/mm/hash_utils_64.c | 114 ++++++++++++++++------- arch/powerpc/mm/hugetlbpage.c | 13 +-- arch/powerpc/mm/pgtable_64.c | 4 +- arch/powerpc/mm/slb.c | 67 ++++++++------ arch/powerpc/mm/slb_low.S | 37 +++++++- arch/powerpc/mm/stab.c | 6 +- arch/powerpc/mm/tlb_64.c | 20 +++-- arch/powerpc/platforms/cell/spu_base.c | 6 +- arch/powerpc/platforms/cell/spufs/switch.c | 2 +- arch/powerpc/platforms/celleb/htab.c | 34 +++---- arch/powerpc/platforms/iseries/htab.c | 13 +-- arch/powerpc/platforms/ps3/htab.c | 14 +-- arch/powerpc/platforms/pseries/lpar.c | 89 +++++++++--------- include/asm-powerpc/cputable.h | 3 +- include/asm-powerpc/machdep.h | 8 +- include/asm-powerpc/mmu-hash64.h | 140 ++++++++++++++++++++--------- include/asm-powerpc/page_64.h | 8 +- include/asm-powerpc/tlbflush.h | 3 +- 23 files changed, 513 insertions(+), 258 deletions(-) (limited to 'arch/powerpc/platforms/cell/spufs') diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S index fbbd3f6f006..0ec13403489 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/entry_64.S @@ -372,9 +372,17 @@ END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) std r6,PACACURRENT(r13) /* Set new 'current' */ ld r8,KSP(r4) /* new stack pointer */ +BEGIN_FTR_SECTION + b 2f +END_FTR_SECTION_IFCLR(CPU_FTR_SLB) BEGIN_FTR_SECTION clrrdi r6,r8,28 /* get its ESID */ clrrdi r9,r1,28 /* get current sp ESID */ +END_FTR_SECTION_IFCLR(CPU_FTR_1T_SEGMENT) +BEGIN_FTR_SECTION + clrrdi r6,r8,40 /* get its 1T ESID */ + clrrdi r9,r1,40 /* get current sp 1T ESID */ +END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT) clrldi. r0,r6,2 /* is new ESID c00000000? */ cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */ cror eq,4*cr1+eq,eq @@ -384,6 +392,11 @@ BEGIN_FTR_SECTION ld r7,KSP_VSID(r4) /* Get new stack's VSID */ oris r0,r6,(SLB_ESID_V)@h ori r0,r0,(SLB_NUM_BOLTED-1)@l +BEGIN_FTR_SECTION + li r9,MMU_SEGSIZE_1T /* insert B field */ + oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h + rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0 +END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT) /* Update the last bolted SLB. No write barriers are needed * here, provided we only update the current CPU's SLB shadow @@ -401,7 +414,6 @@ BEGIN_FTR_SECTION isync 2: -END_FTR_SECTION_IFSET(CPU_FTR_SLB) clrrdi r7,r8,THREAD_SHIFT /* base of new stack */ /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE because we don't need to leave the 288-byte ABI gap at the diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S index 22ac245bd59..97c5857faf0 100644 --- a/arch/powerpc/kernel/head_64.S +++ b/arch/powerpc/kernel/head_64.S @@ -935,7 +935,7 @@ _GLOBAL(do_stab_bolted) /* Calculate VSID */ /* This is a kernel address, so protovsid = ESID */ - ASM_VSID_SCRAMBLE(r11, r9) + ASM_VSID_SCRAMBLE(r11, r9, 256M) rldic r9,r11,12,16 /* r9 = vsid << 12 */ /* Search the primary group for a free entry */ diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index 15998b57767..7949c203cb8 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -564,10 +564,15 @@ int copy_thread(int nr, unsigned long clone_flags, unsigned long usp, #ifdef CONFIG_PPC64 if (cpu_has_feature(CPU_FTR_SLB)) { - unsigned long sp_vsid = get_kernel_vsid(sp); + unsigned long sp_vsid; unsigned long llp = mmu_psize_defs[mmu_linear_psize].sllp; - sp_vsid <<= SLB_VSID_SHIFT; + if (cpu_has_feature(CPU_FTR_1T_SEGMENT)) + sp_vsid = get_kernel_vsid(sp, MMU_SEGSIZE_1T) + << SLB_VSID_SHIFT_1T; + else + sp_vsid = get_kernel_vsid(sp, MMU_SEGSIZE_256M) + << SLB_VSID_SHIFT; sp_vsid |= SLB_VSID_KERNEL | llp; p->thread.ksp_vsid = sp_vsid; } diff --git a/arch/powerpc/mm/hash_low_64.S b/arch/powerpc/mm/hash_low_64.S index 35eabfb5072..ad253b95903 100644 --- a/arch/powerpc/mm/hash_low_64.S +++ b/arch/powerpc/mm/hash_low_64.S @@ -54,7 +54,7 @@ /* * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, - * pte_t *ptep, unsigned long trap, int local) + * pte_t *ptep, unsigned long trap, int local, int ssize) * * Adds a 4K page to the hash table in a segment of 4K pages only */ @@ -66,6 +66,7 @@ _GLOBAL(__hash_page_4K) /* Save all params that we need after a function call */ std r6,STK_PARM(r6)(r1) std r8,STK_PARM(r8)(r1) + std r9,STK_PARM(r9)(r1) /* Add _PAGE_PRESENT to access */ ori r4,r4,_PAGE_PRESENT @@ -117,6 +118,10 @@ _GLOBAL(__hash_page_4K) * r4 (access) is re-useable, we use it for the new HPTE flags */ +BEGIN_FTR_SECTION + cmpdi r9,0 /* check segment size */ + bne 3f +END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT) /* Calc va and put it in r29 */ rldicr r29,r5,28,63-28 rldicl r3,r3,0,36 @@ -126,9 +131,20 @@ _GLOBAL(__hash_page_4K) rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ xor r28,r5,r0 + b 4f + +3: /* Calc VA and hash in r29 and r28 for 1T segment */ + sldi r29,r5,40 /* vsid << 40 */ + clrldi r3,r3,24 /* ea & 0xffffffffff */ + rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ + clrldi r5,r5,40 /* vsid & 0xffffff */ + rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */ + xor r28,r28,r5 + or r29,r3,r29 /* VA */ + xor r28,r28,r0 /* hash */ /* Convert linux PTE bits into HW equivalents */ - andi. r3,r30,0x1fe /* Get basic set of flags */ +4: andi. r3,r30,0x1fe /* Get basic set of flags */ xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ @@ -183,6 +199,7 @@ htab_insert_pte: mr r4,r29 /* Retreive va */ li r7,0 /* !bolted, !secondary */ li r8,MMU_PAGE_4K /* page size */ + ld r9,STK_PARM(r9)(r1) /* segment size */ _GLOBAL(htab_call_hpte_insert1) bl . /* Patched by htab_finish_init() */ cmpdi 0,r3,0 @@ -205,6 +222,7 @@ _GLOBAL(htab_call_hpte_insert1) mr r4,r29 /* Retreive va */ li r7,HPTE_V_SECONDARY /* !bolted, secondary */ li r8,MMU_PAGE_4K /* page size */ + ld r9,STK_PARM(r9)(r1) /* segment size */ _GLOBAL(htab_call_hpte_insert2) bl . /* Patched by htab_finish_init() */ cmpdi 0,r3,0 @@ -273,7 +291,8 @@ htab_modify_pte: /* Call ppc_md.hpte_updatepp */ mr r5,r29 /* va */ li r6,MMU_PAGE_4K /* page size */ - ld r7,STK_PARM(r8)(r1) /* get "local" param */ + ld r7,STK_PARM(r9)(r1) /* segment size */ + ld r8,STK_PARM(r8)(r1) /* get "local" param */ _GLOBAL(htab_call_hpte_updatepp) bl . /* Patched by htab_finish_init() */ @@ -325,6 +344,7 @@ _GLOBAL(__hash_page_4K) /* Save all params that we need after a function call */ std r6,STK_PARM(r6)(r1) std r8,STK_PARM(r8)(r1) + std r9,STK_PARM(r9)(r1) /* Add _PAGE_PRESENT to access */ ori r4,r4,_PAGE_PRESENT @@ -383,18 +403,33 @@ _GLOBAL(__hash_page_4K) /* Load the hidx index */ rldicl r25,r3,64-12,60 +BEGIN_FTR_SECTION + cmpdi r9,0 /* check segment size */ + bne 3f +END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT) /* Calc va and put it in r29 */ rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */ rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */ - or r29,r3,r29 /* r29 = va + or r29,r3,r29 /* r29 = va */ /* Calculate hash value for primary slot and store it in r28 */ rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ xor r28,r5,r0 + b 4f + +3: /* Calc VA and hash in r29 and r28 for 1T segment */ + sldi r29,r5,40 /* vsid << 40 */ + clrldi r3,r3,24 /* ea & 0xffffffffff */ + rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ + clrldi r5,r5,40 /* vsid & 0xffffff */ + rldicl r0,r3,64-12,36 /* (ea >> 12) & 0xfffffff */ + xor r28,r28,r5 + or r29,r3,r29 /* VA */ + xor r28,r28,r0 /* hash */ /* Convert linux PTE bits into HW equivalents */ - andi. r3,r30,0x1fe /* Get basic set of flags */ +4: andi. r3,r30,0x1fe /* Get basic set of flags */ xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ @@ -462,6 +497,7 @@ htab_special_pfn: mr r4,r29 /* Retreive va */ li r7,0 /* !bolted, !secondary */ li r8,MMU_PAGE_4K /* page size */ + ld r9,STK_PARM(r9)(r1) /* segment size */ _GLOBAL(htab_call_hpte_insert1) bl . /* patched by htab_finish_init() */ cmpdi 0,r3,0 @@ -488,6 +524,7 @@ _GLOBAL(htab_call_hpte_insert1) mr r4,r29 /* Retreive va */ li r7,HPTE_V_SECONDARY /* !bolted, secondary */ li r8,MMU_PAGE_4K /* page size */ + ld r9,STK_PARM(r9)(r1) /* segment size */ _GLOBAL(htab_call_hpte_insert2) bl . /* patched by htab_finish_init() */ cmpdi 0,r3,0 @@ -586,7 +623,8 @@ htab_modify_pte: /* Call ppc_md.hpte_updatepp */ mr r5,r29 /* va */ li r6,MMU_PAGE_4K /* page size */ - ld r7,STK_PARM(r8)(r1) /* get "local" param */ + ld r7,STK_PARM(r9)(r1) /* segment size */ + ld r8,STK_PARM(r8)(r1) /* get "local" param */ _GLOBAL(htab_call_hpte_updatepp) bl . /* patched by htab_finish_init() */ @@ -634,6 +672,7 @@ _GLOBAL(__hash_page_64K) /* Save all params that we need after a function call */ std r6,STK_PARM(r6)(r1) std r8,STK_PARM(r8)(r1) + std r9,STK_PARM(r9)(r1) /* Add _PAGE_PRESENT to access */ ori r4,r4,_PAGE_PRESENT @@ -690,6 +729,10 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE) * r4 (access) is re-useable, we use it for the new HPTE flags */ +BEGIN_FTR_SECTION + cmpdi r9,0 /* check segment size */ + bne 3f +END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT) /* Calc va and put it in r29 */ rldicr r29,r5,28,63-28 rldicl r3,r3,0,36 @@ -699,9 +742,20 @@ END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE) rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */ xor r28,r5,r0 + b 4f + +3: /* Calc VA and hash in r29 and r28 for 1T segment */ + sldi r29,r5,40 /* vsid << 40 */ + clrldi r3,r3,24 /* ea & 0xffffffffff */ + rldic r28,r5,25,25 /* (vsid << 25) & 0x7fffffffff */ + clrldi r5,r5,40 /* vsid & 0xffffff */ + rldicl r0,r3,64-16,40 /* (ea >> 16) & 0xffffff */ + xor r28,r28,r5 + or r29,r3,r29 /* VA */ + xor r28,r28,r0 /* hash */ /* Convert linux PTE bits into HW equivalents */ - andi. r3,r30,0x1fe /* Get basic set of flags */ +4: andi. r3,r30,0x1fe /* Get basic set of flags */ xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ @@ -756,6 +810,7 @@ ht64_insert_pte: mr r4,r29 /* Retreive va */ li r7,0 /* !bolted, !secondary */ li r8,MMU_PAGE_64K + ld r9,STK_PARM(r9)(r1) /* segment size */ _GLOBAL(ht64_call_hpte_insert1) bl . /* patched by htab_finish_init() */ cmpdi 0,r3,0 @@ -778,6 +833,7 @@ _GLOBAL(ht64_call_hpte_insert1) mr r4,r29 /* Retreive va */ li r7,HPTE_V_SECONDARY /* !bolted, secondary */ li r8,MMU_PAGE_64K + ld r9,STK_PARM(r9)(r1) /* segment size */ _GLOBAL(ht64_call_hpte_insert2) bl . /* patched by htab_finish_init() */ cmpdi 0,r3,0 @@ -846,7 +902,8 @@ ht64_modify_pte: /* Call ppc_md.hpte_updatepp */ mr r5,r29 /* va */ li r6,MMU_PAGE_64K - ld r7,STK_PARM(r8)(r1) /* get "local" param */ + ld r7,STK_PARM(r9)(r1) /* segment size */ + ld r8,STK_PARM(r8)(r1) /* get "local" param */ _GLOBAL(ht64_call_hpte_updatepp) bl . /* patched by htab_finish_init() */ diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c index 6ba9b47e55a..34e5c0b219b 100644 --- a/arch/powerpc/mm/hash_native_64.c +++ b/arch/powerpc/mm/hash_native_64.c @@ -38,7 +38,7 @@ static DEFINE_SPINLOCK(native_tlbie_lock); -static inline void __tlbie(unsigned long va, unsigned int psize) +static inline void __tlbie(unsigned long va, int psize, int ssize) { unsigned int penc; @@ -48,18 +48,20 @@ static inline void __tlbie(unsigned long va, unsigned int psize) switch (psize) { case MMU_PAGE_4K: va &= ~0xffful; + va |= ssize << 8; asm volatile("tlbie %0,0" : : "r" (va) : "memory"); break; default: penc = mmu_psize_defs[psize].penc; va &= ~((1ul << mmu_psize_defs[psize].shift) - 1); va |= penc << 12; + va |= ssize << 8; asm volatile("tlbie %0,1" : : "r" (va) : "memory"); break; } } -static inline void __tlbiel(unsigned long va, unsigned int psize) +static inline void __tlbiel(unsigned long va, int psize, int ssize) { unsigned int penc; @@ -69,6 +71,7 @@ static inline void __tlbiel(unsigned long va, unsigned int psize) switch (psize) { case MMU_PAGE_4K: va &= ~0xffful; + va |= ssize << 8; asm volatile(".long 0x7c000224 | (%0 << 11) | (0 << 21)" : : "r"(va) : "memory"); break; @@ -76,6 +79,7 @@ static inline void __tlbiel(unsigned long va, unsigned int psize) penc = mmu_psize_defs[psize].penc; va &= ~((1ul << mmu_psize_defs[psize].shift) - 1); va |= penc << 12; + va |= ssize << 8; asm volatile(".long 0x7c000224 | (%0 << 11) | (1 << 21)" : : "r"(va) : "memory"); break; @@ -83,7 +87,7 @@ static inline void __tlbiel(unsigned long va, unsigned int psize) } -static inline void tlbie(unsigned long va, int psize, int local) +static inline void tlbie(unsigned long va, int psize, int ssize, int local) { unsigned int use_local = local && cpu_has_feature(CPU_FTR_TLBIEL); int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE); @@ -94,10 +98,10 @@ static inline void tlbie(unsigned long va, int psize, int local) spin_lock(&native_tlbie_lock); asm volatile("ptesync": : :"memory"); if (use_local) { - __tlbiel(va, psize); + __tlbiel(va, psize, ssize); asm volatile("ptesync": : :"memory"); } else { - __tlbie(va, psize); + __tlbie(va, psize, ssize); asm volatile("eieio; tlbsync; ptesync": : :"memory"); } if (lock_tlbie && !use_local) @@ -126,7 +130,7 @@ static inline void native_unlock_hpte(struct hash_pte *hptep) static long native_hpte_insert(unsigned long hpte_group, unsigned long va, unsigned long pa, unsigned long rflags, - unsigned long vflags, int psize) + unsigned long vflags, int psize, int ssize) { struct hash_pte *hptep = htab_address + hpte_group; unsigned long hpte_v, hpte_r; @@ -153,7 +157,7 @@ static long native_hpte_insert(unsigned long hpte_group, unsigned long va, if (i == HPTES_PER_GROUP) return -1; - hpte_v = hpte_encode_v(va, psize) | vflags | HPTE_V_VALID; + hpte_v = hpte_encode_v(va, psize, ssize) | vflags | HPTE_V_VALID; hpte_r = hpte_encode_r(pa, psize) | rflags; if (!(vflags & HPTE_V_BOLTED)) { @@ -215,13 +219,14 @@ static long native_hpte_remove(unsigned long hpte_group) } static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, - unsigned long va, int psize, int local) + unsigned long va, int psize, int ssize, + int local) { struct hash_pte *hptep = htab_address + slot; unsigned long hpte_v, want_v; int ret = 0; - want_v = hpte_encode_v(va, psize); + want_v = hpte_encode_v(va, psize, ssize); DBG_LOW(" update(va=%016lx, avpnv=%016lx, hash=%016lx, newpp=%x)", va, want_v & HPTE_V_AVPN, slot, newpp); @@ -243,39 +248,32 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, native_unlock_hpte(hptep); /* Ensure it is out of the tlb too. */ - tlbie(va, psize, local); + tlbie(va, psize, ssize, local); return ret; } -static long native_hpte_find(unsigned long va, int psize) +static long native_hpte_find(unsigned long va, int psize, int ssize) { struct hash_pte *hptep; unsigned long hash; - unsigned long i, j; + unsigned long i; long slot; unsigned long want_v, hpte_v; - hash = hpt_hash(va, mmu_psize_defs[psize].shift); - want_v = hpte_encode_v(va, psize); + hash = hpt_hash(va, mmu_psize_defs[psize].shift, ssize); + want_v = hpte_encode_v(va, psize, ssize); - for (j = 0; j < 2; j++) { - slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; - for (i = 0; i < HPTES_PER_GROUP; i++) { - hptep = htab_address + slot; - hpte_v = hptep->v; + /* Bolted mappings are only ever in the primary group */ + slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; + for (i = 0; i < HPTES_PER_GROUP; i++) { + hptep = htab_address + slot; + hpte_v = hptep->v; - if (HPTE_V_COMPARE(hpte_v, want_v) - && (hpte_v & HPTE_V_VALID) - && ( !!(hpte_v & HPTE_V_SECONDARY) == j)) { - /* HPTE matches */ - if (j) - slot = -slot; - return slot; - } - ++slot; - } - hash = ~hash; + if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) + /* HPTE matches */ + return slot; + ++slot; } return -1; @@ -289,16 +287,16 @@ static long native_hpte_find(unsigned long va, int psize) * No need to lock here because we should be the only user. */ static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, - int psize) + int psize, int ssize) { unsigned long vsid, va; long slot; struct hash_pte *hptep; - vsid = get_kernel_vsid(ea); - va = (vsid << 28) | (ea & 0x0fffffff); + vsid = get_kernel_vsid(ea, ssize); + va = hpt_va(ea, vsid, ssize); - slot = native_hpte_find(va, psize); + slot = native_hpte_find(va, psize, ssize); if (slot == -1) panic("could not find page to bolt\n"); hptep = htab_address + slot; @@ -308,11 +306,11 @@ static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, (newpp & (HPTE_R_PP | HPTE_R_N)); /* Ensure it is out of the tlb too. */ - tlbie(va, psize, 0); + tlbie(va, psize, ssize, 0); } static void native_hpte_invalidate(unsigned long slot, unsigned long va, - int psize, int local) + int psize, int ssize, int local) { struct hash_pte *hptep = htab_address + slot; unsigned long hpte_v; @@ -323,7 +321,7 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long va, DBG_LOW(" invalidate(va=%016lx, hash: %x)\n", va, slot); - want_v = hpte_encode_v(va, psize); + want_v = hpte_encode_v(va, psize, ssize); native_lock_hpte(hptep); hpte_v = hptep->v; @@ -335,7 +333,7 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long va, hptep->v = 0; /* Invalidate the TLB */ - tlbie(va, psize, local); + tlbie(va, psize, ssize, local); local_irq_restore(flags); } @@ -345,7 +343,7 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long va, #define LP_MASK(i) ((0xFF >> (i)) << LP_SHIFT) static void hpte_decode(struct hash_pte *hpte, unsigned long slot, - int *psize, unsigned long *va) + int *psize, int *ssize, unsigned long *va) { unsigned long hpte_r = hpte->r; unsigned long hpte_v = hpte->v; @@ -401,6 +399,7 @@ static void hpte_decode(struct hash_pte *hpte, unsigned long slot, *va = avpn; *psize = size; + *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT; } /* @@ -417,7 +416,7 @@ static void native_hpte_clear(void) struct hash_pte *hptep = htab_address; unsigned long hpte_v, va; unsigned long pteg_count; - int psize; + int psize, ssize; pteg_count = htab_hash_mask + 1; @@ -443,9 +442,9 @@ static void native_hpte_clear(void) * already hold the native_tlbie_lock. */ if (hpte_v & HPTE_V_VALID) { - hpte_decode(hptep, slot, &psize, &va); + hpte_decode(hptep, slot, &psize, &ssize, &va); hptep->v = 0; - __tlbie(va, psize); + __tlbie(va, psize, ssize); } } @@ -468,6 +467,7 @@ static void native_flush_hash_range(unsigned long number, int local) real_pte_t pte; struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); unsigned long psize = batch->psize; + int ssize = batch->ssize; int i; local_irq_save(flags); @@ -477,14 +477,14 @@ static void native_flush_hash_range(unsigned long number, int local) pte = batch->pte[i]; pte_iterate_hashed_subpages(pte, psize, va, index, shift) { - hash = hpt_hash(va, shift); + hash = hpt_hash(va, shift, ssize); hidx = __rpte_to_hidx(pte, index); if (hidx & _PTEIDX_SECONDARY) hash = ~hash; slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; slot += hidx & _PTEIDX_GROUP_IX; hptep = htab_address + slot; - want_v = hpte_encode_v(va, psize); + want_v = hpte_encode_v(va, psize, ssize); native_lock_hpte(hptep); hpte_v = hptep->v; if (!HPTE_V_COMPARE(hpte_v, want_v) || @@ -504,7 +504,7 @@ static void native_flush_hash_range(unsigned long number, int local) pte_iterate_hashed_subpages(pte, psize, va, index, shift) { - __tlbiel(va, psize); + __tlbiel(va, psize, ssize); } pte_iterate_hashed_end(); } asm volatile("ptesync":::"memory"); @@ -521,7 +521,7 @@ static void native_flush_hash_range(unsigned long number, int local) pte_iterate_hashed_subpages(pte, psize, va, index, shift) { - __tlbie(va, psize); + __tlbie(va, psize, ssize); } pte_iterate_hashed_end(); } asm volatile("eieio; tlbsync; ptesync":::"memory"); diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index d525f2eba31..611ad084b7e 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -93,6 +93,8 @@ int mmu_linear_psize = MMU_PAGE_4K; int mmu_virtual_psize = MMU_PAGE_4K; int mmu_vmalloc_psize = MMU_PAGE_4K; int mmu_io_psize = MMU_PAGE_4K; +int mmu_kernel_ssize = MMU_SEGSIZE_256M; +int mmu_highuser_ssize = MMU_SEGSIZE_256M; #ifdef CONFIG_HUGETLB_PAGE int mmu_huge_psize = MMU_PAGE_16M; unsigned int HPAGE_SHIFT; @@ -145,7 +147,8 @@ struct mmu_psize_def mmu_psize_defaults_gp[] = { int htab_bolt_mapping(unsigned long vstart, unsigned long vend, - unsigned long pstart, unsigned long mode, int psize) + unsigned long pstart, unsigned long mode, + int psize, int ssize) { unsigned long vaddr, paddr; unsigned int step, shift; @@ -158,8 +161,8 @@ int htab_bolt_mapping(unsigned long vstart, unsigned long vend, for (vaddr = vstart, paddr = pstart; vaddr < vend; vaddr += step, paddr += step) { unsigned long hash, hpteg; - unsigned long vsid = get_kernel_vsid(vaddr); - unsigned long va = (vsid << 28) | (vaddr & 0x0fffffff); + unsigned long vsid = get_kernel_vsid(vaddr, ssize); + unsigned long va = hpt_va(vaddr, vsid, ssize); tmp_mode = mode; @@ -167,14 +170,14 @@ int htab_bolt_mapping(unsigned long vstart, unsigned long vend, if (!in_kernel_text(vaddr)) tmp_mode = mode | HPTE_R_N; - hash = hpt_hash(va, shift); + hash = hpt_hash(va, shift, ssize); hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP); DBG("htab_bolt_mapping: calling %p\n", ppc_md.hpte_insert); BUG_ON(!ppc_md.hpte_insert); ret = ppc_md.hpte_insert(hpteg, va, paddr, - tmp_mode, HPTE_V_BOLTED, psize); + tmp_mode, HPTE_V_BOLTED, psize, ssize); if (ret < 0) break; @@ -186,6 +189,37 @@ int htab_bolt_mapping(unsigned long vstart, unsigned long vend, return ret < 0 ? ret : 0; } +static int __init htab_dt_scan_seg_sizes(unsigned long node, + const char *uname, int depth, + void *data) +{ + char *type = of_get_flat_dt_prop(node, "device_type", NULL); + u32 *prop; + unsigned long size = 0; + + /* We are scanning "cpu" nodes only */ + if (type == NULL || strcmp(type, "cpu") != 0) + return 0; + + prop = (u32 *)of_get_flat_dt_prop(node, "ibm,processor-segment-sizes", + &size); + if (prop == NULL) + return 0; + for (; size >= 4; size -= 4, ++prop) { + if (prop[0] == 40) { + DBG("1T segment support detected\n"); + cur_cpu_spec->cpu_features |= CPU_FTR_1T_SEGMENT; + } + return 1; + } + return 0; +} + +static void __init htab_init_seg_sizes(void) +{ + of_scan_flat_dt(htab_dt_scan_seg_sizes, NULL); +} + static int __init htab_dt_scan_page_sizes(unsigned long node, const char *uname, int depth, void *data) @@ -265,7 +299,6 @@ static int __init htab_dt_scan_page_sizes(unsigned long node, return 0; } - static void __init htab_init_page_sizes(void) { int rc; @@ -398,7 +431,7 @@ void create_section_mapping(unsigned long start, unsigned long end) { BUG_ON(htab_bolt_mapping(start, end, __pa(start), _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_COHERENT | PP_RWXX, - mmu_linear_psize)); + mmu_linear_psize, mmu_kernel_ssize)); } #endif /* CONFIG_MEMORY_HOTPLUG */ @@ -449,9 +482,18 @@ void __init htab_initialize(void) DBG(" -> htab_initialize()\n"); + /* Initialize segment sizes */ + htab_init_seg_sizes(); + /* Initialize page sizes */ htab_init_page_sizes(); + if (cpu_has_feature(CPU_FTR_1T_SEGMENT)) { + mmu_kernel_ssize = MMU_SEGSIZE_1T; + mmu_highuser_ssize = MMU_SEGSIZE_1T; + printk(KERN_INFO "Using 1TB segments\n"); + } + /* * Calculate the required size of the htab. We want the number of * PTEGs to equal one half the number of real pages. @@ -523,18 +565,20 @@ void __init htab_initialize(void) if (base != dart_tablebase) BUG_ON(htab_bolt_mapping(base, dart_tablebase, __pa(base), mode_rw, - mmu_linear_psize)); + mmu_linear_psize, + mmu_kernel_ssize)); if ((base + size) > dart_table_end) BUG_ON(htab_bolt_mapping(dart_tablebase+16*MB, base + size, __pa(dart_table_end), mode_rw, - mmu_linear_psize)); + mmu_linear_psize, + mmu_kernel_ssize)); continue; } #endif /* CONFIG_U3_DART */ BUG_ON(htab_bolt_mapping(base, base + size, __pa(base), - mode_rw, mmu_linear_psize)); + mode_rw, mmu_linear_psize, mmu_kernel_ssize)); } /* @@ -553,7 +597,7 @@ void __init htab_initialize(void) BUG_ON(htab_bolt_mapping(tce_alloc_start, tce_alloc_end, __pa(tce_alloc_start), mode_rw, - mmu_linear_psize)); + mmu_linear_psize, mmu_kernel_ssize)); } htab_finish_init(); @@ -621,7 +665,7 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap) pte_t *ptep; cpumask_t tmp; int rc, user_region = 0, local = 0; - int psize; + int psize, ssize; DBG_LOW("hash_page(ea=%016lx, access=%lx, trap=%lx\n", ea, access, trap); @@ -640,20 +684,22 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap) DBG_LOW(" user region with no mm !\n"); return 1; } - vsid = get_vsid(mm->context.id, ea); #ifdef CONFIG_PPC_MM_SLICES psize = get_slice_psize(mm, ea); #else psize = mm->context.user_psize; #endif + ssize = user_segment_size(ea); + vsid = get_vsid(mm->context.id, ea, ssize); break; case VMALLOC_REGION_ID: mm = &init_mm; - vsid = get_kernel_vsid(ea); + vsid = get_kernel_vsid(ea, mmu_kernel_ssize); if (ea < VMALLOC_END) psize = mmu_vmalloc_psize; else psize = mmu_io_psize; + ssize = mmu_kernel_ssize; break; default: /* Not a valid range @@ -758,10 +804,10 @@ int hash_page(unsigned long ea, unsigned long access, unsigned long trap) #ifdef CONFIG_PPC_HAS_HASH_64K if (psize == MMU_PAGE_64K) - rc = __hash_page_64K(ea, access, vsid, ptep, trap, local); + rc = __hash_page_64K(ea, access, vsid, ptep, trap, local, ssize); else #endif /* CONFIG_PPC_HAS_HASH_64K */ - rc = __hash_page_4K(ea, access, vsid, ptep, trap, local); + rc = __hash_page_4K(ea, access, vsid, ptep, trap, local, ssize); #ifndef CONFIG_PPC_64K_PAGES DBG_LOW(" o-pte: %016lx\n", pte_val(*ptep)); @@ -783,6 +829,7 @@ void hash_preload(struct mm_struct *mm, unsigned long ea, cpumask_t mask; unsigned long flags; int local = 0; + int ssize; BUG_ON(REGION_ID(ea) != USER_REGION_ID); @@ -815,7 +862,8 @@ void hash_preload(struct mm_struct *mm, unsigned long ea, #endif /* CONFIG_PPC_64K_PAGES */ /* Get VSID */ - vsid = get_vsid(mm->context.id, ea); + ssize = user_segment_size(ea); + vsid = get_vsid(mm->context.id, ea, ssize); /* Hash doesn't like irqs */ local_irq_save(flags); @@ -828,28 +876,29 @@ void hash_preload(struct mm_struct *mm, unsigned long ea, /* Hash it in */ #ifdef CONFIG_PPC_HAS_HASH_64K if (mm->context.user_psize == MMU_PAGE_64K) - __hash_page_64K(ea, access, vsid, ptep, trap, local); + __hash_page_64K(ea, access, vsid, ptep, trap, local, ssize); else #endif /* CONFIG_PPC_HAS_HASH_64K */ - __hash_page_4K(ea, access, vsid, ptep, trap, local); + __hash_page_4K(ea, access, vsid, ptep, trap, local, ssize); local_irq_restore(flags); } -void flush_hash_page(unsigned long va, real_pte_t pte, int psize, int local) +void flush_hash_page(unsigned long va, real_pte_t pte, int psize, int ssize, + int local) { unsigned long hash, index, shift, hidx, slot; DBG_LOW("flush_hash_page(va=%016x)\n", va); pte_iterate_hashed_subpages(pte, psize, va, index, shift) { - hash = hpt_hash(va, shift); + hash = hpt_hash(va, shift, ssize); hidx = __rpte_to_hidx(pte, index); if (hidx & _PTEIDX_SECONDARY) hash = ~hash; slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; slot += hidx & _PTEIDX_GROUP_IX; DBG_LOW(" sub %d: hash=%x, hidx=%x\n", index, slot, hidx); - ppc_md.hpte_invalidate(slot, va, psize, local); + ppc_md.hpte_invalidate(slot, va, psize, ssize, local); } pte_iterate_hashed_end(); } @@ -864,7 +913,7 @@ void flush_hash_range(unsigned long number, int local) for (i = 0; i < number; i++) flush_hash_page(batch->vaddr[i], batch->pte[i], - batch->psize, local); + batch->psize, batch->ssize, local); } } @@ -890,17 +939,19 @@ void low_hash_fault(struct pt_regs *regs, unsigned long address) #ifdef CONFIG_DEBUG_PAGEALLOC static void kernel_map_linear_page(unsigned long vaddr, unsigned long lmi) { - unsigned long hash, hpteg, vsid = get_kernel_vsid(vaddr); - unsigned long va = (vsid << 28) | (vaddr & 0x0fffffff); + unsigned long hash, hpteg; + unsigned long vsid = get_kernel_vsid(vaddr, mmu_kernel_ssize); + unsigned long va = hpt_va(vaddr, vsid, mmu_kernel_ssize); unsigned long mode = _PAGE_ACCESSED | _PAGE_DIRTY | _PAGE_COHERENT | PP_RWXX | HPTE_R_N; int ret; - hash = hpt_hash(va, PAGE_SHIFT); + hash = hpt_hash(va, PAGE_SHIFT, mmu_kernel_ssize); hpteg = ((hash & htab_hash_mask) * HPTES_PER_GROUP); ret = ppc_md.hpte_insert(hpteg, va, __pa(vaddr), - mode, HPTE_V_BOLTED, mmu_linear_psize); + mode, HPTE_V_BOLTED, + mmu_linear_psize, mmu_kernel_ssize); BUG_ON (ret < 0); spin_lock(&linear_map_hash_lock); BUG_ON(linear_map_hash_slots[lmi] & 0x80); @@ -910,10 +961,11 @@ static void kernel_map_linear_page(unsigned long vaddr, unsigned long lmi) static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi) { - unsigned long hash, hidx, slot, vsid = get_kernel_vsid(vaddr); - unsigned long va = (vsid << 28) | (vaddr & 0x0fffffff); + unsigned long hash, hidx, slot; + unsigned long vsid = get_kernel_vsid(vaddr, mmu_kernel_ssize); + unsigned long va = hpt_va(vaddr, vsid, mmu_kernel_ssize); - hash = hpt_hash(va, PAGE_SHIFT); + hash = hpt_hash(va, PAGE_SHIFT, mmu_kernel_ssize); spin_lock(&linear_map_hash_lock); BUG_ON(!(linear_map_hash_slots[lmi] & 0x80)); hidx = linear_map_hash_slots[lmi] & 0x7f; @@ -923,7 +975,7 @@ static void kernel_unmap_linear_page(unsigned long vaddr, unsigned long lmi) hash = ~hash; slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; slot += hidx & _PTEIDX_GROUP_IX; - ppc_md.hpte_invalidate(slot, va, mmu_linear_psize, 0); + ppc_md.hpte_invalidate(slot, va, mmu_linear_psize, mmu_kernel_ssize, 0); } void kernel_map_pages(struct page *page, int numpages, int enable) diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index ba5f12a6046..08f0d9ff771 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -403,11 +403,12 @@ int hash_huge_page(struct mm_struct *mm, unsigned long access, unsigned long va, rflags, pa; long slot; int err = 1; + int ssize = user_segment_size(ea); ptep = huge_pte_offset(mm, ea); /* Search the Linux page table for a match with va */ - va = (vsid << 28) | (ea & 0x0fffffff); + va = hpt_va(ea, vsid, ssize); /* * If no pte found or not present, send the problem up to @@ -458,19 +459,19 @@ int hash_huge_page(struct mm_struct *mm, unsigned long access, /* There MIGHT be an HPTE for this pte */ unsigned long hash, slot; - hash = hpt_hash(va, HPAGE_SHIFT); + hash = hpt_hash(va, HPAGE_SHIFT, ssize); if (old_pte & _PAGE_F_SECOND) hash = ~hash; slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; slot += (old_pte & _PAGE_F_GIX) >> 12; if (ppc_md.hpte_updatepp(slot, rflags, va, mmu_huge_psize, - local) == -1) + ssize, local) == -1) old_pte &= ~_PAGE_HPTEFLAGS; } if (likely(!(old_pte & _PAGE_HASHPTE))) { - unsigned long hash = hpt_hash(va, HPAGE_SHIFT); + unsigned long hash = hpt_hash(va, HPAGE_SHIFT, ssize); unsigned long hpte_group; pa = pte_pfn(__pte(old_pte)) << PAGE_SHIFT; @@ -489,7 +490,7 @@ repeat: /* Insert into the hash table, primary slot */ slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, 0, - mmu_huge_psize); + mmu_huge_psize, ssize); /* Primary is full, try the secondary */ if (unlikely(slot == -1)) { @@ -497,7 +498,7 @@ repeat: HPTES_PER_GROUP) & ~0x7UL; slot = ppc_md.hpte_insert(hpte_group, va, pa, rflags, HPTE_V_SECONDARY, - mmu_huge_psize); + mmu_huge_psize, ssize); if (slot == -1) { if (mftb() & 0x1) hpte_group = ((hash & htab_hash_mask) * diff --git a/arch/powerpc/mm/pgtable_64.c b/arch/powerpc/mm/pgtable_64.c index 60fd52cd270..3ef0ad2f9ca 100644 --- a/arch/powerpc/mm/pgtable_64.c +++ b/arch/powerpc/mm/pgtable_64.c @@ -87,8 +87,8 @@ static int map_io_page(unsigned long ea, unsigned long pa, int flags) * entry in the hardware page table. * */ - if (htab_bolt_mapping(ea, (unsigned long)ea + PAGE_SIZE, - pa, flags, mmu_io_psize)) { + if (htab_bolt_mapping(ea, ea + PAGE_SIZE, pa, flags, + mmu_io_psize, mmu_kernel_ssize)) { printk(KERN_ERR "Failed to do bolted mapping IO " "memory at %016lx !\n", pa); return -ENOMEM; diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c index 4bee1cfa9de..6c164cec9d2 100644 --- a/arch/powerpc/mm/slb.c +++ b/arch/powerpc/mm/slb.c @@ -43,17 +43,26 @@ static void slb_allocate(unsigned long ea) slb_allocate_realmode(ea); } -static inline unsigned long mk_esid_data(unsigned long ea, unsigned long slot) +static inline unsigned long mk_esid_data(unsigned long ea, int ssize, + unsigned long slot) { - return (ea & ESID_MASK) | SLB_ESID_V | slot; + unsigned long mask; + + mask = (ssize == MMU_SEGSIZE_256M)? ESID_MASK: ESID_MASK_1T; + return (ea & mask) | SLB_ESID_V | slot; } -static inline unsigned long mk_vsid_data(unsigned long ea, unsigned long flags) +#define slb_vsid_shift(ssize) \ + ((ssize) == MMU_SEGSIZE_256M? SLB_VSID_SHIFT: SLB_VSID_SHIFT_1T) + +static inline unsigned long mk_vsid_data(unsigned long ea, int ssize, + unsigned long flags) { - return (get_kernel_vsid(ea) << SLB_VSID_SHIFT) | flags; + return (get_kernel_vsid(ea, ssize) << slb_vsid_shift(ssize)) | flags | + ((unsigned long) ssize << SLB_VSID_SSIZE_SHIFT); } -static inline void slb_shadow_update(unsigned long ea, +static inline void slb_shadow_update(unsigned long ea, int ssize, unsigned long flags, unsigned long entry) { @@ -63,8 +72,8 @@ static inline void slb_shadow_update(unsigned long ea, * we only update the current CPU's SLB shadow buffer. */ get_slb_shadow()->save_area[entry].esid = 0; - get_slb_shadow()->save_area[entry].vsid = mk_vsid_data(ea, flags); - get_slb_shadow()->save_area[entry].esid = mk_esid_data(ea, entry); + get_slb_shadow()->save_area[entry].vsid = mk_vsid_data(ea, ssize, flags); + get_slb_shadow()->save_area[entry].esid = mk_esid_data(ea, ssize, entry); } static inline void slb_shadow_clear(unsigned long entry) @@ -72,7 +81,8 @@ static inline void slb_shadow_clear(unsigned long entry) get_slb_shadow()->save_area[entry].esid = 0; } -static inline void create_shadowed_slbe(unsigned long ea, unsigned long flags, +static inline void create_shadowed_slbe(unsigned long ea, int ssize, + unsigned long flags, unsigned long entry) { /* @@ -80,11 +90,11 @@ static inline void create_shadowed_slbe(unsigned long ea, unsigned long flags, * we don't get a stale entry here if we get preempted by PHYP * between these two statements. */ - slb_shadow_update(ea, flags, entry); + slb_shadow_update(ea, ssize, flags, entry); asm volatile("slbmte %0,%1" : - : "r" (mk_vsid_data(ea, flags)), - "r" (mk_esid_data(ea, entry)) + : "r" (mk_vsid_data(ea, ssize, flags)), + "r" (mk_esid_data(ea, ssize, entry)) : "memory" ); } @@ -93,7 +103,7 @@ void slb_flush_and_rebolt(void) /* If you change this make sure you change SLB_NUM_BOLTED * appropriately too. */ unsigned long linear_llp, vmalloc_llp, lflags, vflags; - unsigned long ksp_esid_data; + unsigned long ksp_esid_data, ksp_vsid_data; WARN_ON(!irqs_disabled()); @@ -102,13 +112,15 @@ void slb_flush_and_rebolt(void) lflags = SLB_VSID_KERNEL | linear_llp; vflags = SLB_VSID_KERNEL | vmalloc_llp; - ksp_esid_data = mk_esid_data(get_paca()->kstack, 2); - if ((ksp_esid_data & ESID_MASK) == PAGE_OFFSET) { + ksp_esid_data = mk_esid_data(get_paca()->kstack, mmu_kernel_ssize, 2); + if ((ksp_esid_data & ~0xfffffffUL) <= PAGE_OFFSET) { ksp_esid_data &= ~SLB_ESID_V; + ksp_vsid_data = 0; slb_shadow_clear(2); } else { /* Update stack entry; others don't change */ - slb_shadow_update(get_paca()->kstack, lflags, 2); + slb_shadow_update(get_paca()->kstack, mmu_kernel_ssize, lflags, 2); + ksp_vsid_data = get_slb_shadow()->save_area[2].vsid; } /* We need to do this all in asm, so we're sure we don't touch @@ -120,9 +132,9 @@ void slb_flush_and_rebolt(void) /* Slot 2 - kernel stack */ "slbmte %2,%3\n" "isync" - :: "r"(mk_vsid_data(VMALLOC_START, vflags)), - "r"(mk_esid_data(VMALLOC_START, 1)), - "r"(mk_vsid_data(ksp_esid_data, lflags)), + :: "r"(mk_vsid_data(VMALLOC_START, mmu_kernel_ssize, vflags)), + "r"(mk_esid_data(VMALLOC_START, mmu_kernel_ssize, 1)), + "r"(ksp_vsid_data), "r"(ksp_esid_data) : "memory"); } @@ -132,7 +144,7 @@ void slb_vmalloc_update(void) unsigned long vflags; vflags = SLB_VSID_KERNEL | mmu_psize_defs[mmu_vmalloc_psize].sllp; - slb_shadow_update(VMALLOC_START, vflags, 1); + slb_shadow_update(VMALLOC_START, mmu_kernel_ssize, vflags, 1); slb_flush_and_rebolt(); } @@ -140,7 +152,7 @@ void slb_vmalloc_update(void) void switch_slb(struct task_struct *tsk, struct mm_struct *mm) { unsigned long offset = get_paca()->slb_cache_ptr; - unsigned long esid_data = 0; + unsigned long slbie_data = 0; unsigned long pc = KSTK_EIP(tsk); unsigned long stack = KSTK_ESP(tsk); unsigned long unmapped_base; @@ -149,9 +161,12 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm) int i; asm volatile("isync" : : : "memory"); for (i = 0; i < offset; i++) { - esid_data = ((unsigned long)get_paca()->slb_cache[i] - << SID_SHIFT) | SLBIE_C; - asm volatile("slbie %0" : : "r" (esid_data)); + slbie_data = (unsigned long)get_paca()->slb_cache[i] + << SID_SHIFT; /* EA */ + slbie_data |= user_segment_size(slbie_data) + << SLBIE_SSIZE_SHIFT; + slbie_data |= SLBIE_C; /* C set for user addresses */ + asm volatile("slbie %0" : : "r" (slbie_data)); } asm volatile("isync" : : : "memory"); } else { @@ -160,7 +175,7 @@ void switch_slb(struct task_struct *tsk, struct mm_struct *mm) /* Workaround POWER5 < DD2.1 issue */ if (offset == 1 || offset > SLB_CACHE_ENTRIES) - asm volatile("slbie %0" : : "r" (esid_data)); + asm volatile("slbie %0" : : "r" (slbie_data)); get_paca()->slb_cache_ptr = 0; get_paca()->context = mm->context; @@ -243,9 +258,9 @@ void slb_initialize(void) asm volatile("isync":::"memory"); asm volatile("slbmte %0,%0"::"r" (0) : "memory"); asm volatile("isync; slbia; isync":::"memory"); - create_shadowed_slbe(PAGE_OFFSET, lflags, 0); + create_shadowed_slbe(PAGE_OFFSET, mmu_kernel_ssize, lflags, 0); - create_shadowed_slbe(VMALLOC_START, vflags, 1); + create_shadowed_slbe(VMALLOC_START, mmu_kernel_ssize, vflags, 1); /* We don't bolt the stack for the time being - we're in boot, * so the stack is in the bolted segment. By the time it goes diff --git a/arch/powerpc/mm/slb_low.S b/arch/powerpc/mm/slb_low.S index cd1a93d4948..1328a81a84a 100644 --- a/arch/powerpc/mm/slb_low.S +++ b/arch/powerpc/mm/slb_low.S @@ -57,7 +57,10 @@ _GLOBAL(slb_allocate_realmode) */ _GLOBAL(slb_miss_kernel_load_linear) li r11,0 +BEGIN_FTR_SECTION b slb_finish_load +END_FTR_SECTION_IFCLR(CPU_FTR_1T_SEGMENT) + b slb_finish_load_1T 1: /* vmalloc/ioremap mapping encoding bits, the "li" instructions below * will be patched by the kernel at boot @@ -68,13 +71,16 @@ BEGIN_FTR_SECTION cmpldi r11,(VMALLOC_SIZE >> 28) - 1 bgt 5f lhz r11,PACAVMALLOCSLLP(r13) - b slb_finish_load + b 6f 5: END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE) _GLOBAL(slb_miss_kernel_load_io) li r11,0 +6: +BEGIN_FTR_SECTION b slb_finish_load - +END_FTR_SECTION_IFCLR(CPU_FTR_1T_SEGMENT) + b slb_finish_load_1T 0: /* user address: proto-VSID = context << 15 | ESID. First check * if the address is within the boundaries of the user region @@ -122,7 +128,13 @@ _GLOBAL(slb_miss_kernel_load_io) #endif /* CONFIG_PPC_MM_SLICES */ ld r9,PACACONTEXTID(r13) +BEGIN_FTR_SECTION + cmpldi r10,0x1000 +END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT) rldimi r10,r9,USER_ESID_BITS,0 +BEGIN_FTR_SECTION + bge slb_finish_load_1T +END_FTR_SECTION_IFSET(CPU_FTR_1T_SEGMENT) b slb_finish_load 8: /* invalid EA */ @@ -188,7 +200,7 @@ _GLOBAL(slb_allocate_user) * r3 = EA, r10 = proto-VSID, r11 = flags, clobbers r9, cr7 = <> PAGE_OFFSET */ slb_finish_load: - ASM_VSID_SCRAMBLE(r10,r9) + ASM_VSID_SCRAMBLE(r10,r9,256M) rldimi r11,r10,SLB_VSID_SHIFT,16 /* combine VSID and flags */ /* r3 = EA, r11 = VSID data */ @@ -213,7 +225,7 @@ BEGIN_FW_FTR_SECTION END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES) #endif /* CONFIG_PPC_ISERIES */ - ld r10,PACASTABRR(r13) +7: ld r10,PACASTABRR(r13) addi r10,r10,1 /* use a cpu feature mask if we ever change our slb size */ cmpldi r10,SLB_NUM_ENTRIES @@ -259,3 +271,20 @@ END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES) crclr 4*cr0+eq /* set result to "success" */ blr +/* + * Finish loading of a 1T SLB entry (for the kernel linear mapping) and return. + * We assume legacy iSeries will never have 1T segments. + * + * r3 = EA, r10 = proto-VSID, r11 = flags, clobbers r9 + */ +slb_finish_load_1T: + srdi r10,r10,40-28 /* get 1T ESID */ + ASM_VSID_SCRAMBLE(r10,r9,1T) + rldimi r11,r10,SLB_VSID_SHIFT_1T,16 /* combine VSID and flags */ + li r10,MMU_SEGSIZE_1T + rldimi r11,r10,SLB_VSID_SSIZE_SHIFT,0 /* insert segment size */ + + /* r3 = EA, r11 = VSID data */ + clrrdi r3,r3,SID_SHIFT_1T /* clear out non-ESID bits */ + b 7b + diff --git a/arch/powerpc/mm/stab.c b/arch/powerpc/mm/stab.c index 28492bbdee8..9e85bda7621 100644 --- a/arch/powerpc/mm/stab.c +++ b/arch/powerpc/mm/stab.c @@ -122,12 +122,12 @@ static int __ste_allocate(unsigned long ea, struct mm_struct *mm) /* Kernel or user address? */ if (is_kernel_addr(ea)) { - vsid = get_kernel_vsid(ea); + vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M); } else { if ((ea >= TASK_SIZE_USER64) || (! mm)) return 1; - vsid = get_vsid(mm->context.id, ea); + vsid = get_vsid(mm->context.id, ea, MMU_SEGSIZE_256M); } stab_entry = make_ste(get_paca()->stab_addr, GET_ESID(ea), vsid); @@ -261,7 +261,7 @@ void __init stabs_alloc(void) */ void stab_initialize(unsigned long stab) { - unsigned long vsid = get_kernel_vsid(PAGE_OFFSET); + unsigned long vsid = get_kernel_vsid(PAGE_OFFSET, MMU_SEGSIZE_256M); unsigned long stabreal; asm volatile("isync; slbia; isync":::"memory"); diff --git a/arch/powerpc/mm/tlb_64.c b/arch/powerpc/mm/tlb_64.c index cbd34fc813e..eafbca52bff 100644 --- a/arch/powerpc/mm/tlb_64.c +++ b/arch/powerpc/mm/tlb_64.c @@ -132,6 +132,7 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr, struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); unsigned long vsid, vaddr; unsigned int psize; + int ssize; real_pte_t rpte; int i; @@ -161,11 +162,14 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr, /* Build full vaddr */ if (!is_kernel_addr(addr)) { - vsid = get_vsid(mm->context.id, addr); + ssize = user_segment_size(addr); + vsid = get_vsid(mm->context.id, addr, ssize); WARN_ON(vsid == 0); - } else - vsid = get_kernel_vsid(addr); - vaddr = (vsid << 28 ) | (addr & 0x0fffffff); + } else { + vsid = get_kernel_vsid(addr, mmu_kernel_ssize); + ssize = mmu_kernel_ssize; + } + vaddr = hpt_va(addr, vsid, ssize); rpte = __real_pte(__pte(pte), ptep); /* @@ -175,7 +179,7 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr, * and decide to use local invalidates instead... */ if (!batch->active) { - flush_hash_page(vaddr, rpte, psize, 0); + flush_hash_page(vaddr, rpte, psize, ssize, 0); return; } @@ -189,13 +193,15 @@ void hpte_need_flush(struct mm_struct *mm, unsigned long addr, * We also need to ensure only one page size is present in a given * batch */ - if (i != 0 && (mm != batch->mm || batch->psize != psize)) { + if (i != 0 && (mm != batch->mm || batch->psize != psize || + batch->ssize != ssize)) { __flush_tlb_pending(batch); i = 0; } if (i == 0) { batch->mm = mm; batch->psize = psize; + batch->ssize = ssize; } batch->pte[i] = rpte; batch->vaddr[i] = vaddr; @@ -222,7 +228,7 @@ void __flush_tlb_pending(struct ppc64_tlb_batch *batch) local = 1; if (i == 1) flush_hash_page(batch->vaddr[0], batch->pte[0], - batch->psize, local); + batch->psize, batch->ssize, local); else flush_hash_range(i, local); batch->index = 0; diff --git a/arch/powerpc/platforms/cell/spu_base.c b/arch/powerpc/platforms/cell/spu_base.c index b5a21177bb3..c83c3e3f517 100644 --- a/arch/powerpc/platforms/cell/spu_base.c +++ b/arch/powerpc/platforms/cell/spu_base.c @@ -168,7 +168,7 @@ static int __spu_trap_data_seg(struct spu *spu, unsigned long ea) #else psize = mm->context.user_psize; #endif - vsid = (get_vsid(mm->context.id, ea) << SLB_VSID_SHIFT) | + vsid = (get_vsid(mm->context.id, ea, MMU_SEGSIZE_256M) << SLB_VSID_SHIFT) | SLB_VSID_USER; break; case VMALLOC_REGION_ID: @@ -176,12 +176,12 @@ static int __spu_trap_data_seg(struct spu *spu, unsigned long ea) psize = mmu_vmalloc_psize; else psize = mmu_io_psize; - vsid = (get_kernel_vsid(ea) << SLB_VSID_SHIFT) | + vsid = (get_kernel_vsid(ea, MMU_SEGSIZE_256M) << SLB_VSID_SHIFT) | SLB_VSID_KERNEL; break; case KERNEL_REGION_ID: psize = mmu_linear_psize; - vsid = (get_kernel_vsid(ea) << SLB_VSID_SHIFT) | + vsid = (get_kernel_vsid(ea, MMU_SEGSIZE_256M) << SLB_VSID_SHIFT) | SLB_VSID_KERNEL; break; default: diff --git a/arch/powerpc/platforms/cell/spufs/switch.c b/arch/powerpc/platforms/cell/spufs/switch.c index de7e5ee451d..3d64c81cc6e 100644 --- a/arch/powerpc/platforms/cell/spufs/switch.c +++ b/arch/powerpc/platforms/cell/spufs/switch.c @@ -699,7 +699,7 @@ static inline void get_kernel_slb(u64 ea, u64 slb[2]) llp = mmu_psize_defs[mmu_linear_psize].sllp; else llp = mmu_psize_defs[mmu_virtual_psize].sllp; - slb[0] = (get_kernel_vsid(ea) << SLB_VSID_SHIFT) | + slb[0] = (get_kernel_vsid(ea, MMU_SEGSIZE_256M) << SLB_VSID_SHIFT) | SLB_VSID_KERNEL | llp; slb[1] = (ea & ESID_MASK) | SLB_ESID_V; } diff --git a/arch/powerpc/platforms/celleb/htab.c b/arch/powerpc/platforms/celleb/htab.c index 5e75c77ea8f..fbf27c74ebd 100644 --- a/arch/powerpc/platforms/celleb/htab.c +++ b/arch/powerpc/platforms/celleb/htab.c @@ -90,7 +90,7 @@ static inline unsigned int beat_read_mask(unsigned hpte_group) static long beat_lpar_hpte_insert(unsigned long hpte_group, unsigned long va, unsigned long pa, unsigned long rflags, unsigned long vflags, - int psize) + int psize, int ssize) { unsigned long lpar_rc; unsigned long slot; @@ -105,7 +105,8 @@ static long beat_lpar_hpte_insert(unsigned long hpte_group, "rflags=%lx, vflags=%lx, psize=%d)\n", hpte_group, va, pa, rflags, vflags, psize); - hpte_v = hpte_encode_v(va, psize) | vflags | HPTE_V_VALID; + hpte_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M) | + vflags | HPTE_V_VALID; hpte_r = hpte_encode_r(pa, psize) | rflags; if (!(vflags & HPTE_V_BOLTED)) @@ -184,12 +185,12 @@ static void beat_lpar_hptab_clear(void) static long beat_lpar_hpte_updatepp(unsigned long slot, unsigned long newpp, unsigned long va, - int psize, int local) + int psize, int ssize, int local) { unsigned long lpar_rc; unsigned long dummy0, dummy1, want_v; - want_v = hpte_encode_v(va, psize); + want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M); DBG_LOW(" update: " "avpnv=%016lx, slot=%016lx, psize: %d, newpp %016lx ... ", @@ -225,8 +226,8 @@ static long beat_lpar_hpte_find(unsigned long va, int psize) long slot; unsigned long want_v, hpte_v; - hash = hpt_hash(va, mmu_psize_defs[psize].shift); - want_v = hpte_encode_v(va, psize); + hash = hpt_hash(va, mmu_psize_defs[psize].shift, MMU_SEGSIZE_256M); + want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M); for (j = 0; j < 2; j++) { slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; @@ -251,11 +252,11 @@ static long beat_lpar_hpte_find(unsigned long va, int psize) static void beat_lpar_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, - int psize) + int psize, int ssize) { unsigned long lpar_rc, slot, vsid, va, dummy0, dummy1; - vsid = get_kernel_vsid(ea); + vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M); va = (vsid << 28) | (ea & 0x0fffffff); spin_lock(&beat_htab_lock); @@ -270,7 +271,7 @@ static void beat_lpar_hpte_updateboltedpp(unsigned long newpp, } static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long va, - int psize, int local) + int psize, int ssize, int local) { unsigned long want_v; unsigned long lpar_rc; @@ -279,7 +280,7 @@ static void beat_lpar_hpte_invalidate(unsigned long slot, unsigned long va, DBG_LOW(" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n", slot, va, psize, local); - want_v = hpte_encode_v(va, psize); + want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M); spin_lock_irqsave(&beat_htab_lock, flags); dummy1 = beat_lpar_hpte_getword0(slot); @@ -310,7 +311,7 @@ void __init hpte_init_beat(void) static long beat_lpar_hpte_insert_v3(unsigned long hpte_group, unsigned long va, unsigned long pa, unsigned long rflags, unsigned long vflags, - int psize) + int psize, int ssize) { unsigned long lpar_rc; unsigned long slot; @@ -325,7 +326,8 @@ static long beat_lpar_hpte_insert_v3(unsigned long hpte_group, "rflags=%lx, vflags=%lx, psize=%d)\n", hpte_group, va, pa, rflags, vflags, psize); - hpte_v = hpte_encode_v(va, psize) | vflags | HPTE_V_VALID; + hpte_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M) | + vflags | HPTE_V_VALID; hpte_r = hpte_encode_r(pa, psize) | rflags; if (!(vflags & HPTE_V_BOLTED)) @@ -363,13 +365,13 @@ static long beat_lpar_hpte_insert_v3(unsigned long hpte_group, static long beat_lpar_hpte_updatepp_v3(unsigned long slot, unsigned long newpp, unsigned long va, - int psize, int local) + int psize, int ssize, int local) { unsigned long lpar_rc; unsigned long want_v; unsigned long pss; - want_v = hpte_encode_v(va, psize); + want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M); pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc; DBG_LOW(" update: " @@ -391,7 +393,7 @@ static long beat_lpar_hpte_updatepp_v3(unsigned long slot, } static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long va, - int psize, int local) + int psize, int ssize, int local) { unsigned long want_v; unsigned long lpar_rc; @@ -399,7 +401,7 @@ static void beat_lpar_hpte_invalidate_v3(unsigned long slot, unsigned long va, DBG_LOW(" inval : slot=%lx, va=%016lx, psize: %d, local: %d\n", slot, va, psize, local); - want_v = hpte_encode_v(va, psize); + want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M); pss = (psize == MMU_PAGE_4K) ? -1UL : mmu_psize_defs[psize].penc; lpar_rc = beat_invalidate_htab_entry3(0, slot, want_v, pss); diff --git a/arch/powerpc/platforms/iseries/htab.c b/arch/powerpc/platforms/iseries/htab.c index b4e2c7a038e..15a7097e5dd 100644 --- a/arch/powerpc/platforms/iseries/htab.c +++ b/arch/powerpc/platforms/iseries/htab.c @@ -86,7 +86,8 @@ long iSeries_hpte_insert(unsigned long hpte_group, unsigned long va, } - lhpte.v = hpte_encode_v(va, MMU_PAGE_4K) | vflags | HPTE_V_VALID; + lhpte.v = hpte_encode_v(va, MMU_PAGE_4K, MMU_SEGSIZE_256M) | + vflags | HPTE_V_VALID; lhpte.r = hpte_encode_r(phys_to_abs(pa), MMU_PAGE_4K) | rflags; /* Now fill in the actual HPTE */ @@ -142,7 +143,7 @@ static long iSeries_hpte_remove(unsigned long hpte_group) * bits 61..63 : PP2,PP1,PP0 */ static long iSeries_hpte_updatepp(unsigned long slot, unsigned long newpp, - unsigned long va, int psize, int local) + unsigned long va, int psize, int ssize, int local) { struct hash_pte hpte; unsigned long want_v; @@ -150,7 +151,7 @@ static long iSeries_hpte_updatepp(unsigned long slot, unsigned long newpp, iSeries_hlock(slot); HvCallHpt_get(&hpte, slot); - want_v = hpte_encode_v(va, MMU_PAGE_4K); + want_v = hpte_encode_v(va, MMU_PAGE_4K, MMU_SEGSIZE_256M); if (HPTE_V_COMPARE(hpte.v, want_v) && (hpte.v & HPTE_V_VALID)) { /* @@ -205,14 +206,14 @@ static long iSeries_hpte_find(unsigned long vpn) * No need to lock here because we should be the only user. */ static void iSeries_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, - int psize) + int psize, int ssize) { unsigned long vsid,va,vpn; long slot; BUG_ON(psize != MMU_PAGE_4K); - vsid = get_kernel_vsid(ea); + vsid = get_kernel_vsid(ea, MMU_SEGSIZE_256M); va = (vsid << 28) | (ea & 0x0fffffff); vpn = va >> HW_PAGE_SHIFT; slot = iSeries_hpte_find(vpn); @@ -222,7 +223,7 @@ static void iSeries_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, } static void iSeries_hpte_invalidate(unsigned long slot, unsigned long va, - int psize, int local) + int psize, int ssize, int local) { unsigned long hpte_v; unsigned long avpn = va >> 23; diff --git a/arch/powerpc/platforms/ps3/htab.c b/arch/powerpc/platforms/ps3/htab.c index 5d2e176a1b1..7382f195c4f 100644 --- a/arch/powerpc/platforms/ps3/htab.c +++ b/arch/powerpc/platforms/ps3/htab.c @@ -60,7 +60,8 @@ static void _debug_dump_hpte(unsigned long pa, unsigned long va, } static long ps3_hpte_insert(unsigned long hpte_group, unsigned long va, - unsigned long pa, unsigned long rflags, unsigned long vflags, int psize) + unsigned long pa, unsigned long rflags, unsigned long vflags, + int psize, int ssize) { unsigned long slot; struct hash_pte lhpte; @@ -72,7 +73,8 @@ static long ps3_hpte_insert(unsigned long hpte_group, unsigned long va, vflags &= ~HPTE_V_SECONDARY; /* this bit is ignored */ - lhpte.v = hpte_encode_v(va, psize) | vflags | HPTE_V_VALID; + lhpte.v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M) | + vflags | HPTE_V_VALID; lhpte.r = hpte_encode_r(ps3_mm_phys_to_lpar(pa), psize) | rflags; p_pteg = hpte_group / HPTES_PER_GROUP; @@ -167,14 +169,14 @@ static long ps3_hpte_remove(unsigned long hpte_group) } static long ps3_hpte_updatepp(unsigned long slot, unsigned long newpp, - unsigned long va, int psize, int local) + unsigned long va, int psize, int ssize, int local) { unsigned long flags; unsigned long result; unsigned long pteg, bit; unsigned long hpte_v, want_v; - want_v = hpte_encode_v(va, psize); + want_v = hpte_encode_v(va, psize, MMU_SEGSIZE_256M); spin_lock_irqsave(&ps3_bolttab_lock, flags); @@ -205,13 +207,13 @@ static long ps3_hpte_updatepp(unsigned long slot, unsigned long newpp, } static void ps3_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, - int psize) + int psize, int ssize) { panic("ps3_hpte_updateboltedpp() not implemented"); } static void ps3_hpte_invalidate(unsigned long slot, unsigned long va, - int psize, int local) + int psize, int ssize, int local) { unsigned long flags; unsigned long result; diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c index ea327ca345c..9a455d46379 100644 --- a/arch/powerpc/platforms/pseries/lpar.c +++ b/arch/powerpc/platforms/pseries/lpar.c @@ -284,7 +284,7 @@ void vpa_init(int cpu) static long pSeries_lpar_hpte_insert(unsigned long hpte_group, unsigned long va, unsigned long pa, unsigned long rflags, unsigned long vflags, - int psize) + int psize, int ssize) { unsigned long lpar_rc; unsigned long flags; @@ -296,7 +296,7 @@ static long pSeries_lpar_hpte_insert(unsigned long hpte_group, "rflags=%lx, vflags=%lx, psize=%d)\n", hpte_group, va, pa, rflags, vflags, psize); - hpte_v = hpte_encode_v(va, psize) | vflags | HPTE_V_VALID; + hpte_v = hpte_encode_v(va, psize, ssize) | vflags | HPTE_V_VALID; hpte_r = hpte_encode_r(pa, psize) | rflags; if (!(vflags & HPTE_V_BOLTED)) @@ -391,6 +391,22 @@ static void pSeries_lpar_hptab_clear(void) } } +/* + * This computes the AVPN and B fields of the first dword of a HPTE, + * for use when we want to match an existing PTE. The bottom 7 bits + * of the returned value are zero. + */ +static inline unsigned long hpte_encode_avpn(unsigned long va, int psize, + int ssize) +{ + unsigned long v; + + v = (va >> 23) & ~(mmu_psize_defs[psize].avpnm); + v <<= HPTE_V_AVPN_SHIFT; + v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT; + return v; +} + /* * NOTE: for updatepp ops we are fortunate that the linux "newpp" bits and * the low 3 bits of flags happen to line up. So no transform is needed. @@ -400,18 +416,18 @@ static void pSeries_lpar_hptab_clear(void) static long pSeries_lpar_hpte_updatepp(unsigned long slot, unsigned long newpp, unsigned long va, - int psize, int local) + int psize, int ssize, int local) { unsigned long lpar_rc; unsigned long flags = (newpp & 7) | H_AVPN; unsigned long want_v; - want_v = hpte_encode_v(va, psize); + want_v = hpte_encode_avpn(va, psize, ssize); DBG_LOW(" update: avpnv=%016lx, hash=%016lx, f=%x, psize: %d ... ", - want_v & HPTE_V_AVPN, slot, flags, psize); + want_v, slot, flags, psize); - lpar_rc = plpar_pte_protect(flags, slot, want_v & HPTE_V_AVPN); + lpar_rc = plpar_pte_protect(flags, slot, want_v); if (lpar_rc == H_NOT_FOUND) { DBG_LOW("not found !\n"); @@ -444,32 +460,25 @@ static unsigned long pSeries_lpar_hpte_getword0(unsigned long slot) return dword0; } -static long pSeries_lpar_hpte_find(unsigned long va, int psize) +static long pSeries_lpar_hpte_find(unsigned long va, int psize, int ssize) { unsigned long hash; - unsigned long i, j; + unsigned long i; long slot; unsigned long want_v, hpte_v; - hash = hpt_hash(va, mmu_psize_defs[psize].shift); - want_v = hpte_encode_v(va, psize); - - for (j = 0; j < 2; j++) { - slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; - for (i = 0; i < HPTES_PER_GROUP; i++) { - hpte_v = pSeries_lpar_hpte_getword0(slot); - - if (HPTE_V_COMPARE(hpte_v, want_v) - && (hpte_v & HPTE_V_VALID) - && (!!(hpte_v & HPTE_V_SECONDARY) == j)) { - /* HPTE matches */ - if (j) - slot = -slot; - return slot; - } - ++slot; - } - hash = ~hash; + hash = hpt_hash(va, mmu_psize_defs[psize].shift, ssize); + want_v = hpte_encode_avpn(va, psize, ssize); + + /* Bolted entries are always in the primary group */ + slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; + for (i = 0; i < HPTES_PER_GROUP; i++) { + hpte_v = pSeries_lpar_hpte_getword0(slot); + + if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) + /* HPTE matches */ + return slot; + ++slot; } return -1; @@ -477,14 +486,14 @@ static long pSeries_lpar_hpte_find(unsigned long va, int psize) static void pSeries_lpar_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, - int psize) + int psize, int ssize) { unsigned long lpar_rc, slot, vsid, va, flags; - vsid = get_kernel_vsid(ea); - va = (vsid << 28) | (ea & 0x0fffffff); + vsid = get_kernel_vsid(ea, ssize); + va = hpt_va(ea, vsid, ssize); - slot = pSeries_lpar_hpte_find(va, psize); + slot = pSeries_lpar_hpte_find(va, psize, ssize); BUG_ON(slot == -1); flags = newpp & 7; @@ -494,7 +503,7 @@ static void pSeries_lpar_hpte_updateboltedpp(unsigned long newpp, } static void pSeries_lpar_hpte_invalidate(unsigned long slot, unsigned long va, - int psize, int local) + int psize, int ssize, int local) { unsigned long want_v; unsigned long lpar_rc; @@ -503,9 +512,8 @@ static void pSeries_lpar_hpte_invalidate(unsigned long slot, unsigned long va, DBG_LOW(" inval : slot=%lx, va=%016lx, psize: %d, local: %d", slot, va, psize, local); - want_v = hpte_encode_v(va, psize); - lpar_rc = plpar_pte_remove(H_AVPN, slot, want_v & HPTE_V_AVPN, - &dummy1, &dummy2); + want_v = hpte_encode_avpn(va, psize, ssize); + lpar_rc = plpar_pte_remove(H_AVPN, slot, want_v, &dummy1, &dummy2); if (lpar_rc == H_NOT_FOUND) return; @@ -533,18 +541,19 @@ static void pSeries_lpar_flush_hash_range(unsigned long number, int local) unsigned long va; unsigned long hash, index, shift, hidx, slot; real_pte_t pte; - int psize; + int psize, ssize; if (lock_tlbie) spin_lock_irqsave(&pSeries_lpar_tlbie_lock, flags); psize = batch->psize; + ssize = batch->ssize; pix = 0; for (i = 0; i < number; i++) { va = batch->vaddr[i]; pte = batch->pte[i]; pte_iterate_hashed_subpages(pte, psize, va, index, shift) { - hash = hpt_hash(va, shift); + hash = hpt_hash(va, shift, ssize); hidx = __rpte_to_hidx(pte, index); if (hidx & _PTEIDX_SECONDARY) hash = ~hash; @@ -552,11 +561,11 @@ static void pSeries_lpar_flush_hash_range(unsigned long number, int local) slot += hidx & _PTEIDX_GROUP_IX; if (!firmware_has_feature(FW_FEATURE_BULK_REMOVE)) { pSeries_lpar_hpte_invalidate(slot, va, psize, - local); + ssize, local); } else { param[pix] = HBR_REQUEST | HBR_AVPN | slot; - param[pix+1] = hpte_encode_v(va, psize) & - HPTE_V_AVPN; + param[pix+1] = hpte_encode_avpn(va, psize, + ssize); pix += 2; if (pix == 8) { rc = plpar_hcall9(H_BULK_REMOVE, param, diff --git a/include/asm-powerpc/cputable.h b/include/asm-powerpc/cputable.h index d913f460e71..ae093ef6836 100644 --- a/include/asm-powerpc/cputable.h +++ b/include/asm-powerpc/cputable.h @@ -164,6 +164,7 @@ extern void do_feature_fixups(unsigned long value, void *fixup_start, #define CPU_FTR_CELL_TB_BUG LONG_ASM_CONST(0x0000800000000000) #define CPU_FTR_SPURR LONG_ASM_CONST(0x0001000000000000) #define CPU_FTR_DSCR LONG_ASM_CONST(0x0002000000000000) +#define CPU_FTR_1T_SEGMENT LONG_ASM_CONST(0x0004000000000000) #ifndef __ASSEMBLY__ @@ -374,7 +375,7 @@ extern void do_feature_fixups(unsigned long value, void *fixup_start, #define CPU_FTRS_POSSIBLE \ (CPU_FTRS_POWER3 | CPU_FTRS_RS64 | CPU_FTRS_POWER4 | \ CPU_FTRS_PPC970 | CPU_FTRS_POWER5 | CPU_FTRS_POWER6 | \ - CPU_FTRS_CELL | CPU_FTRS_PA6T) + CPU_FTRS_CELL | CPU_FTRS_PA6T | CPU_FTR_1T_SEGMENT) #else enum { CPU_FTRS_POSSIBLE = diff --git a/include/asm-powerpc/machdep.h b/include/asm-powerpc/machdep.h index cc7c17f16a9..6968f4300dc 100644 --- a/include/asm-powerpc/machdep.h +++ b/include/asm-powerpc/machdep.h @@ -51,22 +51,22 @@ struct machdep_calls { #ifdef CONFIG_PPC64 void (*hpte_invalidate)(unsigned long slot, unsigned long va, - int psize, + int psize, int ssize, int local); long (*hpte_updatepp)(unsigned long slot, unsigned long newpp, unsigned long va, - int pize, + int psize, int ssize, int local); void (*hpte_updateboltedpp)(unsigned long newpp, unsigned long ea, - int psize); + int psize, int ssize); long (*hpte_insert)(unsigned long hpte_group, unsigned long va, unsigned long prpn, unsigned long rflags, unsigned long vflags, - int psize); + int psize, int ssize); long (*hpte_remove)(unsigned long hpte_group); void (*flush_hash_range)(unsigned long number, int local); diff --git a/include/asm-powerpc/mmu-hash64.h b/include/asm-powerpc/mmu-hash64.h index b22b0d20e15..82328dec2b5 100644 --- a/include/asm-powerpc/mmu-hash64.h +++ b/include/asm-powerpc/mmu-hash64.h @@ -47,6 +47,8 @@ extern char initial_stab[]; /* Bits in the SLB VSID word */ #define SLB_VSID_SHIFT 12 +#define SLB_VSID_SHIFT_1T 24 +#define SLB_VSID_SSIZE_SHIFT 62 #define SLB_VSID_B ASM_CONST(0xc000000000000000) #define SLB_VSID_B_256M ASM_CONST(0x0000000000000000) #define SLB_VSID_B_1T ASM_CONST(0x4000000000000000) @@ -66,6 +68,7 @@ extern char initial_stab[]; #define SLB_VSID_USER (SLB_VSID_KP|SLB_VSID_KS|SLB_VSID_C) #define SLBIE_C (0x08000000) +#define SLBIE_SSIZE_SHIFT 25 /* * Hash table @@ -77,7 +80,7 @@ extern char initial_stab[]; #define HPTE_V_AVPN_SHIFT 7 #define HPTE_V_AVPN ASM_CONST(0x3fffffffffffff80) #define HPTE_V_AVPN_VAL(x) (((x) & HPTE_V_AVPN) >> HPTE_V_AVPN_SHIFT) -#define HPTE_V_COMPARE(x,y) (!(((x) ^ (y)) & HPTE_V_AVPN)) +#define HPTE_V_COMPARE(x,y) (!(((x) ^ (y)) & 0xffffffffffffff80)) #define HPTE_V_BOLTED ASM_CONST(0x0000000000000010) #define HPTE_V_LOCK ASM_CONST(0x0000000000000008) #define HPTE_V_LARGE ASM_CONST(0x0000000000000004) @@ -164,16 +167,19 @@ struct mmu_psize_def #define MMU_SEGSIZE_256M 0 #define MMU_SEGSIZE_1T 1 + #ifndef __ASSEMBLY__ /* - * The current system page sizes + * The current system page and segment sizes */ extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT]; extern int mmu_linear_psize; extern int mmu_virtual_psize; extern int mmu_vmalloc_psize; extern int mmu_io_psize; +extern int mmu_kernel_ssize; +extern int mmu_highuser_ssize; /* * If the processor supports 64k normal pages but not 64k cache @@ -195,13 +201,15 @@ extern int mmu_huge_psize; * This function sets the AVPN and L fields of the HPTE appropriately * for the page size */ -static inline unsigned long hpte_encode_v(unsigned long va, int psize) +static inline unsigned long hpte_encode_v(unsigned long va, int psize, + int ssize) { - unsigned long v = + unsigned long v; v = (va >> 23) & ~(mmu_psize_defs[psize].avpnm); v <<= HPTE_V_AVPN_SHIFT; if (psize != MMU_PAGE_4K) v |= HPTE_V_LARGE; + v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT; return v; } @@ -226,20 +234,40 @@ static inline unsigned long hpte_encode_r(unsigned long pa, int psize) } /* - * This hashes a virtual address for a 256Mb segment only for now + * Build a VA given VSID, EA and segment size */ +static inline unsigned long hpt_va(unsigned long ea, unsigned long vsid, + int ssize) +{ + if (ssize == MMU_SEGSIZE_256M) + return (vsid << 28) | (ea & 0xfffffffUL); + return (vsid << 40) | (ea & 0xffffffffffUL); +} -static inline unsigned long hpt_hash(unsigned long va, unsigned int shift) +/* + * This hashes a virtual address + */ + +static inline unsigned long hpt_hash(unsigned long va, unsigned int shift, + int ssize) { - return ((va >> 28) & 0x7fffffffffUL) ^ ((va & 0x0fffffffUL) >> shift); + unsigned long hash, vsid; + + if (ssize == MMU_SEGSIZE_256M) { + hash = (va >> 28) ^ ((va & 0x0fffffffUL) >> shift); + } else { + vsid = va >> 40; + hash = vsid ^ (vsid << 25) ^ ((va & 0xffffffffffUL) >> shift); + } + return hash & 0x7fffffffffUL; } extern int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, pte_t *ptep, unsigned long trap, - unsigned int local); + unsigned int local, int ssize); extern int __hash_page_64K(unsigned long ea, unsigned long access, unsigned long vsid, pte_t *ptep, unsigned long trap, - unsigned int local); + unsigned int local, int ssize); struct mm_struct; extern int hash_page(unsigned long ea, unsigned long access, unsigned long trap); extern int hash_huge_page(struct mm_struct *mm, unsigned long access, @@ -248,7 +276,7 @@ extern int hash_huge_page(struct mm_struct *mm, unsigned long access, extern int htab_bolt_mapping(unsigned long vstart, unsigned long vend, unsigned long pstart, unsigned long mode, - int psize); + int psize, int ssize); extern void htab_initialize(void); extern void htab_initialize_secondary(void); @@ -317,12 +345,17 @@ extern void slb_vmalloc_update(void); * which are used by the iSeries firmware. */ -#define VSID_MULTIPLIER ASM_CONST(200730139) /* 28-bit prime */ -#define VSID_BITS 36 -#define VSID_MODULUS ((1UL<= \ @@ -355,7 +388,7 @@ extern void slb_vmalloc_update(void); * doesn't, the answer is the low 36 bits of r3+1. So in all \ * cases the answer is the low 36 bits of (r3 + ((r3+1) >> 36))*/\ addi rx,rt,1; \ - srdi rx,rx,VSID_BITS; /* extract 2^36 bit */ \ + srdi rx,rx,VSID_BITS_##size; /* extract 2^VSID_BITS bit */ \ add rt,rt,rx @@ -377,37 +410,60 @@ typedef struct { } mm_context_t; -static inline unsigned long vsid_scramble(unsigned long protovsid) -{ #if 0 - /* The code below is equivalent to this function for arguments - * < 2^VSID_BITS, which is all this should ever be called - * with. However gcc is not clever enough to compute the - * modulus (2^n-1) without a second multiply. */ - return ((protovsid * VSID_MULTIPLIER) % VSID_MODULUS); -#else /* 1 */ - unsigned long x; +/* + * The code below is equivalent to this function for arguments + * < 2^VSID_BITS, which is all this should ever be called + * with. However gcc is not clever enough to compute the + * modulus (2^n-1) without a second multiply. + */ +#define vsid_scrample(protovsid, size) \ + ((((protovsid) * VSID_MULTIPLIER_##size) % VSID_MODULUS_##size)) - x = protovsid * VSID_MULTIPLIER; - x = (x >> VSID_BITS) + (x & VSID_MODULUS); - return (x + ((x+1) >> VSID_BITS)) & VSID_MODULUS; +#else /* 1 */ +#define vsid_scramble(protovsid, size) \ + ({ \ + unsigned long x; \ + x = (protovsid) * VSID_MULTIPLIER_##size; \ + x = (x >> VSID_BITS_##size) + (x & VSID_MODULUS_##size); \ + (x + ((x+1) >> VSID_BITS_##size)) & VSID_MODULUS_##size; \ + }) #endif /* 1 */ -} /* This is only valid for addresses >= KERNELBASE */ -static inline unsigned long get_kernel_vsid(unsigned long ea) +static inline unsigned long get_kernel_vsid(unsigned long ea, int ssize) { - return vsid_scramble(ea >> SID_SHIFT); + if (ssize == MMU_SEGSIZE_256M) + return vsid_scramble(ea >> SID_SHIFT, 256M); + return vsid_scramble(ea >> SID_SHIFT_1T, 1T); } -/* This is only valid for user addresses (which are below 2^41) */ -static inline unsigned long get_vsid(unsigned long context, unsigned long ea) +/* Returns the segment size indicator for a user address */ +static inline int user_segment_size(unsigned long addr) { - return vsid_scramble((context << USER_ESID_BITS) - | (ea >> SID_SHIFT)); + /* Use 1T segments if possible for addresses >= 1T */ + if (addr >= (1UL << SID_SHIFT_1T)) + return mmu_highuser_ssize; + return MMU_SEGSIZE_256M; } -#define VSID_SCRAMBLE(pvsid) (((pvsid) * VSID_MULTIPLIER) % VSID_MODULUS) +/* This is only valid for user addresses (which are below 2^44) */ +static inline unsigned long get_vsid(unsigned long context, unsigned long ea, + int ssize) +{ + if (ssize == MMU_SEGSIZE_256M) + return vsid_scramble((context << USER_ESID_BITS) + | (ea >> SID_SHIFT), 256M); + return vsid_scramble((context << USER_ESID_BITS_1T) + | (ea >> SID_SHIFT_1T), 1T); +} + +/* + * This is only used on legacy iSeries in lparmap.c, + * hence the 256MB segment assumption. + */ +#define VSID_SCRAMBLE(pvsid) (((pvsid) * VSID_MULTIPLIER_256M) % \ + VSID_MODULUS_256M) #define KERNEL_VSID(ea) VSID_SCRAMBLE(GET_ESID(ea)) /* Physical address used by some IO functions */ diff --git a/include/asm-powerpc/page_64.h b/include/asm-powerpc/page_64.h index 56a2df0f683..4ee82c61e4d 100644 --- a/include/asm-powerpc/page_64.h +++ b/include/asm-powerpc/page_64.h @@ -26,12 +26,18 @@ */ #define PAGE_FACTOR (PAGE_SHIFT - HW_PAGE_SHIFT) -/* Segment size */ +/* Segment size; normal 256M segments */ #define SID_SHIFT 28 #define SID_MASK ASM_CONST(0xfffffffff) #define ESID_MASK 0xfffffffff0000000UL #define GET_ESID(x) (((x) >> SID_SHIFT) & SID_MASK) +/* 1T segments */ +#define SID_SHIFT_1T 40 +#define SID_MASK_1T 0xffffffUL +#define ESID_MASK_1T 0xffffff0000000000UL +#define GET_ESID_1T(x) (((x) >> SID_SHIFT_1T) & SID_MASK_1T) + #ifndef __ASSEMBLY__ #include diff --git a/include/asm-powerpc/tlbflush.h b/include/asm-powerpc/tlbflush.h index 99a0439baa5..a022f806bb2 100644 --- a/include/asm-powerpc/tlbflush.h +++ b/include/asm-powerpc/tlbflush.h @@ -97,6 +97,7 @@ struct ppc64_tlb_batch { real_pte_t pte[PPC64_TLB_BATCH_NR]; unsigned long vaddr[PPC64_TLB_BATCH_NR]; unsigned int psize; + int ssize; }; DECLARE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch); @@ -127,7 +128,7 @@ static inline void arch_leave_lazy_mmu_mode(void) extern void flush_hash_page(unsigned long va, real_pte_t pte, int psize, - int local); + int ssize, int local); extern void flush_hash_range(unsigned long number, int local); -- cgit v1.2.3-70-g09d2