diff options
Diffstat (limited to 'arch/tile/lib')
| -rw-r--r-- | arch/tile/lib/atomic_32.c | 47 | ||||
| -rw-r--r-- | arch/tile/lib/exports.c | 8 | ||||
| -rw-r--r-- | arch/tile/lib/memchr_64.c | 8 | ||||
| -rw-r--r-- | arch/tile/lib/memcpy_64.c | 23 | ||||
| -rw-r--r-- | arch/tile/lib/memcpy_tile64.c | 8 | ||||
| -rw-r--r-- | arch/tile/lib/strchr_64.c | 15 | ||||
| -rw-r--r-- | arch/tile/lib/string-endian.h | 33 | ||||
| -rw-r--r-- | arch/tile/lib/strlen_64.c | 11 | ||||
| -rw-r--r-- | arch/tile/lib/usercopy_32.S | 76 | ||||
| -rw-r--r-- | arch/tile/lib/usercopy_64.S | 49 |
10 files changed, 71 insertions, 207 deletions
diff --git a/arch/tile/lib/atomic_32.c b/arch/tile/lib/atomic_32.c index 771b251b409..f5cada70c3c 100644 --- a/arch/tile/lib/atomic_32.c +++ b/arch/tile/lib/atomic_32.c @@ -18,7 +18,6 @@ #include <linux/module.h> #include <linux/mm.h> #include <linux/atomic.h> -#include <asm/futex.h> #include <arch/chip.h> /* See <asm/atomic_32.h> */ @@ -50,7 +49,7 @@ int atomic_locks[PAGE_SIZE / sizeof(int)] __page_aligned_bss; #endif /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */ -static inline int *__atomic_hashed_lock(volatile void *v) +int *__atomic_hashed_lock(volatile void *v) { /* NOTE: this code must match "sys_cmpxchg" in kernel/intvec_32.S */ #if ATOMIC_LOCKS_FOUND_VIA_TABLE() @@ -191,47 +190,6 @@ u64 _atomic64_cmpxchg(atomic64_t *v, u64 o, u64 n) EXPORT_SYMBOL(_atomic64_cmpxchg); -static inline int *__futex_setup(int __user *v) -{ - /* - * Issue a prefetch to the counter to bring it into cache. - * As for __atomic_setup, but we can't do a read into the L1 - * since it might fault; instead we do a prefetch into the L2. - */ - __insn_prefetch(v); - return __atomic_hashed_lock((int __force *)v); -} - -struct __get_user futex_set(u32 __user *v, int i) -{ - return __atomic_xchg((int __force *)v, __futex_setup(v), i); -} - -struct __get_user futex_add(u32 __user *v, int n) -{ - return __atomic_xchg_add((int __force *)v, __futex_setup(v), n); -} - -struct __get_user futex_or(u32 __user *v, int n) -{ - return __atomic_or((int __force *)v, __futex_setup(v), n); -} - -struct __get_user futex_andn(u32 __user *v, int n) -{ - return __atomic_andn((int __force *)v, __futex_setup(v), n); -} - -struct __get_user futex_xor(u32 __user *v, int n) -{ - return __atomic_xor((int __force *)v, __futex_setup(v), n); -} - -struct __get_user futex_cmpxchg(u32 __user *v, int o, int n) -{ - return __atomic_cmpxchg((int __force *)v, __futex_setup(v), o, n); -} - /* * If any of the atomic or futex routines hit a bad address (not in * the page tables at kernel PL) this routine is called. The futex @@ -323,7 +281,4 @@ void __init __init_atomic_per_cpu(void) BUILD_BUG_ON((PAGE_SIZE >> 3) > ATOMIC_HASH_SIZE); #endif /* ATOMIC_LOCKS_FOUND_VIA_TABLE() */ - - /* The futex code makes this assumption, so we validate it here. */ - BUILD_BUG_ON(sizeof(atomic_t) != sizeof(int)); } diff --git a/arch/tile/lib/exports.c b/arch/tile/lib/exports.c index 2a81d32de0d..dd5f0a33fda 100644 --- a/arch/tile/lib/exports.c +++ b/arch/tile/lib/exports.c @@ -18,14 +18,6 @@ /* arch/tile/lib/usercopy.S */ #include <linux/uaccess.h> -EXPORT_SYMBOL(__get_user_1); -EXPORT_SYMBOL(__get_user_2); -EXPORT_SYMBOL(__get_user_4); -EXPORT_SYMBOL(__get_user_8); -EXPORT_SYMBOL(__put_user_1); -EXPORT_SYMBOL(__put_user_2); -EXPORT_SYMBOL(__put_user_4); -EXPORT_SYMBOL(__put_user_8); EXPORT_SYMBOL(strnlen_user_asm); EXPORT_SYMBOL(strncpy_from_user_asm); EXPORT_SYMBOL(clear_user_asm); diff --git a/arch/tile/lib/memchr_64.c b/arch/tile/lib/memchr_64.c index 84fdc8d8e73..6f867dbf7c5 100644 --- a/arch/tile/lib/memchr_64.c +++ b/arch/tile/lib/memchr_64.c @@ -15,6 +15,7 @@ #include <linux/types.h> #include <linux/string.h> #include <linux/module.h> +#include "string-endian.h" void *memchr(const void *s, int c, size_t n) { @@ -39,11 +40,8 @@ void *memchr(const void *s, int c, size_t n) /* Read the first word, but munge it so that bytes before the array * will not match goal. - * - * Note that this shift count expression works because we know - * shift counts are taken mod 64. */ - before_mask = (1ULL << (s_int << 3)) - 1; + before_mask = MASK(s_int); v = (*p | before_mask) ^ (goal & before_mask); /* Compute the address of the last byte. */ @@ -65,7 +63,7 @@ void *memchr(const void *s, int c, size_t n) /* We found a match, but it might be in a byte past the end * of the array. */ - ret = ((char *)p) + (__insn_ctz(bits) >> 3); + ret = ((char *)p) + (CFZ(bits) >> 3); return (ret <= last_byte_ptr) ? ret : NULL; } EXPORT_SYMBOL(memchr); diff --git a/arch/tile/lib/memcpy_64.c b/arch/tile/lib/memcpy_64.c index 3fab9a6a2bb..c79b8e7c682 100644 --- a/arch/tile/lib/memcpy_64.c +++ b/arch/tile/lib/memcpy_64.c @@ -15,7 +15,6 @@ #include <linux/types.h> #include <linux/string.h> #include <linux/module.h> -#define __memcpy memcpy /* EXPORT_SYMBOL() is in arch/tile/lib/exports.c since this should be asm. */ /* Must be 8 bytes in size. */ @@ -188,6 +187,7 @@ int USERCOPY_FUNC(void *__restrict dstv, const void *__restrict srcv, size_t n) /* n != 0 if we get here. Write out any trailing bytes. */ dst1 = (char *)dst8; +#ifndef __BIG_ENDIAN__ if (n & 4) { ST4((uint32_t *)dst1, final); dst1 += 4; @@ -202,11 +202,30 @@ int USERCOPY_FUNC(void *__restrict dstv, const void *__restrict srcv, size_t n) } if (n) ST1((uint8_t *)dst1, final); +#else + if (n & 4) { + ST4((uint32_t *)dst1, final >> 32); + dst1 += 4; + } + else + { + final >>= 32; + } + if (n & 2) { + ST2((uint16_t *)dst1, final >> 16); + dst1 += 2; + } + else + { + final >>= 16; + } + if (n & 1) + ST1((uint8_t *)dst1, final >> 8); +#endif return RETVAL; } - #ifdef USERCOPY_FUNC #undef ST1 #undef ST2 diff --git a/arch/tile/lib/memcpy_tile64.c b/arch/tile/lib/memcpy_tile64.c index b2fe15e0107..3bc4b4e40d9 100644 --- a/arch/tile/lib/memcpy_tile64.c +++ b/arch/tile/lib/memcpy_tile64.c @@ -160,7 +160,7 @@ retry_source: break; if (get_remote_cache_cpu(src_pte) == smp_processor_id()) break; - src_page = pfn_to_page(hv_pte_get_pfn(src_pte)); + src_page = pfn_to_page(pte_pfn(src_pte)); get_page(src_page); if (pte_val(src_pte) != pte_val(*src_ptep)) { put_page(src_page); @@ -168,7 +168,7 @@ retry_source: } if (pte_huge(src_pte)) { /* Adjust the PTE to correspond to a small page */ - int pfn = hv_pte_get_pfn(src_pte); + int pfn = pte_pfn(src_pte); pfn += (((unsigned long)source & (HPAGE_SIZE-1)) >> PAGE_SHIFT); src_pte = pfn_pte(pfn, src_pte); @@ -188,7 +188,7 @@ retry_dest: put_page(src_page); break; } - dst_page = pfn_to_page(hv_pte_get_pfn(dst_pte)); + dst_page = pfn_to_page(pte_pfn(dst_pte)); if (dst_page == src_page) { /* * Source and dest are on the same page; this @@ -206,7 +206,7 @@ retry_dest: } if (pte_huge(dst_pte)) { /* Adjust the PTE to correspond to a small page */ - int pfn = hv_pte_get_pfn(dst_pte); + int pfn = pte_pfn(dst_pte); pfn += (((unsigned long)dest & (HPAGE_SIZE-1)) >> PAGE_SHIFT); dst_pte = pfn_pte(pfn, dst_pte); diff --git a/arch/tile/lib/strchr_64.c b/arch/tile/lib/strchr_64.c index 617a9273aaa..f39f9dc422b 100644 --- a/arch/tile/lib/strchr_64.c +++ b/arch/tile/lib/strchr_64.c @@ -15,8 +15,7 @@ #include <linux/types.h> #include <linux/string.h> #include <linux/module.h> - -#undef strchr +#include "string-endian.h" char *strchr(const char *s, int c) { @@ -33,13 +32,9 @@ char *strchr(const char *s, int c) * match neither zero nor goal (we make sure the high bit of each * byte is 1, and the low 7 bits are all the opposite of the goal * byte). - * - * Note that this shift count expression works because we know shift - * counts are taken mod 64. */ - const uint64_t before_mask = (1ULL << (s_int << 3)) - 1; - uint64_t v = (*p | before_mask) ^ - (goal & __insn_v1shrsi(before_mask, 1)); + const uint64_t before_mask = MASK(s_int); + uint64_t v = (*p | before_mask) ^ (goal & __insn_v1shrui(before_mask, 1)); uint64_t zero_matches, goal_matches; while (1) { @@ -55,8 +50,8 @@ char *strchr(const char *s, int c) v = *++p; } - z = __insn_ctz(zero_matches); - g = __insn_ctz(goal_matches); + z = CFZ(zero_matches); + g = CFZ(goal_matches); /* If we found c before '\0' we got a match. Note that if c == '\0' * then g == z, and we correctly return the address of the '\0' diff --git a/arch/tile/lib/string-endian.h b/arch/tile/lib/string-endian.h new file mode 100644 index 00000000000..c0eed7ce69c --- /dev/null +++ b/arch/tile/lib/string-endian.h @@ -0,0 +1,33 @@ +/* + * Copyright 2011 Tilera Corporation. All Rights Reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation, version 2. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or + * NON INFRINGEMENT. See the GNU General Public License for + * more details. + * + * Provide a mask based on the pointer alignment that + * sets up non-zero bytes before the beginning of the string. + * The MASK expression works because shift counts are taken mod 64. + * Also, specify how to count "first" and "last" bits + * when the bits have been read as a word. + */ + +#include <asm/byteorder.h> + +#ifdef __LITTLE_ENDIAN +#define MASK(x) (__insn_shl(1ULL, (x << 3)) - 1) +#define NULMASK(x) ((2ULL << x) - 1) +#define CFZ(x) __insn_ctz(x) +#define REVCZ(x) __insn_clz(x) +#else +#define MASK(x) (__insn_shl(-2LL, ((-x << 3) - 1))) +#define NULMASK(x) (-2LL << (63 - x)) +#define CFZ(x) __insn_clz(x) +#define REVCZ(x) __insn_ctz(x) +#endif diff --git a/arch/tile/lib/strlen_64.c b/arch/tile/lib/strlen_64.c index 1c92d46202a..9583fc3361f 100644 --- a/arch/tile/lib/strlen_64.c +++ b/arch/tile/lib/strlen_64.c @@ -15,8 +15,7 @@ #include <linux/types.h> #include <linux/string.h> #include <linux/module.h> - -#undef strlen +#include "string-endian.h" size_t strlen(const char *s) { @@ -24,15 +23,13 @@ size_t strlen(const char *s) const uintptr_t s_int = (uintptr_t) s; const uint64_t *p = (const uint64_t *)(s_int & -8); - /* Read the first word, but force bytes before the string to be nonzero. - * This expression works because we know shift counts are taken mod 64. - */ - uint64_t v = *p | ((1ULL << (s_int << 3)) - 1); + /* Read and MASK the first word. */ + uint64_t v = *p | MASK(s_int); uint64_t bits; while ((bits = __insn_v1cmpeqi(v, 0)) == 0) v = *++p; - return ((const char *)p) + (__insn_ctz(bits) >> 3) - s; + return ((const char *)p) + (CFZ(bits) >> 3) - s; } EXPORT_SYMBOL(strlen); diff --git a/arch/tile/lib/usercopy_32.S b/arch/tile/lib/usercopy_32.S index 979f76d8374..b62d002af00 100644 --- a/arch/tile/lib/usercopy_32.S +++ b/arch/tile/lib/usercopy_32.S @@ -19,82 +19,6 @@ /* Access user memory, but use MMU to avoid propagating kernel exceptions. */ - .pushsection .fixup,"ax" - -get_user_fault: - { move r0, zero; move r1, zero } - { movei r2, -EFAULT; jrp lr } - ENDPROC(get_user_fault) - -put_user_fault: - { movei r0, -EFAULT; jrp lr } - ENDPROC(put_user_fault) - - .popsection - -/* - * __get_user_N functions take a pointer in r0, and return 0 in r2 - * on success, with the value in r0; or else -EFAULT in r2. - */ -#define __get_user_N(bytes, LOAD) \ - STD_ENTRY(__get_user_##bytes); \ -1: { LOAD r0, r0; move r1, zero; move r2, zero }; \ - jrp lr; \ - STD_ENDPROC(__get_user_##bytes); \ - .pushsection __ex_table,"a"; \ - .word 1b, get_user_fault; \ - .popsection - -__get_user_N(1, lb_u) -__get_user_N(2, lh_u) -__get_user_N(4, lw) - -/* - * __get_user_8 takes a pointer in r0, and returns 0 in r2 - * on success, with the value in r0/r1; or else -EFAULT in r2. - */ - STD_ENTRY(__get_user_8); -1: { lw r0, r0; addi r1, r0, 4 }; -2: { lw r1, r1; move r2, zero }; - jrp lr; - STD_ENDPROC(__get_user_8); - .pushsection __ex_table,"a"; - .word 1b, get_user_fault; - .word 2b, get_user_fault; - .popsection - -/* - * __put_user_N functions take a value in r0 and a pointer in r1, - * and return 0 in r0 on success or -EFAULT on failure. - */ -#define __put_user_N(bytes, STORE) \ - STD_ENTRY(__put_user_##bytes); \ -1: { STORE r1, r0; move r0, zero }; \ - jrp lr; \ - STD_ENDPROC(__put_user_##bytes); \ - .pushsection __ex_table,"a"; \ - .word 1b, put_user_fault; \ - .popsection - -__put_user_N(1, sb) -__put_user_N(2, sh) -__put_user_N(4, sw) - -/* - * __put_user_8 takes a value in r0/r1 and a pointer in r2, - * and returns 0 in r0 on success or -EFAULT on failure. - */ -STD_ENTRY(__put_user_8) -1: { sw r2, r0; addi r2, r2, 4 } -2: { sw r2, r1; move r0, zero } - jrp lr - STD_ENDPROC(__put_user_8) - .pushsection __ex_table,"a" - .word 1b, put_user_fault - .word 2b, put_user_fault - .popsection - - /* * strnlen_user_asm takes the pointer in r0, and the length bound in r1. * It returns the length, including the terminating NUL, or zero on exception. diff --git a/arch/tile/lib/usercopy_64.S b/arch/tile/lib/usercopy_64.S index 2ff44f87b78..adb2dbbc70c 100644 --- a/arch/tile/lib/usercopy_64.S +++ b/arch/tile/lib/usercopy_64.S @@ -19,55 +19,6 @@ /* Access user memory, but use MMU to avoid propagating kernel exceptions. */ - .pushsection .fixup,"ax" - -get_user_fault: - { movei r1, -EFAULT; move r0, zero } - jrp lr - ENDPROC(get_user_fault) - -put_user_fault: - { movei r0, -EFAULT; jrp lr } - ENDPROC(put_user_fault) - - .popsection - -/* - * __get_user_N functions take a pointer in r0, and return 0 in r1 - * on success, with the value in r0; or else -EFAULT in r1. - */ -#define __get_user_N(bytes, LOAD) \ - STD_ENTRY(__get_user_##bytes); \ -1: { LOAD r0, r0; move r1, zero }; \ - jrp lr; \ - STD_ENDPROC(__get_user_##bytes); \ - .pushsection __ex_table,"a"; \ - .quad 1b, get_user_fault; \ - .popsection - -__get_user_N(1, ld1u) -__get_user_N(2, ld2u) -__get_user_N(4, ld4u) -__get_user_N(8, ld) - -/* - * __put_user_N functions take a value in r0 and a pointer in r1, - * and return 0 in r0 on success or -EFAULT on failure. - */ -#define __put_user_N(bytes, STORE) \ - STD_ENTRY(__put_user_##bytes); \ -1: { STORE r1, r0; move r0, zero }; \ - jrp lr; \ - STD_ENDPROC(__put_user_##bytes); \ - .pushsection __ex_table,"a"; \ - .quad 1b, put_user_fault; \ - .popsection - -__put_user_N(1, st1) -__put_user_N(2, st2) -__put_user_N(4, st4) -__put_user_N(8, st) - /* * strnlen_user_asm takes the pointer in r0, and the length bound in r1. * It returns the length, including the terminating NUL, or zero on exception. |