From 5a3ea8782c63d3501cb764c176f153c0d9a400e1 Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Thu, 28 Apr 2011 15:55:52 -0400 Subject: flex_array: flex_array_prealloc takes a number of elements, not an end Change flex_array_prealloc to take the number of elements for which space should be allocated instead of the last (inclusive) element. Users and documentation are updated accordingly. flex_arrays got introduced before they had users. When folks started using it, they ended up needing a different API than was coded up originally. This swaps over to the API that folks apparently need. Based-on-patch-by: Steffen Klassert Signed-off-by: Eric Paris Tested-by: Chris Richards Acked-by: Dave Hansen Cc: stable@kernel.org [2.6.38+] --- lib/flex_array.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'lib/flex_array.c') diff --git a/lib/flex_array.c b/lib/flex_array.c index c0ea40ba208..0c33b24498b 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -232,10 +232,10 @@ EXPORT_SYMBOL(flex_array_clear); /** * flex_array_prealloc - guarantee that array space exists - * @fa: the flex array for which to preallocate parts - * @start: index of first array element for which space is allocated - * @end: index of last (inclusive) element for which space is allocated - * @flags: page allocation flags + * @fa: the flex array for which to preallocate parts + * @start: index of first array element for which space is allocated + * @nr_elements: number of elements for which space is allocated + * @flags: page allocation flags * * This will guarantee that no future calls to flex_array_put() * will allocate memory. It can be used if you are expecting to @@ -245,13 +245,16 @@ EXPORT_SYMBOL(flex_array_clear); * Locking must be provided by the caller. */ int flex_array_prealloc(struct flex_array *fa, unsigned int start, - unsigned int end, gfp_t flags) + unsigned int nr_elements, gfp_t flags) { int start_part; int end_part; int part_nr; + unsigned int end; struct flex_array_part *part; + end = start + nr_elements - 1; + if (start >= fa->total_nr_elements || end >= fa->total_nr_elements) return -ENOSPC; if (elements_fit_in_base(fa)) -- cgit v1.2.3-70-g09d2 From 150cdf6ec0ede8d9f102f1817212447727dcf08c Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Thu, 28 Apr 2011 15:55:52 -0400 Subject: flex_arrays: allow zero length flex arrays Just like kmalloc will allow one to allocate a 0 length segment of memory flex arrays should do the same thing. It should bomb if you try to use something, but it should at least allow the allocation. This is needed because when SELinux switched to using flex_arrays in 2.6.38 the inability to allocate a 0 length array resulted in SELinux policy load returning -ENOSPC when previously it worked. Based-on-patch-by: Steffen Klassert Signed-off-by: Eric Paris Tested-by: Chris Richards Cc: stable@kernel.org [2.6.38+] --- lib/flex_array.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'lib/flex_array.c') diff --git a/lib/flex_array.c b/lib/flex_array.c index 0c33b24498b..854b57bd7d9 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -253,9 +253,16 @@ int flex_array_prealloc(struct flex_array *fa, unsigned int start, unsigned int end; struct flex_array_part *part; + if (!start && !nr_elements) + return 0; + if (start >= fa->total_nr_elements) + return -ENOSPC; + if (!nr_elements) + return 0; + end = start + nr_elements - 1; - if (start >= fa->total_nr_elements || end >= fa->total_nr_elements) + if (end >= fa->total_nr_elements) return -ENOSPC; if (elements_fit_in_base(fa)) return 0; @@ -346,6 +353,8 @@ int flex_array_shrink(struct flex_array *fa) int part_nr; int ret = 0; + if (!fa->total_nr_elements) + return 0; if (elements_fit_in_base(fa)) return ret; for (part_nr = 0; part_nr < FLEX_ARRAY_NR_BASE_PTRS; part_nr++) { -- cgit v1.2.3-70-g09d2 From a8d05c81fb238bbb18878ccfae7599ca79448dd3 Mon Sep 17 00:00:00 2001 From: Eric Paris Date: Thu, 28 Apr 2011 15:55:52 -0400 Subject: flex_array: allow 0 length elements flex_arrays are supposed to be a replacement for: kmalloc(num_elements * sizeof(element)) If kmalloc is given 0 num_elements or a 0 size element it will happily return ZERO_SIZE_PTR. Which looks like a valid allocation, but which will explode if something actually try to use it. The current flex_array code will return an equivalent result if num_elements is 0, but will fail to work if sizeof(element) is 0. This patch allows allocation to work even for 0 size elements. It will cause flex_arrays to explode though if they are used. Imitating the kmalloc behavior. Based-on-patch-by: Steffen Klassert Signed-off-by: Eric Paris Acked-by: Dave Hansen --- lib/flex_array.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'lib/flex_array.c') diff --git a/lib/flex_array.c b/lib/flex_array.c index 854b57bd7d9..cab7621f98a 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -88,8 +88,11 @@ struct flex_array *flex_array_alloc(int element_size, unsigned int total, gfp_t flags) { struct flex_array *ret; - int max_size = FLEX_ARRAY_NR_BASE_PTRS * - FLEX_ARRAY_ELEMENTS_PER_PART(element_size); + int max_size = 0; + + if (element_size) + max_size = FLEX_ARRAY_NR_BASE_PTRS * + FLEX_ARRAY_ELEMENTS_PER_PART(element_size); /* max_size will end up 0 if element_size > PAGE_SIZE */ if (total > max_size) @@ -183,15 +186,18 @@ __fa_get_part(struct flex_array *fa, int part_nr, gfp_t flags) int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, gfp_t flags) { - int part_nr = fa_element_to_part_nr(fa, element_nr); + int part_nr; struct flex_array_part *part; void *dst; if (element_nr >= fa->total_nr_elements) return -ENOSPC; + if (!fa->element_size) + return 0; if (elements_fit_in_base(fa)) part = (struct flex_array_part *)&fa->parts[0]; else { + part_nr = fa_element_to_part_nr(fa, element_nr); part = __fa_get_part(fa, part_nr, flags); if (!part) return -ENOMEM; @@ -211,15 +217,18 @@ EXPORT_SYMBOL(flex_array_put); */ int flex_array_clear(struct flex_array *fa, unsigned int element_nr) { - int part_nr = fa_element_to_part_nr(fa, element_nr); + int part_nr; struct flex_array_part *part; void *dst; if (element_nr >= fa->total_nr_elements) return -ENOSPC; + if (!fa->element_size) + return 0; if (elements_fit_in_base(fa)) part = (struct flex_array_part *)&fa->parts[0]; else { + part_nr = fa_element_to_part_nr(fa, element_nr); part = fa->parts[part_nr]; if (!part) return -EINVAL; @@ -264,6 +273,8 @@ int flex_array_prealloc(struct flex_array *fa, unsigned int start, if (end >= fa->total_nr_elements) return -ENOSPC; + if (!fa->element_size) + return 0; if (elements_fit_in_base(fa)) return 0; start_part = fa_element_to_part_nr(fa, start); @@ -291,14 +302,17 @@ EXPORT_SYMBOL(flex_array_prealloc); */ void *flex_array_get(struct flex_array *fa, unsigned int element_nr) { - int part_nr = fa_element_to_part_nr(fa, element_nr); + int part_nr; struct flex_array_part *part; + if (!fa->element_size) + return NULL; if (element_nr >= fa->total_nr_elements) return NULL; if (elements_fit_in_base(fa)) part = (struct flex_array_part *)&fa->parts[0]; else { + part_nr = fa_element_to_part_nr(fa, element_nr); part = fa->parts[part_nr]; if (!part) return NULL; @@ -353,7 +367,7 @@ int flex_array_shrink(struct flex_array *fa) int part_nr; int ret = 0; - if (!fa->total_nr_elements) + if (!fa->total_nr_elements || !fa->element_size) return 0; if (elements_fit_in_base(fa)) return ret; -- cgit v1.2.3-70-g09d2 From 704f15ddb5fc2a7f25a12eb0913302d8ad9ffab3 Mon Sep 17 00:00:00 2001 From: Jesse Gross Date: Thu, 26 May 2011 16:25:02 -0700 Subject: flex_array: avoid divisions when accessing elements On most architectures division is an expensive operation and accessing an element currently requires four of them. This performance penalty effectively precludes flex arrays from being used on any kind of fast path. However, two of these divisions can be handled at creation time and the others can be replaced by a reciprocal divide, completely avoiding real divisions on access. [eparis@redhat.com: rebase on top of changes to support 0 len elements] [eparis@redhat.com: initialize part_nr when array fits entirely in base] Signed-off-by: Jesse Gross Signed-off-by: Eric Paris Cc: Dave Hansen Cc: David Rientjes Signed-off-by: Andrew Morton Signed-off-by: Linus Torvalds --- include/linux/flex_array.h | 2 ++ lib/flex_array.c | 51 ++++++++++++++++++++++++++-------------------- 2 files changed, 31 insertions(+), 22 deletions(-) (limited to 'lib/flex_array.c') diff --git a/include/linux/flex_array.h b/include/linux/flex_array.h index ebeb2f3ad06..6843cf193a4 100644 --- a/include/linux/flex_array.h +++ b/include/linux/flex_array.h @@ -21,6 +21,8 @@ struct flex_array { struct { int element_size; int total_nr_elements; + int elems_per_part; + u32 reciprocal_elems; struct flex_array_part *parts[]; }; /* diff --git a/lib/flex_array.c b/lib/flex_array.c index cab7621f98a..9b8b89458c4 100644 --- a/lib/flex_array.c +++ b/lib/flex_array.c @@ -24,6 +24,7 @@ #include #include #include +#include struct flex_array_part { char elements[FLEX_ARRAY_PART_SIZE]; @@ -70,15 +71,15 @@ static inline int elements_fit_in_base(struct flex_array *fa) * Element size | Objects | Objects | * PAGE_SIZE=4k | 32-bit | 64-bit | * ---------------------------------| - * 1 bytes | 4186112 | 2093056 | - * 2 bytes | 2093056 | 1046528 | - * 3 bytes | 1395030 | 697515 | - * 4 bytes | 1046528 | 523264 | - * 32 bytes | 130816 | 65408 | - * 33 bytes | 126728 | 63364 | - * 2048 bytes | 2044 | 1022 | - * 2049 bytes | 1022 | 511 | - * void * | 1046528 | 261632 | + * 1 bytes | 4177920 | 2088960 | + * 2 bytes | 2088960 | 1044480 | + * 3 bytes | 1392300 | 696150 | + * 4 bytes | 1044480 | 522240 | + * 32 bytes | 130560 | 65408 | + * 33 bytes | 126480 | 63240 | + * 2048 bytes | 2040 | 1020 | + * 2049 bytes | 1020 | 510 | + * void * | 1044480 | 261120 | * * Since 64-bit pointers are twice the size, we lose half the * capacity in the base structure. Also note that no effort is made @@ -88,11 +89,15 @@ struct flex_array *flex_array_alloc(int element_size, unsigned int total, gfp_t flags) { struct flex_array *ret; + int elems_per_part = 0; + int reciprocal_elems = 0; int max_size = 0; - if (element_size) - max_size = FLEX_ARRAY_NR_BASE_PTRS * - FLEX_ARRAY_ELEMENTS_PER_PART(element_size); + if (element_size) { + elems_per_part = FLEX_ARRAY_ELEMENTS_PER_PART(element_size); + reciprocal_elems = reciprocal_value(elems_per_part); + max_size = FLEX_ARRAY_NR_BASE_PTRS * elems_per_part; + } /* max_size will end up 0 if element_size > PAGE_SIZE */ if (total > max_size) @@ -102,6 +107,8 @@ struct flex_array *flex_array_alloc(int element_size, unsigned int total, return NULL; ret->element_size = element_size; ret->total_nr_elements = total; + ret->elems_per_part = elems_per_part; + ret->reciprocal_elems = reciprocal_elems; if (elements_fit_in_base(ret) && !(flags & __GFP_ZERO)) memset(&ret->parts[0], FLEX_ARRAY_FREE, FLEX_ARRAY_BASE_BYTES_LEFT); @@ -112,7 +119,7 @@ EXPORT_SYMBOL(flex_array_alloc); static int fa_element_to_part_nr(struct flex_array *fa, unsigned int element_nr) { - return element_nr / FLEX_ARRAY_ELEMENTS_PER_PART(fa->element_size); + return reciprocal_divide(element_nr, fa->reciprocal_elems); } /** @@ -141,12 +148,12 @@ void flex_array_free(struct flex_array *fa) EXPORT_SYMBOL(flex_array_free); static unsigned int index_inside_part(struct flex_array *fa, - unsigned int element_nr) + unsigned int element_nr, + unsigned int part_nr) { unsigned int part_offset; - part_offset = element_nr % - FLEX_ARRAY_ELEMENTS_PER_PART(fa->element_size); + part_offset = element_nr - part_nr * fa->elems_per_part; return part_offset * fa->element_size; } @@ -186,7 +193,7 @@ __fa_get_part(struct flex_array *fa, int part_nr, gfp_t flags) int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, gfp_t flags) { - int part_nr; + int part_nr = 0; struct flex_array_part *part; void *dst; @@ -202,7 +209,7 @@ int flex_array_put(struct flex_array *fa, unsigned int element_nr, void *src, if (!part) return -ENOMEM; } - dst = &part->elements[index_inside_part(fa, element_nr)]; + dst = &part->elements[index_inside_part(fa, element_nr, part_nr)]; memcpy(dst, src, fa->element_size); return 0; } @@ -217,7 +224,7 @@ EXPORT_SYMBOL(flex_array_put); */ int flex_array_clear(struct flex_array *fa, unsigned int element_nr) { - int part_nr; + int part_nr = 0; struct flex_array_part *part; void *dst; @@ -233,7 +240,7 @@ int flex_array_clear(struct flex_array *fa, unsigned int element_nr) if (!part) return -EINVAL; } - dst = &part->elements[index_inside_part(fa, element_nr)]; + dst = &part->elements[index_inside_part(fa, element_nr, part_nr)]; memset(dst, FLEX_ARRAY_FREE, fa->element_size); return 0; } @@ -302,7 +309,7 @@ EXPORT_SYMBOL(flex_array_prealloc); */ void *flex_array_get(struct flex_array *fa, unsigned int element_nr) { - int part_nr; + int part_nr = 0; struct flex_array_part *part; if (!fa->element_size) @@ -317,7 +324,7 @@ void *flex_array_get(struct flex_array *fa, unsigned int element_nr) if (!part) return NULL; } - return &part->elements[index_inside_part(fa, element_nr)]; + return &part->elements[index_inside_part(fa, element_nr, part_nr)]; } EXPORT_SYMBOL(flex_array_get); -- cgit v1.2.3-70-g09d2