diff options
| -rw-r--r-- | include/linux/slub_def.h | 8 | ||||
| -rw-r--r-- | mm/slub.c | 165 |
2 files changed, 69 insertions, 104 deletions
diff --git a/include/linux/slub_def.h b/include/linux/slub_def.h index 45ca123e800..c8668d161dd 100644 --- a/include/linux/slub_def.h +++ b/include/linux/slub_def.h @@ -37,9 +37,7 @@ enum stat_item { struct kmem_cache_cpu { void **freelist; /* Pointer to next available object */ -#ifdef CONFIG_CMPXCHG_LOCAL unsigned long tid; /* Globally unique transaction id */ -#endif struct page *page; /* The slab from which we are allocating */ int node; /* The node of the page (or -1 for debug) */ #ifdef CONFIG_SLUB_STATS @@ -179,7 +177,8 @@ static __always_inline int kmalloc_index(size_t size) if (size <= 4 * 1024) return 12; /* * The following is only needed to support architectures with a larger page - * size than 4k. + * size than 4k. We need to support 2 * PAGE_SIZE here. So for a 64k page + * size we would have to go up to 128k. */ if (size <= 8 * 1024) return 13; if (size <= 16 * 1024) return 14; @@ -190,7 +189,8 @@ static __always_inline int kmalloc_index(size_t size) if (size <= 512 * 1024) return 19; if (size <= 1024 * 1024) return 20; if (size <= 2 * 1024 * 1024) return 21; - return -1; + BUG(); + return -1; /* Will never be reached */ /* * What we really wanted to do and cannot do because of compiler issues is: diff --git a/mm/slub.c b/mm/slub.c index 9d2e5e46bf0..4ea7f1a22a9 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -261,6 +261,18 @@ static inline void *get_freepointer(struct kmem_cache *s, void *object) return *(void **)(object + s->offset); } +static inline void *get_freepointer_safe(struct kmem_cache *s, void *object) +{ + void *p; + +#ifdef CONFIG_DEBUG_PAGEALLOC + probe_kernel_read(&p, (void **)(object + s->offset), sizeof(p)); +#else + p = get_freepointer(s, object); +#endif + return p; +} + static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) { *(void **)(object + s->offset) = fp; @@ -271,10 +283,6 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp) for (__p = (__addr); __p < (__addr) + (__objects) * (__s)->size;\ __p += (__s)->size) -/* Scan freelist */ -#define for_each_free_object(__p, __s, __free) \ - for (__p = (__free); __p; __p = get_freepointer((__s), __p)) - /* Determine object index from a given position */ static inline int slab_index(void *p, struct kmem_cache *s, void *addr) { @@ -332,6 +340,21 @@ static inline int oo_objects(struct kmem_cache_order_objects x) #ifdef CONFIG_SLUB_DEBUG /* + * Determine a map of object in use on a page. + * + * Slab lock or node listlock must be held to guarantee that the page does + * not vanish from under us. + */ +static void get_map(struct kmem_cache *s, struct page *page, unsigned long *map) +{ + void *p; + void *addr = page_address(page); + + for (p = page->freelist; p; p = get_freepointer(s, p)) + set_bit(slab_index(p, s, addr), map); +} + +/* * Debug settings: */ #ifdef CONFIG_SLUB_DEBUG_ON @@ -1487,7 +1510,7 @@ static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node) int searchnode = (node == NUMA_NO_NODE) ? numa_node_id() : node; page = get_partial_node(get_node(s, searchnode)); - if (page || node != -1) + if (page || node != NUMA_NO_NODE) return page; return get_any_partial(s, flags); @@ -1540,7 +1563,6 @@ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail) } } -#ifdef CONFIG_CMPXCHG_LOCAL #ifdef CONFIG_PREEMPT /* * Calculate the next globally unique transaction for disambiguiation @@ -1600,17 +1622,12 @@ static inline void note_cmpxchg_failure(const char *n, stat(s, CMPXCHG_DOUBLE_CPU_FAIL); } -#endif - void init_kmem_cache_cpus(struct kmem_cache *s) { -#ifdef CONFIG_CMPXCHG_LOCAL int cpu; for_each_possible_cpu(cpu) per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu); -#endif - } /* * Remove the cpu slab @@ -1643,9 +1660,7 @@ static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c) page->inuse--; } c->page = NULL; -#ifdef CONFIG_CMPXCHG_LOCAL c->tid = next_tid(c->tid); -#endif unfreeze_slab(s, page, tail); } @@ -1779,8 +1794,7 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, unsigned long addr, struct kmem_cache_cpu *c) { void **object; - struct page *new; -#ifdef CONFIG_CMPXCHG_LOCAL + struct page *page; unsigned long flags; local_irq_save(flags); @@ -1792,37 +1806,35 @@ static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, */ c = this_cpu_ptr(s->cpu_slab); #endif -#endif /* We handle __GFP_ZERO in the caller */ gfpflags &= ~__GFP_ZERO; - if (!c->page) + page = c->page; + if (!page) goto new_slab; - slab_lock(c->page); + slab_lock(page); if (unlikely(!node_match(c, node))) goto another_slab; stat(s, ALLOC_REFILL); load_freelist: - object = c->page->freelist; + object = page->freelist; if (unlikely(!object)) goto another_slab; if (kmem_cache_debug(s)) goto debug; c->freelist = get_freepointer(s, object); - c->page->inuse = c->page->objects; - c->page->freelist = NULL; - c->node = page_to_nid(c->page); + page->inuse = page->objects; + page->freelist = NULL; + unlock_out: - slab_unlock(c->page); -#ifdef CONFIG_CMPXCHG_LOCAL + slab_unlock(page); c->tid = next_tid(c->tid); local_irq_restore(flags); -#endif stat(s, ALLOC_SLOWPATH); return object; @@ -1830,10 +1842,11 @@ another_slab: deactivate_slab(s, c); new_slab: - new = get_partial(s, gfpflags, node); - if (new) { - c->page = new; + page = get_partial(s, gfpflags, node); + if (page) { stat(s, ALLOC_FROM_PARTIAL); + c->node = page_to_nid(page); + c->page = page; goto load_freelist; } @@ -1841,33 +1854,35 @@ new_slab: if (gfpflags & __GFP_WAIT) local_irq_enable(); - new = new_slab(s, gfpflags, node); + page = new_slab(s, gfpflags, node); if (gfpflags & __GFP_WAIT) local_irq_disable(); - if (new) { + if (page) { c = __this_cpu_ptr(s->cpu_slab); stat(s, ALLOC_SLAB); if (c->page) flush_slab(s, c); - slab_lock(new); - __SetPageSlubFrozen(new); - c->page = new; + + slab_lock(page); + __SetPageSlubFrozen(page); + c->node = page_to_nid(page); + c->page = page; goto load_freelist; } if (!(gfpflags & __GFP_NOWARN) && printk_ratelimit()) slab_out_of_memory(s, gfpflags, node); -#ifdef CONFIG_CMPXCHG_LOCAL local_irq_restore(flags); -#endif return NULL; debug: - if (!alloc_debug_processing(s, c->page, object, addr)) + if (!alloc_debug_processing(s, page, object, addr)) goto another_slab; - c->page->inuse++; - c->page->freelist = get_freepointer(s, object); + page->inuse++; + page->freelist = get_freepointer(s, object); + deactivate_slab(s, c); + c->page = NULL; c->node = NUMA_NO_NODE; goto unlock_out; } @@ -1887,20 +1902,12 @@ static __always_inline void *slab_alloc(struct kmem_cache *s, { void **object; struct kmem_cache_cpu *c; -#ifdef CONFIG_CMPXCHG_LOCAL unsigned long tid; -#else - unsigned long flags; -#endif if (slab_pre_alloc_hook(s, gfpflags)) return NULL; -#ifndef CONFIG_CMPXCHG_LOCAL - local_irq_save(flags); -#else redo: -#endif /* * Must read kmem_cache cpu data via this cpu ptr. Preemption is @@ -1910,7 +1917,6 @@ redo: */ c = __this_cpu_ptr(s->cpu_slab); -#ifdef CONFIG_CMPXCHG_LOCAL /* * The transaction ids are globally unique per cpu and per operation on * a per cpu queue. Thus they can be guarantee that the cmpxchg_double @@ -1919,7 +1925,6 @@ redo: */ tid = c->tid; barrier(); -#endif object = c->freelist; if (unlikely(!object || !node_match(c, node))) @@ -1927,7 +1932,6 @@ redo: object = __slab_alloc(s, gfpflags, node, addr, c); else { -#ifdef CONFIG_CMPXCHG_LOCAL /* * The cmpxchg will only match if there was no additional * operation and if we are on the right processor. @@ -1943,21 +1947,14 @@ redo: if (unlikely(!irqsafe_cpu_cmpxchg_double( s->cpu_slab->freelist, s->cpu_slab->tid, object, tid, - get_freepointer(s, object), next_tid(tid)))) { + get_freepointer_safe(s, object), next_tid(tid)))) { note_cmpxchg_failure("slab_alloc", s, tid); goto redo; } -#else - c->freelist = get_freepointer(s, object); -#endif stat(s, ALLOC_FASTPATH); } -#ifndef CONFIG_CMPXCHG_LOCAL - local_irq_restore(flags); -#endif - if (unlikely(gfpflags & __GFP_ZERO) && object) memset(object, 0, s->objsize); @@ -2034,18 +2031,15 @@ static void __slab_free(struct kmem_cache *s, struct page *page, { void *prior; void **object = (void *)x; -#ifdef CONFIG_CMPXCHG_LOCAL unsigned long flags; local_irq_save(flags); -#endif slab_lock(page); stat(s, FREE_SLOWPATH); - if (kmem_cache_debug(s)) - goto debug; + if (kmem_cache_debug(s) && !free_debug_processing(s, page, x, addr)) + goto out_unlock; -checks_ok: prior = page->freelist; set_freepointer(s, object, prior); page->freelist = object; @@ -2070,9 +2064,7 @@ checks_ok: out_unlock: slab_unlock(page); -#ifdef CONFIG_CMPXCHG_LOCAL local_irq_restore(flags); -#endif return; slab_empty: @@ -2084,17 +2076,9 @@ slab_empty: stat(s, FREE_REMOVE_PARTIAL); } slab_unlock(page); -#ifdef CONFIG_CMPXCHG_LOCAL local_irq_restore(flags); -#endif stat(s, FREE_SLAB); discard_slab(s, page); - return; - -debug: - if (!free_debug_processing(s, page, x, addr)) - goto out_unlock; - goto checks_ok; } /* @@ -2113,20 +2097,11 @@ static __always_inline void slab_free(struct kmem_cache *s, { void **object = (void *)x; struct kmem_cache_cpu *c; -#ifdef CONFIG_CMPXCHG_LOCAL unsigned long tid; -#else - unsigned long flags; -#endif slab_free_hook(s, x); -#ifndef CONFIG_CMPXCHG_LOCAL - local_irq_save(flags); - -#else redo: -#endif /* * Determine the currently cpus per cpu slab. @@ -2136,15 +2111,12 @@ redo: */ c = __this_cpu_ptr(s->cpu_slab); -#ifdef CONFIG_CMPXCHG_LOCAL tid = c->tid; barrier(); -#endif - if (likely(page == c->page && c->node != NUMA_NO_NODE)) { + if (likely(page == c->page)) { set_freepointer(s, object, c->freelist); -#ifdef CONFIG_CMPXCHG_LOCAL if (unlikely(!irqsafe_cpu_cmpxchg_double( s->cpu_slab->freelist, s->cpu_slab->tid, c->freelist, tid, @@ -2153,16 +2125,10 @@ redo: note_cmpxchg_failure("slab_free", s, tid); goto redo; } -#else - c->freelist = object; -#endif stat(s, FREE_FASTPATH); } else __slab_free(s, page, x, addr); -#ifndef CONFIG_CMPXCHG_LOCAL - local_irq_restore(flags); -#endif } void kmem_cache_free(struct kmem_cache *s, void *x) @@ -2673,9 +2639,8 @@ static void list_slab_objects(struct kmem_cache *s, struct page *page, return; slab_err(s, page, "%s", text); slab_lock(page); - for_each_free_object(p, s, page->freelist) - set_bit(slab_index(p, s, addr), map); + get_map(s, page, map); for_each_object(p, s, addr, page->objects) { if (!test_bit(slab_index(p, s, addr), map)) { @@ -3203,7 +3168,7 @@ static void __init kmem_cache_bootstrap_fixup(struct kmem_cache *s) list_for_each_entry(p, &n->partial, lru) p->slab = s; -#ifdef CONFIG_SLAB_DEBUG +#ifdef CONFIG_SLUB_DEBUG list_for_each_entry(p, &n->full, lru) p->slab = s; #endif @@ -3610,10 +3575,11 @@ static int validate_slab(struct kmem_cache *s, struct page *page, /* Now we know that a valid freelist exists */ bitmap_zero(map, page->objects); - for_each_free_object(p, s, page->freelist) { - set_bit(slab_index(p, s, addr), map); - if (!check_object(s, page, p, SLUB_RED_INACTIVE)) - return 0; + get_map(s, page, map); + for_each_object(p, s, addr, page->objects) { + if (test_bit(slab_index(p, s, addr), map)) + if (!check_object(s, page, p, SLUB_RED_INACTIVE)) + return 0; } for_each_object(p, s, addr, page->objects) @@ -3821,8 +3787,7 @@ static void process_slab(struct loc_track *t, struct kmem_cache *s, void *p; bitmap_zero(map, page->objects); - for_each_free_object(p, s, page->freelist) - set_bit(slab_index(p, s, addr), map); + get_map(s, page, map); for_each_object(p, s, addr, page->objects) if (!test_bit(slab_index(p, s, addr), map)) |