From ddbb16bb6f4fb507e2e8fb1f1cfd23deb6e8ff65 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Thu, 26 Jul 2007 19:01:40 -0700 Subject: [PATCH] Simplify slab_free and slab_alloc Make __slab_free and __slab_alloc usable on their own by moving the interrupt disable / enable into the functions itself. Then check in __slab_alloc for cases in which we may be called when a page still contains object on the lockless_freelist. In that case we will have to drain the page. Extract the lockless merge functionality out of deactivate_slab. Use an xchg to get the lockless_list in order to allow future arch specific functionality that may operate on the lockless list without disabling interrupts. Signed-off-by: Christoph Lameter --- mm/slub.c | 103 +++++++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 72 insertions(+), 31 deletions(-) Index: linux-2.6.23-rc1/mm/slub.c =================================================================== --- linux-2.6.23-rc1.orig/mm/slub.c 2007-07-26 21:55:55.000000000 -0700 +++ linux-2.6.23-rc1/mm/slub.c 2007-07-27 15:23:19.000000000 -0700 @@ -1370,29 +1370,41 @@ static void unfreeze_slab(struct kmem_ca } } + /* - * Remove the cpu slab + * Merge cpu freelist into freelist. Typically we get here because both + * freelists are empty. So this is unlikely to occur. + * + * We need to use atomic ops on lockless_freelist since some * arches may + * use atomic ops to update the freelist. + * + * slab must be locked */ -static void deactivate_slab(struct kmem_cache *s, struct page *page, int cpu) +static void noinline merge_lockless_list(struct page *page) { - /* - * Merge cpu freelist into freelist. Typically we get here - * because both freelists are empty. So this is unlikely - * to occur. - */ - while (unlikely(page->lockless_freelist)) { - void **object; - - /* Retrieve object from cpu_freelist */ - object = page->lockless_freelist; - page->lockless_freelist = page->lockless_freelist[page->offset]; + void **ll; + void **object; + + smp_wmb(); /* Make sure cpuslab state is up to date */ + ll = xchg(&page->lockless_freelist, NULL); + while (unlikely(ll)) { + object = ll; + ll = ll[page->offset]; - /* And put onto the regular freelist */ object[page->offset] = page->freelist; page->freelist = object; page->inuse--; } +} + +/* + * Remove the cpu slab + */ +static void deactivate_slab(struct kmem_cache *s, struct page *page, int cpu) +{ s->cpu_slab[cpu] = NULL; + if (unlikely(page->lockless_freelist)) + merge_lockless_list(page); unfreeze_slab(s, page); } @@ -1453,17 +1465,24 @@ static void flush_all(struct kmem_cache * we need to allocate a new slab. This is slowest path since we may sleep. */ static void *__slab_alloc(struct kmem_cache *s, - gfp_t gfpflags, int node, void *addr, struct page *page) + gfp_t gfpflags, int node, void *addr) { void **object; - int cpu = smp_processor_id(); + int cpu; + struct page *page; + unsigned long flags; + local_irq_save(flags); + cpu = smp_processor_id(); + page = s->cpu_slab[cpu]; if (!page) goto new_slab; slab_lock(page); if (unlikely(node != -1 && page_to_nid(page) != node)) goto another_slab; + if (page->lockless_freelist) + goto another_slab; load_freelist: object = page->freelist; if (unlikely(!object)) @@ -1475,7 +1494,12 @@ load_freelist: page->lockless_freelist = object[page->offset]; page->inuse = s->objects; page->freelist = NULL; +out: slab_unlock(page); + local_irq_restore(flags); + if (unlikely((gfpflags & __GFP_ZERO))) + memset(object, 0, s->objsize); + return object; another_slab: @@ -1518,6 +1542,7 @@ new_slab: s->cpu_slab[cpu] = page; goto load_freelist; } + local_irq_restore(flags); return NULL; debug: object = page->freelist; @@ -1526,8 +1551,7 @@ debug: page->inuse++; page->freelist = object[page->offset]; - slab_unlock(page); - return object; + goto out; } /* @@ -1549,21 +1573,28 @@ static void __always_inline *slab_alloc( local_irq_save(flags); page = s->cpu_slab[smp_processor_id()]; - if (unlikely(!page || !page->lockless_freelist || - (node != -1 && page_to_nid(page) != node))) - object = __slab_alloc(s, gfpflags, node, addr, page); + if (unlikely(!page)) + goto slow; - else { - object = page->lockless_freelist; - page->lockless_freelist = object[page->offset]; - } + object = page->lockless_freelist; + + if (unlikely(!object)) + goto slow; + + if (unlikely(node != -1 && page_to_nid(page) != node)) + goto slow; + + page->lockless_freelist = object[page->offset]; local_irq_restore(flags); if (unlikely((gfpflags & __GFP_ZERO) && object)) memset(object, 0, s->objsize); return object; +slow: + local_irq_restore(flags); + return __slab_alloc(s, gfpflags, node, addr); } void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) @@ -1593,7 +1624,9 @@ static void __slab_free(struct kmem_cach { void *prior; void **object = (void *)x; + unsigned long flags; + local_irq_save(flags); slab_lock(page); if (unlikely(SlabDebug(page))) @@ -1619,6 +1652,7 @@ checks_ok: out_unlock: slab_unlock(page); + local_irq_restore(flags); return; slab_empty: @@ -1629,6 +1663,7 @@ slab_empty: remove_partial(s, page); slab_unlock(page); + local_irq_restore(flags); discard_slab(s, page); return; @@ -1656,14 +1691,20 @@ static void __always_inline slab_free(st unsigned long flags; local_irq_save(flags); - if (likely(page == s->cpu_slab[smp_processor_id()] && - !SlabDebug(page))) { - object[page->offset] = page->lockless_freelist; - page->lockless_freelist = object; - } else - __slab_free(s, page, x, addr); + if (unlikely(page != s->cpu_slab[smp_processor_id()])) + goto slow; + + if (unlikely(SlabDebug(page))) + goto slow; + + object[page->offset] = page->lockless_freelist; + page->lockless_freelist = object; + local_irq_restore(flags); + return; +slow: local_irq_restore(flags); + __slab_free(s, page, x, addr); } void kmem_cache_free(struct kmem_cache *s, void *x)