--- mm/slub.c | 159 ++++++++++++++++++++++++++++++++------------------------------ 1 file changed, 83 insertions(+), 76 deletions(-) Index: slub/mm/slub.c =================================================================== --- slub.orig/mm/slub.c 2007-05-05 23:37:59.000000000 -0700 +++ slub/mm/slub.c 2007-05-06 08:23:51.000000000 -0700 @@ -1259,35 +1259,19 @@ static void flush_all(struct kmem_cache * Fastpath is not possible if we need to get a new slab or have * debugging enabled (which means all slabs are marked with PageError) */ -static void *slab_alloc(struct kmem_cache *s, - gfp_t gfpflags, int node, void *addr) +void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node, + void *addr, struct page *page) { - struct page *page; void **object; - unsigned long flags; - int cpu; + int cpu = smp_processor_id(); - local_irq_save(flags); - cpu = smp_processor_id(); - page = s->cpu_slab[cpu]; if (unlikely(!page)) goto new_slab; -#ifdef CONFIG_NUMA - if (unlikely(node != -1 && page_to_nid(page) != node)) { - slab_lock(page); + slab_lock(page); + if (unlikely(node != -1 && page_to_nid(page) != node)) goto another_slab; - } -#endif - if (likely(page->cpu_freelist)) { - object = page->cpu_freelist; - page->cpu_freelist = object[page->offset]; - local_irq_restore(flags); - return object; - } - - slab_lock(page); redo: if (!page->freelist) goto another_slab; @@ -1300,7 +1284,6 @@ redo: page->freelist = NULL; page->inuse = s->objects; slab_unlock(page); - local_irq_restore(flags); return object; another_slab: @@ -1316,35 +1299,34 @@ have_slab: } page = new_slab(s, gfpflags, node); - if (page) { - cpu = smp_processor_id(); - if (s->cpu_slab[cpu]) { + if (!page) + return NULL; + + cpu = smp_processor_id(); + if (s->cpu_slab[cpu]) { + /* + * Someone else populated the cpu_slab while we + * enabled interrupts, or we have gotten scheduled + * on another cpu. The page may not be on the + * requested node even if __GFP_THISNODE was + * specified. So we need to recheck. + */ + if (node == -1 || + page_to_nid(s->cpu_slab[cpu]) == node) { /* - * Someone else populated the cpu_slab while we - * enabled interrupts, or we have gotten scheduled - * on another cpu. The page may not be on the - * requested node even if __GFP_THISNODE was - * specified. So we need to recheck. + * Current cpuslab is acceptable and we + * want the current one since its cache hot */ - if (node == -1 || - page_to_nid(s->cpu_slab[cpu]) == node) { - /* - * Current cpuslab is acceptable and we - * want the current one since its cache hot - */ - discard_slab(s, page); - page = s->cpu_slab[cpu]; - slab_lock(page); - goto redo; - } - /* New slab does not fit our expectations */ - flush_slab(s, s->cpu_slab[cpu], cpu); + discard_slab(s, page); + page = s->cpu_slab[cpu]; + slab_lock(page); + goto redo; } - slab_lock(page); - goto have_slab; + /* New slab does not fit our expectations */ + flush_slab(s, s->cpu_slab[cpu], cpu); } - local_irq_restore(flags); - return NULL; + slab_lock(page); + goto have_slab; debug: object = page->freelist; if (!alloc_object_checks(s, page, object)) @@ -1357,6 +1339,29 @@ debug: page->inuse++; init_object(s, object, 1); slab_unlock(page); + return object; +} + +static void __always_inline *slab_alloc(struct kmem_cache *s, + gfp_t gfpflags, int node, void *addr) +{ + struct page *page; + unsigned long flags; + void **object; + + local_irq_save(flags); + page = s->cpu_slab[smp_processor_id()]; + if (unlikely(!page || + +#ifdef CONFIG_NUMA + (node != -1 && page_to_nid(page) != node) || +#endif + !page->cpu_freelist)) + object = __slab_alloc(s, gfpflags, node, addr, page); + else { + object = page->cpu_freelist; + page->cpu_freelist = object[page->offset]; + } local_irq_restore(flags); return object; } @@ -1382,47 +1387,28 @@ EXPORT_SYMBOL(kmem_cache_alloc_node); * We read the cpu_slab cacheline to check if the slab is the per cpu * slab for this processor. */ -static void slab_free(struct kmem_cache *s, struct page *page, - void *x, void *addr) +void __slab_free(struct kmem_cache *s, struct page *page, + void **object, void *addr) { void *prior; - void **object = (void *)x; - unsigned long flags; - local_irq_save(flags); - /* - * Is this the cpu slab for the current processor? Then we can use - * the cpu_freelist without having to take the slab lock - */ - if (likely(page == s->cpu_slab[smp_processor_id()] && - !DebugSlab(page))) { - object[page->offset] = page->cpu_freelist; - page->cpu_freelist = object; - local_irq_restore(flags); - return; - } - - /* - * Not our current cpu slab. So we have to lock the slab for freeing - * and use the regular freelist for freeing. - */ slab_lock(page); - if (unlikely(DebugSlab(page))) + if (DebugSlab(page)) goto debug; checks_ok: prior = object[page->offset] = page->freelist; page->freelist = object; page->inuse--; - if (unlikely(PageActive(page))) + if (PageActive(page)) /* * Cpu slabs are never on partial lists and are * never freed. */ goto out_unlock; - if (unlikely(!page->inuse)) + if (!page->inuse) goto slab_empty; /* @@ -1430,12 +1416,11 @@ checks_ok: * was not on the partial list before * then add it. */ - if (unlikely(!prior)) + if (!prior) add_partial(get_node(s, page_to_nid(page)), page); out_unlock: slab_unlock(page); - local_irq_restore(flags); return; slab_empty: @@ -1447,21 +1432,43 @@ slab_empty: slab_unlock(page); discard_slab(s, page); - local_irq_restore(flags); return; debug: - if (!free_object_checks(s, page, x)) + if (!free_object_checks(s, page, (void *)object)) goto out_unlock; if (!PageActive(page) && !page->freelist) remove_full(s, page); if (s->flags & SLAB_STORE_USER) - set_track(s, x, TRACK_FREE, addr); + set_track(s, (void *)object, TRACK_FREE, addr); trace(s, page, object, 0); init_object(s, object, 0); goto checks_ok; } +static void __always_inline slab_free(struct kmem_cache *s, struct page *page, + void *x, void *addr) +{ + void **object = (void *)x; + unsigned long flags; + + local_irq_save(flags); + /* + * Is this the cpu slab for the current processor? Then we can use + * the cpu_freelist without having to take the slab lock + */ + if (unlikely(page != s->cpu_slab[smp_processor_id()] || + DebugSlab(page))) + __slab_free(s, page, object, addr); + else { + object[page->offset] = page->cpu_freelist; + page->cpu_freelist = object; + } + + local_irq_restore(flags); + return; +} + void kmem_cache_free(struct kmem_cache *s, void *x) { struct page *page;