Index: linux-2.6.18-rc4-mm3/mm/slabifier.c =================================================================== --- linux-2.6.18-rc4-mm3.orig/mm/slabifier.c 2006-08-30 15:48:12.854306927 -0700 +++ linux-2.6.18-rc4-mm3/mm/slabifier.c 2006-08-30 16:09:55.286573796 -0700 @@ -561,32 +561,20 @@ static __always_inline void *__slab_allo unsigned long flags; int cpu; - if (unlikely(s->objects == 1)) { - struct page *page = new_slab(s, gfpflags, node); - - if (page) - return page_address(page); - else - return NULL; - } + if (unlikely(s->objects == 1)) + goto single_object; local_irq_save(flags); cpu = smp_processor_id(); page = get_and_lock_active(s, cpu); - if (unlikely(!page)) - goto load; - - while (unlikely(!page->freelist || - (node > 0 && page_to_nid(page) != node))) { - - deactivate_slab(s, page, cpu); -load: - page = reload(s, cpu, gfpflags, node); - if (unlikely(!page)) { - local_irq_restore(flags); - return NULL; - } - } +check: + if (unlikely(!page || !page->freelist)) + goto fix_things; + +#ifdef CONFIG_NUMA + if (unlikely(node >= 0) && page_to_nid(page) != node) + goto fix_things; +#endif page->inuse++; object = page->freelist; @@ -597,6 +585,23 @@ load: slab_unlock(page); local_irq_restore(flags); return object; + +fix_things: + if (page) + deactivate_slab(s, page, cpu); + page = reload(s, cpu, gfpflags, node); + if (unlikely(!page)) { + local_irq_restore(flags); + return NULL; + } + goto check; + +single_object: + page = new_slab(s, gfpflags, node); + if (page) + return page_address(page); + else + return NULL; } static void *slab_alloc(struct slab_cache *sc, gfp_t gfpflags) @@ -632,44 +637,16 @@ static void slab_free(struct slab_cache void **object = (void *)x; unsigned long flags; - if (!object) - return; + if (unlikely(!object || !s || s->objects == 1)) + goto fix_things_up_before; page = get_object_page(object); - if (unlikely(!page)) { - printk(KERN_CRIT "slab_free %s size %d: attempt to free object" - "(%p) outside of slab.\n", s->sc.name, s->size, object); - goto dumpret; - } + if (unlikely(page || sc != page->slab)) + goto fix_things_up_after; - if (!s) { - if (unlikely(!page->slab)) { - printk(KERN_CRIT - "slab_free : no slab(NULL) for object %p.\n", - object); - goto dumpret; - } - } else - if (unlikely(sc != page->slab)) { - printk(KERN_CRIT "slab_free %s: object at %p" - " belongs to slab %p\n", - s->sc.name, object, page->slab); - dump_stack(); - s = (struct slab *)page->slab; - } - - if (unlikely(!check_valid_pointer(s, page, object, NULL))) { -dumpret: - dump_stack(); - printk(KERN_CRIT "***** Trying to continue by not" - "freeing object.\n"); - return; - } - - if (unlikely(s->objects == 1)) { - discard_slab(s, page); - return; - } +jumpin: + if (unlikely(!check_valid_pointer(s, page, object, NULL))) + goto dumpret; local_irq_save(flags); slab_lock(page); @@ -689,10 +666,17 @@ dumpret: page->freelist = object; page->inuse--; - if (unlikely(PageActive(page))) - goto out_unlock; + if (!PageActive(page) && (!page->inuse || !prior)) + goto fix_things_after_all; + + slab_unlock(page); +out: + local_irq_restore(flags); + return; - if (unlikely(page->inuse == 0)) { +fix_things_after_all: + + if (!page->inuse) { remove_partial(s, page); check_free_chain(s, page); slab_unlock(page); @@ -709,8 +693,51 @@ dumpret: out_unlock: slab_unlock(page); -out: local_irq_restore(flags); + return; + +dumpret: + dump_stack(); + printk(KERN_CRIT "***** Trying to continue by not" + "freeing object.\n"); + return; + + /* Things were not ideal before check of page */ +fix_things_up_before: + if (!object) + return; + + page = get_object_page(object); + + /* Things were not ideal after we determined the page */ +fix_things_up_after: + if (unlikely(!page)) { + printk(KERN_CRIT "slab_free %s size %d: attempt to free object" + "(%p) outside of slab.\n", s->sc.name, s->size, object); + goto dumpret; + } + + if (!s) { + if (unlikely(!page->slab)) { + printk(KERN_CRIT + "slab_free : no slab(NULL) for object %p.\n", + object); + goto dumpret; + } + } else + if (unlikely(sc != page->slab)) { + printk(KERN_CRIT "slab_free %s: object at %p" + " belongs to slab %p\n", + s->sc.name, object, page->slab); + dump_stack(); + s = (struct slab *)page->slab; + } + + if (unlikely(s->objects == 1)) { + discard_slab(s, page); + return; + } + goto jumpin; } /*