Index: linux-2.6.19-mm1/mm/slub.c =================================================================== --- linux-2.6.19-mm1.orig/mm/slub.c 2006-12-13 17:44:26.000000000 -0800 +++ linux-2.6.19-mm1/mm/slub.c 2006-12-13 19:55:14.000000000 -0800 @@ -14,6 +14,7 @@ #include #include #include +#include #define SLUB_UNIMPLEMENTED (SLAB_DEBUG_FREE | SLAB_DEBUG_INITIAL | \ SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) @@ -57,8 +58,37 @@ #define ARCH_SLAB_MINALIGN sizeof(void *) #endif -static void register_slab(struct kmem_cache *s); -static void unregister_slab(struct kmem_cache *s); +static DECLARE_RWSEM(slabstat_sem); + +LIST_HEAD(slab_caches); + +void for_all_slabs(void (*func)(struct kmem_cache *, int), int cpu) +{ + struct list_head *h; + + down_read(&slabstat_sem); + list_for_each(h, &slab_caches) { + struct kmem_cache *s = + container_of(h, struct kmem_cache, list); + + func(s, cpu); + } + up_read(&slabstat_sem); +} + +void register_slab(struct kmem_cache *s) +{ + down_write(&slabstat_sem); + list_add(&s->list, &slab_caches); + up_write(&slabstat_sem); +} + +void unregister_slab(struct kmem_cache *s) +{ + down_write(&slabstat_sem); + list_add(&s->list, &slab_caches); + up_write(&slabstat_sem); +} #ifdef CONFIG_NUMA @@ -185,13 +215,16 @@ */ static __always_inline void slab_lock(struct page *page) { +#ifdef CONFIG_SMP bit_spin_lock(PG_locked, &page->flags); -// BUG_ON(!bit_spin_trylock(PG_locked, &page->flags)); +#endif } static __always_inline void slab_unlock(struct page *page) { +#ifdef CONFIG_SMP bit_spin_unlock(PG_locked, &page->flags); +#endif } /* @@ -199,6 +232,10 @@ */ static void __always_inline add_partial(struct kmem_cache *s, struct page *page) { + if (page->inuse == s->objects) { + printk("Slab %s page=%p adding fully used slab\n", s->name, page); + dump_stack(); + } spin_lock(&s->list_lock); s->nr_partial++; list_add_tail(&page->lru, &s->partial); @@ -429,6 +466,7 @@ */ static void __always_inline putback_slab(struct kmem_cache *s, struct page *page) { + printk(KERN_CRIT "putback_slab(%s,%p) inuse=%d objects=%d\n",s->name, page,page->inuse, s->objects); if (page->inuse) { if (page->inuse < s->objects) add_partial(s, page); @@ -544,6 +582,8 @@ void **object; unsigned long flags; + printk(KERN_CRIT "allocate(%s,%x,%d)\n", s->name, gfpflags, node); + local_irq_save(flags); a = ACTIVE_SLAB(s, smp_processor_id()); if (unlikely(!a->page)) @@ -605,9 +645,9 @@ } __SetPageActive(a->page); - check_free_chain(s, a->page); switch_freelist: + printk(KERN_CRIT "switch_freelist\n"); a->freelist = a->page->freelist; a->page->freelist = NULL; a->nr_free = s->objects - a->page->inuse; @@ -628,7 +668,12 @@ } #endif out: + check_free_chain(s, a->page); local_irq_restore(flags); + printk(KERN_CRIT "return %p active freelist=%p nr_free=%d page " + "inuse=%d freelist=%p\n", object, a->freelist, a->nr_free, + a->page ? a->page->inuse : -1, + a->page ? a->page->freelist : (void *)-1L); return object; } @@ -742,8 +787,10 @@ */ remove_partial(s, page); slab_unlock(page); + single_object_slab: discard_slab(s, page); + local_irq_restore(flags); return; @@ -863,10 +910,6 @@ } /* - * Boot strap slabs - */ - -/* * We can actually operate slabs any time after the page allocator is up. * slab_is_available() merely means that the kmalloc array is available. * @@ -913,6 +956,9 @@ { int cpu; + printk("kmem_cache_open(%p, %s, %ld, %ld, %lx, %p, %p)\n", + s, name, (long)size, (long)align, flags, ctor, dtor); + BUG_ON(flags & SLUB_UNIMPLEMENTED); memset(s, 0, sizeof(struct kmem_cache)); atomic_long_set(&s->nr_slabs, 0); @@ -967,6 +1013,7 @@ for_each_online_cpu(cpu) alloc_active(s, cpu); + register_slab(s); return 1; @@ -1143,17 +1190,22 @@ return slabs_inuse; } -void release_active(struct kmem_cache *s) +static void free_active(struct kmem_cache *s, int cpu) { #ifdef CONFIG_NUMA + kfree(ACTIVE_SLABS(s, cpu)); + s->active[cpu] = NULL; +#endif +} + +void release_active(struct kmem_cache *s) +{ int cpu; - for_each_online_cpu(cpu) { - kfree(ACTIVE_SLAB(s, cpu)); - s->active[cpu] = NULL; - } -#endif + for_each_online_cpu(cpu) + free_active(s, cpu); } + /* * Release all resources used by slab cache * (Use with caches setup using kmem_cache_setup) @@ -1231,9 +1283,33 @@ (nr_slabs - s->nr_partial - nr_active) * s->objects; } -#ifdef CONFIG_NUMA -/* logic to bring up per cpu portions is missing here */ -#endif +/* + * Use the cpu notifier to insure that the thresholds are recalculated + * when necessary. + */ +static int __cpuinit slab_cpuup_callback(struct notifier_block *nfb, + unsigned long action, + void *hcpu) +{ + long cpu = (long)hcpu; + + switch (action) { + case CPU_UP_PREPARE: + for_all_slabs(alloc_active, cpu); + break; + case CPU_UP_CANCELED: + case CPU_DEAD: + for_all_slabs(free_active, cpu); + break; + default: + break; + } + return NOTIFY_OK; +} + +static struct notifier_block __cpuinitdata slab_notifier = + { &slab_cpuup_callback, NULL, 0 }; + /******************************************************************** * Kmalloc subsystem @@ -1256,6 +1332,8 @@ static struct kmem_cache *create_kmalloc_cache(struct kmem_cache *s, const char *name, int size) { + printk(KERN_CRIT "create_kmalloc_cache(%p,%s,%d)\n", + s, name, size); if (s->size) return s; @@ -1383,6 +1461,8 @@ " Min_order=%d.\n", KMALLOC_SHIFT_HIGH - KMALLOC_SHIFT_LOW + KMALLOC_EXTRAS, slab_min_order); + + register_cpu_notifier(&slab_notifier); } /******************************************************************** @@ -1460,24 +1540,6 @@ * Slab proc interface *******************************************************************/ -static DECLARE_RWSEM(slabstat_sem); - -LIST_HEAD(slab_caches); - -void register_slab(struct kmem_cache *s) -{ - down_write(&slabstat_sem); - list_add(&s->list, &slab_caches); - up_write(&slabstat_sem); -} - -void unregister_slab(struct kmem_cache *s) -{ - down_write(&slabstat_sem); - list_add(&s->list, &slab_caches); - up_write(&slabstat_sem); -} - static void print_slabinfo_header(struct seq_file *m) { /*