Index: linux-2.6.21-rc1/mm/slub.c =================================================================== --- linux-2.6.21-rc1.orig/mm/slub.c 2007-02-24 13:07:42.000000000 -0800 +++ linux-2.6.21-rc1/mm/slub.c 2007-02-24 13:08:10.000000000 -0800 @@ -69,10 +69,15 @@ /* * Set of flags that will prohibit slab merging */ -#define SLUB_NO_MERGE (SLAB_RECLAIM_ACCOUNT | SLAB_DESTROY_BY_RCU | \ - SLAB_CACHE_DMA | SLAB_DEBUG_FREE | SLAB_DEBUG_INITIAL | \ +#define SLUB_NEVER_MERGE (SLAB_DEBUG_FREE | SLAB_DEBUG_INITIAL | \ SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER) +/* + * Set of flags that must be the same for merging + */ +#define SLUB_SAME_MERGE (SLAB_RECLAIM_ACCOUNT | SLAB_DESTROY_BY_RCU | \ + SLAB_CACHE_DMA) + #ifndef ARCH_KMALLOC_MINALIGN #define ARCH_KMALLOC_MINALIGN sizeof(void *) #endif @@ -91,6 +96,10 @@ static struct notifier_block slab_notifier; #endif +static DECLARE_RWSEM(slabstat_sem); + +LIST_HEAD(slab_caches); + /******************************************************************** * Core slab cache functions *******************************************************************/ @@ -1281,20 +1290,53 @@ #endif } -static struct kmem_cache *kmem_cache_dup(struct kmem_cache *s) +static struct kmem_cache *kmem_cache_dup(struct kmem_cache *s, + const char *name) { + char *x; + atomic_inc(&s->refcount); + + if (!s->aliases) + s->aliases = kstrdup(name, GFP_KERNEL); + else { + x = kmalloc(strlen(s->aliases) + strlen(name) + 1, + GFP_KERNEL); + strcpy(x, s->aliases); + strcat(x, " "); + strcat(x, name); + kfree(s->aliases); + s->aliases = x; + } return s; } -static struct kmem_cache *__kmalloc_slab(size_t size) +/* + * Find a mergeable slab cache + */ +static struct kmem_cache *find_mergeable(unsigned long size, unsigned long flags) { - int index = kmalloc_index(size) - KMALLOC_SHIFT_LOW; + struct list_head *h; - if (index < 0) + if (slub_nomerge || (flags & SLUB_NEVER_MERGE)) return NULL; - return &kmalloc_caches[index]; + + down_read(&slabstat_sem); + list_for_each(h, &slab_caches) { + struct kmem_cache *s = + container_of(h, struct kmem_cache, list); + + if (s->size >= size && + s->size - size <= sizeof(void *) && + ((flags & SLUB_SAME_MERGE) == + (s->flags & SLUB_SAME_MERGE))) { + up_read(&slabstat_sem); + return s; + } + } + up_read(&slabstat_sem); + return NULL; } struct kmem_cache *kmem_cache_create(const char *name, size_t size, @@ -1302,29 +1344,18 @@ void (*ctor)(void *, struct kmem_cache *, unsigned long), void (*dtor)(void *, struct kmem_cache *, unsigned long)) { - struct kmem_cache *s; - - if (!slub_nomerge && !ctor && !dtor && !(flags & SLUB_NO_MERGE) && - align <= ARCH_SLAB_MINALIGN) { - int sz = ALIGN(size, calculate_alignment(flags, align)); - - /* Find the kmalloc slab that would be used for this size */ - s = __kmalloc_slab(sz); - if (!s) - return NULL; + struct kmem_cache *s = NULL; - /* - * Check if there would be less than a word difference - * between the size of the slab and the kmalloc slab. - * If so then just use the kmalloc array and avoid creating - * a new slab. - */ - if (s->size - sz <= sizeof(void *)) { - printk(KERN_INFO "SLUB: Merging slab_cache %s size %d" - " into kmalloc array size %d\n", - name, (int)size, (int)s->size); - return kmem_cache_dup(s); - } + if (!ctor && !dtor) + s = find_mergeable( + ALIGN(size, calculate_alignment(flags, align)), + flags); + + if (s) { + printk(KERN_INFO "SLUB: Merging slab_cache %s size %d" + " with slab_cache %s\n", + name, (int)size, s->name); + return kmem_cache_dup(s, name); } s = kmalloc(sizeof(struct kmem_cache), GFP_KERNEL); @@ -1354,10 +1385,6 @@ * Slab proc interface *******************************************************************/ -static DECLARE_RWSEM(slabstat_sem); - -LIST_HEAD(slab_caches); - void for_all_slabs(void (*func)(struct kmem_cache *, int), int cpu) { struct list_head *h; @@ -1367,7 +1394,7 @@ struct kmem_cache *s = container_of(h, struct kmem_cache, list); - func(s, cpu); + func(s, cpu); } up_read(&slabstat_sem); } @@ -1492,6 +1519,10 @@ kfree(x); display_nodes(m, nodes); + if (s->aliases) { + seq_putc(m, ' '); + seq_puts(m, s->aliases); + } seq_putc(m, '\n'); return 0; } Index: linux-2.6.21-rc1/include/linux/slub_def.h =================================================================== --- linux-2.6.21-rc1.orig/include/linux/slub_def.h 2007-02-24 13:08:00.000000000 -0800 +++ linux-2.6.21-rc1/include/linux/slub_def.h 2007-02-24 13:08:13.000000000 -0800 @@ -33,6 +33,7 @@ int objsize; /* The size of an object that is in a chunk */ int inuse; /* Used portion of the chunk */ const char *name; /* Name (only for display!) */ + char *aliases; /* Slabs merged into this one */ struct list_head list; /* List of slabs */ #ifdef CONFIG_SMP struct mutex flushing;