SLUB: kmem_cache_ops instead of constructors / destructors For backward compatibility we provide a kmem_cache_create() emulation that can construct a kmem_cache_ops structure on the fly. The new API's to create slabs are: Without any callbacks: slabhandle = KMEM_CACHE(, ) Creates a slab based on the structure definition with the structure alignment, size and name. This is cleaner because the name showing up in /sys/slab/xxx will be the structure name. One can search the source for the name. The common alignment attributs to the struct can control slab alignment. If the SLAB_HWCACHE_ALIGN flag is specified then the alignments will be consolidated. The alignment will be set to whatever is larger. Create a slabcache with kmem_cache_ops (please use only for special slabs): KMEM_CACHE_OPS(, , ) Old kmem_cache_create() support: It still works. It will create a kmem_cache_ops structure on the fly from the specified constructor and destructor. *Warning* this is a potential memory leak if slabs are repeatedly created and destroyed. In that case switch over to KMEM_CACHE. Signed-off-by: Christoph Lameter --- include/linux/slab.h | 57 ++++++++++++++++++++++++++++----- include/linux/slub_def.h | 3 - mm/slab.c | 55 ++++++++++++++++---------------- mm/slob.c | 26 ++++++++------- mm/slub.c | 80 +++++++++++++++++++++++------------------------ 5 files changed, 132 insertions(+), 89 deletions(-) Index: slub/include/linux/slab.h =================================================================== --- slub.orig/include/linux/slab.h 2007-05-09 16:58:21.000000000 -0700 +++ slub/include/linux/slab.h 2007-05-09 17:31:39.000000000 -0700 @@ -32,19 +32,20 @@ typedef struct kmem_cache kmem_cache_t _ #define SLAB_MEM_SPREAD 0x00100000UL /* Spread some memory over cpuset */ #define SLAB_TRACE 0x00200000UL /* Trace allocations and frees */ -/* Flags passed to a constructor functions */ -#define SLAB_CTOR_CONSTRUCTOR 0x001UL /* If not set, then deconstructor */ - /* * struct kmem_cache related prototypes */ void __init kmem_cache_init(void); int slab_is_available(void); -struct kmem_cache *kmem_cache_create(const char *, size_t, size_t, - unsigned long, - void (*)(void *, struct kmem_cache *, unsigned long), - void (*)(void *, struct kmem_cache *, unsigned long)); +struct kmem_cache_ops { + /* FIXME: Remove flags parameter */ + void (*ctor)(void *, struct kmem_cache *, unsigned long); + void (*dtor)(void *, struct kmem_cache *, unsigned long); +}; + +struct kmem_cache *__kmem_cache_create(const char *, size_t, size_t, + unsigned long, const struct kmem_cache_ops *s); void kmem_cache_destroy(struct kmem_cache *); int kmem_cache_shrink(struct kmem_cache *); void *kmem_cache_alloc(struct kmem_cache *, gfp_t); @@ -62,9 +63,14 @@ int kmem_ptr_validate(struct kmem_cache * f.e. add ____cacheline_aligned_in_smp to the struct declaration * then the objects will be properly aligned in SMP configurations. */ -#define KMEM_CACHE(__struct, __flags) kmem_cache_create(#__struct,\ +#define KMEM_CACHE(__struct, __flags) __kmem_cache_create(#__struct,\ sizeof(struct __struct), __alignof__(struct __struct),\ - (__flags), NULL, NULL) + (__flags), NULL) + +#define KMEM_CACHE_OPS(__struct, __flags, __ops) \ + __kmem_cache_create(#__struct, sizeof(struct __struct), \ + __alignof__(struct __struct), (__flags), (__ops)) + #ifdef CONFIG_NUMA extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); @@ -236,6 +242,39 @@ extern void *__kmalloc_node_track_caller extern const struct seq_operations slabinfo_op; ssize_t slabinfo_write(struct file *, const char __user *, size_t, loff_t *); +/* + * Legacy functions + * + * The sole reason that these definitions are here is because of their + * frequent use. Remove when all call sites have been updated. + */ + +/* + * There is no need to check for this flag. A constructor is only + * called for one purpose. + */ +#define SLAB_CTOR_CONSTRUCTOR 0x001UL + +/* + * We construct a kmem_cache_ops structure on the fly. This is useful to + * support the current way of creating slabs but it means that only + * constructor and destructor callbacks can be used. + */ +static inline struct kmem_cache *kmem_cache_create(const char *s, + size_t size, size_t align, unsigned long flags, + void (*ctor)(void *, struct kmem_cache *, unsigned long), + void (*dtor)(void *, struct kmem_cache *, unsigned long)) +{ + struct kmem_cache_ops *so = NULL; + + if (ctor || dtor) { + so = kzalloc(sizeof(struct kmem_cache_ops), GFP_KERNEL); + so->ctor = ctor; + so->dtor = dtor; + } + return __kmem_cache_create(s, size, align, flags, so); +} + #endif /* __KERNEL__ */ #endif /* _LINUX_SLAB_H */ Index: slub/mm/slub.c =================================================================== --- slub.orig/mm/slub.c 2007-05-09 17:28:01.000000000 -0700 +++ slub/mm/slub.c 2007-05-09 17:31:39.000000000 -0700 @@ -291,6 +291,11 @@ static inline int check_valid_pointer(st return 1; } +struct kmem_cache_ops slub_default_ops = { + NULL, + NULL +}; + /* * Slow version of get and set free pointer. * @@ -936,7 +941,7 @@ static void kmem_cache_open_debug_check( if (s->size >= 65535 * sizeof(void *)) { BUG_ON(s->flags & (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | SLAB_DESTROY_BY_RCU)); - BUG_ON(s->ctor || s->dtor); + BUG_ON(s->ops->ctor || s->ops->dtor); } else /* @@ -1000,8 +1005,8 @@ static void setup_object(struct kmem_cac init_tracking(s, object); } - if (unlikely(s->ctor)) - s->ctor(object, s, SLAB_CTOR_CONSTRUCTOR); + if (unlikely(s->ops->ctor)) + s->ops->ctor(object, s, SLAB_CTOR_CONSTRUCTOR); } static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) @@ -1060,13 +1065,13 @@ static void __free_slab(struct kmem_cach { int pages = 1 << s->order; - if (unlikely(SlabDebug(page) || s->dtor)) { + if (unlikely(SlabDebug(page) || s->ops->dtor)) { void *p; slab_pad_check(s, page); for_each_object(p, s, page_address(page)) { - if (s->dtor) - s->dtor(p, s, 0); + if (s->ops->dtor) + s->ops->dtor(p, s, 0); check_object(s, page, p, 0); } } @@ -1892,7 +1897,7 @@ static int calculate_sizes(struct kmem_c * then we should never poison the object itself. */ if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) && - !s->ctor && !s->dtor) + !s->ops->ctor && !s->ops->dtor) s->flags |= __OBJECT_POISON; else s->flags &= ~__OBJECT_POISON; @@ -1922,7 +1927,7 @@ static int calculate_sizes(struct kmem_c #ifdef CONFIG_SLUB_DEBUG if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) || - s->ctor || s->dtor)) { + s->ops->ctor || s->ops->dtor)) { /* * Relocate free pointer after the object if it is not * permitted to overwrite the first word of the object on @@ -1991,13 +1996,11 @@ static int calculate_sizes(struct kmem_c static int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags, const char *name, size_t size, size_t align, unsigned long flags, - void (*ctor)(void *, struct kmem_cache *, unsigned long), - void (*dtor)(void *, struct kmem_cache *, unsigned long)) + const struct kmem_cache_ops *ops) { memset(s, 0, kmem_size); s->name = name; - s->ctor = ctor; - s->dtor = dtor; + s->ops = ops; s->objsize = size; s->flags = flags; s->align = align; @@ -2182,7 +2185,7 @@ static struct kmem_cache *create_kmalloc down_write(&slub_lock); if (!kmem_cache_open(s, gfp_flags, name, size, ARCH_KMALLOC_MINALIGN, - flags, NULL, NULL)) + flags, &slub_default_ops)) goto panic; list_add(&s->list, &slab_caches); @@ -2493,7 +2496,7 @@ static int slab_unmergeable(struct kmem_ if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE)) return 1; - if (s->ctor || s->dtor) + if (s->ops != &slub_default_ops) return 1; return 0; @@ -2501,15 +2504,14 @@ static int slab_unmergeable(struct kmem_ static struct kmem_cache *find_mergeable(size_t size, size_t align, unsigned long flags, - void (*ctor)(void *, struct kmem_cache *, unsigned long), - void (*dtor)(void *, struct kmem_cache *, unsigned long)) + const struct kmem_cache_ops *ops) { struct list_head *h; if (slub_nomerge || (flags & SLUB_NEVER_MERGE)) return NULL; - if (ctor || dtor) + if (ops != &slub_default_ops) return NULL; size = ALIGN(size, sizeof(void *)); @@ -2544,15 +2546,17 @@ static struct kmem_cache *find_mergeable return NULL; } -struct kmem_cache *kmem_cache_create(const char *name, size_t size, +struct kmem_cache *__kmem_cache_create(const char *name, size_t size, size_t align, unsigned long flags, - void (*ctor)(void *, struct kmem_cache *, unsigned long), - void (*dtor)(void *, struct kmem_cache *, unsigned long)) + const struct kmem_cache_ops *ops) { struct kmem_cache *s; + if (!ops) + ops = &slub_default_ops; + down_write(&slub_lock); - s = find_mergeable(size, align, flags, dtor, ctor); + s = find_mergeable(size, align, flags, ops); if (s) { s->refcount++; /* @@ -2566,7 +2570,7 @@ struct kmem_cache *kmem_cache_create(con } else { s = kmalloc(kmem_size, GFP_KERNEL); if (s && kmem_cache_open(s, GFP_KERNEL, name, - size, align, flags, ctor, dtor)) { + size, align, flags, ops)) { if (sysfs_slab_add(s)) { kfree(s); goto err; @@ -2586,7 +2590,7 @@ err: s = NULL; return s; } -EXPORT_SYMBOL(kmem_cache_create); +EXPORT_SYMBOL(__kmem_cache_create); void *kmem_cache_zalloc(struct kmem_cache *s, gfp_t flags) { @@ -3196,27 +3200,24 @@ static ssize_t order_show(struct kmem_ca } SLAB_ATTR_RO(order); -static ssize_t ctor_show(struct kmem_cache *s, char *buf) +static ssize_t ops_show(struct kmem_cache *s, char *buf) { - if (s->ctor) { - int n = sprint_symbol(buf, (unsigned long)s->ctor); + int x = 0; - return n + sprintf(buf + n, "\n"); + if (s->ops->ctor) { + x += sprintf(buf + x, "ctor : "); + x += sprint_symbol(buf + x, (unsigned long)s->ops->ctor); + x += sprintf(buf + x, "\n"); } - return 0; -} -SLAB_ATTR_RO(ctor); - -static ssize_t dtor_show(struct kmem_cache *s, char *buf) -{ - if (s->dtor) { - int n = sprint_symbol(buf, (unsigned long)s->dtor); - return n + sprintf(buf + n, "\n"); + if (s->ops->dtor) { + x += sprintf(buf + x, "dtor : "); + x += sprint_symbol(buf + x, (unsigned long)s->ops->dtor); + x += sprintf(buf + x, "\n"); } - return 0; + return x; } -SLAB_ATTR_RO(dtor); +SLAB_ATTR_RO(ops); static ssize_t aliases_show(struct kmem_cache *s, char *buf) { @@ -3448,8 +3449,7 @@ static struct attribute * slab_attrs[] = &slabs_attr.attr, &partial_attr.attr, &cpu_slabs_attr.attr, - &ctor_attr.attr, - &dtor_attr.attr, + &ops_attr.attr, &aliases_attr.attr, &align_attr.attr, &sanity_checks_attr.attr, Index: slub/include/linux/slub_def.h =================================================================== --- slub.orig/include/linux/slub_def.h 2007-05-09 16:58:21.000000000 -0700 +++ slub/include/linux/slub_def.h 2007-05-09 17:31:39.000000000 -0700 @@ -39,8 +39,7 @@ struct kmem_cache { /* Allocation and freeing of slabs */ int objects; /* Number of objects in slab */ int refcount; /* Refcount for slab cache destroy */ - void (*ctor)(void *, struct kmem_cache *, unsigned long); - void (*dtor)(void *, struct kmem_cache *, unsigned long); + const struct kmem_cache_ops *ops; int inuse; /* Offset to metadata */ int align; /* Alignment */ const char *name; /* Name (only for display!) */ Index: slub/mm/slab.c =================================================================== --- slub.orig/mm/slab.c 2007-05-09 16:58:21.000000000 -0700 +++ slub/mm/slab.c 2007-05-09 17:31:39.000000000 -0700 @@ -274,6 +274,9 @@ struct array_cache { */ }; +static struct kmem_cache_ops slab_default_ops = { +}; + /* * bootstrap: The caches do not work without cpuarrays anymore, but the * cpuarrays are allocated from the generic caches... @@ -406,11 +409,7 @@ struct kmem_cache { unsigned int slab_size; unsigned int dflags; /* dynamic flags */ - /* constructor func */ - void (*ctor) (void *, struct kmem_cache *, unsigned long); - - /* de-constructor func */ - void (*dtor) (void *, struct kmem_cache *, unsigned long); + const struct kmem_cache_ops *ops; /* 5) cache creation/removal */ const char *name; @@ -1911,18 +1910,19 @@ static void slab_destroy_objs(struct kme slab_error(cachep, "end of a freed object " "was overwritten"); } - if (cachep->dtor && !(cachep->flags & SLAB_POISON)) - (cachep->dtor) (objp + obj_offset(cachep), cachep, 0); + if (cachep->ops->dtor && !(cachep->flags & SLAB_POISON)) + (cachep->ops->dtor) (objp + obj_offset(cachep), + cachep, 0); } } #else static void slab_destroy_objs(struct kmem_cache *cachep, struct slab *slabp) { - if (cachep->dtor) { + if (cachep->ops->dtor) { int i; for (i = 0; i < cachep->num; i++) { void *objp = index_to_obj(cachep, slabp, i); - (cachep->dtor) (objp, cachep, 0); + (cachep->ops->dtor) (objp, cachep, 0); } } } @@ -2123,13 +2123,12 @@ static int setup_cpu_cache(struct kmem_c * @size: The size of objects to be created in this cache. * @align: The required alignment for the objects. * @flags: SLAB flags - * @ctor: A constructor for the objects. - * @dtor: A destructor for the objects. + * @ops: kmem cache operations. * * Returns a ptr to the cache on success, NULL on failure. * Cannot be called within a int, but can be interrupted. - * The @ctor is run when new pages are allocated by the cache - * and the @dtor is run before the pages are handed back. + * The @ops->ctor is run when new pages are allocated by the cache + * and the @ops->dtor is run before the pages are handed back. * * @name must be valid until the cache is destroyed. This implies that * the module calling this has to destroy the cache before getting unloaded. @@ -2147,10 +2146,9 @@ static int setup_cpu_cache(struct kmem_c * as davem. */ struct kmem_cache * -kmem_cache_create (const char *name, size_t size, size_t align, +__kmem_cache_create (const char *name, size_t size, size_t align, unsigned long flags, - void (*ctor)(void*, struct kmem_cache *, unsigned long), - void (*dtor)(void*, struct kmem_cache *, unsigned long)) + const struct kmem_cache_ops *ops) { size_t left_over, slab_size, ralign; struct kmem_cache *cachep = NULL, *pc; @@ -2159,7 +2157,8 @@ kmem_cache_create (const char *name, siz * Sanity checks... these are all serious usage bugs. */ if (!name || in_interrupt() || (size < BYTES_PER_WORD) || - (size > (1 << MAX_OBJ_ORDER) * PAGE_SIZE) || (dtor && !ctor)) { + (size > (1 << MAX_OBJ_ORDER) * PAGE_SIZE) || + (ops->dtor && !ops->ctor)) { printk(KERN_ERR "%s: Early error in slab %s\n", __FUNCTION__, name); BUG(); @@ -2214,7 +2213,7 @@ kmem_cache_create (const char *name, siz BUG_ON(flags & SLAB_POISON); #endif if (flags & SLAB_DESTROY_BY_RCU) - BUG_ON(dtor); + BUG_ON(ops->dtor); /* * Always checks flags, a caller might be expecting debug support which @@ -2369,8 +2368,10 @@ kmem_cache_create (const char *name, siz */ BUG_ON(!cachep->slabp_cache); } - cachep->ctor = ctor; - cachep->dtor = dtor; + if (ops) + cachep->ops = ops; + else + cachep->ops = &slab_default_ops; cachep->name = name; if (setup_cpu_cache(cachep)) { @@ -2648,7 +2649,7 @@ static void cache_init_objs(struct kmem_ * They must also be threaded. */ if (cachep->ctor && !(cachep->flags & SLAB_POISON)) - cachep->ctor(objp + obj_offset(cachep), cachep, + cachep->ops->ctor(objp + obj_offset(cachep), cachep, ctor_flags); if (cachep->flags & SLAB_RED_ZONE) { @@ -2664,8 +2665,8 @@ static void cache_init_objs(struct kmem_ kernel_map_pages(virt_to_page(objp), cachep->buffer_size / PAGE_SIZE, 0); #else - if (cachep->ctor) - cachep->ctor(objp, cachep, ctor_flags); + if (cachep->ops->ctor) + cachep->ops->ctor(objp, cachep, ctor_flags); #endif slab_bufctl(slabp)[i] = i + 1; } @@ -2894,11 +2895,11 @@ static void *cache_free_debugcheck(struc BUG_ON(objnr >= cachep->num); BUG_ON(objp != index_to_obj(cachep, slabp, objnr)); - if (cachep->flags & SLAB_POISON && cachep->dtor) { + if (cachep->flags & SLAB_POISON && cachep->ops->dtor) { /* we want to cache poison the object, * call the destruction callback */ - cachep->dtor(objp + obj_offset(cachep), cachep, 0); + cachep->ops->dtor(objp + obj_offset(cachep), cachep, 0); } #ifdef CONFIG_DEBUG_SLAB_LEAK slab_bufctl(slabp)[objnr] = BUFCTL_FREE; @@ -3098,8 +3099,8 @@ static void *cache_alloc_debugcheck_afte } #endif objp += obj_offset(cachep); - if (cachep->ctor && cachep->flags & SLAB_POISON) - cachep->ctor(objp, cachep, SLAB_CTOR_CONSTRUCTOR); + if (cachep->ops->ctor && cachep->flags & SLAB_POISON) + cachep->ops->ctor(objp, cachep, SLAB_CTOR_CONSTRUCTOR); #if ARCH_SLAB_MINALIGN if ((u32)objp & (ARCH_SLAB_MINALIGN-1)) { printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n", Index: slub/mm/slob.c =================================================================== --- slub.orig/mm/slob.c 2007-05-09 16:58:21.000000000 -0700 +++ slub/mm/slob.c 2007-05-09 17:31:39.000000000 -0700 @@ -62,6 +62,10 @@ static DEFINE_SPINLOCK(block_lock); static void slob_free(void *b, int size); static void slob_timer_cbk(void); +static struct kmem_cache_ops slob_default_ops = { + NULL, + NULL +}; static void *slob_alloc(size_t size, gfp_t gfp, int align) { @@ -267,14 +271,12 @@ size_t ksize(const void *block) struct kmem_cache { unsigned int size, align; const char *name; - void (*ctor)(void *, struct kmem_cache *, unsigned long); - void (*dtor)(void *, struct kmem_cache *, unsigned long); + const struct kmem_cache_ops *ops; }; -struct kmem_cache *kmem_cache_create(const char *name, size_t size, +struct kmem_cache *__kmem_cache_create(const char *name, size_t size, size_t align, unsigned long flags, - void (*ctor)(void*, struct kmem_cache *, unsigned long), - void (*dtor)(void*, struct kmem_cache *, unsigned long)) + const struct kmem_cache_ops *ops) { struct kmem_cache *c; @@ -283,8 +285,10 @@ struct kmem_cache *kmem_cache_create(con if (c) { c->name = name; c->size = size; - c->ctor = ctor; - c->dtor = dtor; + if (ops) + c->ops = ops; + else + c->ops = &slob_default_ops; /* ignore alignment unless it's forced */ c->align = (flags & SLAB_HWCACHE_ALIGN) ? SLOB_ALIGN : 0; if (c->align < align) @@ -311,8 +315,8 @@ void *kmem_cache_alloc(struct kmem_cache else b = (void *)__get_free_pages(flags, get_order(c->size)); - if (c->ctor) - c->ctor(b, c, SLAB_CTOR_CONSTRUCTOR); + if (c->ops->ctor) + c->ops->ctor(b, c, SLAB_CTOR_CONSTRUCTOR); return b; } @@ -330,8 +334,8 @@ EXPORT_SYMBOL(kmem_cache_zalloc); void kmem_cache_free(struct kmem_cache *c, void *b) { - if (c->dtor) - c->dtor(b, c, 0); + if (c->ops->dtor) + c->ops->dtor(b, c, 0); if (c->size < PAGE_SIZE) slob_free(b, c->size);