SL?B: Callbacks instead of constructors / destructors This patch gets rid constructors and destructors and replaces them with callbacks using a new macro based slab creation method. For backward compatibility we provide a kmem_cache_create() emulation that can take a constructor (but no destructor). This is fine because it seems that there are only two locations left where a destructor is used in embedded arch code for frv and sh. FRV uses it like i386 and will have to be converted like i386. Thus its gone. The sole remaining case is sh using destructors for some form of list management (???). That code in the sh arch would have to be converted to the new API. Hmmm... Do we really want to continue supporting destructors? We could just remove it it seems. The new API's to create slabs would be: Without any callbacks: slabhandle = KMEM_CACHE(, ) Creates a slab based on the structure definition with the structure alignment, size and name. This is cleaner because the name showing up in /sys/slab/xxx will be the structure name. One can search the source for the name. The common alignment attributs to the struct can control slab alignmnet. Note: SLAB_HWCACHE_ALIGN is *not* supported as a flag. The flags do *not* specify alignments. The alignment is done to the structure and please nowhere else. Create a slabcache with callback (please use only for special slabs): KMEM_CACHE_CALLBACK(, , ) Callback is a a function that takes the usual parameters for constructors and destructors and has a flag set for eachn type of supported callback. Callbacks returns a result code (required by future support for targeted reclaim). Callback types (more coming to support targeted reclaim): SLAB_CALLBACK_CTOR SLAB_CALLBACK_DTOR These constants must be specified as flags during slab creation in order to enable each callback and they are passed as flags to the callback when it is triggereed. Old kmem_cache_create() support: kmem_cache_create() works as usual with the exception that we do not support destructor specifications. kmem_cache_create also allows the specification of SLAB_HWCACHE_ALIGN *if* there is no other alignment specified. In that case kmem_cache_create will generate a proper alignment depending on the size of the structure. kmem_cache_create also allows the specification of an old constructor that has no return value. It will be cast to the new type. This is safe for now since the return values for constructors are ignored. SLAB_CTOR_CONSTRUCTOR is defined as SLAB_CALLBACK_CTOR. Thus the existing constructors should work fine without modifications (please clean them up and while you are at it think carefully why you would be using a constructor at all. In 95% of the cases the only result is a performance degradation by uselessly writing to memory that is initialized later again and additional overhead in the slab allocators to handle these). Please check if you really need constructors for slab caches. Apart from the use of SLAB_DESTROY_BY_RCU there is no case in which I see a useful purpose in them (We no longer use slabs for page size caches, use quicklists). The best outcome would be if we would use only KMEM_CACHE() throughout. In the future KMEM_CACHE_CALLBACK will have a new use to enable targeted reclaim against slabs. I.e. antifrag / defrag code can kick out a specific slab page by calling a slab function that will use these callbacks to tell the code that a certain object is in an undesirable position. Signed-off-by: Christoph Lameter --- include/linux/slab.h | 66 +++++++++++++++++++++----- include/linux/slub_def.h | 3 - mm/slab.c | 118 +++++++++++++++++++---------------------------- mm/slob.c | 19 +++---- mm/slub.c | 107 +++++++++++++----------------------------- 5 files changed, 147 insertions(+), 166 deletions(-) Index: slub/include/linux/slab.h =================================================================== --- slub.orig/include/linux/slab.h 2007-05-02 16:45:15.000000000 -0700 +++ slub/include/linux/slab.h 2007-05-02 17:56:03.000000000 -0700 @@ -17,13 +17,11 @@ typedef struct kmem_cache kmem_cache_t __deprecated; /* - * Flags to pass to kmem_cache_create(). - * The ones marked DEBUG are only valid if CONFIG_SLAB_DEBUG is set. + * Flags to pass to KMEM_CACHE or KMEM_CACHE_CALLBACK */ #define SLAB_DEBUG_FREE 0x00000100UL /* DEBUG: Perform (expensive) checks on free */ #define SLAB_RED_ZONE 0x00000400UL /* DEBUG: Red zone objs in a cache */ #define SLAB_POISON 0x00000800UL /* DEBUG: Poison objects */ -#define SLAB_HWCACHE_ALIGN 0x00002000UL /* Align objs on cache lines */ #define SLAB_CACHE_DMA 0x00004000UL /* Use GFP_DMA memory */ #define SLAB_STORE_USER 0x00010000UL /* DEBUG: Store the last owner for bug hunting */ #define SLAB_RECLAIM_ACCOUNT 0x00020000UL /* Objects are reclaimable */ @@ -31,9 +29,8 @@ typedef struct kmem_cache kmem_cache_t _ #define SLAB_DESTROY_BY_RCU 0x00080000UL /* Defer freeing slabs to RCU */ #define SLAB_MEM_SPREAD 0x00100000UL /* Spread some memory over cpuset */ #define SLAB_TRACE 0x00200000UL /* Trace allocations and frees */ - -/* Flags passed to a constructor functions */ -#define SLAB_CTOR_CONSTRUCTOR 0x001UL /* If not set, then deconstructor */ +#define SLAB_CALLBACK_CTOR 0x00000200UL /* Use callback for constructor */ +#define SLAB_CALLBACK_DTOR 0x00001000UL /* Use callback for destructor */ /* * struct kmem_cache related prototypes @@ -41,10 +38,9 @@ typedef struct kmem_cache kmem_cache_t _ void __init kmem_cache_init(void); int slab_is_available(void); -struct kmem_cache *kmem_cache_create(const char *, size_t, size_t, - unsigned long, - void (*)(void *, struct kmem_cache *, unsigned long), - void (*)(void *, struct kmem_cache *, unsigned long)); +struct kmem_cache *__kmem_cache_create(const char *, size_t, size_t, + unsigned long, int (*)(void *, struct kmem_cache *, unsigned long)); + void kmem_cache_destroy(struct kmem_cache *); int kmem_cache_shrink(struct kmem_cache *); void *kmem_cache_alloc(struct kmem_cache *, gfp_t); @@ -62,9 +58,55 @@ int kmem_ptr_validate(struct kmem_cache * f.e. add ____cacheline_aligned_in_smp to the struct declaration * then the objects will be properly aligned in SMP configurations. */ -#define KMEM_CACHE(__struct, __flags) kmem_cache_create(#__struct,\ +#define KMEM_CACHE(__struct, __flags) __kmem_cache_create(#__struct,\ sizeof(struct __struct), __alignof__(struct __struct),\ - (__flags), NULL, NULL) + (__flags), NULL) + +#define KMEM_CACHE_CALLBACK(__struct, __flags, __callback) \ + __kmem_cache_create(#__struct,\ + sizeof(struct __struct), __alignof__(struct __struct),\ + (__flags), (__callback)) + +/* + * Legacy function. We can sort of handle a constructor but definitely not a + * destructor. + * + * The sole reason that these definitions are here is because of their + * frequent use. Remove when all call sites have been updated. + */ +#define SLAB_CTOR_CONSTRUCTOR SLAB_CALLBACK_CTOR +#define SLAB_HWCACHE_ALIGN 0x8000000000UL + +static inline struct kmem_cache *kmem_cache_create(const char *s, + size_t size, size_t align, unsigned long flags, + void (*ctor)(void *, struct kmem_cache *, + unsigned long), void *dtor) +{ + /* + * This conversion only works now while we do not check the + * return value of constructors. This is the case for + * constructors. + */ + int (*callback)(void *, struct kmem_cache *, unsigned long) = (void *)ctor; + + BUG_ON(dtor); + + if (ctor) + flags |= SLAB_CALLBACK_CTOR; + + if ((flags & SLAB_HWCACHE_ALIGN) && size > L1_CACHE_BYTES / 2) { + /* Clear the align flag. It is no longer supported */ + flags &= ~SLAB_HWCACHE_ALIGN; + + /* Do not allow conflicting alignment specificiations */ + BUG_ON(align); + + /* And set the cacheline alignment */ + align = L1_CACHE_BYTES; + } + + return __kmem_cache_create(s, size, align, flags, callback); +} #ifdef CONFIG_NUMA extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); Index: slub/include/linux/slub_def.h =================================================================== --- slub.orig/include/linux/slub_def.h 2007-05-02 16:45:15.000000000 -0700 +++ slub/include/linux/slub_def.h 2007-05-02 16:45:17.000000000 -0700 @@ -39,8 +39,7 @@ struct kmem_cache { /* Allocation and freeing of slabs */ int objects; /* Number of objects in slab */ int refcount; /* Refcount for slab cache destroy */ - void (*ctor)(void *, struct kmem_cache *, unsigned long); - void (*dtor)(void *, struct kmem_cache *, unsigned long); + int (*callback)(void *, struct kmem_cache *, unsigned long); int inuse; /* Offset to metadata */ int align; /* Alignment */ const char *name; /* Name (only for display!) */ Index: slub/mm/slab.c =================================================================== --- slub.orig/mm/slab.c 2007-05-02 16:45:15.000000000 -0700 +++ slub/mm/slab.c 2007-05-02 18:35:21.000000000 -0700 @@ -151,7 +151,7 @@ * alignment larger than BYTES_PER_WORD. ARCH_KMALLOC_MINALIGN allows that. * Note that this flag disables some debug features. */ -#define ARCH_KMALLOC_MINALIGN 0 +#define ARCH_KMALLOC_MINALIGN sizeof(void *) #endif #ifndef ARCH_SLAB_MINALIGN @@ -162,26 +162,27 @@ * If possible: Do not enable this flag for CONFIG_DEBUG_SLAB, it disables * some debug features. */ -#define ARCH_SLAB_MINALIGN 0 +#define ARCH_SLAB_MINALIGN sizeof(void *) #endif #ifndef ARCH_KMALLOC_FLAGS -#define ARCH_KMALLOC_FLAGS SLAB_HWCACHE_ALIGN +#define ARCH_KMALLOC_FLAGS 0 #endif /* Legal flag mask for kmem_cache_create(). */ #if DEBUG # define CREATE_MASK (SLAB_RED_ZONE | \ - SLAB_POISON | SLAB_HWCACHE_ALIGN | \ + SLAB_POISON | \ SLAB_CACHE_DMA | \ SLAB_STORE_USER | \ SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \ - SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD) + SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \ + SLAB_CALLBACK_CTOR | SLAB_CALLBACK_DTOR) #else -# define CREATE_MASK (SLAB_HWCACHE_ALIGN | \ - SLAB_CACHE_DMA | \ +# define CREATE_MASK (SLAB_CACHE_DMA | \ SLAB_RECLAIM_ACCOUNT | SLAB_PANIC | \ - SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD) + SLAB_DESTROY_BY_RCU | SLAB_MEM_SPREAD | \ + SLAB_CALLBACK_CTOR | SLAB_CALLBACK_DTOR) #endif /* @@ -405,11 +406,7 @@ struct kmem_cache { unsigned int slab_size; unsigned int dflags; /* dynamic flags */ - /* constructor func */ - void (*ctor) (void *, struct kmem_cache *, unsigned long); - - /* de-constructor func */ - void (*dtor) (void *, struct kmem_cache *, unsigned long); + int (*callback) (void *, struct kmem_cache *, unsigned long); /* 5) cache creation/removal */ const char *name; @@ -1490,19 +1487,19 @@ void __init kmem_cache_init(void) * bug. */ - sizes[INDEX_AC].cs_cachep = kmem_cache_create(names[INDEX_AC].name, + sizes[INDEX_AC].cs_cachep = __kmem_cache_create(names[INDEX_AC].name, sizes[INDEX_AC].cs_size, ARCH_KMALLOC_MINALIGN, ARCH_KMALLOC_FLAGS|SLAB_PANIC, - NULL, NULL); + NULL); if (INDEX_AC != INDEX_L3) { sizes[INDEX_L3].cs_cachep = - kmem_cache_create(names[INDEX_L3].name, + __kmem_cache_create(names[INDEX_L3].name, sizes[INDEX_L3].cs_size, ARCH_KMALLOC_MINALIGN, ARCH_KMALLOC_FLAGS|SLAB_PANIC, - NULL, NULL); + NULL); } slab_early_init = 0; @@ -1516,20 +1513,20 @@ void __init kmem_cache_init(void) * allow tighter packing of the smaller caches. */ if (!sizes->cs_cachep) { - sizes->cs_cachep = kmem_cache_create(names->name, + sizes->cs_cachep = __kmem_cache_create(names->name, sizes->cs_size, ARCH_KMALLOC_MINALIGN, ARCH_KMALLOC_FLAGS|SLAB_PANIC, - NULL, NULL); + NULL); } #ifdef CONFIG_ZONE_DMA - sizes->cs_dmacachep = kmem_cache_create( + sizes->cs_dmacachep = __kmem_cache_create( names->name_dma, sizes->cs_size, ARCH_KMALLOC_MINALIGN, ARCH_KMALLOC_FLAGS|SLAB_CACHE_DMA| SLAB_PANIC, - NULL, NULL); + NULL); #endif sizes++; names++; @@ -1907,18 +1904,20 @@ static void slab_destroy_objs(struct kme slab_error(cachep, "end of a freed object " "was overwritten"); } - if (cachep->dtor && !(cachep->flags & SLAB_POISON)) - (cachep->dtor) (objp + obj_offset(cachep), cachep, 0); + if ((cachep->flags & SLAB_CALLBACK_DTOR) && + !(cachep->flags & SLAB_POISON)) + cachep->callback(objp + obj_offset(cachep), cachep, + SLAB_CALLBACK_DTOR); } } #else static void slab_destroy_objs(struct kmem_cache *cachep, struct slab *slabp) { - if (cachep->dtor) { + if (cachep->flags & SLAB_CALLBACK_DTOR) { int i; for (i = 0; i < cachep->num; i++) { void *objp = index_to_obj(cachep, slabp, i); - (cachep->dtor) (objp, cachep, 0); + cachep->callback(objp, cachep, SLAB_CALLBACK_DTOR); } } } @@ -2114,18 +2113,15 @@ static int setup_cpu_cache(struct kmem_c } /** - * kmem_cache_create - Create a cache. + * __kmem_cache_create - Create a cache. * @name: A string which is used in /proc/slabinfo to identify this cache. * @size: The size of objects to be created in this cache. * @align: The required alignment for the objects. * @flags: SLAB flags - * @ctor: A constructor for the objects. - * @dtor: A destructor for the objects. + * @callback: Callback for constructor and destructor. * * Returns a ptr to the cache on success, NULL on failure. * Cannot be called within a int, but can be interrupted. - * The @ctor is run when new pages are allocated by the cache - * and the @dtor is run before the pages are handed back. * * @name must be valid until the cache is destroyed. This implies that * the module calling this has to destroy the cache before getting unloaded. @@ -2137,16 +2133,11 @@ static int setup_cpu_cache(struct kmem_c * * %SLAB_RED_ZONE - Insert `Red' zones around the allocated memory to check * for buffer overruns. - * - * %SLAB_HWCACHE_ALIGN - Align the objects in this cache to a hardware - * cacheline. This can be beneficial if you're counting cycles as closely - * as davem. */ struct kmem_cache * -kmem_cache_create (const char *name, size_t size, size_t align, +__kmem_cache_create (const char *name, size_t size, size_t align, unsigned long flags, - void (*ctor)(void*, struct kmem_cache *, unsigned long), - void (*dtor)(void*, struct kmem_cache *, unsigned long)) + int (*callback)(void*, struct kmem_cache *, unsigned long)) { size_t left_over, slab_size, ralign; struct kmem_cache *cachep = NULL, *pc; @@ -2155,12 +2146,15 @@ kmem_cache_create (const char *name, siz * Sanity checks... these are all serious usage bugs. */ if (!name || in_interrupt() || (size < BYTES_PER_WORD) || - (size > (1 << MAX_OBJ_ORDER) * PAGE_SIZE) || (dtor && !ctor)) { + (size > (1 << MAX_OBJ_ORDER) * PAGE_SIZE)) { printk(KERN_ERR "%s: Early error in slab %s\n", __FUNCTION__, name); BUG(); } + if (callback && !(flags & (SLAB_CALLBACK_CTOR|SLAB_CALLBACK_DTOR))) + BUG(); + /* * We use cache_chain_mutex to ensure a consistent view of * cpu_online_map as well. Please see cpuup_callback @@ -2210,7 +2204,7 @@ kmem_cache_create (const char *name, siz BUG_ON(flags & SLAB_POISON); #endif if (flags & SLAB_DESTROY_BY_RCU) - BUG_ON(dtor); + BUG_ON(flags & SLAB_CALLBACK_DTOR); /* * Always checks flags, a caller might be expecting debug support which @@ -2228,21 +2222,7 @@ kmem_cache_create (const char *name, siz size &= ~(BYTES_PER_WORD - 1); } - /* calculate the final buffer alignment: */ - - /* 1) arch recommendation: can be overridden for debug */ - if (flags & SLAB_HWCACHE_ALIGN) { - /* - * Default alignment: as specified by the arch code. Except if - * an object is really small, then squeeze multiple objects into - * one cacheline. - */ - ralign = cache_line_size(); - while (size <= ralign / 2) - ralign /= 2; - } else { - ralign = BYTES_PER_WORD; - } + ralign = BYTES_PER_WORD; /* * Redzoning and user store require word alignment. Note this will be @@ -2365,8 +2345,7 @@ kmem_cache_create (const char *name, siz */ BUG_ON(!cachep->slabp_cache); } - cachep->ctor = ctor; - cachep->dtor = dtor; + cachep->callback = callback; cachep->name = name; if (setup_cpu_cache(cachep)) { @@ -2384,7 +2363,7 @@ oops: mutex_unlock(&cache_chain_mutex); return cachep; } -EXPORT_SYMBOL(kmem_cache_create); +EXPORT_SYMBOL(__kmem_cache_create); #if DEBUG static void check_irq_off(void) @@ -2621,7 +2600,7 @@ static inline kmem_bufctl_t *slab_bufctl } static void cache_init_objs(struct kmem_cache *cachep, - struct slab *slabp, unsigned long ctor_flags) + struct slab *slabp) { int i; @@ -2643,9 +2622,10 @@ static void cache_init_objs(struct kmem_ * cache which they are a constructor for. Otherwise, deadlock. * They must also be threaded. */ - if (cachep->ctor && !(cachep->flags & SLAB_POISON)) - cachep->ctor(objp + obj_offset(cachep), cachep, - ctor_flags); + if ((cachep->flags & SLAB_CALLBACK_CTOR) + && !(cachep->flags & SLAB_POISON)) + cachep->callback(objp + obj_offset(cachep), cachep, + SLAB_CALLBACK_CTOR); if (cachep->flags & SLAB_RED_ZONE) { if (*dbg_redzone2(cachep, objp) != RED_INACTIVE) @@ -2660,8 +2640,8 @@ static void cache_init_objs(struct kmem_ kernel_map_pages(virt_to_page(objp), cachep->buffer_size / PAGE_SIZE, 0); #else - if (cachep->ctor) - cachep->ctor(objp, cachep, ctor_flags); + if (cachep->flags & SLAB_CALLBACK_CTOR) + cachep->callback(objp, cachep, SLAB_CALLBACK_CTOR); #endif slab_bufctl(slabp)[i] = i + 1; } @@ -2750,7 +2730,6 @@ static int cache_grow(struct kmem_cache struct slab *slabp; size_t offset; gfp_t local_flags; - unsigned long ctor_flags; struct kmem_list3 *l3; /* @@ -2759,7 +2738,6 @@ static int cache_grow(struct kmem_cache */ BUG_ON(flags & ~(GFP_DMA | GFP_LEVEL_MASK)); - ctor_flags = SLAB_CTOR_CONSTRUCTOR; local_flags = (flags & GFP_LEVEL_MASK); /* Take the l3 list lock to change the colour_next on this node */ check_irq_off(); @@ -2804,7 +2782,7 @@ static int cache_grow(struct kmem_cache slabp->nodeid = nodeid; slab_map_pages(cachep, slabp, objp); - cache_init_objs(cachep, slabp, ctor_flags); + cache_init_objs(cachep, slabp); if (local_flags & __GFP_WAIT) local_irq_disable(); @@ -2890,11 +2868,11 @@ static void *cache_free_debugcheck(struc BUG_ON(objnr >= cachep->num); BUG_ON(objp != index_to_obj(cachep, slabp, objnr)); - if (cachep->flags & SLAB_POISON && cachep->dtor) { + if (cachep->flags & SLAB_POISON && cachep->flags & SLAB_CALLBACK_DTOR) { /* we want to cache poison the object, * call the destruction callback */ - cachep->dtor(objp + obj_offset(cachep), cachep, 0); + cachep->callback(objp + obj_offset(cachep), cachep, SLAB_CALLBACK_DTOR); } #ifdef CONFIG_DEBUG_SLAB_LEAK slab_bufctl(slabp)[objnr] = BUFCTL_FREE; @@ -3094,8 +3072,8 @@ static void *cache_alloc_debugcheck_afte } #endif objp += obj_offset(cachep); - if (cachep->ctor && cachep->flags & SLAB_POISON) - cachep->ctor(objp, cachep, SLAB_CTOR_CONSTRUCTOR); + if (cachep->flags & SLAB_CALLBACK_CTOR && cachep->flags & SLAB_POISON) + cachep->callback(objp, cachep, SLAB_CALLBACK_CTOR); #if ARCH_SLAB_MINALIGN if ((u32)objp & (ARCH_SLAB_MINALIGN-1)) { printk(KERN_ERR "0x%p: not aligned to ARCH_SLAB_MINALIGN=%d\n", Index: slub/mm/slob.c =================================================================== --- slub.orig/mm/slob.c 2007-05-02 16:45:15.000000000 -0700 +++ slub/mm/slob.c 2007-05-02 16:45:17.000000000 -0700 @@ -276,14 +276,13 @@ size_t ksize(const void *block) struct kmem_cache { unsigned int size, align; const char *name; - void (*ctor)(void *, struct kmem_cache *, unsigned long); - void (*dtor)(void *, struct kmem_cache *, unsigned long); + unsigned long flags; + int (*callback)(void *, struct kmem_cache *, unsigned long); }; struct kmem_cache *kmem_cache_create(const char *name, size_t size, size_t align, unsigned long flags, - void (*ctor)(void*, struct kmem_cache *, unsigned long), - void (*dtor)(void*, struct kmem_cache *, unsigned long)) + int (*callback)(void*, struct kmem_cache *, unsigned long)) { struct kmem_cache *c; @@ -292,8 +291,8 @@ struct kmem_cache *kmem_cache_create(con if (c) { c->name = name; c->size = size; - c->ctor = ctor; - c->dtor = dtor; + c->flags = flags; + c->callback = callback; /* ignore alignment unless it's forced */ c->align = (flags & SLAB_HWCACHE_ALIGN) ? SLOB_ALIGN : 0; if (c->align < align) @@ -320,8 +319,8 @@ void *kmem_cache_alloc(struct kmem_cache else b = (void *)__get_free_pages(flags, find_order(c->size)); - if (c->ctor) - c->ctor(b, c, SLAB_CTOR_CONSTRUCTOR); + if (c->flags & SLAB_CALLBACK_CTOR) + c->callback(b, c, SLAB_CALLBACK_CTOR); return b; } @@ -339,8 +338,8 @@ EXPORT_SYMBOL(kmem_cache_zalloc); void kmem_cache_free(struct kmem_cache *c, void *b) { - if (c->dtor) - c->dtor(b, c, 0); + if (c->flags & SLAB_CALLBACK_DTOR) + c->callback(b, c, SLAB_CALLBACK_DTOR); if (c->size < PAGE_SIZE) slob_free(b, c->size); Index: slub/mm/slub.c =================================================================== --- slub.orig/mm/slub.c 2007-05-02 16:45:15.000000000 -0700 +++ slub/mm/slub.c 2007-05-02 16:45:17.000000000 -0700 @@ -148,7 +148,8 @@ * Set of flags that will prevent slab merging */ #define SLUB_NEVER_MERGE (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | \ - SLAB_TRACE | SLAB_DESTROY_BY_RCU) + SLAB_TRACE | SLAB_DESTROY_BY_RCU | \ + SLAB_CALLBACK_CTOR | SLAB_CALLBACK_DTOR) #define SLUB_MERGE_SAME (SLAB_DEBUG_FREE | SLAB_RECLAIM_ACCOUNT | \ SLAB_CACHE_DMA) @@ -809,8 +810,8 @@ static void setup_object(struct kmem_cac init_tracking(s, object); } - if (unlikely(s->ctor)) - s->ctor(object, s, SLAB_CTOR_CONSTRUCTOR); + if (unlikely(s->flags & SLAB_CALLBACK_CTOR)) + s->callback(object, s, SLAB_CALLBACK_CTOR); } static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) @@ -868,15 +869,17 @@ static void __free_slab(struct kmem_cach { int pages = 1 << s->order; - if (unlikely(PageError(page) || s->dtor)) { + if (unlikely(PageError(page) || + (s->flags & SLAB_CALLBACK_DTOR))) { + void *start = page_address(page); void *end = start + (pages << PAGE_SHIFT); void *p; slab_pad_check(s, page); for (p = start; p <= end - s->size; p += s->size) { - if (s->dtor) - s->dtor(p, s, 0); + if (s->flags & SLAB_CALLBACK_DTOR) + s->callback(p, s, SLAB_CALLBACK_DTOR); check_object(s, page, p, 0); } } @@ -1482,19 +1485,6 @@ static int calculate_order(int size) static unsigned long calculate_alignment(unsigned long flags, unsigned long align, unsigned long size) { - /* - * If the user wants hardware cache aligned objects then - * follow that suggestion if the object is sufficiently - * large. - * - * The hardware cache alignment cannot override the - * specified alignment though. If that is greater - * then use it. - */ - if ((flags & SLAB_HWCACHE_ALIGN) && - size > L1_CACHE_BYTES / 2) - return max_t(unsigned long, align, L1_CACHE_BYTES); - if (align < ARCH_SLAB_MINALIGN) return ARCH_SLAB_MINALIGN; @@ -1617,8 +1607,9 @@ static int calculate_sizes(struct kmem_c * the slab may touch the object after free or before allocation * then we should never poison the object itself. */ - if ((flags & SLAB_POISON) && !(flags & SLAB_DESTROY_BY_RCU) && - !s->ctor && !s->dtor) + if ((flags & SLAB_POISON) && + !(flags & (SLAB_DESTROY_BY_RCU | + SLAB_CALLBACK_CTOR | SLAB_CALLBACK_DTOR))) s->flags |= __OBJECT_POISON; else s->flags &= ~__OBJECT_POISON; @@ -1646,8 +1637,8 @@ static int calculate_sizes(struct kmem_c */ s->inuse = size; - if (((flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON)) || - s->ctor || s->dtor)) { + if (flags & (SLAB_DESTROY_BY_RCU | SLAB_POISON | + SLAB_CALLBACK_CTOR | SLAB_CALLBACK_DTOR)) { /* * Relocate free pointer after the object if it is not * permitted to overwrite the first word of the object on @@ -1731,17 +1722,18 @@ static int __init finish_bootstrap(void) static int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags, const char *name, size_t size, size_t align, unsigned long flags, - void (*ctor)(void *, struct kmem_cache *, unsigned long), - void (*dtor)(void *, struct kmem_cache *, unsigned long)) + int (*callback)(void *, struct kmem_cache *, unsigned long)) { memset(s, 0, kmem_size); s->name = name; - s->ctor = ctor; - s->dtor = dtor; + s->callback = callback; s->objsize = size; s->flags = flags; s->align = align; + if (!!callback ^ !!(s->flags & (SLAB_CALLBACK_CTOR|SLAB_CALLBACK_DTOR))) + goto error; + /* * The page->offset field is only 16 bit wide. This is an offset * in units of words from the beginning of an object. If the slab @@ -1751,14 +1743,13 @@ static int kmem_cache_open(struct kmem_c * On 32 bit platforms the limit is 256k. On 64bit platforms * the limit is 512k. * - * Debugging or ctor/dtors may create a need to move the free + * Debugging or callbacks may create a need to move the free * pointer. Fail if this happens. */ - if (s->size >= 65535 * sizeof(void *)) { + if (s->size >= 65535 * sizeof(void *)) BUG_ON(flags & (SLAB_RED_ZONE | SLAB_POISON | - SLAB_STORE_USER | SLAB_DESTROY_BY_RCU)); - BUG_ON(ctor || dtor); - } + SLAB_STORE_USER | SLAB_DESTROY_BY_RCU | + SLAB_CALLBACK_CTOR | SLAB_CALLBACK_DTOR)); else /* * Enable debugging if selected on the kernel commandline. @@ -1992,7 +1983,7 @@ static struct kmem_cache *create_kmalloc down_write(&slub_lock); if (!kmem_cache_open(s, gfp_flags, name, size, ARCH_KMALLOC_MINALIGN, - flags, NULL, NULL)) + flags, NULL)) goto panic; list_add(&s->list, &slab_caches); @@ -2313,25 +2304,17 @@ static int slab_unmergeable(struct kmem_ if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE)) return 1; - if (s->ctor || s->dtor) - return 1; - return 0; } static struct kmem_cache *find_mergeable(size_t size, - size_t align, unsigned long flags, - void (*ctor)(void *, struct kmem_cache *, unsigned long), - void (*dtor)(void *, struct kmem_cache *, unsigned long)) + size_t align, unsigned long flags) { struct list_head *h; if (slub_nomerge || (flags & SLUB_NEVER_MERGE)) return NULL; - if (ctor || dtor) - return NULL; - size = ALIGN(size, sizeof(void *)); align = calculate_alignment(flags, align, size); size = ALIGN(size, align); @@ -2364,15 +2347,14 @@ static struct kmem_cache *find_mergeable return NULL; } -struct kmem_cache *kmem_cache_create(const char *name, size_t size, +struct kmem_cache *__kmem_cache_create(const char *name, size_t size, size_t align, unsigned long flags, - void (*ctor)(void *, struct kmem_cache *, unsigned long), - void (*dtor)(void *, struct kmem_cache *, unsigned long)) + int (*callback)(void *, struct kmem_cache *, unsigned long)) { struct kmem_cache *s; down_write(&slub_lock); - s = find_mergeable(size, align, flags, dtor, ctor); + s = find_mergeable(size, align, flags); if (s) { s->refcount++; /* @@ -2386,7 +2368,7 @@ struct kmem_cache *kmem_cache_create(con } else { s = kmalloc(kmem_size, GFP_KERNEL); if (s && kmem_cache_open(s, GFP_KERNEL, name, - size, align, flags, ctor, dtor)) { + size, align, flags, callback)) { if (sysfs_slab_add(s)) { kfree(s); goto err; @@ -2406,7 +2388,7 @@ err: s = NULL; return s; } -EXPORT_SYMBOL(kmem_cache_create); +EXPORT_SYMBOL(__kmem_cache_create); void *kmem_cache_zalloc(struct kmem_cache *s, gfp_t flags) { @@ -2961,27 +2943,16 @@ static ssize_t order_show(struct kmem_ca } SLAB_ATTR_RO(order); -static ssize_t ctor_show(struct kmem_cache *s, char *buf) +static ssize_t callback_show(struct kmem_cache *s, char *buf) { - if (s->ctor) { - int n = sprint_symbol(buf, (unsigned long)s->ctor); + if (s->callback) { + int n = sprint_symbol(buf, (unsigned long)s->callback); return n + sprintf(buf + n, "\n"); } return 0; } -SLAB_ATTR_RO(ctor); - -static ssize_t dtor_show(struct kmem_cache *s, char *buf) -{ - if (s->dtor) { - int n = sprint_symbol(buf, (unsigned long)s->dtor); - - return n + sprintf(buf + n, "\n"); - } - return 0; -} -SLAB_ATTR_RO(dtor); +SLAB_ATTR_RO(callback); static ssize_t aliases_show(struct kmem_cache *s, char *buf) { @@ -3058,12 +3029,6 @@ static ssize_t reclaim_account_store(str } SLAB_ATTR(reclaim_account); -static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf) -{ - return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN)); -} -SLAB_ATTR_RO(hwcache_align); - #ifdef CONFIG_ZONE_DMA static ssize_t cache_dma_show(struct kmem_cache *s, char *buf) { @@ -3213,13 +3178,11 @@ static struct attribute * slab_attrs[] = &slabs_attr.attr, &partial_attr.attr, &cpu_slabs_attr.attr, - &ctor_attr.attr, - &dtor_attr.attr, + &callback_attr.attr, &aliases_attr.attr, &align_attr.attr, &sanity_checks_attr.attr, &trace_attr.attr, - &hwcache_align_attr.attr, &reclaim_account_attr.attr, &destroy_by_rcu_attr.attr, &red_zone_attr.attr,