The slab emulation layer. This provides a layer that implements the existing slab API. Put a hook into slab.h to redirect include for slab.h. kmem_cache_create dynamically derives page allocators with the proper features requested. Signed-off-by: Christoph Lameter Index: linux-2.6.18-rc4/mm/slabulator.c =================================================================== --- /dev/null 1970-01-01 00:00:00.000000000 +0000 +++ linux-2.6.18-rc4/mm/slabulator.c 2006-08-23 12:10:11.813432336 -0700 @@ -0,0 +1,186 @@ +/* + * Slabulator = Emulate the old Slab API and provide various debugging helps. + * + * (C) 2006 Silicon Graphics, Inc. Christoph Lameter + * + * Important missing things: (breaking stuff!) + * 1. Support 2 second cache reaper calls (no pcp draining and VM stat + * update currently!). + */ +#include +#include +#include +#include +#include + +#define SLAB_MAX_ORDER 4 + +// #define SLABMULATOR_DEBUG +// #define SLABMULATOR_MERGE + +#ifndef ARCH_SLAB_MINALIGN +#define ARCH_SLAB_MINALIGN sizeof(void *) +#endif + +static int calculate_order(int size) +{ + int order; + int rem; + + for(order = max(0, fls(size - 1) - PAGE_SHIFT); + order < MAX_ORDER; order++) { + unsigned long slab_size = PAGE_SIZE << order; + + if (slab_size < size) + continue; + + rem = slab_size % size; + + if (rem * 8 <= PAGE_SIZE << order) + break; + + } + if (order >= MAX_ORDER) + return -E2BIG; + return order; +} + +/* + * We can actually operate slabs any time after the page allocator is up. + * slab_is_available() merely means that the kmalloc array is available. + */ +int slabulator_up = 0; + +int slab_is_available(void) { + return slabulator_up; +} + +void kmem_cache_init(void) +{ + extern void kmalloc_init(void); + + kmalloc_init(); + slabulator_up = 1; +} + +struct slab_cache *kmem_cache_create(const char *name, size_t size, + size_t align, unsigned long flags, + void (*ctor)(void *, struct slab_cache *, unsigned long), + void (*dtor)(void *, struct slab_cache *, unsigned long)) +{ + const struct page_allocator *a = &page_allocator; + struct slab_cache s; + struct slab_cache *rs; + struct slab_control *x; + + s.offset = 0; + s.align = max(ARCH_SLAB_MINALIGN, ALIGN(align, sizeof(void *))); + + if (flags & (SLAB_MUST_HWCACHE_ALIGN|SLAB_HWCACHE_ALIGN)) + s.align = L1_CACHE_BYTES; + + s.inuse = size; + s.objsize = size; + s.size = ALIGN(size, s.align); + + /* Pick the right allocator for our purposes */ + if (flags & SLAB_RECLAIM_ACCOUNT) + a = reclaim_allocator(a); + + if (flags & SLAB_CACHE_DMA) + a = dmaify_page_allocator(a); + + if (flags & SLAB_DESTROY_BY_RCU) + a = rcuify_page_allocator(a); + + if ((flags & SLAB_DESTROY_BY_RCU) || ctor || dtor) { + /* + * For RCU processing and constructors / destructors: + * The object must remain intact even if it is free. + * The free pointer would hurt us there. + * Relocate the free object pointer out of + * the space used by the object. + */ + s.offset = s.size - sizeof(void *); + if (s.offset < s.objsize) { + /* + * Would overlap the object. We need to waste some + * more space to make the object RCU safe + */ + s.offset = s.size; + s.size += s.align; + } + s.inuse = s.size; + } + + s.order = calculate_order(s.size); + + if (s.order < 0) + goto error; + +#ifdef SLABMULATOR_MERGE + /* + * This works but is this really something we want? + */ + if (((s.realsize & (s.realsize - 1))==0) && !ctor && !dtor && + !(flags & (SLAB_DESTROY_BY_RCU|SLAB_RECLAIM_ACCOUNT))) { + a->destructor((struct page_allocator *)a); + + printk(KERN_CRIT "Merging slab %s size %d to kmalloc\n", + name, s.realsize); + return kmalloc_allocator(....); + } +#endif + s.name = name; + s.node = -1; + + x = kmalloc(sizeof(struct slab_control), GFP_KERNEL); + + if (!x) + return NULL; + s.page_alloc = a; + s.slab_alloc = &SLABULATOR_ALLOCATOR; + rs = SLABULATOR_ALLOCATOR.create(x, &s); +#ifdef SLABMULATOR_DEBUG + printk(KERN_CRIT "Creating slab %s size=%ld realsize=%d " + "order=%d offset=%d flags=%lx\n", + s.name, size, s.size, s.order, s.offset, flags); +#endif + if (!rs) + goto error; + + /* + * Now deal with constuctors and destructors. We need to know the + * slab_cache address in order to be able to pass the slab_cache + * address down the chain. + */ + if (ctor || dtor) + rs->page_alloc = + ctor_and_dtor_for_page_allocator(rs->page_alloc, + rs->size, rs, + (void *)ctor, (void *)dtor); + + return rs; + +error: + a->destructor((struct page_allocator *)a); + if (flags & SLAB_PANIC) + panic("Cannot create slab %s size=%ld realsize=%d " + "order=%d offset=%d flags=%lx\n", + s.name, size, s.size, s.order, s.offset, flags); + + + return NULL; +} +EXPORT_SYMBOL(kmem_cache_create); + +void *kmem_cache_zalloc(struct slab_cache *s, gfp_t flags) +{ + void *x; + + x = kmem_cache_alloc(s, flags); + if (x) + memset(x, 0, s->objsize); + return x; +} + Index: linux-2.6.18-rc4/include/linux/slabulator.h =================================================================== --- /dev/null 1970-01-01 00:00:00.000000000 +0000 +++ linux-2.6.18-rc4/include/linux/slabulator.h 2006-08-23 12:10:11.814408838 -0700 @@ -0,0 +1,119 @@ +/* + * Slabulator: Emulate the existing Slab API. + * + * (C) 2006 Silicon Graphics, Inc. + * Christoph Lameter + * + * The use of this API necessarily restrict the user to a single type + * of Slab allocator and the functions available. + */ + +#include +#include + +#define kmem_cache_t struct slab_cache +#define kmem_cache slab_cache + +#ifndef SLABULATOR_ALLOCATOR +#define SLABULATOR_ALLOCATOR slabifier_allocator +#endif + +/* We really should be getting rid of these */ +#define SLAB_KERNEL GFP_KERNEL +#define SLAB_ATOMIC GFP_ATOMIC +#define SLAB_NOFS GFP_NOFS +#define SLAB_NOIO GFP_NOIO + +/* No debug features for now */ +#define SLAB_HWCACHE_ALIGN 0x00002000UL /* align objs on a h/w cache lines */ +#define SLAB_CACHE_DMA 0x00004000UL /* use GFP_DMA memory */ +#define SLAB_MUST_HWCACHE_ALIGN 0x00008000UL /* force alignment */ +#define SLAB_RECLAIM_ACCOUNT 0x00020000UL /* track pages allocated to indicate + what is reclaimable later*/ +#define SLAB_PANIC 0x00040000UL /* panic if kmem_cache_create() fails */ +#define SLAB_DESTROY_BY_RCU 0x00080000UL /* defer freeing pages to RCU */ +#define SLAB_MEM_SPREAD 0x00100000UL /* Spread some memory over cpuset */ + +/* flags passed to a constructor func */ +#define SLAB_CTOR_CONSTRUCTOR 0x001UL /* if not set, then deconstructor */ +#define SLAB_CTOR_ATOMIC 0x002UL /* tell constructor it can't sleep */ +#define SLAB_CTOR_VERIFY 0x004UL /* tell constructor it's a verify call */ + +/* + * slab_allocators are always available after the page allocator + * has been brought up. kmem_cache_init creates the kmalloc array: + */ +extern int slab_is_available(void); +extern void kmem_cache_init(void); + +/* System wide caches (Should these be really here?) */ +extern struct slab_cache *vm_area_cachep; +extern struct slab_cache *names_cachep; +extern struct slab_cache *files_cachep; +extern struct slab_cache *filp_cachep; +extern struct slab_cache *fs_cachep; +extern struct slab_cache *sighand_cachep; +extern struct slab_cache *bio_cachep; + +extern struct slab_cache *kmem_cache_create(const char *name, size_t size, size_t align, + unsigned long flags, + void (*ctor)(void *, struct slab_cache *, unsigned long), + void (*dtor)(void *, struct slab_cache *, unsigned long)); + +static inline unsigned int kmem_cache_size(struct slab_cache *s) +{ + return s->objsize; +} + +static inline const char *kmem_cache_name(struct slab_cache *s) +{ + return s->name; +} + +static inline void *kmem_cache_alloc(struct slab_cache *s, gfp_t flags) +{ + return SLABULATOR_ALLOCATOR.alloc(s, flags); +} + +static inline void *kmem_cache_alloc_node(struct slab_cache *s, + gfp_t flags, int node) +{ + return SLABULATOR_ALLOCATOR.alloc_node(s, flags, node); +} + +extern void *kmem_cache_zalloc(struct slab_cache *s, gfp_t flags); + +static inline void kmem_cache_free(struct slab_cache *s, const void *x) +{ + SLABULATOR_ALLOCATOR.free(s, x); +} + +static inline int kmem_ptr_validate(struct slab_cache *s, void *x) +{ + return SLABULATOR_ALLOCATOR.valid_pointer(s, x); +} + +static inline int kmem_cache_destroy(struct slab_cache *s) +{ + return SLABULATOR_ALLOCATOR.destroy(s); +} + +static inline int kmem_cache_shrink(struct slab_cache *s) +{ + return SLABULATOR_ALLOCATOR.shrink(s, NULL); +} + +/** + * kcalloc - allocate memory for an array. The memory is set to zero. + * @n: number of elements. + * @size: element size. + * @flags: the type of memory to allocate. + */ +static inline void *kcalloc(size_t n, size_t size, gfp_t flags) +{ + if (n != 0 && size > ULONG_MAX / n) + return NULL; + return kzalloc(n * size, flags); +} + + Index: linux-2.6.18-rc4/mm/Makefile =================================================================== --- linux-2.6.18-rc4.orig/mm/Makefile 2006-08-23 12:08:55.771249631 -0700 +++ linux-2.6.18-rc4/mm/Makefile 2006-08-23 12:10:11.815385340 -0700 @@ -24,5 +24,5 @@ obj-$(CONFIG_SLAB) += slab.o obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o obj-$(CONFIG_FS_XIP) += filemap_xip.o obj-$(CONFIG_MIGRATION) += migrate.o -obj-$(CONFIG_MODULAR_SLAB) += allocator.o slabifier.o kmalloc.o +obj-$(CONFIG_MODULAR_SLAB) += allocator.o slabifier.o kmalloc.o slabulator.o Index: linux-2.6.18-rc4/init/Kconfig =================================================================== --- linux-2.6.18-rc4.orig/init/Kconfig 2006-08-06 11:20:11.000000000 -0700 +++ linux-2.6.18-rc4/init/Kconfig 2006-08-23 12:10:11.816361842 -0700 @@ -281,6 +281,24 @@ config CC_OPTIMIZE_FOR_SIZE If unsure, say N. +config SLAB + default y + bool "Monolithic SLAB allocator" + help + Disabling this allows the user of the other slab allocators + with less overhead such as SLOB (very simple) or the + use the slabifier with the module allocator framework. + +config MODULAR_SLAB + default y + bool "Use the modular allocator framework" + depends on EXPERIMENTAL && !SLAB + help + The modular allocator framework allows the flexible use + of different slab allocators and page allocators for memory + allocation. This will completely replace the existing + slab allocator. Beware this is experimental code. + menuconfig EMBEDDED bool "Configure standard kernel features (for small systems)" help @@ -319,7 +337,6 @@ config KALLSYMS_EXTRA_PASS reported. KALLSYMS_EXTRA_PASS is only a temporary workaround while you wait for kallsyms to be fixed. - config HOTPLUG bool "Support for hot-pluggable devices" if EMBEDDED default y @@ -394,15 +411,6 @@ config SHMEM option replaces shmem and tmpfs with the much simpler ramfs code, which may be appropriate on small systems without swap. -config SLAB - default y - bool "Use full SLAB allocator" if EMBEDDED - help - Disabling this replaces the advanced SLAB allocator and - kmalloc support with the drastically simpler SLOB allocator. - SLOB is more space efficient but does not scale well and is - more susceptible to fragmentation. - config VM_EVENT_COUNTERS default y bool "Enable VM event counters for /proc/vmstat" if EMBEDDED @@ -424,7 +432,7 @@ config BASE_SMALL default 1 if !BASE_FULL config SLOB - default !SLAB + default !SLAB && !MODULAR_SLAB bool menu "Loadable module support" Index: linux-2.6.18-rc4/include/linux/slab.h =================================================================== --- linux-2.6.18-rc4.orig/include/linux/slab.h 2006-08-06 11:20:11.000000000 -0700 +++ linux-2.6.18-rc4/include/linux/slab.h 2006-08-23 12:10:11.817338345 -0700 @@ -9,6 +9,10 @@ #if defined(__KERNEL__) +#ifdef CONFIG_MODULAR_SLAB +#include +#else + typedef struct kmem_cache kmem_cache_t; #include @@ -265,6 +269,8 @@ extern kmem_cache_t *bio_cachep; extern atomic_t slab_reclaim_pages; +#endif /* CONFIG_SLABULATOR */ + #endif /* __KERNEL__ */ #endif /* _LINUX_SLAB_H */