Kmalloc layer for the modular slab New ideas: 1. Generate slabs on the fly (May have issues with GFP_KERNEL). 2. use fls to calculate array position. Signed-off-by: Christoph Lameter Index: linux-2.6.18-rc4/include/linux/kmalloc.h =================================================================== --- /dev/null 1970-01-01 00:00:00.000000000 +0000 +++ linux-2.6.18-rc4/include/linux/kmalloc.h 2006-08-23 12:10:08.540196817 -0700 @@ -0,0 +1,113 @@ +#ifndef _LINUX_KMALLOC_H +#define _LINUX_KMALLOC_H +/* + * In kernel dynamic memory allocator. + * + * (C) 2006 Silicon Graphics, Inc, + * Christoph Lameter + */ + +#include +#include +#include + +#ifndef KMALLOC_ALLOCATOR +#define KMALLOC_ALLOCATOR slabifier_allocator +#endif + +/* + * We keep the general caches in an array of slab caches that are used for + * 2^x bytes of allocations. For each size we generate a DMA and a + * non DMA cache (Do not get confused: DMA simply means memory for + * legacy I/O. The regular caches can be used for devices that can + * do DMA to all of memory). + */ +extern struct slab_control kmalloc_caches[2][16]; + +/* + * Sorry that the following has to be that ugly but GCC has trouble + * with constant propagation and loops. + */ +static inline int kmalloc_index(int size) +{ + if (size <= 32) return 0; + if (size <= 64) return 1; + if (size <= 96) return 14; + if (size <= 128) return 2; + if (size <= 192) return 15; + if (size <= 256) return 3; + if (size <= 512) return 4; + if (size <= 1024) return 5; + if (size <= 2048) return 6; + if (size <= 4096) return 7; + if (size <= 8 * 1024) return 8; + if (size <= 16 * 1024) return 9; + if (size <= 32 * 1024) return 10; + if (size <= 64 * 1024) return 11; + if (size <= 128 * 1024) return 12; + if (size <= 256 * 1024) return 13; + return -1; +} + +/* + * Find the slab cache for a given combination of allocation flags and size. + * + * This ought to end up with a global pointer to the right cache + * in kmalloc_caches. + */ +static inline struct slab_cache *kmalloc_slab(size_t size, gfp_t flags) +{ + int index = kmalloc_index(size); + + if (index < 0) { + /* + * Generate a link failure. Would be great if we could + * do something to stop the compile here. + */ + extern void __kmalloc_size_too_large(void); + __kmalloc_size_too_large(); + } + return &kmalloc_caches[!!(flags & __GFP_DMA)][index].sc; +} + +extern void *__kmalloc(size_t, gfp_t); +#define ____kmalloc __kmalloc + +static inline void *kmalloc(size_t size, gfp_t flags) +{ + if (__builtin_constant_p(size)) { + struct slab_cache *s = kmalloc_slab(size, flags); + + return KMALLOC_ALLOCATOR.alloc(s, flags); + } else + return __kmalloc(size, flags); +} + +#ifdef CONFIG_NUMA +extern void *__kmalloc_node(size_t, gfp_t, int); +static inline void *kmalloc_node(size_t size, gfp_t flags, int node) +{ + if (__builtin_constant_p(size)) { + struct slab_cache *s = kmalloc_slab(size, flags); + + return KMALLOC_ALLOCATOR.alloc_node(s, flags, node); + } else + return __kmalloc_node(size, flags, node); +} +#else +#define kmalloc_node(__size, __flags, __node) kmalloc((__size), (__flags)) +#endif + +/* Free an object */ +static inline void kfree(const void *x) +{ + return KMALLOC_ALLOCATOR.free(NULL, x); +} + +/* Allocate and zero the specified number of bytes */ +extern void *kzalloc(size_t, gfp_t); + +/* Figure out what size the chunk is */ +extern size_t ksize(const void *); + +#endif /* _LINUX_KMALLOC_H */ Index: linux-2.6.18-rc4/mm/kmalloc.c =================================================================== --- /dev/null 1970-01-01 00:00:00.000000000 +0000 +++ linux-2.6.18-rc4/mm/kmalloc.c 2006-08-23 12:10:08.538243813 -0700 @@ -0,0 +1,154 @@ +/* + * Implement basic generic caches for memory allocation. + * + * This is an on demand implementation. General slabs + * are only created if there is an actual need for allocations + * of that size. + * + * (C) 2006 Silicon Graphics. Inc. Christoph Lameter + */ + +#include +#include +#include +#include + +#define KMALLOC_SHIFT_LOW 5 +#define KMALLOC_SHIFT_HIGH 18 +#define KMALLOC_EXTRA 2 + +struct slab_control kmalloc_caches[2] + [KMALLOC_SHIFT_HIGH - KMALLOC_SHIFT_LOW + 1 + KMALLOC_EXTRA] __cacheline_aligned; +EXPORT_SYMBOL(kmalloc_caches); + + +static struct slab_cache *get_slab(size_t size, gfp_t flags) +{ + int index = kmalloc_index(size); + + BUG_ON(size < 0); + + return &kmalloc_caches[!!(flags & __GFP_DMA)] [index].sc; + +} + +void *__kmalloc(size_t size, gfp_t flags) +{ + return KMALLOC_ALLOCATOR.alloc(get_slab(size, flags), flags); +} +EXPORT_SYMBOL(__kmalloc); + +#ifdef CONFIG_NUMA +void *__kmalloc_node(size_t size, gfp_t flags, int node) +{ + return KMALLOC_ALLOCATOR.alloc_node(get_slab(size, flags), + flags, node); +} +EXPORT_SYMBOL(__kmalloc_node); +#endif + +void *kzalloc(size_t size, gfp_t flags) +{ + void *x = __kmalloc(size, flags); + + if (x) + memset(x, 0, size); + return x; +} +EXPORT_SYMBOL(kzalloc); + +size_t ksize(const void *object) +{ + return KMALLOC_ALLOCATOR.object_size(NULL, object); +}; +EXPORT_SYMBOL(ksize); + +/* + * Given a slab size find the correct order to use. + * We only support powers of two so there is really + * no need for anything special. Objects will always + * fit exactly into the slabs with no overhead. + */ +static __init int order(size_t size) +{ + if (size >= PAGE_SIZE) + /* One object per slab */ + return fls(size -1) - PAGE_SHIFT + 1; + + /* Multiple objects but let slab_create() figure out how much */ + return 0; +} + +void __init create_kmalloc_cache(struct slab_control *x, + const char *name, + const struct page_allocator *p, + int size) +{ + struct slab_cache s; + struct slab_cache *rs; + +// printk(KERN_CRIT "create_kmalloc_cache(%p,%s,%p,%d)\n", x, name, p, size); + + s.page_alloc = p; + s.slab_alloc = &KMALLOC_ALLOCATOR; + s.size = size; + s.align = 0; + s.offset = 0; + s.objsize = size; + s.inuse = size; + s.node = -1l; + s.order = order(size); + s.name = "kmalloc"; + rs = KMALLOC_ALLOCATOR.create(x, &s); + if (!rs) + panic("Creation of kmalloc slab %s size=%d failed.\n", + name, size); +} + +struct slab_allocator kmalloc_slab_allocator; +/* Export the fixed kmalloc array in another way */ +struct slab_cache *kmalloc_create(struct slab_control *x, + const struct slab_cache *s) +{ + int index = max(0, fls(s->size -1) - KMALLOC_SHIFT_LOW); + + if (index > KMALLOC_SHIFT_HIGH || s->offset) + return NULL; + + return KMALLOC_ALLOCATOR.dup(&kmalloc_caches[0][index].sc); +} + +void __init kmalloc_init_array(int dma, const char *name, + const struct page_allocator *pa) +{ + int i; + + for (i = 0; i <= 13; i++) { + create_kmalloc_cache( + &kmalloc_caches[dma][i], + name, pa, 1 << (i + KMALLOC_SHIFT_LOW)); + } + /* Non-power of two caches */ + create_kmalloc_cache(&kmalloc_caches[dma][14], name, pa, 96); + create_kmalloc_cache(&kmalloc_caches[dma][15], name, pa, 192); +} + +void __init kmalloc_init(void) +{ + + kmalloc_init_array(0, "kmalloc", &page_allocator); + /* + * The above must be done first. Deriving a page allocator requires + * a working (normal) kmalloc array. + */ + + kmalloc_init_array(1, "kmalloc-DMA", + dmaify_page_allocator(&page_allocator)); + + /* And deal with the kmalloc_cache_allocator */ + memcpy(&kmalloc_slab_allocator, &KMALLOC_ALLOCATOR, + sizeof(struct slab_allocator)); + kmalloc_slab_allocator.create = kmalloc_create; + kmalloc_slab_allocator.destructor = null_slab_allocator_destructor; +} + Index: linux-2.6.18-rc4/mm/Makefile =================================================================== --- linux-2.6.18-rc4.orig/mm/Makefile 2006-08-23 12:08:00.279564703 -0700 +++ linux-2.6.18-rc4/mm/Makefile 2006-08-23 12:08:55.771249631 -0700 @@ -24,5 +24,5 @@ obj-$(CONFIG_SLAB) += slab.o obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o obj-$(CONFIG_FS_XIP) += filemap_xip.o obj-$(CONFIG_MIGRATION) += migrate.o -obj-$(CONFIG_MODULAR_SLAB) += allocator.o slabifier.o +obj-$(CONFIG_MODULAR_SLAB) += allocator.o slabifier.o kmalloc.o