A generic kmalloc layer for the modular slab Signed-off-by: Christoph Lameter Index: linux-2.6.18-rc4-mm2/include/linux/kmalloc.h =================================================================== --- /dev/null 1970-01-01 00:00:00.000000000 +0000 +++ linux-2.6.18-rc4-mm2/include/linux/kmalloc.h 2006-08-25 19:19:43.168661964 -0700 @@ -0,0 +1,136 @@ +#ifndef _LINUX_KMALLOC_H +#define _LINUX_KMALLOC_H +/* + * In kernel dynamic memory allocator. + * + * (C) 2006 Silicon Graphics, Inc, + * Christoph Lameter + */ + +#include +#include +#include + +#ifndef KMALLOC_ALLOCATOR +#define KMALLOC_ALLOCATOR slabifier_allocator +#endif + +#ifdef ARCH_NEEDS_SMALL_SLABS +#define KMALLOC_SHIFT_LOW 3 +#else +#define KMALLOC_SHIFT_LOW 7 +#endif + +#define KMALLOC_SHIFT_HIGH 20 + +#ifdef ARCH_NEEDS_SMALL_SLABS +#define KMALLOC_EXTRAS 2 +#else +#define KMALLOC_EXTRAS 0 +#endif + +#define KMALLOC_NR_CACHES (KMALLOC_SHIFT_HIGH - KMALLOC_SHIFT_LOW \ + + 1 + KMALLOC_EXTRAS) +/* + * We keep the general caches in an array of slab caches that are used for + * 2^x bytes of allocations. For each size we generate a DMA and a + * non DMA cache (DMA simply means memory for legacy I/O. The regular + * caches can be used for devices that can DMA to all of memory). + */ +extern struct slab_control kmalloc_caches[2][KMALLOC_NR_CACHES]; + +/* + * Sorry that the following has to be that ugly but GCC has trouble + * with constant propagation and loops. + */ +static inline int kmalloc_index(int size) +{ +#ifdef ARCH_NEEDS_SMALL_SLABS + if (size <= 8) return 3; + if (size <= 16) return 4; + if (size <= 32) return 5; + if (size <= 64) return 6; + if (size <= 96) return KMALLOC_SHIFT_HIGH + 1; +#endif + if (size <= 128) return 7; +#ifdef ARCH_NEEDS_SMALL_SLABS + if (size <= 192) return KMALLOC_SHIFT_HIGH + 2; +#endif + if (size <= 256) return 8; + if (size <= 512) return 9; + if (size <= 1024) return 10; + if (size <= 2048) return 11; + if (size <= 4096) return 12; + if (size <= 8 * 1024) return 13; + if (size <= 16 * 1024) return 14; + if (size <= 32 * 1024) return 15; + if (size <= 64 * 1024) return 16; + if (size <= 128 * 1024) return 17; + if (size <= 256 * 1024) return 18; + if (size <= 512 * 1024) return 19; + if (size <=1024 * 1024) return 20; + return -1; +} + +/* + * Find the slab cache for a given combination of allocation flags and size. + * + * This ought to end up with a global pointer to the right cache + * in kmalloc_caches. + */ +static inline struct slab_cache *kmalloc_slab(size_t size, gfp_t flags) +{ + int index = kmalloc_index(size) - KMALLOC_SHIFT_LOW; + + if (index < 0) { + /* + * Generate a link failure. Would be great if we could + * do something to stop the compile here. + */ + extern void __kmalloc_size_too_large(void); + __kmalloc_size_too_large(); + } + return &kmalloc_caches[!!(flags & __GFP_DMA)][index].sc; +} + +extern void *__kmalloc(size_t, gfp_t); +#define ____kmalloc __kmalloc + +static inline void *kmalloc(size_t size, gfp_t flags) +{ + if (__builtin_constant_p(size)) { + struct slab_cache *s = kmalloc_slab(size, flags); + + return KMALLOC_ALLOCATOR.alloc(s, flags); + } else + return __kmalloc(size, flags); +} + +#ifdef CONFIG_NUMA +extern void *__kmalloc_node(size_t, gfp_t, int); +static inline void *kmalloc_node(size_t size, gfp_t flags, int node) +{ + if (__builtin_constant_p(size)) { + struct slab_cache *s = kmalloc_slab(size, flags); + + return KMALLOC_ALLOCATOR.alloc_node(s, flags, node); + } else + return __kmalloc_node(size, flags, node); +} +#else +#define kmalloc_node(__size, __flags, __node) kmalloc((__size), (__flags)) +#endif + +/* Free an object */ +static inline void kfree(const void *x) +{ + return KMALLOC_ALLOCATOR.free(NULL, x); +} + +/* Allocate and zero the specified number of bytes */ +extern void *kzalloc(size_t, gfp_t); + +/* Figure out what size the chunk is */ +extern size_t ksize(const void *); + +#endif /* _LINUX_KMALLOC_H */ Index: linux-2.6.18-rc4-mm2/mm/kmalloc.c =================================================================== --- /dev/null 1970-01-01 00:00:00.000000000 +0000 +++ linux-2.6.18-rc4-mm2/mm/kmalloc.c 2006-08-25 19:22:04.922595245 -0700 @@ -0,0 +1,166 @@ +/* + * Create generic slab caches for memory allocation. + * + * (C) 2006 Silicon Graphics. Inc. Christoph Lameter + */ + +#include +#include +#include +#include +#include + +struct slab_control kmalloc_caches[2][KMALLOC_NR_CACHES] __cacheline_aligned; +EXPORT_SYMBOL(kmalloc_caches); + +static struct slab_cache *get_slab(size_t size, gfp_t flags) +{ + int index = kmalloc_index(size) - KMALLOC_SHIFT_LOW; + + BUG_ON(size < 0); + + return &kmalloc_caches[!!(flags & __GFP_DMA)][index].sc; + +} + +void *__kmalloc(size_t size, gfp_t flags) +{ + return KMALLOC_ALLOCATOR.alloc(get_slab(size, flags), flags); +} +EXPORT_SYMBOL(__kmalloc); + +#ifdef CONFIG_NUMA +void *__kmalloc_node(size_t size, gfp_t flags, int node) +{ + return KMALLOC_ALLOCATOR.alloc_node(get_slab(size, flags), + flags, node); +} +EXPORT_SYMBOL(__kmalloc_node); +#endif + +void *kzalloc(size_t size, gfp_t flags) +{ + void *x = __kmalloc(size, flags); + + if (x) + memset(x, 0, size); + return x; +} +EXPORT_SYMBOL(kzalloc); + +size_t ksize(const void *object) +{ + return KMALLOC_ALLOCATOR.object_size(NULL, object); +}; +EXPORT_SYMBOL(ksize); + +/* + * Given a slab size find the correct order to use. + * We only support powers of two so there is really + * no need for anything special. Objects will always + * fit exactly into the slabs with no overhead. + */ +static __init int order(size_t size) +{ + if (size >= PAGE_SIZE) + /* One object per slab */ + return fls(size -1) - PAGE_SHIFT; + + /* Multiple objects per page which will fit neatly */ + return 0; +} +/* + * Provide the kmalloc array as regular slab allocator for the + * generic allocator framework. + */ +struct slab_allocator kmalloc_slab_allocator; + +struct slab_cache *kmalloc_create(struct slab_control *x, + const struct slab_cache *s) +{ + struct slab_cache *km; + + int index = max(0, fls(s->size - 1) - KMALLOC_SHIFT_LOW); + + if (index > KMALLOC_SHIFT_HIGH - KMALLOC_SHIFT_LOW + 1 + || s->offset) + return NULL; + + km = &kmalloc_caches[0][index].sc; + + BUG_ON(s->size > km->size); + + return KMALLOC_ALLOCATOR.dup(km); +} + +#ifndef ARCH_KMALLOC_MINALIGN +#define ARCH_KMALLOC_MINALIGN sizeof(void *) +#endif + +void __init create_kmalloc_cache(struct slab_control *x, + const char *name, + const struct page_allocator *p, + int size) +{ + struct slab_cache s; + struct slab_cache *rs; + + s.page_alloc = p; + s.slab_alloc = &KMALLOC_ALLOCATOR; + s.size = size; + s.align = ARCH_KMALLOC_MINALIGN; + s.offset = 0; + s.objsize = size; + s.inuse = size; + s.node = -1; + s.order = order(size); + s.name = "kmalloc"; + rs = KMALLOC_ALLOCATOR.create(x, &s); + if (!rs) + panic("Creation of kmalloc slab %s size=%d failed.\n", + name, size); + register_slab(rs); +} + +void __init kmalloc_init_array(int dma, const char *name, + const struct page_allocator *pa) +{ + int i; + + for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) { + create_kmalloc_cache( + &kmalloc_caches[dma][i - KMALLOC_SHIFT_LOW], + name, pa, 1 << i); + } +#ifdef ARCH_NEEDS_SMALL_SLABS + /* Non-power of two caches */ + create_kmalloc_cache(&kmalloc_caches[dma] + [KMALLOC_SHIFT_HIGH - KMALLOC_SHIFT_LOW + 1], name, pa, 96); + create_kmalloc_cache(&kmalloc_caches[dma] + [KMALLOC_SHIFT_HIGH - KMALLOC_SHIFT_LOW + 2], name, pa, 192); +#endif +} + +void __init kmalloc_init(void) +{ + + kmalloc_init_array(0, "kmalloc", &page_allocator); + /* + * The above must be done first. Deriving a page allocator requires + * a working (normal) kmalloc array. + */ + + /* + * On all my machines the DMA array is always empty. I wish we + * could get rid of it. + */ + kmalloc_init_array(1, "kmalloc-DMA", + dmaify_page_allocator(&page_allocator)); + + /* And deal with the kmalloc_cache_allocator */ + memcpy(&kmalloc_slab_allocator, &KMALLOC_ALLOCATOR, + sizeof(struct slab_allocator)); + kmalloc_slab_allocator.create = kmalloc_create; + kmalloc_slab_allocator.destructor = null_slab_allocator_destructor; +} + Index: linux-2.6.18-rc4-mm2/mm/Makefile =================================================================== --- linux-2.6.18-rc4-mm2.orig/mm/Makefile 2006-08-25 19:16:57.338094817 -0700 +++ linux-2.6.18-rc4-mm2/mm/Makefile 2006-08-25 19:18:23.408944973 -0700 @@ -25,4 +25,4 @@ obj-$(CONFIG_MEMORY_HOTPLUG) += memory_h obj-$(CONFIG_FS_XIP) += filemap_xip.o obj-$(CONFIG_MIGRATION) += migrate.o obj-$(CONFIG_SMP) += allocpercpu.o -obj-$(CONFIG_MODULAR_SLAB) += allocator.o slabifier.o slabstat.o +obj-$(CONFIG_MODULAR_SLAB) += allocator.o slabifier.o slabstat.o kmalloc.o Index: linux-2.6.18-rc4-mm2/include/asm-i386/page.h =================================================================== --- linux-2.6.18-rc4-mm2.orig/include/asm-i386/page.h 2006-08-06 11:20:11.000000000 -0700 +++ linux-2.6.18-rc4-mm2/include/asm-i386/page.h 2006-08-25 19:18:23.408944973 -0700 @@ -37,6 +37,7 @@ #define alloc_zeroed_user_highpage(vma, vaddr) alloc_page_vma(GFP_HIGHUSER | __GFP_ZERO, vma, vaddr) #define __HAVE_ARCH_ALLOC_ZEROED_USER_HIGHPAGE +#define ARCH_NEEDS_SMALL_SLABS /* * These are used to make use of C type-checking..