From lethal@linux-sh.org Thu Jun 14 20:35:45 2007 Date: Fri, 15 Jun 2007 12:34:12 +0900 From: Paul Mundt To: Matt Mackall Cc: Nick Piggin , Christoph Lameter , Andrew Morton , linux-mm@kvack.org Subject: [PATCH] slob: poor man's NUMA, take 5. Updated version of the SLOB NUMA support. This version adds in a slob_def.h and reorders a bit of the slab.h definitions. This should take in to account all of the outstanding comments so far on the earlier versions. Tested on all of SLOB/SLUB/SLAB with and without CONFIG_NUMA. Signed-off-by: Paul Mundt -- include/linux/slab.h | 85 ++++------------------------------------------- include/linux/slab_def.h | 4 ++ include/linux/slob_def.h | 83 +++++++++++++++++++++++++++++++++++++++++++++ include/linux/slub_def.h | 6 ++- mm/slob.c | 51 ++++++++++++++++++++-------- 5 files changed, 137 insertions(+), 92 deletions(-) Index: linux-2.6/include/linux/slab.h =================================================================== --- linux-2.6.orig/include/linux/slab.h 2007-05-17 10:34:13.000000000 -0700 +++ linux-2.6/include/linux/slab.h 2007-06-14 21:25:44.000000000 -0700 @@ -44,7 +44,6 @@ struct kmem_cache *kmem_cache_create(con void (*)(void *, struct kmem_cache *, unsigned long)); void kmem_cache_destroy(struct kmem_cache *); int kmem_cache_shrink(struct kmem_cache *); -void *kmem_cache_alloc(struct kmem_cache *, gfp_t); void *kmem_cache_zalloc(struct kmem_cache *, gfp_t); void kmem_cache_free(struct kmem_cache *, void *); unsigned int kmem_cache_size(struct kmem_cache *); @@ -63,9 +62,9 @@ int kmem_ptr_validate(struct kmem_cache sizeof(struct __struct), __alignof__(struct __struct),\ (__flags), NULL, NULL) -#ifdef CONFIG_NUMA -extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); -#else +#if !defined(CONFIG_NUMA) && !defined(CONFIG_SLOB) +void *kmem_cache_alloc(struct kmem_cache *, gfp_t); + static inline void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int node) { @@ -91,7 +90,6 @@ static inline void *kmem_cache_alloc_nod /* * Common kmalloc functions provided by all allocators */ -void *__kmalloc(size_t, gfp_t); void *__kzalloc(size_t, gfp_t); void * __must_check krealloc(const void *, size_t, gfp_t); void kfree(const void *); @@ -112,85 +110,19 @@ static inline void *kcalloc(size_t n, si /* * Allocator specific definitions. These are mainly used to establish optimized - * ways to convert kmalloc() calls to kmem_cache_alloc() invocations by selecting - * the appropriate general cache at compile time. + * ways to convert kmalloc() calls to kmem_cache_alloc() invocations by + * selecting the appropriate general cache at compile time. */ -#if defined(CONFIG_SLAB) || defined(CONFIG_SLUB) #ifdef CONFIG_SLUB #include +#elif defined(CONFIG_SLOB) +#include #else #include -#endif /* !CONFIG_SLUB */ -#else - -/* - * Fallback definitions for an allocator not wanting to provide - * its own optimized kmalloc definitions (like SLOB). - */ - -/** - * kmalloc - allocate memory - * @size: how many bytes of memory are required. - * @flags: the type of memory to allocate. - * - * kmalloc is the normal method of allocating memory - * in the kernel. - * - * The @flags argument may be one of: - * - * %GFP_USER - Allocate memory on behalf of user. May sleep. - * - * %GFP_KERNEL - Allocate normal kernel ram. May sleep. - * - * %GFP_ATOMIC - Allocation will not sleep. - * For example, use this inside interrupt handlers. - * - * %GFP_HIGHUSER - Allocate pages from high memory. - * - * %GFP_NOIO - Do not do any I/O at all while trying to get memory. - * - * %GFP_NOFS - Do not make any fs calls while trying to get memory. - * - * Also it is possible to set different flags by OR'ing - * in one or more of the following additional @flags: - * - * %__GFP_COLD - Request cache-cold pages instead of - * trying to return cache-warm pages. - * - * %__GFP_DMA - Request memory from the DMA-capable zone. - * - * %__GFP_HIGH - This allocation has high priority and may use emergency pools. - * - * %__GFP_HIGHMEM - Allocated memory may be from highmem. - * - * %__GFP_NOFAIL - Indicate that this allocation is in no way allowed to fail - * (think twice before using). - * - * %__GFP_NORETRY - If memory is not immediately available, - * then give up at once. - * - * %__GFP_NOWARN - If allocation fails, don't issue any warnings. - * - * %__GFP_REPEAT - If allocation fails initially, try once more before failing. - */ -static inline void *kmalloc(size_t size, gfp_t flags) -{ - return __kmalloc(size, flags); -} - -/** - * kzalloc - allocate memory. The memory is set to zero. - * @size: how many bytes of memory are required. - * @flags: the type of memory to allocate (see kmalloc). - */ -static inline void *kzalloc(size_t size, gfp_t flags) -{ - return __kzalloc(size, flags); -} #endif -#ifndef CONFIG_NUMA +#if !defined(CONFIG_NUMA) && !defined(CONFIG_SLOB) static inline void *kmalloc_node(size_t size, gfp_t flags, int node) { return kmalloc(size, flags); @@ -247,4 +179,3 @@ extern void *__kmalloc_node_track_caller #endif /* __KERNEL__ */ #endif /* _LINUX_SLAB_H */ - Index: linux-2.6/include/linux/slab_def.h =================================================================== --- linux-2.6.orig/include/linux/slab_def.h 2007-05-17 10:34:13.000000000 -0700 +++ linux-2.6/include/linux/slab_def.h 2007-06-14 21:25:44.000000000 -0700 @@ -25,6 +25,9 @@ struct cache_sizes { }; extern struct cache_sizes malloc_sizes[]; +void *kmem_cache_alloc(struct kmem_cache *, gfp_t); +void *__kmalloc(size_t size, gfp_t flags); + static inline void *kmalloc(size_t size, gfp_t flags) { if (__builtin_constant_p(size)) { @@ -79,6 +82,7 @@ found: #ifdef CONFIG_NUMA extern void *__kmalloc_node(size_t size, gfp_t flags, int node); +extern void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); static inline void *kmalloc_node(size_t size, gfp_t flags, int node) { Index: linux-2.6/include/linux/slob_def.h =================================================================== --- /dev/null 1970-01-01 00:00:00.000000000 +0000 +++ linux-2.6/include/linux/slob_def.h 2007-06-14 21:25:44.000000000 -0700 @@ -0,0 +1,83 @@ +#ifndef __LINUX_SLOB_DEF_H +#define __LINUX_SLOB_DEF_H + +void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); + +static inline void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags) +{ + return kmem_cache_alloc_node(cachep, flags, -1); +} + +void *__kmalloc_node(size_t size, gfp_t flags, int node); + +static inline void *kmalloc_node(size_t size, gfp_t flags, int node) +{ + return __kmalloc_node(size, flags, node); +} + +static inline void *__kmalloc(size_t size, gfp_t flags) +{ + return __kmalloc_node(size, flags, -1); +} + +/** + * kmalloc - allocate memory + * @size: how many bytes of memory are required. + * @flags: the type of memory to allocate. + * + * kmalloc is the normal method of allocating memory + * in the kernel. + * + * The @flags argument may be one of: + * + * %GFP_USER - Allocate memory on behalf of user. May sleep. + * + * %GFP_KERNEL - Allocate normal kernel ram. May sleep. + * + * %GFP_ATOMIC - Allocation will not sleep. + * For example, use this inside interrupt handlers. + * + * %GFP_HIGHUSER - Allocate pages from high memory. + * + * %GFP_NOIO - Do not do any I/O at all while trying to get memory. + * + * %GFP_NOFS - Do not make any fs calls while trying to get memory. + * + * Also it is possible to set different flags by OR'ing + * in one or more of the following additional @flags: + * + * %__GFP_COLD - Request cache-cold pages instead of + * trying to return cache-warm pages. + * + * %__GFP_DMA - Request memory from the DMA-capable zone. + * + * %__GFP_HIGH - This allocation has high priority and may use emergency pools. + * + * %__GFP_HIGHMEM - Allocated memory may be from highmem. + * + * %__GFP_NOFAIL - Indicate that this allocation is in no way allowed to fail + * (think twice before using). + * + * %__GFP_NORETRY - If memory is not immediately available, + * then give up at once. + * + * %__GFP_NOWARN - If allocation fails, don't issue any warnings. + * + * %__GFP_REPEAT - If allocation fails initially, try once more before failing. + */ +static inline void *kmalloc(size_t size, gfp_t flags) +{ + return __kmalloc(size, flags); +} + +/** + * kzalloc - allocate memory. The memory is set to zero. + * @size: how many bytes of memory are required. + * @flags: the type of memory to allocate (see kmalloc). + */ +static inline void *kzalloc(size_t size, gfp_t flags) +{ + return __kzalloc(size, flags); +} + +#endif /* __LINUX_SLOB_DEF_H */ Index: linux-2.6/include/linux/slub_def.h =================================================================== --- linux-2.6.orig/include/linux/slub_def.h 2007-06-11 09:24:30.000000000 -0700 +++ linux-2.6/include/linux/slub_def.h 2007-06-14 21:25:44.000000000 -0700 @@ -162,6 +162,9 @@ static inline struct kmem_cache *kmalloc #define ZERO_SIZE_PTR ((void *)16) +void *kmem_cache_alloc(struct kmem_cache *, gfp_t); +void *__kmalloc(size_t size, gfp_t flags); + static inline void *kmalloc(size_t size, gfp_t flags) { if (__builtin_constant_p(size) && !(flags & SLUB_DMA)) { @@ -189,7 +192,8 @@ static inline void *kzalloc(size_t size, } #ifdef CONFIG_NUMA -extern void *__kmalloc_node(size_t size, gfp_t flags, int node); +void *__kmalloc_node(size_t size, gfp_t flags, int node); +void *kmem_cache_alloc_node(struct kmem_cache *, gfp_t flags, int node); static inline void *kmalloc_node(size_t size, gfp_t flags, int node) { Index: linux-2.6/mm/slob.c =================================================================== --- linux-2.6.orig/mm/slob.c 2007-05-17 10:34:13.000000000 -0700 +++ linux-2.6/mm/slob.c 2007-06-14 21:25:44.000000000 -0700 @@ -54,6 +54,23 @@ struct bigblock { }; typedef struct bigblock bigblock_t; +static void *slob_new_page(gfp_t gfp, int order, int node) +{ + void *page; + +#ifdef CONFIG_NUMA + if (node != -1) + page = alloc_pages_node(node, gfp, order); + else +#endif + page = alloc_pages(gfp, order); + + if (!page) + return NULL; + + return page_address(page); +} + /* * struct slob_rcu is inserted at the tail of allocated slob blocks, which * were created with a SLAB_DESTROY_BY_RCU slab. slob_rcu is used to free @@ -190,7 +207,7 @@ void *__kmalloc(size_t size, gfp_t gfp) slob_free(bb, sizeof(bigblock_t)); return 0; } -EXPORT_SYMBOL(__kmalloc); +EXPORT_SYMBOL(__kmalloc_node); /** * krealloc - reallocate memory. The contents will remain unchanged. @@ -251,7 +268,6 @@ void kfree(const void *block) slob_free((slob_t *)block - 1, 0); return; } - EXPORT_SYMBOL(kfree); size_t ksize(const void *block) @@ -289,7 +305,7 @@ struct kmem_cache *kmem_cache_create(con { struct kmem_cache *c; - c = slob_alloc(sizeof(struct kmem_cache), flags, 0); + c = slob_alloc(sizeof(struct kmem_cache), flags, 0, -1); if (c) { c->name = name; @@ -317,21 +333,21 @@ void kmem_cache_destroy(struct kmem_cach } EXPORT_SYMBOL(kmem_cache_destroy); -void *kmem_cache_alloc(struct kmem_cache *c, gfp_t flags) +void *kmem_cache_alloc_node(struct kmem_cache *c, gfp_t flags, int node) { void *b; if (c->size < PAGE_SIZE) - b = slob_alloc(c->size, flags, c->align); + b = slob_alloc(c->size, flags, c->align, node); else - b = (void *)__get_free_pages(flags, get_order(c->size)); + b = slob_new_page(flags, get_order(c->size), node); if (c->ctor) c->ctor(b, c, 0); return b; } -EXPORT_SYMBOL(kmem_cache_alloc); +EXPORT_SYMBOL(kmem_cache_alloc_node); void *kmem_cache_zalloc(struct kmem_cache *c, gfp_t flags) {