SLUB mm-only: Right align kmalloc objects to trigger overwrite detection Right align kmalloc objects if they are less than the full kmalloc slab size. This will move the object to be flush with the end of the object in order to allow the easy detection of writes / reads after the end of the kmalloc object. Without slub_debug overwrites will destroy the free pointer of the next object or the next object. Read will yield garbage that is likely zero. With slub_debug redzone checks will be triggered. Reads will read redzone poison. This patch is only for checking things out. There are issues: 1. Alignment of kmalloc objects may now be different. In particular objects whose size is not a multiple of wordsize may be not word alignmed. 2. __kmalloc and kfree need to touch an additional cacheline in struct kmem_cache thereby reducing performance. 3. An object allocated via kmalloc may no longer be freed via kmem_cache_free. So we need to figure out some may to make this configurable. Preferably runtime configurable. Signed-off-by: Christoph Lameter --- include/linux/slub_def.h | 31 +++++++++++++++++++++++++++---- mm/slub.c | 32 ++++++++++++++++++++++++++------ 2 files changed, 53 insertions(+), 10 deletions(-) Index: slub/include/linux/slub_def.h =================================================================== --- slub.orig/include/linux/slub_def.h 2007-06-02 10:59:16.000000000 -0700 +++ slub/include/linux/slub_def.h 2007-06-02 13:38:53.000000000 -0700 @@ -123,6 +123,29 @@ static inline struct kmem_cache *kmalloc return &kmalloc_caches[index]; } +static inline unsigned long kmalloc_align(unsigned long size) +{ + return ALIGN(size, sizeof(void *)); +} +/* + * Determine offset to right align an object in a power of two slab cache. + */ +static inline unsigned long kmalloc_offset(size_t size) +{ + int index = kmalloc_index(size); + unsigned long p2size; + + if (index >= KMALLOC_SHIFT_LOW) + p2size = 1 << index; + else if (index == 2) + p2size = 192; + else + p2size = 96; + + return p2size - kmalloc_align(size); +} + + #ifdef CONFIG_ZONE_DMA #define SLUB_DMA __GFP_DMA #else @@ -130,7 +153,6 @@ static inline struct kmem_cache *kmalloc #define SLUB_DMA 0 #endif - /* * ZERO_SIZE_PTR will be returned for zero sized kmalloc requests. * @@ -150,7 +172,7 @@ static inline void *kmalloc(size_t size, if (!s) return ZERO_SIZE_PTR; - return kmem_cache_alloc(s, flags); + return kmem_cache_alloc(s, flags) + kmalloc_offset(size); } else return __kmalloc(size, flags); } @@ -163,7 +185,7 @@ static inline void *kzalloc(size_t size, if (!s) return ZERO_SIZE_PTR; - return kmem_cache_zalloc(s, flags); + return kmem_cache_zalloc(s, flags) + kmalloc_offset(size); } else return __kzalloc(size, flags); } @@ -179,7 +201,8 @@ static inline void *kmalloc_node(size_t if (!s) return ZERO_SIZE_PTR; - return kmem_cache_alloc_node(s, flags, node); + return kmem_cache_alloc_node(s, flags, node) + + kmalloc_offset(size); } else return __kmalloc_node(size, flags, node); } Index: slub/mm/slub.c =================================================================== --- slub.orig/mm/slub.c 2007-06-02 12:41:23.000000000 -0700 +++ slub/mm/slub.c 2007-06-02 13:38:53.000000000 -0700 @@ -662,7 +662,13 @@ static int slab_pad_check(struct kmem_ca end--; slab_err(s, page, "Padding overwritten. 0x%p-0x%p", fault, end - 1); - print_section("Padding", start, length); + if (length < 128) + print_section("Padding", start, length); + else { + print_section("Padding", start, 64); + printk(KERN_ERR " .....\n"); + print_section("Padding", start + length - 64, 64); + } restore_bytes(s, "slab padding", POISON_INUSE, start, end); return 0; @@ -1670,7 +1676,9 @@ void kmem_cache_free(struct kmem_cache * { struct page *page; + BUG_ON(!x); page = virt_to_head_page(x); + BUG_ON(!page); slab_free(s, page, x, __builtin_return_address(0)); } @@ -2308,7 +2316,8 @@ void *__kmalloc(size_t size, gfp_t flags struct kmem_cache *s = get_slab(size, flags); if (s) - return slab_alloc(s, flags, -1, __builtin_return_address(0)); + return slab_alloc(s, flags, -1, __builtin_return_address(0)) + + (s->size - kmalloc_align(size)); return ZERO_SIZE_PTR; } EXPORT_SYMBOL(__kmalloc); @@ -2319,7 +2328,8 @@ void *__kmalloc_node(size_t size, gfp_t struct kmem_cache *s = get_slab(size, flags); if (s) - return slab_alloc(s, flags, node, __builtin_return_address(0)); + return slab_alloc(s, flags, node, __builtin_return_address(0)) + + (s->size - kmalloc_align(size)); return ZERO_SIZE_PTR; } EXPORT_SYMBOL(__kmalloc_node); @@ -2334,6 +2344,9 @@ size_t ksize(const void *object) s = page->slab; BUG_ON(!s); + if (object == ZERO_SIZE_PTR) + return 0; + /* * Debugging requires use of the padding between object * and whatever may come after it. @@ -2360,6 +2373,8 @@ void kfree(const void *x) { struct kmem_cache *s; struct page *page; + unsigned long page_offset; + unsigned long item_offset; /* * This has to be an unsigned comparison. According to Linus @@ -2373,7 +2388,10 @@ void kfree(const void *x) page = virt_to_head_page(x); s = page->slab; - slab_free(s, page, (void *)x, __builtin_return_address(0)); + page_offset = (unsigned long)x & ((PAGE_SIZE << s->order) - 1); + item_offset = page_offset % s->size; + + slab_free(s, page, (void *)x - item_offset, __builtin_return_address(0)); } EXPORT_SYMBOL(kfree); @@ -2738,7 +2756,8 @@ void *__kmalloc_track_caller(size_t size if (!s) return ZERO_SIZE_PTR; - return slab_alloc(s, gfpflags, -1, caller); + return slab_alloc(s, gfpflags, -1, caller) + + (s->size - kmalloc_align(size)); } void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags, @@ -2749,7 +2768,8 @@ void *__kmalloc_node_track_caller(size_t if (!s) return ZERO_SIZE_PTR; - return slab_alloc(s, gfpflags, node, caller); + return slab_alloc(s, gfpflags, node, caller) + + (s->size - kmalloc_align(size)); } #if defined(CONFIG_SYSFS) && defined(CONFIG_SLUB_DEBUG)