From b1c009a7f23830807dc6cad9aba41c5b8f906c31 Mon Sep 17 00:00:00 2001 From: Christoph Lameter Date: Mon, 7 Jan 2008 23:20:30 -0800 Subject: [PATCH] SLUB: Alternate fast paths using cmpxchg_local Provide an alternate implementation of the SLUB fast paths for alloc and free using cmpxchg_local. The cmpxchg_local fast path is selected for arches that have CONFIG_FAST_CMPXCHG_LOCAL set. An arch should only set CONFIG_FAST_CMPXCHG_LOCAL if the cmpxchg_local is faster than an interrupt enable/disable sequence. This is known to be true for both x86 platforms so set FAST_CMPXCHG_LOCAL for both arches. Not all arches can support fast cmpxchg operations. Typically the architecture must have an optimized cmpxchg instruction. The cmpxchg fast path makes no sense on platforms whose cmpxchg is slower than interrupt enable/disable (like f.e. IA64). The advantages of a cmpxchg_local based fast path are: 1. Lower cycle count (30%-60% faster) 2. There is no need to disable and enable interrupts on the fast path. Currently interrupts have to be disabled and enabled on every slab operation. This is likely saving a significant percentage of interrupt off / on sequences in the kernel. 3. The disposal of freed slabs can occur with interrupts enabled. The alternate path is realized using #ifdef's. Several attempts to do the same with macros and in line functions resulted in a mess (in particular due to the strange way that local_interrupt_save() handles its argument and due to the need to define macros/functions that sometimes disable interrupts and sometimes do something else. The macro based approaches made it also #ifdef seems to be the way to go here to have a readable source. Signed-off-by: Christoph Lameter Reviewed-by: Pekka Enberg Cc: Signed-off-by: Andrew Morton --- arch/x86/Kconfig | 4 ++ mm/slub.c | 79 +++++++++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 78 insertions(+), 5 deletions(-) Index: linux-2.6/arch/x86/Kconfig =================================================================== --- linux-2.6.orig/arch/x86/Kconfig 2008-01-19 10:09:07.545926228 -0800 +++ linux-2.6/arch/x86/Kconfig 2008-01-19 19:14:09.716401586 -0800 @@ -53,6 +53,10 @@ config SEMAPHORE_SLEEPERS bool default y +config FAST_CMPXCHG_LOCAL + bool + default y + config MMU bool default y Index: linux-2.6/mm/slub.c =================================================================== --- linux-2.6.orig/mm/slub.c 2008-01-19 18:56:37.525781523 -0800 +++ linux-2.6/mm/slub.c 2008-01-19 19:20:35.982921158 -0800 @@ -1493,7 +1493,12 @@ static void *__slab_alloc(struct kmem_ca { void **object; struct page *new; +#ifdef CONFIG_FAST_CMPXCHG_LOCAL + unsigned long flags; + local_irq_save(flags); + preempt_enable_no_resched(); +#endif if (!c->page) goto new_slab; @@ -1512,7 +1517,13 @@ load_freelist: c->page->inuse = s->objects; c->page->freelist = c->page->end; c->node = page_to_nid(c->page); +unlock_out: slab_unlock(c->page); +out: +#ifdef CONFIG_FAST_CMPXCHG_LOCAL + preempt_disable(); + local_irq_restore(flags); +#endif return object; another_slab: @@ -1542,7 +1553,8 @@ new_slab: c->page = new; goto load_freelist; } - return NULL; + object = NULL; + goto out; debug: object = c->page->freelist; if (!alloc_debug_processing(s, c->page, object, addr)) @@ -1551,8 +1563,7 @@ debug: c->page->inuse++; c->page->freelist = object[c->offset]; c->node = -1; - slab_unlock(c->page); - return object; + goto unlock_out; } /* @@ -1569,9 +1580,26 @@ static __always_inline void *slab_alloc( gfp_t gfpflags, int node, void *addr) { void **object; - unsigned long flags; struct kmem_cache_cpu *c; +#ifdef CONFIG_FAST_CMPXCHG_LOCAL + c = get_cpu_slab(s, get_cpu()); + do { + object = c->freelist; + if (unlikely(is_end(object) || !node_match(c, node))) { + object = __slab_alloc(s, gfpflags, node, addr, c); + if (unlikely(!object)) { + put_cpu(); + return NULL; + } + break; + } + } while (cmpxchg_local(&c->freelist, object, object[c->offset]) + != object); + put_cpu(); +#else + unsigned long flags; + local_irq_save(flags); c = get_cpu_slab(s, smp_processor_id()); if (unlikely(is_end(c->freelist) || !node_match(c, node))) @@ -1583,6 +1611,7 @@ static __always_inline void *slab_alloc( c->freelist = object[c->offset]; } local_irq_restore(flags); +#endif if (unlikely((gfpflags & __GFP_ZERO) && object)) memset(object, 0, c->objsize); @@ -1618,6 +1647,11 @@ static void __slab_free(struct kmem_cach void *prior; void **object = (void *)x; +#ifdef CONFIG_FAST_CMPXCHG_LOCAL + unsigned long flags; + + local_irq_save(flags); +#endif slab_lock(page); if (unlikely(SlabDebug(page))) @@ -1643,6 +1677,9 @@ checks_ok: out_unlock: slab_unlock(page); +#ifdef CONFIG_FAST_CMPXCHG_LOCAL + local_irq_restore(flags); +#endif return; slab_empty: @@ -1653,6 +1690,9 @@ slab_empty: remove_partial(s, page); slab_unlock(page); +#ifdef CONFIG_FAST_CMPXCHG_LOCAL + local_irq_restore(flags); +#endif discard_slab(s, page); return; @@ -1677,9 +1717,37 @@ static __always_inline void slab_free(st struct page *page, void *x, void *addr) { void **object = (void *)x; - unsigned long flags; struct kmem_cache_cpu *c; +#ifdef CONFIG_FAST_CMPXCHG_LOCAL + void **freelist; + + c = get_cpu_slab(s, get_cpu()); + debug_check_no_locks_freed(object, s->objsize); + do { + freelist = c->freelist; + barrier(); + /* + * If the compiler would reorder the retrieval of c->page to + * come before c->freelist then an interrupt could + * change the cpu slab before we retrieve c->freelist. We + * could be matching on a page no longer active and put the + * object onto the freelist of the wrong slab. + * + * On the other hand: If we already have the freelist pointer + * then any change of cpu_slab will cause the cmpxchg to fail + * since the freelist pointers are unique per slab. + */ + if (unlikely(page != c->page || c->node < 0)) { + __slab_free(s, page, x, addr, c->offset); + break; + } + object[c->offset] = freelist; + } while (cmpxchg_local(&c->freelist, freelist, object) != freelist); + put_cpu(); +#else + unsigned long flags; + local_irq_save(flags); debug_check_no_locks_freed(object, s->objsize); c = get_cpu_slab(s, smp_processor_id()); @@ -1690,6 +1758,7 @@ static __always_inline void slab_free(st __slab_free(s, page, x, addr, c->offset); local_irq_restore(flags); +#endif } void kmem_cache_free(struct kmem_cache *s, void *x)