SLUB: prefetch object cachelines

SLUB uses linked list to build chains of objects. This is a classical
case in which cacheline prefetches are useful.

Prefetch the cacheline of the next object at the end of allocations and
prefetch the cacheline of object to free at the start of the free code.

Signed-off-by: Christoph Lameter <clameter@sgi.com>

---
 mm/slub.c |    4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c	2008-02-08 06:59:26.000000000 -0800
+++ linux-2.6/mm/slub.c	2008-02-08 07:11:23.000000000 -0800
@@ -1538,7 +1538,6 @@ load_freelist:
 	if (unlikely(SlabDebug(c->page)))
 		goto debug;
 
-	object = c->page->freelist;
 	c->freelist = object[c->offset];
 	c->page->inuse = s->objects;
 	c->page->freelist = c->page->end;
@@ -1656,6 +1655,7 @@ static __always_inline void *slab_alloc(
 	if (unlikely((gfpflags & __GFP_ZERO) && object))
 		memset(object, 0, c->objsize);
 
+	prefetchw(c->freelist[c->offset]);
 	return object;
 }
 
@@ -1771,6 +1771,7 @@ static __always_inline void slab_free(st
 #ifdef SLUB_FASTPATH
 	void **freelist;
 
+	prefetchw(object);	/* Assumes free pointer in first cacheline */
 	c = get_cpu_slab(s, raw_smp_processor_id());
 	debug_check_no_locks_freed(object, s->objsize);
 	do {
@@ -1797,6 +1798,7 @@ static __always_inline void slab_free(st
 #else
 	unsigned long flags;
 
+	prefetchw(object);	/* Assumes freepointer in first cacheline */
 	local_irq_save(flags);
 	debug_check_no_locks_freed(object, s->objsize);
 	c = get_cpu_slab(s, smp_processor_id());