SLUB: prefetch object cachelines

SLUB uses linked list to build chains of objects. This is a classical
case in which cacheline prefetches are useful.

Prefetch the cacheline of the next object at the end of allocations and
prefetch the cacheline of object to free at the start of the free code.

a tbench showed a slight regression with this patch:

with  2228.37 MB/sec 8 procs
without 2239.7 MB/sec 8 procs

Signed-off-by: Christoph Lameter <clameter@sgi.com>

---
 mm/slub.c |    5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c	2008-02-09 10:57:02.990170061 -0800
+++ linux-2.6/mm/slub.c	2008-02-09 10:57:09.722205757 -0800
@@ -1143,6 +1143,7 @@ static struct page *new_slab(struct kmem
 
 	last = start;
 	for_each_object(p, s, start) {
+		prefetchw(p + s->size);
 		setup_object(s, page, last);
 		set_freepointer(s, last, p);
 		last = p;
@@ -1538,7 +1539,6 @@ load_freelist:
 	if (unlikely(SlabDebug(c->page)))
 		goto debug;
 
-	object = c->page->freelist;
 	c->freelist = object[c->offset];
 	c->page->inuse = s->objects;
 	c->page->freelist = c->page->end;
@@ -1656,6 +1656,7 @@ static __always_inline void *slab_alloc(
 	if (unlikely((gfpflags & __GFP_ZERO) && object))
 		memset(object, 0, c->objsize);
 
+	prefetchw(c->freelist);
 	return object;
 }
 
@@ -1815,6 +1816,7 @@ void kmem_cache_free(struct kmem_cache *
 {
 	struct page *page;
 
+	prefetchw(x);
 	page = virt_to_head_page(x);
 
 	slab_free(s, page, x, __builtin_return_address(0));
@@ -2758,6 +2760,7 @@ void kfree(const void *x)
 	if (unlikely(ZERO_OR_NULL_PTR(x)))
 		return;
 
+	prefetchw(x);
 	page = virt_to_head_page(x);
 	if (unlikely(!PageSlab(page))) {
 		put_page(page);