---
 include/linux/slub_def.h |   13 ++++++++-
 mm/slub.c                |   63 ++++++++++++++++++++++++++---------------------
 2 files changed, 48 insertions(+), 28 deletions(-)

Index: slub/include/linux/slub_def.h
===================================================================
--- slub.orig/include/linux/slub_def.h	2007-05-04 17:22:15.000000000 -0700
+++ slub/include/linux/slub_def.h	2007-05-04 17:25:16.000000000 -0700
@@ -19,6 +19,13 @@ struct kmem_cache_node {
 	struct list_head full;
 };
 
+struct kmem_cach_cpu {
+	struct page *page;
+	void **freelist;
+	void **fixuppointer;
+	int remaining;
+}
+
 /*
  * Slab cache management.
  */
@@ -50,7 +57,11 @@ struct kmem_cache {
 	int defrag_ratio;
 	struct kmem_cache_node *node[MAX_NUMNODES];
 #endif
-	struct page *cpu_slab[NR_CPUS];
+	/*
+	 * Temporary definition. In the NUMA case this should really be on the node
+	 * local to the processor.
+	 */
+	struct kmem_cache_cpu cpu_slab[NR_CPUS];
 };
 
 /*
Index: slub/mm/slub.c
===================================================================
--- slub.orig/mm/slub.c	2007-05-04 17:23:29.000000000 -0700
+++ slub/mm/slub.c	2007-05-04 17:39:21.000000000 -0700
@@ -1135,9 +1135,15 @@ static void putback_slab(struct kmem_cac
 /*
  * Remove the cpu slab
  */
-static void deactivate_slab(struct kmem_cache *s, struct page *page, int cpu)
+static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c, int cpu)
 {
-	s->cpu_slab[cpu] = NULL;
+	struct page = c->page;
+
+	c->page = NULL;
+	if (c->freelist) {
+		*fixuploc = c->freelist;
+		page->inuse += c->remaining;
+	}
 	ClearPageActive(page);
 
 	putback_slab(s, page);
@@ -1201,65 +1207,68 @@ static void *slab_alloc(struct kmem_cach
 	struct page *page;
 	void **object;
 	unsigned long flags;
-	int cpu;
+	struct kmem_cache_cpu *c;
 
 	local_irq_save(flags);
-	cpu = smp_processor_id();
-	page = s->cpu_slab[cpu];
+	c = s->cpuslabs + smp_processor_id();
 	if (!page)
 		goto new_slab;
 
-	slab_lock(page);
-	if (unlikely(node != -1 && page_to_nid(page) != node))
+	if (likely(c->freelist)) {
+fast_object:
+		object = c->freelist;
+		c->freelist = c->freelist[c->page->offset];
+		local_irq_restore(flags);
+		return;
+	}
+
+	/* Replenish free pages from slab */
+	slab_lock(c->page);
+	if (unlikely(node != -1 && page_to_nid(c->page) != node))
 		goto another_slab;
 redo:
-	object = page->freelist;
-	if (unlikely(!object))
-		goto another_slab;
-	if (unlikely(PageError(page)))
-		goto debug;
+	if (!c->page->freelist)
+		goto another slab;
 
 have_object:
-	page->inuse++;
-	page->freelist = object[page->offset];
-	slab_unlock(page);
-	local_irq_restore(flags);
-	return object;
+	c->freelist = c->page->freelist;
+	c->page->inuse = s->objects;
+	slab_unlock(c->page);
+	goto fastobject;
 
 another_slab:
-	deactivate_slab(s, page, cpu);
+	deactivate_slab(s, page, c);
 
 new_slab:
 	page = get_partial(s, gfpflags, node);
 	if (likely(page)) {
 have_slab:
-		s->cpu_slab[cpu] = page;
+		c->page = page;
 		SetPageActive(page);
 		goto redo;
 	}
 
 	page = new_slab(s, gfpflags, node);
 	if (page) {
-		cpu = smp_processor_id();
-		if (s->cpu_slab[cpu]) {
+		c = s->cpu_slab + smp_processor_id();
+		if (c->page) {
 			/*
 			 * Someone else populated the cpu_slab while we enabled
 			 * interrupts, or we have got scheduled on another cpu.
 			 * The page may not be on the requested node.
 			 */
 			if (node == -1 ||
-				page_to_nid(s->cpu_slab[cpu]) == node) {
+				page_to_nid(c->page) == node) {
 				/*
 				 * Current cpuslab is acceptable and we
 				 * want the current one since its cache hot
 				 */
 				discard_slab(s, page);
-				page = s->cpu_slab[cpu];
-				slab_lock(page);
+				slab_lock(c->page);
 				goto redo;
 			}
 			/* Dump the current slab */
-			flush_slab(s, s->cpu_slab[cpu], cpu);
+			flush_slab(s, c);
 		}
 		slab_lock(page);
 		goto have_slab;
@@ -2999,7 +3008,7 @@ static unsigned long slab_objects(struct
 	per_cpu = nodes + nr_node_ids;
 
 	for_each_possible_cpu(cpu) {
-		struct page *page = s->cpu_slab[cpu];
+		struct page *page = s->cpu_slab[cpu].page;
 		int node;
 
 		if (page) {
@@ -3061,7 +3070,7 @@ static int any_slab_objects(struct kmem_
 	int cpu;
 
 	for_each_possible_cpu(cpu)
-		if (s->cpu_slab[cpu])
+		if (s->cpu_slab[cpu]->page)
 			return 1;
 
 	for_each_node(node) {