---
 include/linux/slub_def.h |    6 +-
 mm/slub.c                |  129 ++++++++++++++++++++++++-----------------------
 2 files changed, 71 insertions(+), 64 deletions(-)

Index: slub/include/linux/slub_def.h
===================================================================
--- slub.orig/include/linux/slub_def.h	2007-05-13 14:45:15.000000000 -0700
+++ slub/include/linux/slub_def.h	2007-05-13 21:56:33.000000000 -0700
@@ -11,6 +11,10 @@
 #include <linux/workqueue.h>
 #include <linux/kobject.h>
 
+struct kmem_cache_cpu {
+	struct page *page;
+};
+
 struct kmem_cache_node {
 	spinlock_t list_lock;	/* Protect partial list and nr_partial */
 	unsigned long nr_partial;
@@ -51,7 +55,7 @@ struct kmem_cache {
 	int defrag_ratio;
 	struct kmem_cache_node *node[MAX_NUMNODES];
 #endif
-	struct page *cpu_slab[NR_CPUS];
+	struct kmem_cache_cpu cpu[NR_CPUS];
 };
 
 /*
Index: slub/mm/slub.c
===================================================================
--- slub.orig/mm/slub.c	2007-05-13 14:45:54.000000000 -0700
+++ slub/mm/slub.c	2007-05-13 21:56:33.000000000 -0700
@@ -274,6 +274,11 @@ static inline struct kmem_cache_node *ge
 #endif
 }
 
+static inline struct kmem_cache_cpu *get_kcpu(struct kmem_cache *s, int cpu)
+{
+	return &s->cpu[cpu];
+}
+
 static inline int check_valid_pointer(struct kmem_cache *s,
 				struct page *page, const void *object)
 {
@@ -1340,53 +1345,50 @@ static void unfreeze_slab(struct kmem_ca
 /*
  * Remove the cpu slab
  */
-static void deactivate_slab(struct kmem_cache *s, struct page *page, int cpu)
+static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
 {
 	/*
 	 * Merge cpu freelist into freelist. Typically we get here
 	 * because both freelists are empty. So this is unlikely
 	 * to occur.
 	 */
-	while (unlikely(page->lockless_freelist)) {
+	while (unlikely(c->page->lockless_freelist)) {
 		void **object;
 
 		/* Retrieve object from cpu_freelist */
-		object = page->lockless_freelist;
-		page->lockless_freelist = page->lockless_freelist[page->offset];
+		object = c->page->lockless_freelist;
+		c->page->lockless_freelist = c->page->lockless_freelist[c->page->offset];
 
 		/* And put onto the regular freelist */
-		object[page->offset] = page->freelist;
-		page->freelist = object;
-		page->inuse--;
+		object[c->page->offset] = c->page->freelist;
+		c->page->freelist = object;
+		c->page->inuse--;
 	}
-	s->cpu_slab[cpu] = NULL;
-	unfreeze_slab(s, page);
+	c->page = NULL;
+	unfreeze_slab(s, c->page);
 }
 
-static void flush_slab(struct kmem_cache *s, struct page *page, int cpu)
+static void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
 {
-	slab_lock(page);
-	deactivate_slab(s, page, cpu);
+	slab_lock(c->page);
+	deactivate_slab(s, c);
 }
 
 /*
  * Flush cpu slab.
  * Called from IPI handler with interrupts disabled.
  */
-static void __flush_cpu_slab(struct kmem_cache *s, int cpu)
+static void __flush_cpu_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
 {
-	struct page *page = s->cpu_slab[cpu];
-
-	if (likely(page))
-		flush_slab(s, page, cpu);
+	if (likely(c->page))
+		flush_slab(s, c);
 }
 
 static void flush_cpu_slab(void *d)
 {
 	struct kmem_cache *s = d;
-	int cpu = smp_processor_id();
 
-	__flush_cpu_slab(s, cpu);
+	__flush_cpu_slab(s, s->cpu + smp_processor_id());
 }
 
 static void flush_all(struct kmem_cache *s)
@@ -1420,45 +1422,45 @@ static void flush_all(struct kmem_cache 
  * we need to allocate a new slab. This is slowest path since we may sleep.
  */
 static void *__slab_alloc(struct kmem_cache *s,
-		gfp_t gfpflags, int node, void *addr, struct page *page)
+		gfp_t gfpflags, int node, void *addr, struct kmem_cache_cpu *c)
 {
 	void **object;
-	int cpu = smp_processor_id();
+	struct page *page;
 
-	if (!page)
+	if (!c->page)
 		goto new_slab;
 
-	slab_lock(page);
-	if (unlikely(node != -1 && page_to_nid(page) != node))
+	slab_lock(c->page);
+	if (unlikely(node != -1 && page_to_nid(c->page) != node))
 		goto another_slab;
 load_freelist:
-	object = page->freelist;
+	object = c->page->freelist;
 	if (unlikely(!object))
 		goto another_slab;
-	if (unlikely(SlabDebug(page)))
+	if (unlikely(SlabDebug(c->page)))
 		goto debug;
 
-	object = page->freelist;
-	page->lockless_freelist = object[page->offset];
-	page->inuse = s->objects;
-	page->freelist = NULL;
-	slab_unlock(page);
+	object = c->page->freelist;
+	c->page->lockless_freelist = object[c->page->offset];
+	c->page->inuse = s->objects;
+	c->page->freelist = NULL;
+	slab_unlock(c->page);
 	return object;
 
 another_slab:
-	deactivate_slab(s, page, cpu);
+	deactivate_slab(s, c);
 
 new_slab:
 	page = get_partial(s, gfpflags, node);
 	if (page) {
-		s->cpu_slab[cpu] = page;
+		c->page = page;
 		goto load_freelist;
 	}
 
 	page = new_slab(s, gfpflags, node);
 	if (page) {
-		cpu = smp_processor_id();
-		if (s->cpu_slab[cpu]) {
+		c = get_kcpu(s, smp_processor_id());
+		if (c->page) {
 			/*
 			 * Someone else populated the cpu_slab while we
 			 * enabled interrupts, or we have gotten scheduled
@@ -1467,33 +1469,32 @@ new_slab:
 			 * specified. So we need to recheck.
 			 */
 			if (node == -1 ||
-				page_to_nid(s->cpu_slab[cpu]) == node) {
+				page_to_nid(c->page) == node) {
 				/*
 				 * Current cpuslab is acceptable and we
 				 * want the current one since its cache hot
 				 */
-				discard_slab(s, page);
-				page = s->cpu_slab[cpu];
+				discard_slab(s, c->page);
 				slab_lock(page);
 				goto load_freelist;
 			}
 			/* New slab does not fit our expectations */
-			flush_slab(s, s->cpu_slab[cpu], cpu);
+			flush_slab(s, c);
 		}
 		slab_lock(page);
 		SetSlabFrozen(page);
-		s->cpu_slab[cpu] = page;
+		c->page = page;
 		goto load_freelist;
 	}
 	return NULL;
 debug:
-	object = page->freelist;
-	if (!alloc_object_checks(s, page, object, addr))
+	object = c->page->freelist;
+	if (!alloc_object_checks(s, c->page, object, addr))
 		goto another_slab;
 
-	page->inuse++;
-	page->freelist = object[page->offset];
-	slab_unlock(page);
+	c->page->inuse++;
+	c->page->freelist = object[c->page->offset];
+	slab_unlock(c->page);
 	return object;
 }
 
@@ -1510,20 +1511,20 @@ debug:
 static void __always_inline *slab_alloc(struct kmem_cache *s,
 				gfp_t gfpflags, int node, void *addr)
 {
-	struct page *page;
+	struct kmem_cache_cpu *c;
 	void **object;
 	unsigned long flags;
 
 	local_irq_save(flags);
-	page = s->cpu_slab[smp_processor_id()];
-	if (unlikely(!page || !page->lockless_freelist ||
-			(node != -1 && page_to_nid(page) != node)))
+	c = get_kcpu(s, smp_processor_id());
+	if (unlikely(!c->page || !c->page->lockless_freelist ||
+			(node != -1 && page_to_nid(c->page) != node)))
 
-		object = __slab_alloc(s, gfpflags, node, addr, page);
+		object = __slab_alloc(s, gfpflags, node, addr, c);
 
 	else {
-		object = page->lockless_freelist;
-		page->lockless_freelist = object[page->offset];
+		object = c->page->lockless_freelist;
+		c->page->lockless_freelist = object[c->page->offset];
 	}
 	local_irq_restore(flags);
 	return object;
@@ -1617,10 +1618,11 @@ static void __always_inline slab_free(st
 {
 	void **object = (void *)x;
 	unsigned long flags;
+	struct kmem_cache_cpu *c;
 
 	local_irq_save(flags);
-	if (likely(page == s->cpu_slab[smp_processor_id()] &&
-						!SlabDebug(page))) {
+	c = get_kcpu(s, smp_processor_id());
+	if (likely(page == c->page && !SlabDebug(page))) {
 		object[page->offset] = page->lockless_freelist;
 		page->lockless_freelist = object;
 	} else
@@ -2663,16 +2665,16 @@ long kmem_cache_count(struct kmem_cache 
 
 
 	for_each_possible_cpu(cpu) {
-		struct page *page = s->cpu_slab[cpu];
+		struct kmem_cache_cpu *c = get_kcpu(s, cpu);
 		int node;
 
-		if (page) {
-			node = page_to_nid(page);
+		if (c->page) {
+			node = page_to_nid(c->page);
 			if (flags & KMEM_CACHE_PER_CPU) {
 				int x = 0;
 
 				if (flags & KMEM_CACHE_OBJECTS)
-					x = page->inuse;
+					x = c->page->inuse;
 				else
 					x = 1;
 				total += x;
@@ -2823,8 +2825,8 @@ void __init kmem_cache_init(void)
 	register_cpu_notifier(&slab_notifier);
 #endif
 
-	kmem_size = offsetof(struct kmem_cache, cpu_slab) +
-				nr_cpu_ids * sizeof(struct page *);
+	kmem_size = offsetof(struct kmem_cache, cpu) +
+				nr_cpu_ids * sizeof(struct kmem_cache_cpu);
 
 	printk(KERN_INFO "SLUB: Genslabs=%d, HWalign=%d, Order=%d-%d, MinObjects=%d,"
 		" Processors=%d, Nodes=%d\n",
@@ -2957,7 +2959,8 @@ void *kmem_cache_zalloc(struct kmem_cach
 EXPORT_SYMBOL(kmem_cache_zalloc);
 
 #ifdef CONFIG_SMP
-static void for_all_slabs(void (*func)(struct kmem_cache *, int), int cpu)
+static void for_all_slabs(void (*func)(struct kmem_cache *,
+				struct kmem_cache_cpu *), int cpu)
 {
 	struct list_head *h;
 
@@ -2966,7 +2969,7 @@ static void for_all_slabs(void (*func)(s
 		struct kmem_cache *s =
 			container_of(h, struct kmem_cache, list);
 
-		func(s, cpu);
+		func(s, get_kcpu(s, cpu));
 	}
 	up_read(&slub_lock);
 }
@@ -3437,7 +3440,7 @@ static int any_slab_objects(struct kmem_
 	int cpu;
 
 	for_each_possible_cpu(cpu)
-		if (s->cpu_slab[cpu])
+		if (get_kcpu(s, cpu)->page)
 			return 1;
 
 	for_each_node(node) {