---
 mm/slub.c |   78 +++++++++++++++++++++++++++++---------------------------------
 1 file changed, 37 insertions(+), 41 deletions(-)

Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c	2007-10-18 13:34:18.000000000 -0700
+++ linux-2.6/mm/slub.c	2007-10-18 13:48:13.000000000 -0700
@@ -1515,7 +1515,8 @@ static inline void **get_freelist(struct
 }
 
 /* Free an object to the por cpu structure and return the old freelist pointer */
-static inline void **free_cpu_object(struct kmem_cache_cpu *c, void **object)
+static inline void **free_cpu_object(struct kmem_cache_cpu *c,
+				void **object, struct page *page)
 {
 	void **prior;
 
@@ -1523,31 +1524,55 @@ static inline void **free_cpu_object(str
 	do {
 		prior = c->freelist;
 		object[c->offset] = prior;
+		/*
+		 * Potential issue with prior == NULL which may match
+		 * multiple slabs?
+		 */
+		smp_rmb();
+		if (c->page != page)
+			return NULL;
 	} while (cmpxchg_local(&c->freelist, prior, object) != prior);
 
 #else
 	unsigned long flags;
 
 	local_irq_save(flags);
-	prior = c->freelist;
-	object[c->offset] = prior;
-	c->freelist = object;
+	if (c->page == page) {
+		prior = c->freelist;
+		object[c->offset] = prior;
+		c->freelist = object;
+	} else
+		prior = NULL;
 	local_irq_restore(flags);
 #endif
 	return prior;
 }
 
 /*
+ * Check if the objects in a per cpu structure fit numa
+ * locality expectations.
+ */
+static inline int node_match(struct kmem_cache_cpu *c, int node)
+{
+#ifdef CONFIG_NUMA
+	if (node != -1 && c->node != node)
+		return 0;
+#endif
+	return 1;
+}
+
+/*
  * Allocate object from the per cpu structure.
  */
-static inline void **alloc_cpu_object(struct kmem_cache_cpu *c)
+static inline void **alloc_cpu_object(struct kmem_cache_cpu *c, int node)
 {
 	void **object;
 
 #ifdef SLUB_ATOMIC
 	do {
 		object = c->freelist;
-		if (!object)
+		smp_rmb();
+		if (!object || !node_match(c, node))
 			return NULL;
 	} while (cmpxchg_local(&c->freelist, object, object[c->offset]) != object);
 #else
@@ -1555,8 +1580,10 @@ static inline void **alloc_cpu_object(st
 
 	local_irq_save(flags);
 	object = c->freelist;
-	if (object)
+	if (object && node_match(c, node))
 		c->freelist = object[c->offset];
+	else
+		object = NULL;
 	local_irq_restore(flags);
 #endif
 	return object;
@@ -1627,19 +1654,6 @@ static void flush_all(struct kmem_cache 
 }
 
 /*
- * Check if the objects in a per cpu structure fit numa
- * locality expectations.
- */
-static inline int node_match(struct kmem_cache_cpu *c, int node)
-{
-#ifdef CONFIG_NUMA
-	if (node != -1 && c->node != node)
-		return 0;
-#endif
-	return 1;
-}
-
-/*
  * Slow path. The lockless freelist is empty or we need to perform
  * debugging duties.
  *
@@ -1760,14 +1774,7 @@ static void __always_inline *slab_alloc(
 	struct kmem_cache_cpu *c;
 
 	c = get_cpu_slab(s, get_cpu());
-	/*
-	 * Node may change after this test. That is only relevant to
-	 * kmalloc_node though
-	 */
-	if (unlikely(!node_match(c, node)))
-		goto slow;
-
-	object = alloc_cpu_object(c);
+	object = alloc_cpu_object(c, node);
 	if (unlikely(!object))
 		goto slow;
 back:
@@ -1815,25 +1822,20 @@ static void __always_inline slab_free(st
 {
 	void **object = (void *)x;
 	struct kmem_cache_cpu *c;
+	void **prior;
 
 	c = get_cpu_slab(s, get_cpu());
 	debug_check_no_locks_freed(object, s->objsize);
 	if (likely(c->node >= 0)) {
-		if (likely(page == c->page))
-			/* FIXME: May change if interrupt alloc exhausts cpuslab! */
-			free_cpu_object(c, object);
-		else
-			goto offcpuslab;
-	} else {
-		void **prior;
-
+		if (likely(free_cpu_object(c, object, page)))
+			goto out;
+	} else
 		if (!free_debug_processing(s, page, x, addr))
 			goto out;
-offcpuslab:
-		prior = free_object(page, object, c->offset);
-		if (unlikely(!prior))
-			add_partial(s, page, 0);
-	}
+
+	prior = free_object(page, object, c->offset);
+	if (unlikely(!prior))
+		add_partial(s, page, 0);
 out:
 	put_cpu();
 }