---
 mm/slub.c |   86 +++++++++++++++++++++++++++++++++++++++++++++-----------------
 1 file changed, 63 insertions(+), 23 deletions(-)

Index: linux-2.6.22-rc6-mm1/mm/slub.c
===================================================================
--- linux-2.6.22-rc6-mm1.orig/mm/slub.c	2007-07-04 00:25:08.000000000 -0700
+++ linux-2.6.22-rc6-mm1/mm/slub.c	2007-07-04 00:36:37.000000000 -0700
@@ -143,6 +143,20 @@ static inline void ClearSlabDebug(struct
  */
 #define CPU_FREELIST_OFF (void **)(16)
 
+static inline int freelist_off(void **x)
+{
+	return x == CPU_FREELIST_OFF;
+}
+
+static inline int freelist_off_or_empty(void *x)
+{
+	return (unsigned long)x <= (unsigned long)CPU_FREELIST_OFF;
+}
+
+static inline void **freelist_get_and_clear(void ***x)
+{
+	return xchg(x, CPU_FREELIST_OFF);
+}
 /*
  * Issues still to be resolved:
  *
@@ -1372,34 +1386,39 @@ static void unfreeze_slab(struct kmem_ca
 /*
  * Remove the cpu slab
  */
-static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
+static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c,
+			void **freelist)
 {
 	struct page *page = c->page;
+
+	c->page = NULL;
+
+	VM_BUG_ON(freelist_off(freelist));
+
+	slab_lock(page);
 	/*
 	 * Merge cpu freelist into freelist. Typically we get here
 	 * because both freelists are empty. So this is unlikely
 	 * to occur.
 	 */
-	while (unlikely(c->freelist)) {
+	while (unlikely(freelist)) {
 		void **object;
 
 		/* Retrieve object from cpu_freelist */
-		object = c->freelist;
-		c->freelist = c->freelist[c->offset];
+		object = freelist;
+		freelist = freelist[c->offset];
 
 		/* And put onto the regular freelist */
 		object[c->offset] = page->freelist;
 		page->freelist = object;
 		page->inuse--;
 	}
-	c->page = NULL;
 	unfreeze_slab(s, page);
 }
 
 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
 {
-	slab_lock(c->page);
-	deactivate_slab(s, c);
+	deactivate_slab(s, c, freelist_get_and_clear(&c->freelist));
 }
 
 /*
@@ -1465,17 +1484,29 @@ static inline int node_match(struct kmem
  * we need to allocate a new slab. This is slowest path since we may sleep.
  */
 static void *__slab_alloc(struct kmem_cache *s,
-		gfp_t gfpflags, int node, void *addr, struct kmem_cache_cpu *c)
+		gfp_t gfpflags, int node, void *addr)
 {
 	void **object;
 	struct page *new;
+	struct kmem_cache_cpu *c;
+	void **cpu_freelist = NULL;
+	unsigned long flags;
+
+	local_irq_save(flags);
+
+	c = get_cpu_slab(s, smp_processor_id());
 
 	if (!c->page)
+		/* Slab was flushed */
 		goto new_slab;
 
-	slab_lock(c->page);
+	cpu_freelist = freelist_get_and_clear(&c->freelist);
+	if (unlikely(freelist_off_or_empty(cpu_freelist)))
+		goto another_slab;
+
 	if (unlikely(!node_match(c, node)))
 		goto another_slab;
+	slab_lock(c->page);
 load_freelist:
 	object = c->page->freelist;
 	if (unlikely(!object))
@@ -1489,10 +1520,11 @@ load_freelist:
 	c->page->freelist = NULL;
 	c->node = page_to_nid(c->page);
 	slab_unlock(c->page);
+	local_irq_restore(flags);
 	return object;
 
 another_slab:
-	deactivate_slab(s, c);
+	deactivate_slab(s, c, cpu_freelist);
 
 new_slab:
 	new = get_partial(s, gfpflags, node);
@@ -1529,9 +1561,9 @@ new_slab:
 		c->page = new;
 		goto load_freelist;
 	}
+	local_irq_restore(flags);
 	return NULL;
 debug:
-	c->freelist = CPU_FREELIST_OFF;
 	object = c->page->freelist;
 	if (!alloc_debug_processing(s, c->page, object, addr))
 		goto another_slab;
@@ -1539,6 +1571,7 @@ debug:
 	c->page->inuse++;
 	c->page->freelist = object[c->offset];
 	slab_unlock(c->page);
+	local_irq_restore(flags);
 	return object;
 }
 
@@ -1556,26 +1589,33 @@ static void __always_inline *slab_alloc(
 		gfp_t gfpflags, int node, void *addr, int length)
 {
 	void **object;
-	unsigned long flags;
 	struct kmem_cache_cpu *c;
 
-	local_irq_save(flags);
-	c = get_cpu_slab(s, smp_processor_id());
-	if (unlikely(!c->page || !c->freelist ||
-					!node_match(c, node)))
+redo:
+	c = get_cpu_slab(s, raw_smp_processor_id());
+	object = c->freelist;
 
-		object = __slab_alloc(s, gfpflags, node, addr, c);
+	if (unlikely(freelist_off_or_empty(object)))
+		goto slow;
 
-	else {
-		object = c->freelist;
-		c->freelist = object[c->offset];
-	}
-	local_irq_restore(flags);
+	if (unlikely(!node_match(c, node)))
+		goto slow;
+
+	if (cmpxchg(&c->freelist, object,
+				object[c->offset]) != object)
+		goto redo;
 
-	if (unlikely((gfpflags & __GFP_ZERO) && object))
+zerotest:
+	if (unlikely((gfpflags & __GFP_ZERO)))
 		memset(object, 0, length);
 
 	return object;
+
+slow:
+	object = __slab_alloc(s, gfpflags, node, addr);
+	if (object)
+		goto zerotest;
+	return object;
 }
 
 void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)