Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c	2007-07-02 09:53:42.000000000 -0700
+++ linux-2.6/mm/slub.c	2007-07-03 08:39:27.000000000 -0700
@@ -1315,24 +1315,44 @@
  */
 static void deactivate_slab(struct kmem_cache *s, struct page *page, int cpu)
 {
+	void **lockless_freelist;
+
+	/*
+	 * Careful with the lockless freelist.  Allocations occur without
+	 * disabling interrupts and preemption. So we need to disable
+	 * the use of the lockess freelist with an atomic operation before
+	 * we start merging objects back into the slab
+	 *
+	 * Clearing s->cpu_slab should occur as early as possible in order
+	 * to avoid repeats in slab_alloc but clearing is not necessary
+	 * for correctness since allocations cannot occur if the lockless
+	 * freelist is empty.
+	 */
+	s->cpu_slab[cpu] = NULL;
+	if (likely(!page->lockless_freelist)) {
+		unfreeze_slab(s, page);
+		return;
+	}
+
+	lockless_freelist = xchg(&page->lockless_freelist, NULL);
+
 	/*
 	 * Merge cpu freelist into freelist. Typically we get here
 	 * because both freelists are empty. So this is unlikely
 	 * to occur.
 	 */
-	while (unlikely(page->lockless_freelist)) {
+	while (lockless_freelist) {
 		void **object;
 
 		/* Retrieve object from cpu_freelist */
-		object = page->lockless_freelist;
-		page->lockless_freelist = page->lockless_freelist[page->offset];
+		object = lockless_freelist;
+		lockless_freelist = lockless_freelist[page->offset];
 
 		/* And put onto the regular freelist */
 		object[page->offset] = page->freelist;
 		page->freelist = object;
 		page->inuse--;
 	}
-	s->cpu_slab[cpu] = NULL;
 	unfreeze_slab(s, page);
 }
 
@@ -1393,16 +1413,29 @@
  * we need to allocate a new slab. This is slowest path since we may sleep.
  */
 static void *__slab_alloc(struct kmem_cache *s,
-		gfp_t gfpflags, int node, void *addr, struct page *page)
+		gfp_t gfpflags, int node, void *addr)
 {
 	void **object;
-	int cpu = smp_processor_id();
+	int cpu;
+	unsigned long flags;
+	struct page *page;
 
+	local_irq_save(flags);
+	cpu = smp_processor_id();
+	page = s->cpu_slab[cpu];
 	if (!page)
 		goto new_slab;
-
 	slab_lock(page);
-	if (unlikely(node != -1 && page_to_nid(page) != node))
+
+	/*
+	 * Check for a slab that must be deactivated. This may occur because
+	 *
+	 * 1. Page locality is not right for the NUMA case.
+	 * 2. A race has left us handling a Frozen Slab which we only
+	 *    handle in slab_alloc(). Unfreeze the frozen slab.
+	 */
+	if (unlikely(SlabFrozen(page) ||
+			(node != -1 && page_to_nid(page) != node)))
 		goto another_slab;
 load_freelist:
 	object = page->freelist;
@@ -1416,6 +1449,7 @@
 	page->inuse = s->objects;
 	page->freelist = NULL;
 	slab_unlock(page);
+	local_irq_restore(flags);
 	return object;
 
 another_slab:
@@ -1458,6 +1492,7 @@
 		s->cpu_slab[cpu] = page;
 		goto load_freelist;
 	}
+	local_irq_restore(flags);
 	return NULL;
 debug:
 	object = page->freelist;
@@ -1467,6 +1502,7 @@
 	page->inuse++;
 	page->freelist = object[page->offset];
 	slab_unlock(page);
+	local_irq_restore(flags);
 	return object;
 }
 
@@ -1485,20 +1521,20 @@
 {
 	struct page *page;
 	void **object;
-	unsigned long flags;
 
-	local_irq_save(flags);
-	page = s->cpu_slab[smp_processor_id()];
+redo:
+	page = s->cpu_slab[raw_smp_processor_id()];
 	if (unlikely(!page || !page->lockless_freelist ||
 			(node != -1 && page_to_nid(page) != node)))
 
-		object = __slab_alloc(s, gfpflags, node, addr, page);
+		object = __slab_alloc(s, gfpflags, node, addr);
 
 	else {
 		object = page->lockless_freelist;
-		page->lockless_freelist = object[page->offset];
+		if (cmpxchg(&page->lockless_freelist, object,
+				object[page->offset]) != object)
+			goto redo;
 	}
-	local_irq_restore(flags);
 	return object;
 }