---
 mm/slub.c |   90 +++++++++++++++++++++++++++++++++++---------------------------
 1 file changed, 51 insertions(+), 39 deletions(-)

Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c	2007-10-18 13:31:40.000000000 -0700
+++ linux-2.6/mm/slub.c	2007-10-18 13:34:18.000000000 -0700
@@ -1451,11 +1451,15 @@ static inline void **free_object(struct 
 	} while (cmpxchg(&page->freelist, prior, object) != prior);
 
 #else
+	unsigned long flags;
+
+	local_irq_save(flags);
 	slab_lock(page);
 	prior = page->freelist;
 	object[offset] = prior;
 	page->freelist = object;
 	slab_unlock(page);
+	local_irq_restore(flags);
 #endif
 
 	return prior;
@@ -1475,11 +1479,15 @@ static inline void **alloc_object(struct
 			return NULL;
 	} while (cmpxchg(&page->freelist, object, object[offset]) != object);
 #else
+	unsigned long flags;
+
+	local_irq_save(flags);
 	slab_lock(page);
 	object = page->freelist;
 	if (object)
 		page->freelist = object[offset];
 	slab_unlock(page);
+	local_irq_restore(flags);
 #endif
 	return object;
 }
@@ -1494,10 +1502,14 @@ static inline void **get_freelist(struct
 #ifdef SLUB_ATOMIC
 	list = xchg(&page->freelist, NULL);
 #else
+	unsigned long flags;
+
+	local_irq_save(flags);
 	slab_lock(page);
 	list = page->freelist;
 	page->freelist = NULL;
 	slab_unlock(page);
+	local_irq_restore(flags);
 #endif
 	return list;
 }
@@ -1514,9 +1526,13 @@ static inline void **free_cpu_object(str
 	} while (cmpxchg_local(&c->freelist, prior, object) != prior);
 
 #else
+	unsigned long flags;
+
+	local_irq_save(flags);
 	prior = c->freelist;
 	object[c->offset] = prior;
 	c->freelist = object;
+	local_irq_restore(flags);
 #endif
 	return prior;
 }
@@ -1535,9 +1551,13 @@ static inline void **alloc_cpu_object(st
 			return NULL;
 	} while (cmpxchg_local(&c->freelist, object, object[c->offset]) != object);
 #else
+	unsigned long flags;
+
+	local_irq_save(flags);
 	object = c->freelist;
 	if (object)
 		c->freelist = object[c->offset];
+	local_irq_restore(flags);
 #endif
 	return object;
 }
@@ -1641,7 +1661,10 @@ static void *__slab_alloc(struct kmem_ca
 {
 	void **object;
 	struct page *new;
+	unsigned long flags;
 
+	/* Must have consistent view of kmem_cache_cpu contents */
+	local_irq_save(flags);
 	if (!c->page)
 		goto new_slab;
 
@@ -1657,6 +1680,7 @@ load_freelist:
 
 	c->freelist = object[c->offset];
 	c->node = page_to_nid(c->page);
+	local_irq_restore(flags);
 	return object;
 
 another_slab:
@@ -1670,9 +1694,11 @@ new_slab:
 	}
 
 	put_cpu();
+	local_irq_restore(flags);
 
 	new = new_slab(s, gfpflags, node);
 
+	local_irq_save(flags);
 	if (new) {
 		c = get_cpu_slab(s, get_cpu());
 		if (c->page) {
@@ -1699,9 +1725,11 @@ new_slab:
 		goto load_freelist;
 	}
 	get_cpu();
+	local_irq_restore(flags);
 	return NULL;
 debug:
-	/* FIXME: Thjs onlu works on UP. There needs to be some way
+	/*
+	 * FIXME: Thjs onlu works on UP. There needs to be some way
 	 * to guarantee slab consistency.
 	 */
 	if (!c->page->freelist)
@@ -1711,6 +1739,7 @@ debug:
 
 	object = alloc_object(c->page, c->offset);
 	c->node = -1;
+	local_irq_restore(flags);
 	return object;
 }
 
@@ -1731,6 +1760,10 @@ static void __always_inline *slab_alloc(
 	struct kmem_cache_cpu *c;
 
 	c = get_cpu_slab(s, get_cpu());
+	/*
+	 * Node may change after this test. That is only relevant to
+	 * kmalloc_node though
+	 */
 	if (unlikely(!node_match(c, node)))
 		goto slow;
 
@@ -1767,39 +1800,6 @@ EXPORT_SYMBOL(kmem_cache_alloc_node);
 #endif
 
 /*
- * Slow patch handling. This may still be called frequently since objects
- * have a longer lifetime than the cpu slabs in most processing loads.
- *
- * So we still attempt to reduce cache line usage. Just take the slab
- * lock and free the item. If there is no additional partial page
- * handling required then we can return immediately.
- */
-static void __slab_free(struct kmem_cache *s, struct page *page,
-				void *x, void *addr, unsigned int offset)
-{
-	void *prior;
-	void **object = (void *)x;
-
-	if (unlikely(SlabDebug(page)))
-		goto debug;
-checks_ok:
-	prior = free_object(page, object, offset);
-
-	/*
-	 * The slab is now guaranteed to have free objects.
-	 * We may have to move it to the partial list.
-	 */
-	if (unlikely(!prior))
-		add_partial(s, page, 0);
-	return;
-
-debug:
-	if (free_debug_processing(s, page, x, addr))
-		goto checks_ok;
-	return;
-}
-
-/*
  * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
  * can perform fastpath freeing without additional function calls.
  *
@@ -1818,11 +1818,23 @@ static void __always_inline slab_free(st
 
 	c = get_cpu_slab(s, get_cpu());
 	debug_check_no_locks_freed(object, s->objsize);
-	if (likely(page == c->page && c->node >= 0))
-		free_cpu_object(c, object);
-	else
-		__slab_free(s, page, x, addr, c->offset);
+	if (likely(c->node >= 0)) {
+		if (likely(page == c->page))
+			/* FIXME: May change if interrupt alloc exhausts cpuslab! */
+			free_cpu_object(c, object);
+		else
+			goto offcpuslab;
+	} else {
+		void **prior;
 
+		if (!free_debug_processing(s, page, x, addr))
+			goto out;
+offcpuslab:
+		prior = free_object(page, object, c->offset);
+		if (unlikely(!prior))
+			add_partial(s, page, 0);
+	}
+out:
 	put_cpu();
 }