---
 mm/slub.c |  136 ++++++++++++++++++++++++++++++++++----------------------------
 1 file changed, 76 insertions(+), 60 deletions(-)

Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c	2007-10-28 15:45:54.000000000 -0700
+++ linux-2.6/mm/slub.c	2007-10-28 17:15:53.000000000 -0700
@@ -265,6 +265,12 @@ void *slab_address(struct page *page)
 	return page->end - PAGE_MAPPING_ANON;
 }
 
+static inline void zap_cpu_slab(struct kmem_cache_cpu *c)
+{
+	c->page = NULL;
+	c->freelist = (void *)PAGE_MAPPING_ANON;
+}
+
 static inline int check_valid_pointer(struct kmem_cache *s,
 				struct page *page, const void *object)
 {
@@ -1281,7 +1287,7 @@ static inline unsigned long lock_and_fre
 		list_del(&page->lru);
 		n->nr_partial--;
 		c->page = page;
-		return state | FROZEN;
+		return state;
 	}
 	return 0;
 }
@@ -1383,86 +1389,95 @@ static noinline unsigned long get_partia
 	return state;
 }
 
-/*
- * Move a page back to the lists.
- *
- * Must be called with the slab lock held.
- *
- * On exit the slab lock will have been dropped.
- */
-static void unfreeze_slab(struct kmem_cache *s, struct page *page,
-				int tail, unsigned long state)
+static void update_lists_empty(struct kmem_cache *s, struct page *page,
+	unsigned long state, int on_partial)
 {
-	state &= ~FROZEN;
-	if (page->inuse) {
-
-		if (page->freelist != page->end)
-			add_partial(s, page, tail);
-		else
-			add_full(s, page, state);
+	if (get_node(s, page_to_nid(page))->nr_partial
+						< MIN_PARTIAL) {
+		/*
+		 * Adding an empty slab to the partial slabs in order
+		 * to avoid page allocator overhead. This slab needs
+		 * to come after the other slabs with objects in
+		 * order to fill them up. That way the size of the
+		 * partial list stays small. kmem_cache_shrink can
+		 * reclaim empty slabs from the partial list.
+		 */
+		if (!on_partial)
+			add_partial(s, page, 1);
 		slab_unlock(page, state);
-
 	} else {
-		if (get_node(s, page_to_nid(page))->nr_partial
-							< MIN_PARTIAL) {
-			/*
-			 * Adding an empty slab to the partial slabs in order
-			 * to avoid page allocator overhead. This slab needs
-			 * to come after the other slabs with objects in
-			 * order to fill them up. That way the size of the
-			 * partial list stays small. kmem_cache_shrink can
-			 * reclaim empty slabs from the partial list.
-			 */
-			add_partial(s, page, 1);
-			slab_unlock(page, state);
-		} else {
-			slab_unlock(page, state);
-			discard_slab(s, page);
-		}
+		slab_unlock(page, state);
+		discard_slab(s, page);
 	}
 }
 
-/*
- * Remove the cpu slab
- */
-static void deactivate_slab(struct kmem_cache *s, struct kmem_cache_cpu *c,
-				unsigned long state)
+static void update_lists_objects(struct kmem_cache *s, struct page *page,
+	unsigned long state, int on_partial, int full, int tail)
+{
+	if (!full) {
+		if (!on_partial)
+			add_partial(s, page, tail);
+	} else
+		add_full(s, page, state);
+	slab_unlock(page, state);
+}
+
+
+static void merge_objects_to_slab(struct kmem_cache *s, struct page *page,
+		void **freelist, int offset)
 {
-	struct page *page = c->page;
 	int tail = 1;
+	unsigned long state = slab_lock(page);
+	int on_partial = !(state & FROZEN) && page->freelist != page->end;
+	int inuse = page->inuse;
+	void **page_freelist = page->freelist;
+
+	state &= ~FROZEN;
+
 	/*
-	 * Merge cpu freelist into freelist. Typically we get here
-	 * because both freelists are empty. So this is unlikely
-	 * to occur.
+	 * Merge freelist into page freelist.
 	 *
 	 * We need to use _is_end here because deactivate slab may
 	 * be called for a debug slab. Then c->freelist may contain
 	 * a dummy pointer.
 	 */
-	while (unlikely(!is_end(c->freelist))) {
+	while (unlikely(!is_end(freelist))) {
 		void **object;
 
 		tail = 0;	/* Hot objects. Put the slab first */
 
 		/* Retrieve object from cpu_freelist */
-		object = c->freelist;
-		c->freelist = c->freelist[c->offset];
+		object = freelist;
+		freelist = freelist[offset];
 
 		/* And put onto the regular freelist */
-		object[c->offset] = page->freelist;
-		page->freelist = object;
-		page->inuse--;
+		object[offset] = page_freelist;
+		page_freelist = object;
+		inuse--;
 	}
-	c->page = NULL;
-	unfreeze_slab(s, page, tail, state);
+	page->inuse = inuse;
+	page->freelist = page_freelist;
+
+	if (inuse)
+		update_lists_empty(s, page, state, on_partial);
+	else
+		update_lists_objects(s, page, state, on_partial,
+			page_freelist == page->end, tail);
 }
 
-static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
+/*
+ * Remove the cpu slab
+ */
+static void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
 {
-	unsigned long state;
+	struct page *page = c->page;
+	void **freelist = c->freelist;
+
+	if (!c->page || (c->node >= 0 && is_end(freelist)))
+		return;
 
-	state = slab_lock(c->page);
-	deactivate_slab(s, c, state);
+	zap_cpu_slab(c);
+	merge_objects_to_slab(s, page, freelist, c->offset);
 }
 
 /*
@@ -1549,7 +1564,7 @@ static noinline unsigned long get_new_sl
 		flush_slab(s, c);
 	}
 	c->page = page;
-	return slab_lock(page) | FROZEN;
+	return slab_lock(page);
 }
 
 /*
@@ -1581,13 +1596,13 @@ static void *__slab_alloc(struct kmem_ca
 	preempt_enable_no_resched();
 #endif
 	if (likely(c->page)) {
-		state = slab_lock(c->page);
 
 		if (unlikely(node_match(c, node) &&
-			c->page->freelist != c->page->end))
+			c->page->freelist != c->page->end)) {
+				state = slab_lock(c->page);
 				goto load_freelist;
-
-		deactivate_slab(s, c, state);
+		}
+		flush_slab(s, c);
 	}
 
 another_slab:
@@ -1631,6 +1646,7 @@ grow_slab:
 /* Perform debugging */
 debug:
 	object = c->page->freelist;
+	state |= FROZEN;
 	if (!alloc_debug_processing(s, c->page, object, addr))
 		goto another_slab;