---
 mm/slub.c |  120 +++++++++++++++++++++++++++++++++++++++++++++++---------------
 1 file changed, 92 insertions(+), 28 deletions(-)

Index: slub/mm/slub.c
===================================================================
--- slub.orig/mm/slub.c	2007-05-13 21:58:29.000000000 -0700
+++ slub/mm/slub.c	2007-05-13 22:46:03.000000000 -0700
@@ -1200,7 +1200,7 @@ static void remove_partial(struct kmem_c
 static inline int lock_and_freeze_slab(struct kmem_cache_node *n, struct page *page)
 {
 	if (slab_trylock(page)) {
-		list_del(&page->lru);
+		list_del_init(&page->lru);
 		n->nr_partial--;
 		SetSlabFrozen(page);
 		return 1;
@@ -1338,29 +1338,34 @@ static void unfreeze_slab(struct kmem_ca
 }
 
 /*
- * Remove the cpu slab
+ * Return the slabs used for a cpu back to the lists
  */
 static void deactivate_slab(struct kmem_cache *s, struct page *page, int cpu)
 {
-	/*
-	 * Merge cpu freelist into freelist. Typically we get here
-	 * because both freelists are empty. So this is unlikely
-	 * to occur.
-	 */
-	while (unlikely(page->lockless_freelist)) {
-		void **object;
-
-		/* Retrieve object from cpu_freelist */
-		object = page->lockless_freelist;
-		page->lockless_freelist = page->lockless_freelist[page->offset];
-
-		/* And put onto the regular freelist */
-		object[page->offset] = page->freelist;
-		page->freelist = object;
-		page->inuse--;
-	}
+	struct page *p, *p2;
+	int offset = page->offset;
+
 	s->cpu_slab[cpu] = NULL;
-	unfreeze_slab(s, page);
+	list_for_each_entry_safe(p, p2, &page->lru, lru) {
+		/*
+		 * Merge cpu freelist into freelist. Typically we get here
+		 * because both freelists are empty. So this is unlikely
+		 * to occur.
+		 */
+		while (unlikely(p->lockless_freelist)) {
+			void **object;
+
+			/* Retrieve object from cpu_freelist */
+			object = p->lockless_freelist;
+			p->lockless_freelist = p->lockless_freelist[offset];
+
+			/* And put onto the regular freelist */
+			object[offset] = p->freelist;
+			p->freelist = object;
+			p->inuse--;
+		}
+		unfreeze_slab(s, p);
+	}
 }
 
 static void flush_slab(struct kmem_cache *s, struct page *page, int cpu)
@@ -1402,6 +1407,30 @@ static void flush_all(struct kmem_cache 
 #endif
 }
 
+static inline int add_cpu_slab(struct kmem_cache *s, gfp_t gfpflags,
+			struct page *page, int node)
+{
+	int nr;
+	struct page *p;
+
+	p = get_partial(s, gfpflags, node);
+	if (!p) {
+		p = new_slab(s, gfpflags, node);
+		if (!p)
+			return 0;
+		slab_lock(p);
+		SetSlabFrozen(p);
+	}
+
+	list_add(&p->lru, &page->lru);
+	nr = s->objects - p->inuse;
+	p->lockless_freelist = p->freelist;
+	p->freelist = NULL;
+	p->inuse = s->objects;
+	slab_unlock(page);
+	return nr;
+}
+
 /*
  * Slow path. The lockless freelist is empty or we need to perform
  * debugging duties.
@@ -1423,25 +1452,52 @@ static void *__slab_alloc(struct kmem_ca
 		gfp_t gfpflags, int node, void *addr, struct page *page)
 {
 	void **object;
+	int objects = 0;
 	int cpu = smp_processor_id();
 
 	if (!page)
 		goto new_slab;
 
+	/* Drop eventual remaining auxiliary pages */
+	list_for_each_entry(p, page->lru) {
+		BUG_ON(p->lockless_freelist);
+		slab_lock(p);
+		if (page->inuse < s->objects) {
+			/* Ohh we got objects still left */
+			page->lockless_freelist = page->freelist;
+			objects += s->objects - p->inuse;
+			p->inuse = s->objects;
+			slab_unlock(slab);
+		else {
+			if (p != page) {
+				list_del_init(p->lru);
+			} else {
+				if (p != p->lru.next)
+					page = p->lru.next;
+				else
+					page = NULL;
+			}
+			unfreeze_slab(s, p);
+		}
+	}
+
 	slab_lock(page);
 	if (unlikely(node != -1 && page_to_nid(page) != node))
 		goto another_slab;
 load_freelist:
 	object = page->freelist;
-	if (unlikely(!object))
+	if (unlikely(!page->freelist))
 		goto another_slab;
 	if (unlikely(SlabDebug(page)))
 		goto debug;
 
 	object = page->freelist;
 	page->lockless_freelist = object[page->offset];
+	objects = s->objects - page->inuse;
 	page->inuse = s->objects;
 	page->freelist = NULL;
+	while (objects < s->min_objects)
+		objects += add_cpu_slab(s, gfpflags, page, node);
 	slab_unlock(page);
 	return object;
 
@@ -1516,15 +1572,22 @@ static void __always_inline *slab_alloc(
 
 	local_irq_save(flags);
 	page = s->cpu_slab[smp_processor_id()];
-	if (unlikely(!page || !page->lockless_freelist ||
-			(node != -1 && page_to_nid(page) != node)))
-
-		object = __slab_alloc(s, gfpflags, node, addr, page);
+	if (unlikely(!page || (node != -1 && page_to_nid(page) != node)))
+		goto slow;
 
-	else {
-		object = page->lockless_freelist;
-		page->lockless_freelist = object[page->offset];
+	if (unlikely(!page->lockless_freelist)) {
+		/* Try next page in the cpu active set */
+		page = container_of(page->lru.next, struct page, lru);
+		if (!page->lockless_freelist)
+			goto slow;
+		s->cpu_slab[smp_processor_id()] = page;
 	}
+	object = page->lockless_freelist;
+	page->lockless_freelist = object[page->offset];
+	local_irq_restore(flags);
+	return object;
+slow:
+	object = __slab_alloc(s, gfpflags, node, addr, page);
 	local_irq_restore(flags);
 	return object;
 }
@@ -2030,6 +2093,7 @@ static int kmem_cache_open(struct kmem_c
 	s->objsize = size;
 	s->flags = flags;
 	s->align = align;
+	s->min_objects = slub_min_objects;
 	kmem_cache_open_debug_check(s);
 
 	if (!calculate_sizes(s))