From ddbb16bb6f4fb507e2e8fb1f1cfd23deb6e8ff65 Mon Sep 17 00:00:00 2001
From: Christoph Lameter <clameter@schroedinger.engr.sgi.com>
Date: Thu, 26 Jul 2007 19:01:40 -0700
Subject: [PATCH] Simplify slab_free and slab_alloc

---
 mm/slub.c |   50 ++++++++++++++++++++++++++++++++++----------------
 1 files changed, 34 insertions(+), 16 deletions(-)

diff --git a/mm/slub.c b/mm/slub.c
index 9b2d617..7df9f2b 100644
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -1453,10 +1453,11 @@ static void flush_all(struct kmem_cache *s)
  * we need to allocate a new slab. This is slowest path since we may sleep.
  */
 static void *__slab_alloc(struct kmem_cache *s,
-		gfp_t gfpflags, int node, void *addr, struct page *page)
+		gfp_t gfpflags, int node, void *addr, unsigned long flags)
 {
 	void **object;
 	int cpu = smp_processor_id();
+	struct page *page = s->cpu_slab[cpu];
 
 	if (!page)
 		goto new_slab;
@@ -1475,7 +1476,12 @@ load_freelist:
 	page->lockless_freelist = object[page->offset];
 	page->inuse = s->objects;
 	page->freelist = NULL;
+out:
 	slab_unlock(page);
+	local_irq_restore(flags);
+	if (unlikely((gfpflags & __GFP_ZERO)))
+		memset(object, 0, s->objsize);
+
 	return object;
 
 another_slab:
@@ -1518,6 +1524,7 @@ new_slab:
 		s->cpu_slab[cpu] = page;
 		goto load_freelist;
 	}
+	local_irq_restore(flags);
 	return NULL;
 debug:
 	object = page->freelist;
@@ -1526,8 +1533,7 @@ debug:
 
 	page->inuse++;
 	page->freelist = object[page->offset];
-	slab_unlock(page);
-	return object;
+	goto out;
 }
 
 /*
@@ -1549,21 +1555,27 @@ static void __always_inline *slab_alloc(struct kmem_cache *s,
 
 	local_irq_save(flags);
 	page = s->cpu_slab[smp_processor_id()];
-	if (unlikely(!page || !page->lockless_freelist ||
-			(node != -1 && page_to_nid(page) != node)))
 
-		object = __slab_alloc(s, gfpflags, node, addr, page);
+	if (unlikely(!page))
+		goto slow;
 
-	else {
-		object = page->lockless_freelist;
-		page->lockless_freelist = object[page->offset];
-	}
+	object = page->lockless_freelist;
+
+	if (unlikely(!object))
+		goto slow;
+
+	if (unlikely(node != -1 && page_to_nid(page) != node))
+		goto slow;
+
+	page->lockless_freelist = object[page->offset];
 	local_irq_restore(flags);
 
 	if (unlikely((gfpflags & __GFP_ZERO) && object))
 		memset(object, 0, s->objsize);
 
 	return object;
+slow:
+	return __slab_alloc(s, gfpflags, node, addr, flags);
 }
 
 void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
@@ -1656,13 +1668,19 @@ static void __always_inline slab_free(struct kmem_cache *s,
 	unsigned long flags;
 
 	local_irq_save(flags);
-	if (likely(page == s->cpu_slab[smp_processor_id()] &&
-						!SlabDebug(page))) {
-		object[page->offset] = page->lockless_freelist;
-		page->lockless_freelist = object;
-	} else
-		__slab_free(s, page, x, addr);
+	if (unlikely(page != s->cpu_slab[smp_processor_id()]))
+		goto slow;
+
+	if (unlikely(SlabDebug(page)))
+		goto slow;
+
+	object[page->offset] = page->lockless_freelist;
+	page->lockless_freelist = object;
+	local_irq_restore(flags);
+	return;
 
+slow:
+	__slab_free(s, page, x, addr);
 	local_irq_restore(flags);
 }
 
-- 
1.4.4.4