From: Christoph Lameter <clameter@sgi.com>
Subject: slub: Large allocs for other slab sizes that do not fit in order 0

Expand the scheme used for kmalloc-2048 and kmalloc-4096 to all slab
caches. That means that kmem_cache_free() must now be able to handle
a fallback object that was allocated from the page allocator. This is
touching the fastpath costing us 1/2 % of performance (pretty small
so within variance). Kind of hacky though.

If we can handle fallback for all slab caches then we can also
merge to kmalloc-2048 and kmalloc-4096 again. So remove the
exclusion from slab_unmergeable().

Reviewed-by: Pekka Enberg <penberg@cs.helsinki.fi>
Signed-off-by: Christoph Lameter <clameter@sgi.com>

---
 mm/slub.c |   30 +++++++++++++++---------------
 1 file changed, 15 insertions(+), 15 deletions(-)

Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c	2008-02-14 12:18:03.578942195 -0800
+++ linux-2.6/mm/slub.c	2008-02-14 14:14:31.677251487 -0800
@@ -211,7 +211,6 @@ static inline void ClearSlabDebug(struct
 /* Internal SLUB flags */
 #define __OBJECT_POISON		0x80000000 /* Poison object */
 #define __SYSFS_ADD_DEFERRED	0x40000000 /* Not yet visible via sysfs */
-#define __KMALLOC_CACHE		0x20000000 /* objects freed using kfree */
 #define __PAGE_ALLOC_FALLBACK	0x10000000 /* Allow fallback to page alloc */
 
 /* Not all arches define cache_line_size */
@@ -1827,7 +1826,10 @@ void kmem_cache_free(struct kmem_cache *
 
 	page = virt_to_head_page(x);
 
-	slab_free(s, page, x, __builtin_return_address(0));
+	if (likely(PageSlab(page)))
+		slab_free(s, page, x, __builtin_return_address(0));
+	else
+		put_page(page);
 }
 EXPORT_SYMBOL(kmem_cache_free);
 
@@ -2338,23 +2340,24 @@ static int calculate_sizes(struct kmem_c
 	 */
 	size = ALIGN(size, align);
 	s->size = size;
+	s->order = calculate_order(size);
+	if (s->order < 0)
+		return 0;
 
-	if ((flags & __KMALLOC_CACHE) &&
-			PAGE_SIZE / size < slub_min_objects) {
+	if (s->order && !s->ctor && s->size < 2 * PAGE_SIZE &&
+		!(flags & (SLAB_DEBUG_FREE | SLAB_RED_ZONE | SLAB_STORE_USER
+			| __OBJECT_POISON | SLAB_DESTROY_BY_RCU | SLAB_RECLAIM_ACCOUNT
+			| SLAB_TRACE))) {
 		/*
-		 * Kmalloc cache that would not have enough objects in
-		 * an order 0 page. Kmalloc slabs can fallback to
+		 * Cache that would not have enough objects in
+		 * an order 0 page. Slabs can fallback to
 		 * page allocator order 0 allocs so take a reasonably large
 		 * order that will allows us a good number of objects.
 		 */
 		s->order = max(slub_max_order, PAGE_ALLOC_COSTLY_ORDER);
 		s->flags |= __PAGE_ALLOC_FALLBACK;
 		s->allocflags |= __GFP_NOWARN;
-	} else
-		s->order = calculate_order(size);
-
-	if (s->order < 0)
-		return 0;
+	}
 
 	s->allocflags = 0;
 	if (s->order)
@@ -2569,7 +2572,7 @@ static struct kmem_cache *create_kmalloc
 
 	down_write(&slub_lock);
 	if (!kmem_cache_open(s, gfp_flags, name, size, ARCH_KMALLOC_MINALIGN,
-			flags | __KMALLOC_CACHE, NULL))
+			flags, NULL))
 		goto panic;
 
 	list_add(&s->list, &slab_caches);
@@ -3088,9 +3091,6 @@ static int slab_unmergeable(struct kmem_
 	if (slub_nomerge || (s->flags & SLUB_NEVER_MERGE))
 		return 1;
 
-	if ((s->flags & __PAGE_ALLOC_FALLBACK))
-		return 1;
-
 	if (s->ctor)
 		return 1;