Avoid node pointer indirection for !CONFIG_NUMA and default cpu.


Index: linux-2.6.21-rc1/include/linux/slub_def.h
===================================================================
--- linux-2.6.21-rc1.orig/include/linux/slub_def.h	2007-02-25 16:08:16.000000000 -0800
+++ linux-2.6.21-rc1/include/linux/slub_def.h	2007-02-25 16:08:31.000000000 -0800
@@ -26,6 +26,7 @@ struct kmem_cache {
 	unsigned long flags;
 	int size;		/* Total size of an object */
 	int objects;		/* Number of objects in slab */
+	struct kmem_cache_node local_node;
 	atomic_t refcount;	/* Refcount for destroy */
 	void (*ctor)(void *, struct kmem_cache *, unsigned long);
 	void (*dtor)(void *, struct kmem_cache *, unsigned long);
Index: linux-2.6.21-rc1/mm/slub.c
===================================================================
--- linux-2.6.21-rc1.orig/mm/slub.c	2007-02-25 16:06:09.000000000 -0800
+++ linux-2.6.21-rc1/mm/slub.c	2007-02-25 16:17:04.000000000 -0800
@@ -239,13 +239,21 @@ static __always_inline int slab_trylock(
 	return rc;
 }
 
+struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
+{
+#ifdef CONFIG_NUMA
+	return s->node[node];
+#else
+	return &s->local_node;
+#endif
+}
+
 /*
  * Management of partially allocated slabs
  */
 static void __always_inline add_partial(struct kmem_cache *s, struct page *page)
 {
-	int node = page_to_nid(page);
-	struct kmem_cache_node *n = s->node[node];
+	struct kmem_cache_node *n = get_node(s, page_to_nid(page));
 
 	spin_lock(&n->list_lock);
 	n->nr_partial++;
@@ -256,8 +264,7 @@ static void __always_inline add_partial(
 static void __always_inline remove_partial(struct kmem_cache *s,
 						struct page *page)
 {
-	int node = page_to_nid(page);
-	struct kmem_cache_node *n = s->node[node];
+	struct kmem_cache_node *n = get_node(s, page_to_nid(page));
 
 	spin_lock(&n->list_lock);
 	list_del(&page->lru);
@@ -313,7 +320,6 @@ static struct page *get_any_partial(stru
 					->node_zonelists[gfp_zone(flags)];
 	struct zone **z;
 	struct page *page;
-	int nid;
 
 	/*
 	 * Look through allowed nodes for objects available
@@ -322,8 +328,7 @@ static struct page *get_any_partial(stru
 	for (z = zonelist->zones; *z; z++) {
 		struct kmem_cache_node *n;
 
-		nid = zone_to_nid(*z);
-		n = s->node[nid];
+		n = get_node(s, zone_to_nid(*z));
 
 		if (n && cpuset_zone_allowed_hardwall(*z, flags) &&
 				n->nr_partial) {
@@ -344,7 +349,7 @@ static struct page *get_partial(struct k
 	struct page *page;
 	int searchnode = (node == -1) ? numa_node_id() : node;
 
-	page = get_partial_node(s->node[searchnode]);
+	page = get_partial_node(get_node(s, searchnode));
 	if (page || (flags & __GFP_THISNODE))
 		return page;
 
@@ -440,8 +445,7 @@ static void check_free_chain(struct kmem
 
 static void discard_slab(struct kmem_cache *s, struct page *page)
 {
-	int node = page_to_nid(page);
-	struct kmem_cache_node *n = s->node[node];
+	struct kmem_cache_node *n = get_node(s, page_to_nid(page));
 
 	atomic_long_dec(&n->nr_slabs);
 
@@ -469,8 +473,7 @@ static struct page *new_slab(struct kmem
 	if (!page)
 		goto out;
 
-	node = page_to_nid(page);
-	n = s->node[node];
+	n = get_node(s, page_to_nid(page));
 	if (n)
 		atomic_long_inc(&n->nr_slabs);
 	page->offset = s->offset;
@@ -904,6 +907,20 @@ static unsigned long calculate_alignment
 	return ALIGN(align, sizeof(void *));
 }
 
+void free_kmem_cache_nodes(struct kmem_cache *s)
+{
+#ifdef CONFIG_NUMA
+	int node;
+
+	for_each_online_node(node) {
+		struct kmem_cache_node *n = s->node[node];
+		if (n && n != &s->local_node)
+			kfree(n);
+		s->node[node] = NULL;
+	}
+#endif
+}
+
 static void init_kmem_cache_node(struct kmem_cache_node *n)
 {
 	memset(n, 0, sizeof(struct kmem_cache_node));
@@ -912,14 +929,63 @@ static void init_kmem_cache_node(struct 
 	INIT_LIST_HEAD(&n->partial);
 }
 
+int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags)
+{
+#ifdef CONFIG_NUMA
+	int node;
+	int local_node = page_to_nid(virt_to_page(s));
+
+	for_each_online_node(node) {
+		struct kmem_cache_node *n;
+
+		if (local_node == node)
+			n = &s->local_node;
+		else
+		if (slab_state == DOWN) {
+			/*
+			 * No kmalloc_node yet so do it by hand.
+			 * We know that this is the first slab on the
+			 * node for this slabcache. There are no concurrent
+			 * accesses possible. Which simplifies things.
+			 */
+			unsigned long flags;
+			struct page *page;
+
+			local_irq_save(flags);
+			page = new_slab(s, gfpflags, node);
+
+			BUG_ON(!page);
+			n = page->freelist;
+			page->freelist = *(void **)page->freelist;
+			page->inuse++;
+			local_irq_restore(flags);
+		} else
+			n = kmalloc_node(sizeof(struct kmem_cache_node),
+				gfpflags, node);
+
+		if (!n) {
+			free_kmem_cache_nodes(s);
+			return 0;
+		}
+
+		s->node[node] = n;
+		init_kmem_cache_node(n);
+
+		if (slab_state == DOWN)
+			atomic_long_inc(&n->nr_slabs);
+	}
+#else
+	init_kmem_cache_node(&s->local_node);
+#endif
+	return 1;
+}
+
 int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags,
 		const char *name, size_t size,
 		size_t align, unsigned long flags,
 		void (*ctor)(void *, struct kmem_cache *, unsigned long),
 		void (*dtor)(void *, struct kmem_cache *, unsigned long))
 {
-	int node;
-
 	BUG_ON(flags & SLUB_UNIMPLEMENTED);
 	memset(s, 0, sizeof(struct kmem_cache));
 	s->name = name;
@@ -969,48 +1035,10 @@ int kmem_cache_open(struct kmem_cache *s
 	atomic_set(&s->cpu_slabs, 0);
 	INIT_DELAYED_WORK(&s->flush, flusher);
 #endif
-
-	for_each_online_node(node) {
-		struct kmem_cache_node *n;
-
-		if (slab_state == DOWN) {
-			/*
-			 * No kmalloc_node yet so do it by hand.
-			 * We know that this is the first slab on the
-			 * node for this slabcache. There are no concurrent
-			 * accesses possible. Which simplifies things.
-			 */
-			unsigned long flags;
-			struct page *page;
-
-			local_irq_save(flags);
-			page = new_slab(s, gfpflags, node);
-
-			BUG_ON(!page);
-			n = page->freelist;
-			page->freelist = *(void **)page->freelist;
-			page->inuse++;
-			local_irq_restore(flags);
-		} else
-			n = kmalloc_node(sizeof(struct kmem_cache_node),
-				gfpflags, node);
-
-		if (!n)
-			goto undo_alloc_err;
-
-		s->node[node] = n;
-		init_kmem_cache_node(n);
-
-		if (slab_state == DOWN)
-			atomic_long_inc(&n->nr_slabs);
+	if (init_kmem_cache_nodes(s, gfpflags)) {
+		register_slab(s);
+		return 1;
 	}
-	register_slab(s);
-	return 1;
-
-undo_alloc_err:
-	for_each_online_node(node)
-		kfree(s->node[node]);
-
 error:
 	if (flags & SLAB_PANIC)
 		panic("Cannot create slab %s size=%ld realsize=%d "
@@ -1105,12 +1133,7 @@ int kmem_cache_close(struct kmem_cache *
 		if (atomic_long_read(&n->nr_slabs))
 			return 1;
 	}
-
-	/* Free allocated metadata */
-	for_each_online_node(node) {
-		kfree(s->node[node]);
-		s->node[node] = NULL;
-	}
+	free_kmem_cache_nodes(s);
 	unregister_slab(s);
 	return 0;
 }
@@ -1145,7 +1168,7 @@ static unsigned long slab_objects(struct
 	struct page *page;
 
 	for_each_online_node(node) {
-		struct kmem_cache_node *n = s->node[node];
+		struct kmem_cache_node *n = get_node(s, node);
 
 		nr_slabs += atomic_read(&n->nr_slabs);
 		nr_partial_slabs += n->nr_partial;
@@ -1262,7 +1285,7 @@ static struct kmem_cache *get_slab(size_
 		}
 #endif
 
-		text = kasprintf(flags, "kmalloc_dma-%ld", realsize);
+		text = kasprintf(flags, "kmalloc_dma-%d", (unsigned int)realsize);
 		s = create_kmalloc_cache(x, text, realsize, flags);
 		kfree(text);
 		kmalloc_caches_dma[index] = s;