Add free_block_lock and free_block_lock_irq

Add a free_block_lock and a free_block_lock_irq function and remove
the node parameter from free_block. That one can be determined from slabp.

Signed-off-by: Christoph Lameter <clameter@sgi.com>

Index: linux-2.6.16-rc5-mm2/mm/slab.c
===================================================================
--- linux-2.6.16-rc5-mm2.orig/mm/slab.c	2006-03-06 17:41:06.000000000 -0800
+++ linux-2.6.16-rc5-mm2/mm/slab.c	2006-03-06 19:02:42.000000000 -0800
@@ -701,8 +701,7 @@ static enum {
 
 static DEFINE_PER_CPU(struct work_struct, reap_work);
 
-static void free_block(struct kmem_cache *cachep, void **objpp, int len,
-			int node);
+static void free_block(struct kmem_cache *cachep, void **objpp, int len);
 static void enable_cpucache(struct kmem_cache *cachep);
 static void cache_reap(void *unused);
 static int __node_shrink(struct kmem_cache *cachep, int node);
@@ -900,6 +899,55 @@ static struct array_cache *alloc_arrayca
 	return nc;
 }
 
+/*
+ * Drain an array completely if it contains any elements.
+ * The l3 lock is only taken if necessary.
+ * The edition here is for slab internal calls where we know that interrupts are
+ * off or when we know that corresponding lock has already been obtained.
+ */
+static void __drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
+			 struct array_cache *ac)
+{
+//	check_irq_off();
+	if (!ac)
+		return;
+
+	if (ac->avail) {
+		spin_lock(&l3->list_lock);
+		free_block(cachep, ac->entry, ac->avail);
+		spin_unlock(&l3->list_lock);
+		ac->avail = 0;
+	}
+}
+
+/*
+ * For when we know that interrupts are on. This may be used for per
+ * cpu caches.
+ */
+static void drain_array_irq(struct kmem_cache *cachep, struct kmem_list3 *l3,
+			struct array_cache *ac)
+{
+	local_irq_disable();
+	__drain_array(cachep, l3, ac);
+	local_irq_enable();
+}
+
+/*
+ * Drain an array protected by its lock.
+ */
+static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
+			struct array_cache *ac)
+{
+	unsigned long flags;
+
+	if (!ac || !ac->avail)
+		return;
+
+	spin_lock_irqsave(&ac->lock, flags);
+	__drain_array(cachep, l3, ac);
+	spin_unlock_irqrestore(&ac->lock, flags);
+}
+
 #ifdef CONFIG_NUMA
 static void *__cache_alloc_node(struct kmem_cache *, gfp_t, int);
 static void *alternate_node_alloc(struct kmem_cache *, gfp_t);
@@ -942,19 +990,6 @@ static void free_alien_cache(struct arra
 	kfree(ac_ptr);
 }
 
-static void __drain_alien_cache(struct kmem_cache *cachep,
-				struct array_cache *ac, int node)
-{
-	struct kmem_list3 *rl3 = cachep->nodelists[node];
-
-	if (ac->avail) {
-		spin_lock(&rl3->list_lock);
-		free_block(cachep, ac->entry, ac->avail, node);
-		ac->avail = 0;
-		spin_unlock(&rl3->list_lock);
-	}
-}
-
 /*
  * Called from cache_reap() to regularly drain alien caches round robin.
  */
@@ -962,31 +997,17 @@ static void reap_alien(struct kmem_cache
 {
 	int node = __get_cpu_var(reap_node);
 
-	if (l3->alien) {
-		struct array_cache *ac = l3->alien[node];
-		if (ac && ac->avail) {
-			spin_lock_irq(&ac->lock);
-			__drain_alien_cache(cachep, ac, node);
-			spin_unlock_irq(&ac->lock);
-		}
-	}
+	if (l3->alien)
+		drain_array(cachep, cachep->nodelists[node], l3->alien[node]);
 }
 
 static void drain_alien_cache(struct kmem_cache *cachep,
 				struct array_cache **alien)
 {
-	int i = 0;
-	struct array_cache *ac;
-	unsigned long flags;
+	int i;
 
-	for_each_online_node(i) {
-		ac = alien[i];
-		if (ac) {
-			spin_lock_irqsave(&ac->lock, flags);
-			__drain_alien_cache(cachep, ac, i);
-			spin_unlock_irqrestore(&ac->lock, flags);
-		}
-	}
+	for_each_online_node(i)
+		drain_array(cachep, cachep->nodelists[numa_node_id()], alien[i]);
 }
 #else
 
@@ -1135,7 +1156,7 @@ static int __devinit cpuup_callback(stru
 			/* Free limit for this kmem_list3 */
 			l3->free_limit -= cachep->batchcount;
 			if (nc)
-				free_block(cachep, nc->entry, nc->avail, node);
+				free_block(cachep, nc->entry, nc->avail);
 
 			if (!cpus_empty(mask)) {
 				spin_unlock_irq(&l3->list_lock);
@@ -1145,7 +1166,7 @@ static int __devinit cpuup_callback(stru
 			shared = l3->shared;
 			if (shared) {
 				free_block(cachep, l3->shared->entry,
-					   l3->shared->avail, node);
+					   l3->shared->avail);
 				l3->shared = NULL;
 			}
 
@@ -2125,22 +2146,13 @@ static void check_spinlock_acquired_node
 #define check_spinlock_acquired_node(x, y) do { } while(0)
 #endif
 
-static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
-			struct array_cache *ac,
-			int force, int node);
-
 static void do_drain(void *arg)
 {
 	struct kmem_cache *cachep = arg;
-	struct array_cache *ac;
-	int node = numa_node_id();
 
 	check_irq_off();
-	ac = cpu_cache_get(cachep);
-	spin_lock(&cachep->nodelists[node]->list_lock);
-	free_block(cachep, ac->entry, ac->avail, node);
-	spin_unlock(&cachep->nodelists[node]->list_lock);
-	ac->avail = 0;
+	__drain_array(cachep, cachep->nodelists[numa_node_id()],
+			cpu_cache_get(cachep));
 }
 
 static void drain_cpu_caches(struct kmem_cache *cachep)
@@ -2153,7 +2165,7 @@ static void drain_cpu_caches(struct kmem
 	for_each_online_node(node) {
 		l3 = cachep->nodelists[node];
 		if (l3) {
-			drain_array(cachep, l3, l3->shared, 1, node);
+			drain_array(cachep, l3, l3->shared);
 			if (l3->alien)
 				drain_alien_cache(cachep, l3->alien);
 		}
@@ -2945,8 +2957,8 @@ done:
 /*
  * Caller needs to acquire correct kmem_list's list_lock
  */
-static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects,
-		       int node)
+static void free_block(struct kmem_cache *cachep, void **objpp,
+			 int nr_objects)
 {
 	int i;
 	struct kmem_list3 *l3;
@@ -2954,8 +2966,10 @@ static void free_block(struct kmem_cache
 	for (i = 0; i < nr_objects; i++) {
 		void *objp = objpp[i];
 		struct slab *slabp;
+		int node;
 
 		slabp = virt_to_slab(objp);
+		node = slabp->nodeid;
 		l3 = cachep->nodelists[node];
 		list_del(&slabp->list);
 		check_spinlock_acquired_node(cachep, node);
@@ -3009,7 +3023,7 @@ static void cache_flusharray(struct kmem
 		}
 	}
 
-	free_block(cachep, ac->entry, batchcount, node);
+	free_block(cachep, ac->entry, batchcount);
 free_done:
 #if STATS
 	{
@@ -3063,18 +3077,17 @@ static inline void __cache_free(struct k
 				alien = l3->alien[nodeid];
 				spin_lock(&alien->lock);
 				if (unlikely(alien->avail == alien->limit))
-					__drain_alien_cache(cachep,
-							    alien, nodeid);
+					__drain_array(cachep, l3, alien);
 				alien->entry[alien->avail++] = objp;
 				spin_unlock(&alien->lock);
 			} else {
-				spin_lock(&(cachep->nodelists[nodeid])->
-					  list_lock);
-				free_block(cachep, &objp, 1, nodeid);
-				spin_unlock(&(cachep->nodelists[nodeid])->
-					    list_lock);
+				struct kmem_list3 *rl3 = cachep->nodelists[nodeid];
+
+				spin_lock(&rl3->list_lock);
+				free_block(cachep, &objp, 1);
+				spin_unlock(&rl3->list_lock);
+				return;
 			}
-			return;
 		}
 	}
 #endif
@@ -3402,7 +3415,7 @@ static int alloc_kmemlist(struct kmem_ca
 
 			nc = cachep->nodelists[node]->shared;
 			if (nc)
-				free_block(cachep, nc->entry, nc->avail, node);
+				free_block(cachep, nc->entry, nc->avail);
 
 			l3->shared = new;
 			if (!cachep->nodelists[node]->alien) {
@@ -3480,11 +3493,12 @@ static int do_tune_cpucache(struct kmem_
 
 	for_each_online_cpu(i) {
 		struct array_cache *ccold = new.new[i];
+		int node = cpu_to_node(i);
+
 		if (!ccold)
 			continue;
-		spin_lock_irq(&cachep->nodelists[cpu_to_node(i)]->list_lock);
-		free_block(cachep, ccold->entry, ccold->avail, cpu_to_node(i));
-		spin_unlock_irq(&cachep->nodelists[cpu_to_node(i)]->list_lock);
+
+		drain_array_irq(cachep, cachep->nodelists[node], ccold);
 		kfree(ccold);
 	}
 
@@ -3553,26 +3567,28 @@ static void enable_cpucache(struct kmem_
 }
 
 /*
- * Drain an array if it contains any elements taking the l3 lock only if
+ * Drain an array partially if it contains any elements taking the l3 lock only if
  * necessary.
  */
-void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
-			 struct array_cache *ac, int force, int node)
+static void drain_array_partial(struct kmem_cache *cachep, struct kmem_list3 *l3,
+			 struct array_cache *ac)
 {
 	int tofree;
 
-	if (!ac || !ac->avail)
+	if (!ac)
 		return;
 
-	if (ac->touched && !force) {
+	if (ac->touched)
 		ac->touched = 0;
-	} else if (ac->avail) {
-		tofree = force ? ac->avail : (ac->limit + 4) / 5;
+	else if (ac->avail) {
+		tofree = (ac->limit + 4) / 5;
 		if (tofree > ac->avail)
 			tofree = (ac->avail + 1) / 2;
+
 		spin_lock_irq(&l3->list_lock);
-		free_block(cachep, ac->entry, tofree, node);
+		free_block(cachep, ac->entry, tofree);
 		spin_unlock_irq(&l3->list_lock);
+
 		ac->avail -= tofree;
 		memmove(ac->entry, &(ac->entry[tofree]),
 			sizeof(void *) * ac->avail);
@@ -3622,7 +3638,7 @@ static void cache_reap(void *unused)
 
 		reap_alien(searchp, l3);
 
-		drain_array(searchp, l3, cpu_cache_get(searchp), 0, node);
+		drain_array_partial(searchp, l3, cpu_cache_get(searchp));
 
 		/*
 		 * These are racy checks but it does not matter
@@ -3633,7 +3649,7 @@ static void cache_reap(void *unused)
 
 		l3->next_reap = jiffies + REAPTIMEOUT_LIST3;
 
-		drain_array(searchp, l3, l3->shared, 0, node);
+		drain_array_partial(searchp, l3, l3->shared);
 
 		if (l3->free_touched) {
 			l3->free_touched = 0;