slab: Reorganize draining of arrays

With the latest chance to cache reap the draining functions now
line up nicely.

1. Provide a generic drain_array that only takes the lock
   if necessary to drain a cache and that also will obtain
   the necessary l3 lock for free. There is no need for any
   function to obtain the l3 lock separately anymore.

2. drain_array_locked now means obtain the lock for the respective
   array_cache not the lock for the l3 structure.

3. We can drop the node parameter from free_block since the nodeid
   can be determined from the slabp.

Signed-off-by: Christoph Lameter <clameter@sgi.com>

Index: linux-2.6.16-rc5-mm2/mm/slab.c
===================================================================
--- linux-2.6.16-rc5-mm2.orig/mm/slab.c	2006-03-03 15:37:39.000000000 -0800
+++ linux-2.6.16-rc5-mm2/mm/slab.c	2006-03-03 15:57:19.000000000 -0800
@@ -701,12 +701,20 @@ static enum {
 
 static DEFINE_PER_CPU(struct work_struct, reap_work);
 
-static void free_block(struct kmem_cache *cachep, void **objpp, int len,
-			int node);
+static void free_block(struct kmem_cache *cachep, void **objpp, int len);
 static void enable_cpucache(struct kmem_cache *cachep);
 static void cache_reap(void *unused);
 static int __node_shrink(struct kmem_cache *cachep, int node);
 
+static void free_block_locked(struct kmem_cache *cachep,
+		struct kmem_list3 *l3, void **objs, int nr)
+{
+	spin_lock_irq(&l3->list_lock);
+	free_block(cachep, objs, nr);
+	spin_unlock_irq(&l3->list_lock);
+}
+
+
 static inline struct array_cache *cpu_cache_get(struct kmem_cache *cachep)
 {
 	return cachep->array[smp_processor_id()];
@@ -942,16 +950,20 @@ static void free_alien_cache(struct arra
 	kfree(ac_ptr);
 }
 
-static void __drain_alien_cache(struct kmem_cache *cachep,
-				struct array_cache *ac, int node)
+static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
+			struct array_cache *ac, int force);
+
+static void drain_array_locked(struct kmem_cache *cachep,
+			struct kmem_list3 *l3,
+			struct array_cache *ac,
+			int force)
 {
-	struct kmem_list3 *rl3 = cachep->nodelists[node];
+	unsigned long flags;
 
-	if (ac->avail) {
-		spin_lock(&rl3->list_lock);
-		free_block(cachep, ac->entry, ac->avail, node);
-		ac->avail = 0;
-		spin_unlock(&rl3->list_lock);
+	if (ac && ac->avail) {
+		spin_lock_irqsave(&ac->lock, flags);
+		drain_array(cachep, l3, ac, 0);
+		spin_unlock_irqrestore(&ac->lock, flags);
 	}
 }
 
@@ -962,35 +974,23 @@ static void reap_alien(struct kmem_cache
 {
 	int node = __get_cpu_var(reap_node);
 
-	if (l3->alien) {
-		struct array_cache *ac = l3->alien[node];
-		if (ac && ac->avail) {
-			spin_lock_irq(&ac->lock);
-			__drain_alien_cache(cachep, ac, node);
-			spin_unlock_irq(&ac->lock);
-		}
-	}
+	drain_array_locked(cachep, l3, l3->alien[node], 0);
 }
 
 static void drain_alien_cache(struct kmem_cache *cachep,
 				struct array_cache **alien)
 {
-	int i = 0;
-	struct array_cache *ac;
-	unsigned long flags;
+	int i;
+	struct kmem_list3 *l3 = cachep->nodelists[numa_node_id()];
 
-	for_each_online_node(i) {
-		ac = alien[i];
-		if (ac) {
-			spin_lock_irqsave(&ac->lock, flags);
-			__drain_alien_cache(cachep, ac, i);
-			spin_unlock_irqrestore(&ac->lock, flags);
-		}
-	}
+	if (!l3->alien)
+		return;
+
+	for_each_online_node(i)
+		drain_array_locked(cachep, l3, alien[i], 1);
 }
 #else
 
-#define drain_alien_cache(cachep, alien) do { } while (0)
 #define reap_alien(cachep, l3) do { } while (0)
 
 static inline struct array_cache **alloc_alien_cache(int node, int limit)
@@ -1135,7 +1135,7 @@ static int __devinit cpuup_callback(stru
 			/* Free limit for this kmem_list3 */
 			l3->free_limit -= cachep->batchcount;
 			if (nc)
-				free_block(cachep, nc->entry, nc->avail, node);
+				free_block(cachep, nc->entry, nc->avail);
 
 			if (!cpus_empty(mask)) {
 				spin_unlock_irq(&l3->list_lock);
@@ -1145,7 +1145,7 @@ static int __devinit cpuup_callback(stru
 			shared = l3->shared;
 			if (shared) {
 				free_block(cachep, l3->shared->entry,
-					   l3->shared->avail, node);
+					   l3->shared->avail);
 				l3->shared = NULL;
 			}
 
@@ -1155,10 +1155,8 @@ static int __devinit cpuup_callback(stru
 			spin_unlock_irq(&l3->list_lock);
 
 			kfree(shared);
-			if (alien) {
-				drain_alien_cache(cachep, alien);
-				free_alien_cache(alien);
-			}
+			drain_alien_cache(cachep, alien);
+			free_alien_cache(alien);
 free_array_cache:
 			kfree(nc);
 		}
@@ -2125,21 +2123,13 @@ static void check_spinlock_acquired_node
 #define check_spinlock_acquired_node(x, y) do { } while(0)
 #endif
 
-static void drain_array_locked(struct kmem_cache *cachep,
-			struct array_cache *ac, int force, int node);
-
 static void do_drain(void *arg)
 {
 	struct kmem_cache *cachep = arg;
-	struct array_cache *ac;
-	int node = numa_node_id();
 
 	check_irq_off();
-	ac = cpu_cache_get(cachep);
-	spin_lock(&cachep->nodelists[node]->list_lock);
-	free_block(cachep, ac->entry, ac->avail, node);
-	spin_unlock(&cachep->nodelists[node]->list_lock);
-	ac->avail = 0;
+	drain_array(cachep, cachep->nodelists[numa_node_id()],
+			 cpu_cache_get(cachep), 1);
 }
 
 static void drain_cpu_caches(struct kmem_cache *cachep)
@@ -2152,11 +2142,8 @@ static void drain_cpu_caches(struct kmem
 	for_each_online_node(node) {
 		l3 = cachep->nodelists[node];
 		if (l3) {
-			spin_lock_irq(&l3->list_lock);
-			drain_array_locked(cachep, l3->shared, 1, node);
-			spin_unlock_irq(&l3->list_lock);
-			if (l3->alien)
-				drain_alien_cache(cachep, l3->alien);
+			drain_array(cachep, l3, l3->shared, 1);
+			drain_alien_cache(cachep, l3->alien);
 		}
 	}
 }
@@ -2946,8 +2933,7 @@ done:
 /*
  * Caller needs to acquire correct kmem_list's list_lock
  */
-static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects,
-		       int node)
+static void free_block(struct kmem_cache *cachep, void **objpp, int nr_objects)
 {
 	int i;
 	struct kmem_list3 *l3;
@@ -2955,8 +2941,10 @@ static void free_block(struct kmem_cache
 	for (i = 0; i < nr_objects; i++) {
 		void *objp = objpp[i];
 		struct slab *slabp;
+		int node;
 
 		slabp = virt_to_slab(objp);
+		node = slabp->nodeid;
 		l3 = cachep->nodelists[node];
 		list_del(&slabp->list);
 		check_spinlock_acquired_node(cachep, node);
@@ -3010,7 +2998,7 @@ static void cache_flusharray(struct kmem
 		}
 	}
 
-	free_block(cachep, ac->entry, batchcount, node);
+	free_block(cachep, ac->entry, batchcount);
 free_done:
 #if STATS
 	{
@@ -3064,17 +3052,13 @@ static inline void __cache_free(struct k
 				alien = l3->alien[nodeid];
 				spin_lock(&alien->lock);
 				if (unlikely(alien->avail == alien->limit))
-					__drain_alien_cache(cachep,
-							    alien, nodeid);
+					drain_array(cachep, l3, alien, 0);
 				alien->entry[alien->avail++] = objp;
 				spin_unlock(&alien->lock);
-			} else {
-				spin_lock(&(cachep->nodelists[nodeid])->
-					  list_lock);
-				free_block(cachep, &objp, 1, nodeid);
-				spin_unlock(&(cachep->nodelists[nodeid])->
-					    list_lock);
-			}
+			} else
+				free_block_locked(cachep,
+					cachep->nodelists[nodeid],
+					&objp, 1);
 			return;
 		}
 	}
@@ -3403,7 +3387,7 @@ static int alloc_kmemlist(struct kmem_ca
 
 			nc = cachep->nodelists[node]->shared;
 			if (nc)
-				free_block(cachep, nc->entry, nc->avail, node);
+				free_block(cachep, nc->entry, nc->avail);
 
 			l3->shared = new;
 			if (!cachep->nodelists[node]->alien) {
@@ -3484,7 +3468,7 @@ static int do_tune_cpucache(struct kmem_
 		if (!ccold)
 			continue;
 		spin_lock_irq(&cachep->nodelists[cpu_to_node(i)]->list_lock);
-		free_block(cachep, ccold->entry, ccold->avail, cpu_to_node(i));
+		free_block(cachep, ccold->entry, ccold->avail);
 		spin_unlock_irq(&cachep->nodelists[cpu_to_node(i)]->list_lock);
 		kfree(ccold);
 	}
@@ -3553,41 +3537,28 @@ static void enable_cpucache(struct kmem_
 		       cachep->name, -err);
 }
 
-static void drain_array_locked(struct kmem_cache *cachep,
-				struct array_cache *ac, int force, int node)
+static void drain_array(struct kmem_cache *cachep, struct kmem_list3 *l3,
+			struct array_cache *ac, int force)
 {
 	int tofree;
 
-	check_spinlock_acquired_node(cachep, node);
+	/* Skip draining if the array has no elements */
+	if (!ac || !ac->avail)
+		return;
+
 	if (ac->touched && !force) {
 		ac->touched = 0;
 	} else if (ac->avail) {
 		tofree = force ? ac->avail : (ac->limit + 4) / 5;
 		if (tofree > ac->avail)
 			tofree = (ac->avail + 1) / 2;
-		free_block(cachep, ac->entry, tofree, node);
+		free_block_locked(cachep, l3, ac->entry, tofree);
 		ac->avail -= tofree;
 		memmove(ac->entry, &(ac->entry[tofree]),
 			sizeof(void *) * ac->avail);
 	}
 }
 
-
-/*
- * Drain an array if it contains any elements taking the l3 lock only if
- * necessary.
- */
-void drain_array(struct kmem_cache *searchp, struct kmem_list3 *l3,
-					 struct array_cache *ac)
-{
-	if (ac && ac->avail) {
-		spin_lock_irq(&l3->list_lock);
-		drain_array_locked(searchp, ac, 0,
-				   numa_node_id());
-		spin_unlock_irq(&l3->list_lock);
-	}
-}
-
 /**
  * cache_reap - Reclaim memory from caches.
  * @unused: unused parameter
@@ -3630,7 +3601,7 @@ static void cache_reap(void *unused)
 
 		reap_alien(searchp, l3);
 
-		drain_array(searchp, l3, cpu_cache_get(searchp));
+		drain_array(searchp, l3, cpu_cache_get(searchp), 0);
 
 		/*
 		 * These are racy checks but it does not matter
@@ -3641,7 +3612,7 @@ static void cache_reap(void *unused)
 
 		l3->next_reap = jiffies + REAPTIMEOUT_LIST3;
 
-		drain_array(searchp, l3, l3->shared);
+		drain_array(searchp, l3, l3->shared, 0);
 
 		if (l3->free_touched) {
 			l3->free_touched = 0;