From: Christoph Lameter <clameter@sgi.com>
Subject: NUMA Quicklists: Fix off node freeing behavior

[NUMA] Quicklists: Do not free off node pages before the TLB flush has been done

The current quicklist code can free a page before the TLB flush has been
done which is wrong.

Delay the quicklist freeing for off node pages until the trimming function is
called. At that point we know that it is okay to free pages.

Signed-off-by: Christoph Lameter <clameter@sgi.com>

---
 include/linux/quicklist.h |   23 +++++++++++++----------
 mm/quicklist.c            |   29 +++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 10 deletions(-)

Index: mm/include/linux/quicklist.h
===================================================================
--- mm.orig/include/linux/quicklist.h	2007-11-16 21:16:36.000000000 -0800
+++ mm/include/linux/quicklist.h	2007-11-29 13:00:44.695562556 -0800
@@ -15,6 +15,9 @@
 
 struct quicklist {
 	void *page;
+#ifdef CONFIG_NUMA
+	void *offnode;
+#endif
 	int nr_pages;
 };
 
@@ -56,18 +59,18 @@ static inline void __quicklist_free(int 
 	struct page *page)
 {
 	struct quicklist *q;
-	int nid = page_to_nid(page);
-
-	if (unlikely(nid != numa_node_id())) {
-		if (dtor)
-			dtor(p);
-		__free_page(page);
-		return;
-	}
 
 	q = &get_cpu_var(quicklist)[nr];
-	*(void **)p = q->page;
-	q->page = p;
+#ifdef CONFIG_NUMA
+	if (page_to_nid(page) != numa_node_id()) {
+	 	*(void **)p = q->offnode;
+		q->offnode = p;
+	} else
+#endif
+	{
+	 	*(void **)p = q->page;
+		q->page = p;
+	}
 	q->nr_pages++;
 	put_cpu_var(quicklist);
 }
Index: mm/mm/quicklist.c
===================================================================
--- mm.orig/mm/quicklist.c	2007-11-16 21:16:36.000000000 -0800
+++ mm/mm/quicklist.c	2007-11-29 13:00:44.750812452 -0800
@@ -44,6 +44,34 @@ static long min_pages_to_free(struct qui
 }
 
 /*
+ * Drop off node pages from the quicklists
+ */
+static inline void free_offnode_pages(struct quicklist *q,
+			int min_pages, int max_free, void (*dtor)(void *))
+{
+#ifdef CONFIG_NUMA
+	void **p;
+	void **prev = NULL;
+
+	if (q->nr_pages <= min_pages)
+		return;
+
+	for (p = q->offnode; p && max_free; p = *p) {
+		if (prev)
+			*prev = *p;
+		else
+			q->page = *p;
+
+		if (dtor)
+			dtor(p);
+		free_page((unsigned long)p);
+		q->nr_pages--;
+		max_free--;
+	}
+#endif
+}
+
+/*
  * Trim down the number of pages in the quicklist
  */
 void quicklist_trim(int nr, void (*dtor)(void *),
@@ -53,6 +81,7 @@ void quicklist_trim(int nr, void (*dtor)
 	struct quicklist *q;
 
 	q = &get_cpu_var(quicklist)[nr];
+	free_offnode_pages(q, min_pages, max_free, dtor);
 	if (q->nr_pages > min_pages) {
 		pages_to_free = min_pages_to_free(q, min_pages, max_free);