---
 mm/vmstat.c |   95 +++++++++++++++++++++++++++++++++++++++---------------------
 1 file changed, 63 insertions(+), 32 deletions(-)

Index: linux-2.6.22-rc6-mm1/mm/vmstat.c
===================================================================
--- linux-2.6.22-rc6-mm1.orig/mm/vmstat.c	2007-07-12 19:46:02.000000000 -0700
+++ linux-2.6.22-rc6-mm1/mm/vmstat.c	2007-07-12 19:49:50.000000000 -0700
@@ -159,15 +159,25 @@ void __mod_zone_page_state(struct zone *
 {
 	struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id());
 	s8 *p = pcp->vm_stat_diff + item;
-	long x;
+	s8 o;
+	unsigned long n;
+	unsigned long add;
 
-	x = delta + *p;
+redo:
+	add = 0;
+	o = *p;
+	n = delta + *p;
 
-	if (unlikely(x > pcp->stat_threshold || x < -pcp->stat_threshold)) {
-		zone_page_state_add(x, zone, item);
-		x = 0;
+	if (unlikely(n > pcp->stat_threshold || n < -pcp->stat_threshold)) {
+		add = n;
+		n = 0;
 	}
-	*p = x;
+
+	if (cmpxchg_local(p, o, n) != o)
+		goto redo;
+
+	if (add)
+		zone_page_state_add(add, zone, item);
 }
 EXPORT_SYMBOL(__mod_zone_page_state);
 
@@ -179,9 +189,9 @@ void mod_zone_page_state(struct zone *zo
 {
 	unsigned long flags;
 
-	local_irq_save(flags);
+	__local_begin(flags);
 	__mod_zone_page_state(zone, item, delta);
-	local_irq_restore(flags);
+	__local_end(flags);
 }
 EXPORT_SYMBOL(mod_zone_page_state);
 
@@ -212,15 +222,25 @@ void __inc_zone_state(struct zone *zone,
 {
 	struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id());
 	s8 *p = pcp->vm_stat_diff + item;
+	int overstep;
+	s8 o,n;
 
-	(*p)++;
-
-	if (unlikely(*p > pcp->stat_threshold)) {
-		int overstep = pcp->stat_threshold / 2;
+redo:
+	overstep = 0;
+	o = *p;
+	n = o + 1;
 
-		zone_page_state_add(*p + overstep, zone, item);
-		*p = -overstep;
+	if (unlikely(n > pcp->stat_threshold)) {
+		overstep = pcp->stat_threshold / 2;
+		n = -overstep;
 	}
+
+	if (cmpxchg_local(p, o, n) != o)
+		goto redo;
+
+	if (overstep)
+		zone_page_state_add(o + 1 + overstep,
+						zone, item);
 }
 
 void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
@@ -233,15 +253,25 @@ void __dec_zone_state(struct zone *zone,
 {
 	struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id());
 	s8 *p = pcp->vm_stat_diff + item;
+	int overstep;
+	s8 o,n;
 
-	(*p)--;
-
-	if (unlikely(*p < - pcp->stat_threshold)) {
-		int overstep = pcp->stat_threshold / 2;
+redo:
+	overstep = 0;
+	o = *p;
+	n = o - 1;
 
-		zone_page_state_add(*p - overstep, zone, item);
-		*p = overstep;
+	if (unlikely(n < - pcp->stat_threshold)) {
+		overstep = pcp->stat_threshold / 2;
+		n = overstep;
 	}
+
+	if (cmpxchg_local(p, o, n) != 0)
+		goto redo;
+
+	if (overstep)
+		zone_page_state_add(o - 1 - overstep,
+						zone, item);
 }
 
 void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
@@ -254,20 +284,20 @@ void inc_zone_state(struct zone *zone, e
 {
 	unsigned long flags;
 
-	local_irq_save(flags);
+	__local_begin(flags);
 	__inc_zone_state(zone, item);
-	local_irq_restore(flags);
+	__local_end(flags);
 }
 
 void inc_zone_page_state(struct page *page, enum zone_stat_item item)
 {
-	unsigned long flags;
 	struct zone *zone;
+	unsigned long flags;
 
 	zone = page_zone(page);
-	local_irq_save(flags);
+	__local_begin(flags);
 	__inc_zone_state(zone, item);
-	local_irq_restore(flags);
+	__local_end(flags);
 }
 EXPORT_SYMBOL(inc_zone_page_state);
 
@@ -275,9 +305,9 @@ void dec_zone_page_state(struct page *pa
 {
 	unsigned long flags;
 
-	local_irq_save(flags);
+	__local_begin(flags);
 	__dec_zone_page_state(page, item);
-	local_irq_restore(flags);
+	__local_end(flags);
 }
 EXPORT_SYMBOL(dec_zone_page_state);
 
@@ -311,15 +341,16 @@ void refresh_cpu_vm_stats(int cpu)
 
 		for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
 			if (p->vm_stat_diff[i]) {
-				local_irq_save(flags);
-				zone_page_state_add(p->vm_stat_diff[i],
-					zone, i);
-				p->vm_stat_diff[i] = 0;
+				int diff;
+
+				__local_begin(flags);
+				diff = __local_xchg(&p->vm_stat_diff[i], 0);
+				zone_page_state_add(diff, zone, i);
 #ifdef CONFIG_NUMA
 				/* 3 seconds idle till flush */
 				p->expire = 3;
 #endif
-				local_irq_restore(flags);
+				__local_end(flags);
 			}
 #ifdef CONFIG_NUMA
 		/*