Optimize Index: linux-2.6.18-rc4/mm/vmstat.c =================================================================== --- linux-2.6.18-rc4.orig/mm/vmstat.c 2006-08-18 14:52:03.478999664 -0700 +++ linux-2.6.18-rc4/mm/vmstat.c 2006-08-18 15:14:06.191296910 -0700 @@ -168,17 +168,13 @@ static s8 calculate_threshold(struct zon static void refresh_zone_stat_thresholds(void) { struct zone *zone; - int cpu; - int threshold; for_each_zone(zone) { if (!zone->present_pages) continue; - threshold = calculate_threshold(zone); - - for_each_online_cpu(cpu) - zone_pcp(zone, cpu)->stat_threshold = threshold; + zone->stat_threshold = + calculate_threshold(zone); } } @@ -194,7 +190,7 @@ void __mod_zone_page_state(struct zone * x = delta + *p; - if (unlikely(x > pcp->stat_threshold || x < -pcp->stat_threshold)) { + if (unlikely(x > zone->stat_threshold || x < -zone->stat_threshold)) { zone_page_state_add(x, zone, item); x = 0; } @@ -244,8 +240,8 @@ static void __inc_zone_state(struct zone (*p)++; - if (unlikely(*p > pcp->stat_threshold)) { - int overstep = pcp->stat_threshold / 2; + if (unlikely(*p > zone->stat_threshold)) { + int overstep = zone->stat_threshold / 2; zone_page_state_add(*p + overstep, zone, item); *p = - overstep; @@ -266,8 +262,8 @@ void __dec_zone_page_state(struct page * (*p)--; - if (unlikely(*p < - pcp->stat_threshold)) { - int overstep = pcp->stat_threshold / 2; + if (unlikely(*p < - zone->stat_threshold)) { + int overstep = zone->stat_threshold / 2; zone_page_state_add(*p - overstep, zone, item); *p = overstep; @@ -312,7 +308,7 @@ EXPORT_SYMBOL(dec_zone_page_state); void refresh_cpu_vm_stats(int cpu) { struct zone *zone; - int i; + int i,j; unsigned long flags; for_each_zone(zone) { @@ -320,14 +316,25 @@ void refresh_cpu_vm_stats(int cpu) pcp = zone_pcp(zone, cpu); - for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) - if (pcp->vm_stat_diff[i]) { + for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i += sizeof(long)) { + /* + * Use a long fetch to check a group of counters + * at a time. This works because the pcp structure + * is cacheline aligned. Extraneous counters at the + * end are always zero. + */ + if (! *(long *)(pcp->vm_stat_diff + i)) + continue; + + for(j = i; j < i + sizeof(long); j++) + if (pcp->vm_stat_diff[j]) { local_irq_save(flags); - zone_page_state_add(pcp->vm_stat_diff[i], - zone, i); - pcp->vm_stat_diff[i] = 0; + zone_page_state_add(pcp->vm_stat_diff[j], + zone, j); + pcp->vm_stat_diff[j] = 0; local_irq_restore(flags); } + } } } @@ -575,8 +582,6 @@ static int zoneinfo_show(struct seq_file pageset->pcp[j].high, pageset->pcp[j].batch); } - seq_printf(m, "\n vm stats threshold: %d", - pageset->stat_threshold); } seq_printf(m, "\n all_unreclaimable: %u" @@ -588,7 +593,8 @@ static int zoneinfo_show(struct seq_file zone->temp_priority, zone->zone_start_pfn); spin_unlock_irqrestore(&zone->lock, flags); - seq_putc(m, '\n'); + seq_printf(m, "\n vm stats threshold: %ld\n", + zone->stat_threshold); } return 0; } Index: linux-2.6.18-rc4/include/linux/mmzone.h =================================================================== --- linux-2.6.18-rc4.orig/include/linux/mmzone.h 2006-08-18 14:52:03.479976166 -0700 +++ linux-2.6.18-rc4/include/linux/mmzone.h 2006-08-18 15:26:11.950215397 -0700 @@ -77,7 +77,6 @@ struct per_cpu_pages { struct per_cpu_pageset { struct per_cpu_pages pcp[2]; /* 0: hot. 1: cold */ #ifdef CONFIG_SMP - s8 stat_threshold; /* maximum diff before update */ s8 vm_stat_diff[NR_VM_ZONE_STAT_ITEMS]; #endif } ____cacheline_aligned_in_smp; @@ -154,7 +153,12 @@ struct zone { /* * zone reclaim becomes active if more unmapped pages exist. */ - unsigned long min_unmapped_ratio; + unsigned int min_unmapped_ratio; + /* + * Maximum per processor diff before global update + */ + unsigned int stat_threshold; + struct per_cpu_pageset *pageset[NR_CPUS]; #else struct per_cpu_pageset pageset[NR_CPUS];