ZVC: Scale thresholds depending on the size of the system The ZVC counter update threshold is currently set to a fixed value of 32. This patch sets up the threshold depending on the number of processors and the sizes of the zones in the system. With the current threshold of 32, I was only able to observe slight contention when more than 130-140 processors concurrently updated the counters. The contention vanished when I either increased the threshold to 64 or used Andrew's idea of overstepping the interval (see next patch). However, we saw contention again at 220-230 processors. So we need a higher value for larger systems. But the current default is a bit of an overkill for smaller systems. A lesser threshold means less counter update deferrals and therefore the counters are more precise. Small (this patch only affects SMP) systems may not have gobs of memory and so the additional precision may be useful. Some systems have tiny zones where precision matters. For example i386 and x86_64 have 16M DMA zones and either 900M ZONE_NORMAL or ZONE_DMA32. The patch here sets up a threshold based on the number of processors in the system and the size of the zone that these counters are used for. The threshold should grow logarithmically, so we use fls() as an easy approximation. On a system with 16 processors and a threshold of 32 we currently defer updates for 16*32 pages which covers 2M (1/8th counter variance). Such precious memory needs to be frugally managed and it is unlikely to be touched frequently. This patch reduces that threshold to 10. With 16 processors we defer at maximum 16*10 = 640k. In the typical dual processor case we use a threshold of 4 for a maximum counter variance of 16k. For the 900M i386/x86_64 zone this scheme will yield a threshold of 16 for a dual processor configuration. Maximum variance is 128k then. For the 16 processor case we reach a threshold of 32 yielding a maximum counter variance of 2Mb. Results of tests on a system with 1024 processors (4TB Ram) The following output is from a test allocating 1GB of memory concurrently on each processor (Forking the process. So contention on mmap_sem and the pte locks is not a factor): X MIN TYPE: CPUS WALL WALL SYS USER TOTCPU fork 1 0.552 0.552 0.540 0.012 0.552 fork 4 0.552 0.548 2.164 0.036 2.200 fork 16 0.564 0.548 8.812 0.164 8.976 fork 128 0.580 0.572 72.204 1.208 73.412 fork 256 1.300 0.660 310.400 2.160 312.560 fork 512 3.512 0.696 1526.836 4.816 1531.652 fork 1020 20.024 0.700 17243.176 6.688 17249.863 So a threshold of 32 scales is fine up to 128 processors. At 256 processors contention becomes a factor. Overstepping the counter (earlier patch) improves the numbers a bit: fork 4 0.552 0.548 2.164 0.040 2.204 fork 16 0.552 0.548 8.640 0.148 8.788 fork 128 0.556 0.548 69.676 0.956 70.632 fork 256 0.876 0.636 212.468 2.108 214.576 fork 512 2.276 0.672 997.324 4.260 1001.584 fork 1020 13.564 0.680 11586.436 6.088 11592.523 Still contention at 512 and 1020. 1020 contention is down by a third. 256 still has a slight bit of contention. After this patch we get: Signed-off-by: Christoph Lameter Index: linux-2.6.18-rc4/mm/vmstat.c =================================================================== --- linux-2.6.18-rc4.orig/mm/vmstat.c 2006-08-18 20:33:34.259428315 -0700 +++ linux-2.6.18-rc4/mm/vmstat.c 2006-08-19 14:39:16.505041889 -0700 @@ -12,6 +12,7 @@ #include #include #include +#include void __get_zone_counts(unsigned long *active, unsigned long *inactive, unsigned long *free, struct pglist_data *pgdat) @@ -114,17 +115,67 @@ EXPORT_SYMBOL(vm_stat); #ifdef CONFIG_SMP -#define STAT_THRESHOLD 32 +static int calculate_threshold(struct zone *zone) +{ + int threshold; + int mem; /* memory in 128 MB units */ + + /* + * The threshold scales with the number of processors and the amount + * of memory per zone. More memory means that we can defer updates for + * longer, more processors could lead to more contention. + * fls() is used to have a cheap way of logarithmic scaling. + * + * Some sample thresholds: + * + * Threshold Processors (fls) Zonesize fls(mem+1) + * ------------------------------------------------------------------ + * 8 1 1 0.9-1 GB 4 + * 16 2 2 0.9-1 GB 4 + * 20 2 2 1-2 GB 5 + * 24 2 2 2-4 GB 6 + * 28 2 2 4-8 GB 7 + * 32 2 2 8-16 GB 8 + * 4 2 2 <128M 1 + * 30 4 3 2-4 GB 5 + * 48 4 3 8-16 GB 8 + * 32 8 4 1-2 GB 4 + * 32 8 4 0.9-1GB 4 + * 10 16 5 <128M 1 + * 40 16 5 900M 4 + * 70 64 7 2-4 GB 5 + * 84 64 7 4-8 GB 6 + * 108 512 9 4-8 GB 6 + * 125 1024 10 8-16 GB 8 + * 125 1024 10 16-32 GB 9 + */ + + mem = zone->present_pages >> (27 - PAGE_SHIFT); + + threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem)); + + /* + * Maximum threshold is 120 + */ + threshold = min(120, threshold); + + return threshold; +} /* - * Determine pointer to currently valid differential byte given a zone and - * the item number. - * - * Preemption must be off + * Refresh the thresholds for each zone. */ -static inline s8 *diff_pointer(struct zone *zone, enum zone_stat_item item) +static void refresh_zone_stat_thresholds(void) { - return &zone_pcp(zone, smp_processor_id())->vm_stat_diff[item]; + struct zone *zone; + + for_each_zone(zone) { + if (!zone->present_pages) + continue; + + zone->stat_threshold = + calculate_threshold(zone); + } } /* @@ -133,17 +184,16 @@ static inline s8 *diff_pointer(struct zo void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item, int delta) { - s8 *p; + struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id()); + s8 *p = pcp->vm_stat_diff + item; long x; - p = diff_pointer(zone, item); x = delta + *p; - if (unlikely(x > STAT_THRESHOLD || x < -STAT_THRESHOLD)) { + if (unlikely(x > zone->stat_threshold || x < -zone->stat_threshold)) { zone_page_state_add(x, zone, item); x = 0; } - *p = x; } EXPORT_SYMBOL(__mod_zone_page_state); @@ -185,13 +235,16 @@ EXPORT_SYMBOL(mod_zone_page_state); */ static void __inc_zone_state(struct zone *zone, enum zone_stat_item item) { - s8 *p = diff_pointer(zone, item); + struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id()); + s8 *p = pcp->vm_stat_diff + item; (*p)++; - if (unlikely(*p > STAT_THRESHOLD)) { - zone_page_state_add(*p + STAT_THRESHOLD / 2, zone, item); - *p = -STAT_THRESHOLD / 2; + if (unlikely(*p > zone->stat_threshold)) { + int overstep = zone->stat_threshold / 2; + + zone_page_state_add(*p + overstep, zone, item); + *p = -overstep; } } @@ -204,13 +257,16 @@ EXPORT_SYMBOL(__inc_zone_page_state); void __dec_zone_page_state(struct page *page, enum zone_stat_item item) { struct zone *zone = page_zone(page); - s8 *p = diff_pointer(zone, item); + struct per_cpu_pageset *pcp = zone_pcp(zone, smp_processor_id()); + s8 *p = pcp->vm_stat_diff + item; (*p)--; - if (unlikely(*p < -STAT_THRESHOLD)) { - zone_page_state_add(*p - STAT_THRESHOLD / 2, zone, item); - *p = STAT_THRESHOLD /2; + if (unlikely(*p < - zone->stat_threshold)) { + int overstep = zone->stat_threshold / 2; + + zone_page_state_add(*p - overstep, zone, item); + *p = overstep; } } EXPORT_SYMBOL(__dec_zone_page_state); @@ -252,7 +308,7 @@ EXPORT_SYMBOL(dec_zone_page_state); void refresh_cpu_vm_stats(int cpu) { struct zone *zone; - int i; + int i,j; unsigned long flags; for_each_zone(zone) { @@ -263,14 +319,25 @@ void refresh_cpu_vm_stats(int cpu) pcp = zone_pcp(zone, cpu); - for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) - if (pcp->vm_stat_diff[i]) { - local_irq_save(flags); - zone_page_state_add(pcp->vm_stat_diff[i], - zone, i); - pcp->vm_stat_diff[i] = 0; - local_irq_restore(flags); - } + for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i += sizeof(long)) { + /* + * Use a long fetch to check a group of counters + * at a time. This works because the pcp structure + * is cacheline aligned. Extraneous counters at the + * end are always zero. + */ + if (! *(long *)(pcp->vm_stat_diff + i)) + continue; + + local_irq_save(flags); + for (j = i; j < i + sizeof(long); j++) + if (pcp->vm_stat_diff[j]) { + zone_page_state_add(pcp->vm_stat_diff[j], + zone, j); + pcp->vm_stat_diff[j] = 0; + } + local_irq_restore(flags); + } } } @@ -529,7 +596,8 @@ static int zoneinfo_show(struct seq_file zone->temp_priority, zone->zone_start_pfn); spin_unlock_irqrestore(&zone->lock, flags); - seq_putc(m, '\n'); + seq_printf(m, "\n vm stats threshold: %d\n", + zone->stat_threshold); } return 0; } @@ -606,3 +674,34 @@ struct seq_operations vmstat_op = { #endif /* CONFIG_PROC_FS */ +/* + * Use the cpu notifier to insure that the thresholds are recalculated + * when necessary. + */ +static int __cpuinit vmstat_cpuup_callback(struct notifier_block *nfb, + unsigned long action, + void *hcpu) +{ + switch (action) { + case CPU_UP_PREPARE: + case CPU_UP_CANCELED: + case CPU_DEAD: + refresh_zone_stat_thresholds(); + break; + default: + break; + } + return NOTIFY_OK; +} + +static struct notifier_block __cpuinitdata vmstat_notifier = + { &vmstat_cpuup_callback, NULL, 0 }; + +int __init setup_vmstat(void) +{ + refresh_zone_stat_thresholds(); + register_cpu_notifier(&vmstat_notifier); + return 0; +} +module_init(setup_vmstat) + Index: linux-2.6.18-rc4/include/linux/mmzone.h =================================================================== --- linux-2.6.18-rc4.orig/include/linux/mmzone.h 2006-08-06 11:20:11.000000000 -0700 +++ linux-2.6.18-rc4/include/linux/mmzone.h 2006-08-19 14:21:27.597385437 -0700 @@ -47,6 +47,7 @@ struct zone_padding { #endif enum zone_stat_item { + /* First group of 8 counters */ NR_ANON_PAGES, /* Mapped anonymous pages */ NR_FILE_MAPPED, /* pagecache pages mapped into pagetables. only modified from process context */ @@ -56,6 +57,7 @@ enum zone_stat_item { NR_FILE_DIRTY, NR_WRITEBACK, NR_UNSTABLE_NFS, /* NFS unstable pages */ + /* Second group of 7 counters */ NR_BOUNCE, #ifdef CONFIG_NUMA NUMA_HIT, /* allocated in intended node */ @@ -153,7 +155,12 @@ struct zone { /* * zone reclaim becomes active if more unmapped pages exist. */ - unsigned long min_unmapped_ratio; + unsigned int min_unmapped_ratio; + /* + * Maximum per processor diff before global update + */ + unsigned int stat_threshold; + struct per_cpu_pageset *pageset[NR_CPUS]; #else struct per_cpu_pageset pageset[NR_CPUS];