percpu_counter: new function percpu_counter_sum_and_set From: Mingming Cao Delayed allocation need to check free blocks at every write time. percpu_counter_read_positive() is not quit accurate. delayed allocation need a more accurate accounting, but using percpu_counter_sum_positive() is frequently is quite expensive. This patch added a new function to update center counter when sum per-cpu counter, to increase the accurate rate for next percpu_counter_read() and require less calling expensive percpu_counter_sum(). Signed-off-by: Mingming Cao Signed-off-by: "Theodore Ts'o" --- fs/ext4/balloc.c | 2 +- include/linux/percpu_counter.h | 12 +++++++++--- lib/percpu_counter.c | 7 ++++++- 3 files changed, 16 insertions(+), 5 deletions(-) Index: linux-2.6.26-rc9/fs/ext4/balloc.c =================================================================== --- linux-2.6.26-rc9.orig/fs/ext4/balloc.c 2008-07-11 16:05:06.000000000 -0700 +++ linux-2.6.26-rc9/fs/ext4/balloc.c 2008-07-11 16:05:12.000000000 -0700 @@ -1621,7 +1621,7 @@ ext4_fsblk_t ext4_has_free_blocks(struct #ifdef CONFIG_SMP if (free_blocks - root_blocks < FBC_BATCH) free_blocks = - percpu_counter_sum_positive(&sbi->s_freeblocks_counter); + percpu_counter_sum_and_set(&sbi->s_freeblocks_counter); #endif if (free_blocks - root_blocks < nblocks) return free_blocks - root_blocks; Index: linux-2.6.26-rc9/include/linux/percpu_counter.h =================================================================== --- linux-2.6.26-rc9.orig/include/linux/percpu_counter.h 2008-07-11 16:04:27.000000000 -0700 +++ linux-2.6.26-rc9/include/linux/percpu_counter.h 2008-07-11 16:05:12.000000000 -0700 @@ -35,7 +35,7 @@ int percpu_counter_init_irq(struct percp void percpu_counter_destroy(struct percpu_counter *fbc); void percpu_counter_set(struct percpu_counter *fbc, s64 amount); void __percpu_counter_add(struct percpu_counter *fbc, s64 amount, s32 batch); -s64 __percpu_counter_sum(struct percpu_counter *fbc); +s64 __percpu_counter_sum(struct percpu_counter *fbc, int set); static inline void percpu_counter_add(struct percpu_counter *fbc, s64 amount) { @@ -44,13 +44,19 @@ static inline void percpu_counter_add(st static inline s64 percpu_counter_sum_positive(struct percpu_counter *fbc) { - s64 ret = __percpu_counter_sum(fbc); + s64 ret = __percpu_counter_sum(fbc, 0); return ret < 0 ? 0 : ret; } +static inline s64 percpu_counter_sum_and_set(struct percpu_counter *fbc) +{ + return __percpu_counter_sum(fbc, 1); +} + + static inline s64 percpu_counter_sum(struct percpu_counter *fbc) { - return __percpu_counter_sum(fbc); + return __percpu_counter_sum(fbc, 0); } static inline s64 percpu_counter_read(struct percpu_counter *fbc) Index: linux-2.6.26-rc9/lib/percpu_counter.c =================================================================== --- linux-2.6.26-rc9.orig/lib/percpu_counter.c 2008-07-11 16:04:27.000000000 -0700 +++ linux-2.6.26-rc9/lib/percpu_counter.c 2008-07-11 16:05:12.000000000 -0700 @@ -52,7 +52,7 @@ EXPORT_SYMBOL(__percpu_counter_add); * Add up all the per-cpu counts, return the result. This is a more accurate * but much slower version of percpu_counter_read_positive() */ -s64 __percpu_counter_sum(struct percpu_counter *fbc) +s64 __percpu_counter_sum(struct percpu_counter *fbc, int set) { s64 ret; int cpu; @@ -62,7 +62,12 @@ s64 __percpu_counter_sum(struct percpu_c for_each_online_cpu(cpu) { s32 *pcount = per_cpu_ptr(fbc->counters, cpu); ret += *pcount; + if (set) + *pcount = 0; } + if (set) + fbc->count = ret; + spin_unlock(&fbc->lock); return ret; }