From: Daniel Walker -ENOCHANGELOG! Creates /proc/likely_prof. [randy.dunlap@oracle.com: profile_likely: export do_check_likely] Signed-off-by: Daniel Walker Signed-off-by: Hua Zhong Cc: Andi Kleen DESC profile-likely-unlikely-macros-fix EDESC From: Andrew Morton Cc: Andi Kleen Cc: Daniel Walker Cc: Hua Zhong Cc: Valdis.Kletnieks@vt.edu Signed-off-by: Andrew Morton --- arch/x86/kernel/vsyscall_64.c | 2 arch/x86/vdso/vclock_gettime.c | 2 include/linux/compiler.h | 36 +++++++ lib/Kconfig.debug | 8 + lib/Makefile | 2 lib/likely_prof.c | 146 +++++++++++++++++++++++++++++++ 6 files changed, 196 insertions(+) diff -puN arch/x86/kernel/vsyscall_64.c~profile-likely-unlikely-macros arch/x86/kernel/vsyscall_64.c --- a/arch/x86/kernel/vsyscall_64.c~profile-likely-unlikely-macros +++ a/arch/x86/kernel/vsyscall_64.c @@ -17,6 +17,8 @@ * want per guest time just set the kernel.vsyscall64 sysctl to 0. */ +#define SUPPRESS_LIKELY_PROFILING + #include #include #include diff -puN include/linux/compiler.h~profile-likely-unlikely-macros include/linux/compiler.h --- a/include/linux/compiler.h~profile-likely-unlikely-macros +++ a/include/linux/compiler.h @@ -51,6 +51,41 @@ extern void __chk_io_ptr(const volatile # include #endif +#if defined(CONFIG_PROFILE_LIKELY) && !defined(SUPPRESS_LIKELY_PROFILING) && \ + !(defined(CONFIG_MODULE_UNLOAD) && defined(MODULE)) +struct likeliness { + const char *func; + char *file; + int line; + int type; + unsigned int count[2]; + struct likeliness *next; +}; + +extern int do_check_likely(struct likeliness *likeliness, int exp); + +#define LP_UNSEEN 4 + +#define __check_likely(exp, is_likely) \ + ({ \ + static struct likeliness likeliness = { \ + .func = __func__, \ + .file = __FILE__, \ + .line = __LINE__, \ + .type = is_likely | LP_UNSEEN, \ + }; \ + do_check_likely(&likeliness, !!(exp)); \ + }) + +/* + * We check for constant values with __builtin_constant_p() since + * it's not interesting to profile them, and there is a compiler + * bug in gcc 3.x which blows up during constant evalution when + * CONFIG_PROFILE_LIKELY is turned on. + */ +#define likely(x) (__builtin_constant_p(x) ? (!!(x)) : __check_likely((x), 1)) +#define unlikely(x) (__builtin_constant_p(x) ? (!!(x)) : __check_likely((x), 0)) +#else /* * Generic compiler-dependent macros required for kernel * build go below this comment. Actual compiler/compiler version @@ -59,6 +94,7 @@ extern void __chk_io_ptr(const volatile #define likely(x) __builtin_expect(!!(x), 1) #define unlikely(x) __builtin_expect(!!(x), 0) +#endif /* Optimization barrier */ #ifndef barrier diff -puN lib/Kconfig.debug~profile-likely-unlikely-macros lib/Kconfig.debug --- a/lib/Kconfig.debug~profile-likely-unlikely-macros +++ a/lib/Kconfig.debug @@ -559,6 +559,14 @@ config DEBUG_SYNCHRO_TEST See Documentation/synchro-test.txt. +config PROFILE_LIKELY + bool "Record return values from likely/unlikely macros" + default n + help + Adds profiling on likely/unlikly macros . To see the + results of the profiling you can view the following, + /proc/likely_prof + config BOOT_PRINTK_DELAY bool "Delay each boot printk message by N milliseconds" depends on DEBUG_KERNEL && PRINTK && GENERIC_CALIBRATE_DELAY diff -puN lib/Makefile~profile-likely-unlikely-macros lib/Makefile --- a/lib/Makefile~profile-likely-unlikely-macros +++ a/lib/Makefile @@ -83,6 +83,8 @@ obj-$(CONFIG_HAVE_LMB) += lmb.o obj-$(CONFIG_TRACE) += trace.o +obj-$(CONFIG_PROFILE_LIKELY) += likely_prof.o + hostprogs-y := gen_crc32table clean-files := crc32table.h diff -puN /dev/null lib/likely_prof.c --- /dev/null +++ a/lib/likely_prof.c @@ -0,0 +1,146 @@ +/* + * This code should enable profiling the likely and unlikely macros. + * + * Output goes in /proc/likely_prof + * + * Authors: + * Daniel Walker + * Hua Zhong + * Andrew Morton + */ + +#include +#include +#include +#include +#include +#include + +#include +#include + +static struct likeliness *likeliness_head; + +int do_check_likely(struct likeliness *likeliness, int ret) +{ + static unsigned long likely_lock; + + if (ret) + likeliness->count[1]++; + else + likeliness->count[0]++; + + if (likeliness->type & LP_UNSEEN) { + /* + * We don't simple use a spinlock because internally to the + * spinlock there is a call to unlikely which causes recursion. + * We opted for this method because we didn't need a preempt/irq + * disable and it was a bit cleaner then using internal __raw + * spinlock calls. + */ + if (!test_and_set_bit(0, &likely_lock)) { + if (likeliness->type & LP_UNSEEN) { + likeliness->type &= (~LP_UNSEEN); + likeliness->next = likeliness_head; + likeliness_head = likeliness; + } + smp_mb__before_clear_bit(); + clear_bit(0, &likely_lock); + } + } + + return ret; +} +EXPORT_SYMBOL(do_check_likely); + +static void * lp_seq_start(struct seq_file *out, loff_t *pos) +{ + + if (!*pos) { + + seq_printf(out, "Likely Profiling Results\n"); + seq_printf(out, " --------------------------------------------" + "------------------------\n"); + seq_printf(out, "[+- ] Type | # True | # False | Function:" + "Filename@Line\n"); + + out->private = likeliness_head; + } + + return out->private; +} + +static void *lp_seq_next(struct seq_file *out, void *p, loff_t *pos) +{ + struct likeliness *entry = p; + + if (entry->next) { + ++(*pos); + out->private = entry->next; + } else + out->private = NULL; + + return out->private; +} + +static int lp_seq_show(struct seq_file *out, void *p) +{ + struct likeliness *entry = p; + unsigned int true = entry->count[1]; + unsigned int false = entry->count[0]; + + if (!entry->type) { + if (true > false) + seq_printf(out, "+"); + else + seq_printf(out, " "); + + seq_printf(out, "unlikely "); + } else { + if (true < false) + seq_printf(out, "-"); + else + seq_printf(out, " "); + + seq_printf(out, "likely "); + } + + seq_printf(out, "|%9u|%9u\t%s()@:%s@%d\n", true, false, + entry->func, entry->file, entry->line); + + return 0; +} + +static void lp_seq_stop(struct seq_file *m, void *p) +{ +} + +struct seq_operations likely_profiling_ops = { + .start = lp_seq_start, + .next = lp_seq_next, + .stop = lp_seq_stop, + .show = lp_seq_show +}; + +static int lp_results_open(struct inode *inode, struct file *file) +{ + return seq_open(file, &likely_profiling_ops); +} + +static struct file_operations proc_likely_operations = { + .open = lp_results_open, + .read = seq_read, + .llseek = seq_lseek, + .release = seq_release, +}; + +static int __init init_likely(void) +{ + struct proc_dir_entry *entry; + entry = create_proc_entry("likely_prof", 0, &proc_root); + if (entry) + entry->proc_fops = &proc_likely_operations; + + return 0; +} +__initcall(init_likely); diff -puN arch/x86/vdso/vclock_gettime.c~profile-likely-unlikely-macros arch/x86/vdso/vclock_gettime.c --- a/arch/x86/vdso/vclock_gettime.c~profile-likely-unlikely-macros +++ a/arch/x86/vdso/vclock_gettime.c @@ -9,6 +9,8 @@ * Also alternative() doesn't work. */ +#define SUPPRESS_LIKELY_PROFILING + #include #include #include _