From: john stultz Signed-off-by: John Stultz Cc: Ingo Molnar Cc: Thomas Gleixner Cc: Roman Zippel Signed-off-by: Andrew Morton --- arch/i386/kernel/setup.c | 1 arch/i386/kernel/tsc.c | 194 ++++++++++++++-------------- drivers/acpi/processor_idle.c | 4 include/asm-i386/mach-default/mach_timer.h | 4 include/asm-i386/mach-summit/mach_mpparse.h | 3 include/asm-i386/tsc.h | 11 + 6 files changed, 116 insertions(+), 101 deletions(-) diff -puN arch/i386/kernel/setup.c~time-i386-conversion-part-3-rework-tsc-support arch/i386/kernel/setup.c --- devel/arch/i386/kernel/setup.c~time-i386-conversion-part-3-rework-tsc-support 2006-01-03 18:16:32.000000000 -0800 +++ devel-akpm/arch/i386/kernel/setup.c 2006-01-03 18:16:32.000000000 -0800 @@ -1628,6 +1628,7 @@ void __init setup_arch(char **cmdline_p) conswitchp = &dummy_con; #endif #endif + tsc_init(); } #include "setup_arch_post.h" diff -puN arch/i386/kernel/tsc.c~time-i386-conversion-part-3-rework-tsc-support arch/i386/kernel/tsc.c --- devel/arch/i386/kernel/tsc.c~time-i386-conversion-part-3-rework-tsc-support 2006-01-03 18:16:32.000000000 -0800 +++ devel-akpm/arch/i386/kernel/tsc.c 2006-01-03 18:17:27.000000000 -0800 @@ -6,13 +6,23 @@ #include #include +#include #include +#include #include #include "mach_timer.h" +/* + * On some systems the TSC frequency does not + * change with the cpu frequency. So we need + * an extra value to store the TSC freq + */ +unsigned int tsc_khz; + int tsc_disable __initdata = 0; + #ifdef CONFIG_X86_TSC static int __init tsc_setup(char *str) { @@ -38,15 +48,32 @@ __setup("notsc", tsc_setup); int read_current_timer(unsigned long *timer_val) { - if (cur_timer->read_timer) { - *timer_val = cur_timer->read_timer(); + if (!tsc_disable && cpu_khz) { + rdtscl(*timer_val); return 0; } return -1; } +/* + * code to mark and check if the TSC is unstable + * due to cpufreq or due to unsynced TSCs + */ +static int tsc_unstable; + +static inline int check_tsc_unstable(void) +{ + return tsc_unstable; +} + +void mark_tsc_unstable(void) +{ + tsc_unstable = 1; +} +EXPORT_SYMBOL_GPL(mark_tsc_unstable); -/* convert from cycles(64bits) => nanoseconds (64bits) +/* Accellerators for sched_clock() + * convert from cycles(64bits) => nanoseconds (64bits) * basic equation: * ns = cycles / (freq / ns_per_sec) * ns = cycles * (ns_per_sec / freq) @@ -92,10 +119,10 @@ unsigned long long sched_clock(void) * synchronized across all CPUs. */ #ifndef CONFIG_NUMA - if (!use_tsc) + if (!cpu_khz || check_tsc_unstable()) #endif /* no locking but a rare wrong value is not a big deal */ - return jiffies_64 * (1000000000 / HZ); + return (jiffies_64 - INITIAL_JIFFIES) * (1000000000 / HZ); /* read the Time Stamp Counter: */ rdtscll(this_offset); @@ -104,64 +131,43 @@ unsigned long long sched_clock(void) return cycles_2_ns(this_offset); } -/* ------ Calibrate the TSC ------- - * Return 2^32 * (1 / (TSC clocks per usec)) for do_fast_gettimeoffset(). - * Too much 64-bit arithmetic here to do this cleanly in C, and for - * accuracy's sake we want to keep the overhead on the CTC speaker (channel 2) - * output busy loop as low as possible. We avoid reading the CTC registers - * directly because of the awkward 8-bit access mechanism of the 82C54 - * device. - */ - -#define CALIBRATE_TIME (5 * 1000020/HZ) - -unsigned long calibrate_tsc(void) +static unsigned long calculate_cpu_khz(void) { - mach_prepare_counter(); - - { - unsigned long startlow, starthigh; - unsigned long endlow, endhigh; - unsigned long count; - - rdtsc(startlow,starthigh); + unsigned long long start, end; + unsigned long count; + u64 delta64; + int i; + + /* run 3 times to ensure the cache is warm */ + for (i = 0; i < 3; i++) { + mach_prepare_counter(); + rdtscll(start); mach_countup(&count); - rdtsc(endlow,endhigh); - - - /* Error: ECTCNEVERSET */ - if (count <= 1) - goto bad_ctc; - - /* 64-bit subtract - gcc just messes up with long longs */ - __asm__("subl %2,%0\n\t" - "sbbl %3,%1" - :"=a" (endlow), "=d" (endhigh) - :"g" (startlow), "g" (starthigh), - "0" (endlow), "1" (endhigh)); - - /* Error: ECPUTOOFAST */ - if (endhigh) - goto bad_ctc; - - /* Error: ECPUTOOSLOW */ - if (endlow <= CALIBRATE_TIME) - goto bad_ctc; - - __asm__("divl %2" - :"=a" (endlow), "=d" (endhigh) - :"r" (endlow), "0" (0), "1" (CALIBRATE_TIME)); - - return endlow; + rdtscll(end); } - /* + * Error: ECTCNEVERSET * The CTC wasn't reliable: we got a hit on the very first read, * or the CPU was so fast/slow that the quotient wouldn't fit in * 32 bits.. */ -bad_ctc: - return 0; + if (count <= 1) + return 0; + + delta64 = end - start; + + /* cpu freq too fast: */ + if (delta64 > (1ULL<<32)) + return 0; + + /* cpu freq too slow: */ + if (delta64 <= CALIBRATE_TIME_MSEC) + return 0; + + delta64 += CALIBRATE_TIME_MSEC/2; /* round for do_div */ + do_div(delta64,CALIBRATE_TIME_MSEC); + + return (unsigned long)delta64; } int recalibrate_cpu_khz(void) @@ -171,12 +177,12 @@ int recalibrate_cpu_khz(void) if (cpu_has_tsc) { local_irq_disable(); - init_cpu_khz(); + cpu_khz = calculate_cpu_khz(); local_irq_enable(); + tsc_khz = cpu_khz; cpu_data[0].loops_per_jiffy = - cpufreq_scale(cpu_data[0].loops_per_jiffy, - cpu_khz_old, - cpu_khz); + cpufreq_scale(cpu_data[0].loops_per_jiffy, + cpu_khz_old, cpu_khz); return 0; } else return -ENODEV; @@ -184,28 +190,25 @@ int recalibrate_cpu_khz(void) return -ENODEV; #endif } -EXPORT_SYMBOL(recalibrate_cpu_khz); +EXPORT_SYMBOL(recalibrate_cpu_khz); -/* calculate cpu_khz */ -void init_cpu_khz(void) +void tsc_init(void) { - if (cpu_has_tsc) { - unsigned long tsc_quotient = calibrate_tsc(); - if (tsc_quotient) { - /* report CPU clock rate in Hz. - * The formula is (10^6 * 2^32) / (2^32 * 1 / (clocks/us)) = - * clock/second. Our precision is about 100 ppm. - */ - { unsigned long eax=0, edx=1000; - __asm__("divl %2" - :"=a" (cpu_khz), "=d" (edx) - :"r" (tsc_quotient), - "0" (eax), "1" (edx)); - printk("Detected %lu.%03lu MHz processor.\n", cpu_khz / 1000, cpu_khz % 1000); - } - } - } + if (!cpu_has_tsc || tsc_disable) + return; + + cpu_khz = calculate_cpu_khz(); + tsc_khz = cpu_khz; + + if (!cpu_khz) + return; + + printk("Detected %lu.%03lu MHz processor.\n", + (unsigned long)cpu_khz / 1000, + (unsigned long)cpu_khz % 1000); + + set_cyc2ns_scale(cpu_khz); } #ifdef CONFIG_CPU_FREQ @@ -225,15 +228,15 @@ static void handle_cpufreq_delayed_get(v } /* - * if we notice lost ticks, schedule a call to cpufreq_get() as it tries + * if we notice cpufreq oddness, schedule a call to cpufreq_get() as it tries * to verify the CPU frequency the timing core thinks the CPU is running * at is still correct. */ -void cpufreq_delayed_get(void) +static inline void cpufreq_delayed_get(void) { if (cpufreq_init && !cpufreq_delayed_issched) { cpufreq_delayed_issched = 1; - printk(KERN_DEBUG "Losing some ticks... checking if CPU frequency changed.\n"); + printk(KERN_DEBUG "Checking if CPU frequency changed.\n"); schedule_work(&cpufreq_delayed_get_work); } } @@ -242,14 +245,9 @@ void cpufreq_delayed_get(void) * if the CPU frequency is scaled, TSC-based delays will need a different * loops_per_jiffy value to function properly. */ - static unsigned int ref_freq = 0; static unsigned long loops_per_jiffy_ref = 0; - -#ifndef CONFIG_SMP -static unsigned long fast_gettimeoffset_ref = 0; static unsigned long cpu_khz_ref = 0; -#endif static int time_cpufreq_notifier(struct notifier_block *nb, unsigned long val, void *data) @@ -262,10 +260,7 @@ time_cpufreq_notifier(struct notifier_bl if (!ref_freq) { ref_freq = freq->old; loops_per_jiffy_ref = cpu_data[freq->cpu].loops_per_jiffy; -#ifndef CONFIG_SMP - fast_gettimeoffset_ref = fast_gettimeoffset_quotient; cpu_khz_ref = cpu_khz; -#endif } if ((val == CPUFREQ_PRECHANGE && freq->old < freq->new) || @@ -273,16 +268,22 @@ time_cpufreq_notifier(struct notifier_bl (val == CPUFREQ_RESUMECHANGE)) { if (!(freq->flags & CPUFREQ_CONST_LOOPS)) cpu_data[freq->cpu].loops_per_jiffy = cpufreq_scale(loops_per_jiffy_ref, ref_freq, freq->new); -#ifndef CONFIG_SMP - if (cpu_khz) - cpu_khz = cpufreq_scale(cpu_khz_ref, ref_freq, freq->new); - if (use_tsc) { + + if (cpu_khz) { + + if (num_online_cpus() == 1) + cpu_khz = cpufreq_scale(cpu_khz_ref, + ref_freq, freq->new); if (!(freq->flags & CPUFREQ_CONST_LOOPS)) { - fast_gettimeoffset_quotient = cpufreq_scale(fast_gettimeoffset_ref, freq->new, ref_freq); + tsc_khz = cpu_khz; set_cyc2ns_scale(cpu_khz); + /* + * TSC based sched_clock turns + * to junk w/ cpufreq + */ + mark_tsc_unstable(); } } -#endif } if (val != CPUFREQ_RESUMECHANGE) @@ -304,11 +305,10 @@ static int __init cpufreq_tsc(void) CPUFREQ_TRANSITION_NOTIFIER); if (!ret) cpufreq_init = 1; + return ret; } core_initcall(cpufreq_tsc); -#else /* CONFIG_CPU_FREQ */ -void cpufreq_delayed_get(void) { return; } #endif diff -puN drivers/acpi/processor_idle.c~time-i386-conversion-part-3-rework-tsc-support drivers/acpi/processor_idle.c --- devel/drivers/acpi/processor_idle.c~time-i386-conversion-part-3-rework-tsc-support 2006-01-03 18:16:32.000000000 -0800 +++ devel-akpm/drivers/acpi/processor_idle.c 2006-01-03 18:16:32.000000000 -0800 @@ -459,6 +459,10 @@ static void acpi_processor_idle(void) ACPI_MTX_DO_NOT_LOCK); } +#ifdef CONFIG_GENERIC_TIME + /* TSC halts in C3, so notify users */ + mark_tsc_unstable(); +#endif /* Re-enable interrupts */ local_irq_enable(); set_thread_flag(TIF_POLLING_NRFLAG); diff -puN include/asm-i386/mach-default/mach_timer.h~time-i386-conversion-part-3-rework-tsc-support include/asm-i386/mach-default/mach_timer.h --- devel/include/asm-i386/mach-default/mach_timer.h~time-i386-conversion-part-3-rework-tsc-support 2006-01-03 18:16:32.000000000 -0800 +++ devel-akpm/include/asm-i386/mach-default/mach_timer.h 2006-01-03 18:16:32.000000000 -0800 @@ -15,7 +15,9 @@ #ifndef _MACH_TIMER_H #define _MACH_TIMER_H -#define CALIBRATE_LATCH (5 * LATCH) +#define CALIBRATE_TIME_MSEC 30 /* 30 msecs */ +#define CALIBRATE_LATCH \ + ((CLOCK_TICK_RATE * CALIBRATE_TIME_MSEC + 1000/2)/1000) static inline void mach_prepare_counter(void) { diff -puN include/asm-i386/mach-summit/mach_mpparse.h~time-i386-conversion-part-3-rework-tsc-support include/asm-i386/mach-summit/mach_mpparse.h --- devel/include/asm-i386/mach-summit/mach_mpparse.h~time-i386-conversion-part-3-rework-tsc-support 2006-01-03 18:16:32.000000000 -0800 +++ devel-akpm/include/asm-i386/mach-summit/mach_mpparse.h 2006-01-03 18:16:32.000000000 -0800 @@ -2,6 +2,7 @@ #define __ASM_MACH_MPPARSE_H #include +#include extern int use_cyclone; @@ -29,6 +30,7 @@ static inline int mps_oem_check(struct m (!strncmp(productid, "VIGIL SMP", 9) || !strncmp(productid, "EXA", 3) || !strncmp(productid, "RUTHLESS SMP", 12))){ + mark_tsc_unstable(); use_cyclone = 1; /*enable cyclone-timer*/ setup_summit(); return 1; @@ -42,6 +44,7 @@ static inline int acpi_madt_oem_check(ch if (!strncmp(oem_id, "IBM", 3) && (!strncmp(oem_table_id, "SERVIGIL", 8) || !strncmp(oem_table_id, "EXA", 3))){ + mark_tsc_unstable(); use_cyclone = 1; /*enable cyclone-timer*/ setup_summit(); return 1; diff -puN include/asm-i386/tsc.h~time-i386-conversion-part-3-rework-tsc-support include/asm-i386/tsc.h --- devel/include/asm-i386/tsc.h~time-i386-conversion-part-3-rework-tsc-support 2006-01-03 18:16:32.000000000 -0800 +++ devel-akpm/include/asm-i386/tsc.h 2006-01-03 18:16:32.000000000 -0800 @@ -25,9 +25,12 @@ */ typedef unsigned long long cycles_t; -static inline cycles_t get_cycles (void) +extern unsigned int cpu_khz; +extern unsigned int tsc_khz; + +static inline cycles_t get_cycles(void) { - unsigned long long ret=0; + unsigned long long ret = 0; #ifndef CONFIG_X86_TSC if (!cpu_has_tsc) @@ -40,5 +43,7 @@ static inline cycles_t get_cycles (void) return ret; } -extern unsigned int cpu_khz; +extern void tsc_init(void); +extern void mark_tsc_unstable(void); + #endif _