Add idle notifiers This adds a new notifier chain that is called with IDLE_START when a CPU goes idle and IDLE_END when it goes out of idle. The context can be idle thread or interrupt context. Since we cannot rely on MONITOR/MWAIT existing the idle end check currently has to be done in all interrupt handlers. They were originally inspired by the similar s390 implementation. They have a variety of applications: - They will be needed for CONFIG_NO_IDLE_HZ - They can be used for oprofile to fix up the missing time in idle when performance counters don't tick. - They can be used for better C state management in ACPI - They could be used for microstate accounting. This is just infrastructure so far, no users. Signed-off-by: Andi Kleen Index: linux/include/asm-x86_64/idle.h =================================================================== --- /dev/null +++ linux/include/asm-x86_64/idle.h @@ -0,0 +1,14 @@ +#ifndef _ASM_X86_64_IDLE_H +#define _ASM_X86_64_IDLE_H 1 + +#define IDLE_START 1 +#define IDLE_END 2 + +struct notifier_block; +void idle_notifier_register(struct notifier_block *n); +void idle_notifier_unregister(struct notifier_block *n); + +void enter_idle(void); +void exit_idle(void); + +#endif Index: linux/arch/x86_64/kernel/apic.c =================================================================== --- linux.orig/arch/x86_64/kernel/apic.c +++ linux/arch/x86_64/kernel/apic.c @@ -34,6 +34,7 @@ #include #include #include +#include int apic_verbosity; @@ -922,6 +923,7 @@ void smp_apic_timer_interrupt(struct pt_ * Besides, if we don't timer interrupts ignore the global * interrupt lock, which is the WrongThing (tm) to do. */ + exit_idle(); irq_enter(); smp_local_timer_interrupt(regs); irq_exit(); @@ -981,6 +983,7 @@ __init int oem_force_hpet_timer(void) asmlinkage void smp_spurious_interrupt(void) { unsigned int v; + exit_idle(); irq_enter(); /* * Check if this really is a spurious interrupt and ACK it @@ -1016,6 +1019,7 @@ asmlinkage void smp_error_interrupt(void { unsigned int v, v1; + exit_idle(); irq_enter(); /* First tickle the hardware, only then report what went on. -- REW */ v = apic_read(APIC_ESR); Index: linux/arch/x86_64/kernel/irq.c =================================================================== --- linux.orig/arch/x86_64/kernel/irq.c +++ linux/arch/x86_64/kernel/irq.c @@ -17,6 +17,7 @@ #include #include #include +#include atomic_t irq_err_count; #ifdef CONFIG_X86_IO_APIC @@ -98,6 +99,7 @@ asmlinkage unsigned int do_IRQ(struct pt /* high bits used in ret_from_ code */ unsigned irq = regs->orig_rax & 0xff; + exit_idle(); irq_enter(); __do_IRQ(irq, regs); Index: linux/arch/x86_64/kernel/mce_intel.c =================================================================== --- linux.orig/arch/x86_64/kernel/mce_intel.c +++ linux/arch/x86_64/kernel/mce_intel.c @@ -10,6 +10,7 @@ #include #include #include +#include static DEFINE_PER_CPU(unsigned long, next_check); @@ -19,6 +20,7 @@ asmlinkage void smp_thermal_interrupt(vo ack_APIC_irq(); + exit_idle(); irq_enter(); if (time_before(jiffies, __get_cpu_var(next_check))) goto done; Index: linux/arch/x86_64/kernel/process.c =================================================================== --- linux.orig/arch/x86_64/kernel/process.c +++ linux/arch/x86_64/kernel/process.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -50,6 +51,7 @@ #include #include #include +#include asmlinkage extern void ret_from_fork(void); @@ -64,6 +66,50 @@ EXPORT_SYMBOL(boot_option_idle_override) void (*pm_idle)(void); static DEFINE_PER_CPU(unsigned int, cpu_idle_state); +static struct notifier_block *idle_notifier; +static DEFINE_SPINLOCK(idle_notifier_lock); + +void idle_notifier_register(struct notifier_block *n) +{ + unsigned long flags; + spin_lock_irqsave(&idle_notifier_lock, flags); + notifier_chain_register(&idle_notifier, n); + spin_unlock_irqrestore(&idle_notifier_lock, flags); +} +EXPORT_SYMBOL_GPL(idle_notifier_register); + +void idle_notifier_unregister(struct notifier_block *n) +{ + unsigned long flags; + spin_lock_irqsave(&idle_notifier_lock, flags); + notifier_chain_unregister(&idle_notifier, n); + spin_unlock_irqrestore(&idle_notifier_lock, flags); +} +EXPORT_SYMBOL(idle_notifier_unregister); + +enum idle_state { CPU_IDLE, CPU_NOT_IDLE }; +static DEFINE_PER_CPU(enum idle_state, idle_state) = CPU_NOT_IDLE; + +void enter_idle(void) +{ + __get_cpu_var(idle_state) = CPU_IDLE; + notifier_call_chain(&idle_notifier, IDLE_START, NULL); +} + +static void __exit_idle(void) +{ + __get_cpu_var(idle_state) = CPU_NOT_IDLE; + notifier_call_chain(&idle_notifier, IDLE_END, NULL); +} + +/* Called from interrupts to signify idle end */ +void exit_idle(void) +{ + if (current->pid | read_pda(irqcount)) + return; + __exit_idle(); +} + /* * We use this if we don't have any better * idle routine.. @@ -179,7 +225,9 @@ void cpu_idle (void) idle = default_idle; if (cpu_is_offline(smp_processor_id())) play_dead(); + enter_idle(); idle(); + __exit_idle(); } preempt_enable_no_resched(); Index: linux/arch/x86_64/kernel/smp.c =================================================================== --- linux.orig/arch/x86_64/kernel/smp.c +++ linux/arch/x86_64/kernel/smp.c @@ -27,6 +27,7 @@ #include #include #include +#include /* * Smarter SMP flushing macros. @@ -512,6 +513,7 @@ asmlinkage void smp_call_function_interr /* * At this point the info structure may be out of scope unless wait==1 */ + exit_idle(); irq_enter(); (*func)(info); irq_exit(); Index: linux/arch/x86_64/kernel/mce_amd.c =================================================================== --- linux.orig/arch/x86_64/kernel/mce_amd.c +++ linux/arch/x86_64/kernel/mce_amd.c @@ -27,6 +27,7 @@ #include #include #include +#include #define PFX "mce_threshold: " #define VERSION "version 1.00.9" @@ -140,6 +141,7 @@ asmlinkage void mce_threshold_interrupt( struct mce m; ack_APIC_irq(); + exit_idle(); irq_enter(); memset(&m, 0, sizeof(m));