Signed-off-by: Andrew Morton --- arch/i386/kernel/nmi.c | 16 +++++++++++++--- arch/i386/kernel/traps.c | 24 +++++++++++------------- arch/x86_64/kernel/nmi.c | 26 +++++++++++++++++++------- arch/x86_64/kernel/traps.c | 8 ++++---- include/asm-i386/nmi.h | 2 +- include/asm-x86_64/nmi.h | 10 +++++++++- 6 files changed, 57 insertions(+), 29 deletions(-) diff -puN arch/i386/kernel/nmi.c~x86_64-mm-cleanup-nmi-interrupt-path arch/i386/kernel/nmi.c --- a/arch/i386/kernel/nmi.c~x86_64-mm-cleanup-nmi-interrupt-path +++ a/arch/i386/kernel/nmi.c @@ -781,7 +781,7 @@ EXPORT_SYMBOL(touch_nmi_watchdog); extern void die_nmi(struct pt_regs *, const char *msg); -void nmi_watchdog_tick (struct pt_regs * regs, unsigned reason) +int nmi_watchdog_tick (struct pt_regs * regs, unsigned reason) { /* @@ -794,10 +794,12 @@ void nmi_watchdog_tick (struct pt_regs * int cpu = smp_processor_id(); struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); u64 dummy; + int rc=0; /* check for other users first */ if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT) == NOTIFY_STOP) { + rc = 1; touched = 1; } @@ -850,10 +852,18 @@ void nmi_watchdog_tick (struct pt_regs * } /* start the cycle over again */ write_watchdog_counter(wd->perfctr_msr, NULL); - } + rc = 1; + } else if (nmi_watchdog == NMI_IO_APIC) { + /* don't know how to accurately check for this. + * just assume it was a watchdog timer interrupt + * This matches the old behaviour. + */ + rc = 1; + } else + printk(KERN_WARNING "Unknown enabled NMI hardware?!\n"); } done: - return; + return rc; } #ifdef CONFIG_SYSCTL diff -puN arch/i386/kernel/traps.c~x86_64-mm-cleanup-nmi-interrupt-path arch/i386/kernel/traps.c --- a/arch/i386/kernel/traps.c~x86_64-mm-cleanup-nmi-interrupt-path +++ a/arch/i386/kernel/traps.c @@ -704,6 +704,13 @@ void die_nmi (struct pt_regs *regs, cons do_exit(SIGSEGV); } +static int dummy_nmi_callback(struct pt_regs * regs, int cpu) +{ + return 0; +} + +static nmi_callback_t nmi_callback = dummy_nmi_callback; + static void default_do_nmi(struct pt_regs * regs) { unsigned char reason = 0; @@ -721,12 +728,11 @@ static void default_do_nmi(struct pt_reg * Ok, so this is none of the documented NMI sources, * so it must be the NMI watchdog. */ - if (nmi_watchdog) { - nmi_watchdog_tick(regs, reason); + if (nmi_watchdog_tick(regs, reason)) return; - } #endif - unknown_nmi_error(reason, regs); + if (!rcu_dereference(nmi_callback)(regs, smp_processor_id())) + unknown_nmi_error(reason, regs); return; } if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT) == NOTIFY_STOP) @@ -742,13 +748,6 @@ static void default_do_nmi(struct pt_reg reassert_nmi(); } -static int dummy_nmi_callback(struct pt_regs * regs, int cpu) -{ - return 0; -} - -static nmi_callback_t nmi_callback = dummy_nmi_callback; - fastcall void do_nmi(struct pt_regs * regs, long error_code) { int cpu; @@ -759,8 +758,7 @@ fastcall void do_nmi(struct pt_regs * re ++nmi_count(cpu); - if (!rcu_dereference(nmi_callback)(regs, cpu)) - default_do_nmi(regs); + default_do_nmi(regs); nmi_exit(); } diff -puN arch/x86_64/kernel/nmi.c~x86_64-mm-cleanup-nmi-interrupt-path arch/x86_64/kernel/nmi.c --- a/arch/x86_64/kernel/nmi.c~x86_64-mm-cleanup-nmi-interrupt-path +++ a/arch/x86_64/kernel/nmi.c @@ -682,16 +682,18 @@ void touch_nmi_watchdog (void) touch_softlockup_watchdog(); } -void __kprobes nmi_watchdog_tick(struct pt_regs * regs, unsigned reason) +int __kprobes nmi_watchdog_tick(struct pt_regs * regs, unsigned reason) { int sum; int touched = 0; struct nmi_watchdog_ctlblk *wd = &__get_cpu_var(nmi_watchdog_ctlblk); u64 dummy; + int rc=0; /* check for other users first */ if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT) == NOTIFY_STOP) { + rc = 1; touched = 1; } @@ -746,10 +748,18 @@ void __kprobes nmi_watchdog_tick(struct } /* start the cycle over again */ wrmsrl(wd->perfctr_msr, -((u64)cpu_khz * 1000 / nmi_hz)); - } + rc = 1; + } else if (nmi_watchdog == NMI_IO_APIC) { + /* don't know how to accurately check for this. + * just assume it was a watchdog timer interrupt + * This matches the old behaviour. + */ + rc = 1; + } else + printk(KERN_WARNING "Unknown enabled NMI hardware?!\n"); } done: - return; + return rc; } static __kprobes int dummy_nmi_callback(struct pt_regs * regs, int cpu) @@ -761,15 +771,17 @@ static nmi_callback_t nmi_callback = dum asmlinkage __kprobes void do_nmi(struct pt_regs * regs, long error_code) { - int cpu = safe_smp_processor_id(); - nmi_enter(); add_pda(__nmi_count,1); - if (!rcu_dereference(nmi_callback)(regs, cpu)) - default_do_nmi(regs); + default_do_nmi(regs); nmi_exit(); } +int do_nmi_callback(struct pt_regs * regs, int cpu) +{ + return rcu_dereference(nmi_callback)(regs, cpu); +} + void set_nmi_callback(nmi_callback_t callback) { vmalloc_sync_all(); diff -puN arch/x86_64/kernel/traps.c~x86_64-mm-cleanup-nmi-interrupt-path arch/x86_64/kernel/traps.c --- a/arch/x86_64/kernel/traps.c~x86_64-mm-cleanup-nmi-interrupt-path +++ a/arch/x86_64/kernel/traps.c @@ -778,12 +778,12 @@ asmlinkage __kprobes void default_do_nmi * Ok, so this is none of the documented NMI sources, * so it must be the NMI watchdog. */ - if (nmi_watchdog > 0) { - nmi_watchdog_tick(regs,reason); + if (nmi_watchdog_tick(regs,reason)) return; - } + if (!do_nmi_callback(regs,cpu)) #endif - unknown_nmi_error(reason, regs); + unknown_nmi_error(reason, regs); + return; } if (notify_die(DIE_NMI, "nmi", regs, reason, 2, SIGINT) == NOTIFY_STOP) diff -puN include/asm-i386/nmi.h~x86_64-mm-cleanup-nmi-interrupt-path include/asm-i386/nmi.h --- a/include/asm-i386/nmi.h~x86_64-mm-cleanup-nmi-interrupt-path +++ a/include/asm-i386/nmi.h @@ -37,7 +37,7 @@ extern int reserve_lapic_nmi(void); extern void release_lapic_nmi(void); extern void disable_timer_nmi_watchdog(void); extern void enable_timer_nmi_watchdog(void); -extern void nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); +extern int nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); extern atomic_t nmi_active; extern unsigned int nmi_watchdog; diff -puN include/asm-x86_64/nmi.h~x86_64-mm-cleanup-nmi-interrupt-path include/asm-x86_64/nmi.h --- a/include/asm-x86_64/nmi.h~x86_64-mm-cleanup-nmi-interrupt-path +++ a/include/asm-x86_64/nmi.h @@ -26,6 +26,14 @@ void set_nmi_callback(nmi_callback_t cal */ void unset_nmi_callback(void); +/** + * do_nmi_callback + * + * Check to see if a callback exists and execute it. Return 1 + * if the handler exists and was handled successfully. + */ +int do_nmi_callback(struct pt_regs *regs, int cpu); + #ifdef CONFIG_PM /** Replace the PM callback routine for NMI. */ @@ -68,7 +76,7 @@ extern int reserve_lapic_nmi(void); extern void release_lapic_nmi(void); extern void disable_timer_nmi_watchdog(void); extern void enable_timer_nmi_watchdog(void); -extern void nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); +extern int nmi_watchdog_tick (struct pt_regs * regs, unsigned reason); extern void nmi_watchdog_default(void); extern int setup_nmi_watchdog(char *); _