[RFC] sched_tick with interrupts enabled

scheduler_tick() has the potential of running for some time if f.e.
sched_domains for a system with 1024 processors have to be balanced.
We currently do all of that with interrupts disabled. So we may be unable
to service interrupts for some time.

I wonder if it would be possible to put the sched_tick() into a tasklet and
allow interrupts to be enabled? Preemption is still disabled and so we
are stuck on a cpu.

This can only work if we are sure that no scheduler activity is performed
from interrupt code or from other tasklets that may potentially run.

sched_tick() takes the lock on a request queue without disabling interrupts.
So any other attempt to invoke scheduler functions from interrupts or
tasklets will cause a deadlock.

I tested this patch with AIM7 and things seem to be fine.

Signed-off-by: Christoph Lameter <clameter@sgi.com>

Index: linux-2.6.19-rc2-mm1/kernel/timer.c
===================================================================
--- linux-2.6.19-rc2-mm1.orig/kernel/timer.c	2006-10-18 12:32:42.074852129 -0500
+++ linux-2.6.19-rc2-mm1/kernel/timer.c	2006-10-18 12:39:55.261816514 -0500
@@ -1210,8 +1210,16 @@ static void update_wall_time(void)
 	}
 }
 
+static void sched_tick_action(unsigned long state)
+{
+	int cpu = smp_processor_id();
+
+	rebalance_tick(cpu, cpu_rq(cpu), state);
+}
+
+static DECLARE_TASKLET(scheduler_tick_tasklet, sched_tick_action, 0);
 /*
- * Called from the timer interrupt handler to charge one tick to the current 
+ * Called from the timer interrupt handler to charge one tick to the current
  * process.  user_tick is 1 if the tick is user time, 0 for system.
  */
 void update_process_times(int user_tick)
@@ -1227,7 +1235,8 @@ void update_process_times(int user_tick)
 	run_local_timers();
 	if (rcu_pending(cpu))
 		rcu_check_callbacks(cpu, user_tick);
-	scheduler_tick();
+	state = scheduler_tick();
+	tasklet_schedule(&scheduler_tick_tasklet, state);
  	run_posix_cpu_timers(p);
 }
 
Index: linux-2.6.19-rc2-mm1/kernel/sched.c
===================================================================
--- linux-2.6.19-rc2-mm1.orig/kernel/sched.c	2006-10-18 12:33:21.873225336 -0500
+++ linux-2.6.19-rc2-mm1/kernel/sched.c	2006-10-18 12:38:09.007491433 -0500
@@ -1602,15 +1602,18 @@ void fastcall sched_fork(struct task_str
 	current->time_slice >>= 1;
 	p->timestamp = sched_clock();
 	if (unlikely(!current->time_slice)) {
+		enum idle_type state;
 		/*
 		 * This case is rare, it happens when the parent has only
 		 * a single jiffy left from its timeslice. Taking the
 		 * runqueue lock is not a problem.
 		 */
 		current->time_slice = 1;
-		scheduler_tick();
-	}
-	local_irq_enable();
+		state = scheduler_tick();
+		local_irq_enable();
+		rebalance_tick(cpu, cpu_rq(cpu), state);
+	} else
+		local_irq_enable();
 	put_cpu();
 }
 
@@ -3040,12 +3043,13 @@ void account_steal_time(struct task_stru
 
 /*
  * This function gets called by the timer code, with HZ frequency.
- * We call it with interrupts disabled.
+ * We call it with interrupts enabled. However, the thread is
+ * pinned to a specific cpu.
  *
  * It also gets called by the fork code, when changing the parent's
  * timeslices.
  */
-void scheduler_tick(void)
+enum idle_type scheduler_tick(void)
 {
 	unsigned long long now = sched_clock();
 	struct task_struct *p = current;
@@ -3135,6 +3139,7 @@ void scheduler_tick(void)
 out_unlock:
 	spin_unlock(&rq->lock);
 out:
+	return state;
 	rebalance_tick(cpu, rq, state);
 }
 
Index: linux-2.6.19-rc2-mm1/include/linux/sched.h
===================================================================
--- linux-2.6.19-rc2-mm1.orig/include/linux/sched.h	2006-10-18 12:32:42.097315014 -0500
+++ linux-2.6.19-rc2-mm1/include/linux/sched.h	2006-10-18 12:33:31.078125105 -0500
@@ -220,7 +220,7 @@ long io_schedule_timeout(long timeout);
 extern void cpu_init (void);
 extern void trap_init(void);
 extern void update_process_times(int user);
-extern void scheduler_tick(void);
+extern enum idle_type scheduler_tick(void);
 
 #ifdef CONFIG_DETECT_SOFTLOCKUP
 extern void softlockup_tick(void);