From: Ingo Molnar RCU tasklist_lock and RCU signal handling: send signals RCU-read-locked instead of tasklist_lock read-locked. This is a scalability improvement on SMP and a preemption-latency improvement under PREEMPT_RCU. Signed-off-by: Paul E. McKenney Signed-off-by: Ingo Molnar Signed-off-by: Andrew Morton --- fs/exec.c | 2 - include/linux/list.h | 20 +++++++++++ include/linux/sched.h | 35 ++++++++++++++++++- kernel/exit.c | 1 kernel/fork.c | 8 ++++ kernel/pid.c | 20 +++++------ kernel/rcupdate.c | 1 kernel/sched.c | 7 +++ kernel/signal.c | 89 ++++++++++++++++++++++++++++++++++++++++++-------- 9 files changed, 156 insertions(+), 27 deletions(-) diff -puN fs/exec.c~rcu-signal-handling fs/exec.c --- devel/fs/exec.c~rcu-signal-handling 2005-11-05 02:23:24.000000000 -0800 +++ devel-akpm/fs/exec.c 2005-11-05 02:23:24.000000000 -0800 @@ -774,7 +774,7 @@ no_thread_group: write_unlock_irq(&tasklist_lock); if (atomic_dec_and_test(&oldsighand->count)) - kmem_cache_free(sighand_cachep, oldsighand); + sighand_free(oldsighand); } BUG_ON(!thread_group_leader(current)); diff -puN include/linux/list.h~rcu-signal-handling include/linux/list.h --- devel/include/linux/list.h~rcu-signal-handling 2005-11-05 02:23:24.000000000 -0800 +++ devel-akpm/include/linux/list.h 2005-11-05 02:23:24.000000000 -0800 @@ -208,6 +208,7 @@ static inline void list_replace_rcu(stru smp_wmb(); new->next->prev = new; new->prev->next = new; + old->prev = LIST_POISON2; } /** @@ -590,6 +591,25 @@ static inline void hlist_del_init(struct } } +/* + * hlist_replace_rcu - replace old entry by new one + * @old : the element to be replaced + * @new : the new element to insert + * + * The old entry will be replaced with the new entry atomically. + */ +static inline void hlist_replace_rcu(struct hlist_node *old, struct hlist_node *new){ + struct hlist_node *next = old->next; + + new->next = next; + new->pprev = old->pprev; + smp_wmb(); + if (next) + new->next->pprev = &new->next; + *new->pprev = new; + old->pprev = LIST_POISON2; +} + static inline void hlist_add_head(struct hlist_node *n, struct hlist_head *h) { struct hlist_node *first = h->first; diff -puN include/linux/sched.h~rcu-signal-handling include/linux/sched.h --- devel/include/linux/sched.h~rcu-signal-handling 2005-11-05 02:23:24.000000000 -0800 +++ devel-akpm/include/linux/sched.h 2005-11-05 02:23:24.000000000 -0800 @@ -34,6 +34,7 @@ #include #include #include +#include #include /* For AT_VECTOR_SIZE */ @@ -364,8 +365,16 @@ struct sighand_struct { atomic_t count; struct k_sigaction action[_NSIG]; spinlock_t siglock; + struct rcu_head rcu; }; +static inline void sighand_free(struct sighand_struct *sp) +{ + extern void sighand_free_cb(struct rcu_head *rhp); + + call_rcu(&sp->rcu, sighand_free_cb); +} + /* * NOTE! "signal_struct" does not have it's own * locking, because a shared signal_struct always @@ -858,6 +867,7 @@ struct task_struct { int cpuset_mems_generation; #endif atomic_t fs_excl; /* holding fs exclusive resources */ + struct rcu_head rcu; }; static inline pid_t process_group(struct task_struct *tsk) @@ -881,8 +891,29 @@ static inline int pid_alive(struct task_ extern void free_task(struct task_struct *tsk); extern void __put_task_struct(struct task_struct *tsk); #define get_task_struct(tsk) do { atomic_inc(&(tsk)->usage); } while(0) -#define put_task_struct(tsk) \ -do { if (atomic_dec_and_test(&(tsk)->usage)) __put_task_struct(tsk); } while(0) + +static inline int get_task_struct_rcu(struct task_struct *t) +{ + int oldusage; + + do { + oldusage = atomic_read(&t->usage); + if (oldusage == 0) { + return 0; + } + } while (cmpxchg(&t->usage.counter, + oldusage, oldusage + 1) != oldusage); + return 1; +} + +extern void __put_task_struct_cb(struct rcu_head *rhp); + +static inline void put_task_struct(struct task_struct *t) +{ + if (atomic_dec_and_test(&t->usage)) { + call_rcu(&t->rcu, __put_task_struct_cb); + } +} /* * Per process flags diff -puN kernel/exit.c~rcu-signal-handling kernel/exit.c --- devel/kernel/exit.c~rcu-signal-handling 2005-11-05 02:23:24.000000000 -0800 +++ devel-akpm/kernel/exit.c 2005-11-05 02:23:24.000000000 -0800 @@ -73,7 +73,6 @@ repeat: __ptrace_unlink(p); BUG_ON(!list_empty(&p->ptrace_list) || !list_empty(&p->ptrace_children)); __exit_signal(p); - __exit_sighand(p); /* * Note that the fastpath in sys_times depends on __exit_signal having * updated the counters before a task is removed from the tasklist of diff -puN kernel/fork.c~rcu-signal-handling kernel/fork.c --- devel/kernel/fork.c~rcu-signal-handling 2005-11-05 02:23:24.000000000 -0800 +++ devel-akpm/kernel/fork.c 2005-11-05 02:23:24.000000000 -0800 @@ -752,6 +752,14 @@ int unshare_files(void) EXPORT_SYMBOL(unshare_files); +void sighand_free_cb(struct rcu_head *rhp) +{ + struct sighand_struct *sp = + container_of(rhp, struct sighand_struct, rcu); + + kmem_cache_free(sighand_cachep, sp); +} + static inline int copy_sighand(unsigned long clone_flags, struct task_struct * tsk) { struct sighand_struct *sig; diff -puN kernel/pid.c~rcu-signal-handling kernel/pid.c --- devel/kernel/pid.c~rcu-signal-handling 2005-11-05 02:23:24.000000000 -0800 +++ devel-akpm/kernel/pid.c 2005-11-05 02:23:24.000000000 -0800 @@ -136,7 +136,7 @@ struct pid * fastcall find_pid(enum pid_ struct hlist_node *elem; struct pid *pid; - hlist_for_each_entry(pid, elem, + hlist_for_each_entry_rcu(pid, elem, &pid_hash[type][pid_hashfn(nr)], pid_chain) { if (pid->nr == nr) return pid; @@ -151,12 +151,12 @@ int fastcall attach_pid(task_t *task, en task_pid = &task->pids[type]; pid = find_pid(type, nr); if (pid == NULL) { - hlist_add_head(&task_pid->pid_chain, - &pid_hash[type][pid_hashfn(nr)]); INIT_LIST_HEAD(&task_pid->pid_list); + hlist_add_head_rcu(&task_pid->pid_chain, + &pid_hash[type][pid_hashfn(nr)]); } else { INIT_HLIST_NODE(&task_pid->pid_chain); - list_add_tail(&task_pid->pid_list, &pid->pid_list); + list_add_tail_rcu(&task_pid->pid_list, &pid->pid_list); } task_pid->nr = nr; @@ -170,20 +170,20 @@ static fastcall int __detach_pid(task_t pid = &task->pids[type]; if (!hlist_unhashed(&pid->pid_chain)) { - hlist_del(&pid->pid_chain); - if (list_empty(&pid->pid_list)) + if (list_empty(&pid->pid_list)) { nr = pid->nr; - else { + hlist_del_rcu(&pid->pid_chain); + } else { pid_next = list_entry(pid->pid_list.next, struct pid, pid_list); /* insert next pid from pid_list to hash */ - hlist_add_head(&pid_next->pid_chain, - &pid_hash[type][pid_hashfn(pid_next->nr)]); + hlist_replace_rcu(&pid->pid_chain, + &pid_next->pid_chain); } } - list_del(&pid->pid_list); + list_del_rcu(&pid->pid_list); pid->nr = 0; return nr; diff -puN kernel/rcupdate.c~rcu-signal-handling kernel/rcupdate.c --- devel/kernel/rcupdate.c~rcu-signal-handling 2005-11-05 02:23:24.000000000 -0800 +++ devel-akpm/kernel/rcupdate.c 2005-11-05 02:23:24.000000000 -0800 @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include diff -puN kernel/sched.c~rcu-signal-handling kernel/sched.c --- devel/kernel/sched.c~rcu-signal-handling 2005-11-05 02:23:24.000000000 -0800 +++ devel-akpm/kernel/sched.c 2005-11-05 02:23:24.000000000 -0800 @@ -176,6 +176,13 @@ static unsigned int task_timeslice(task_ #define task_hot(p, now, sd) ((long long) ((now) - (p)->last_ran) \ < (long long) (sd)->cache_hot_time) +void __put_task_struct_cb(struct rcu_head *rhp) +{ + __put_task_struct(container_of(rhp, struct task_struct, rcu)); +} + +EXPORT_SYMBOL_GPL(__put_task_struct_cb); + /* * These are the runqueue data structures: */ diff -puN kernel/signal.c~rcu-signal-handling kernel/signal.c --- devel/kernel/signal.c~rcu-signal-handling 2005-11-05 02:23:24.000000000 -0800 +++ devel-akpm/kernel/signal.c 2005-11-05 02:23:24.000000000 -0800 @@ -329,13 +329,20 @@ void __exit_sighand(struct task_struct * /* Ok, we're done with the signal handlers */ tsk->sighand = NULL; if (atomic_dec_and_test(&sighand->count)) - kmem_cache_free(sighand_cachep, sighand); + sighand_free(sighand); } void exit_sighand(struct task_struct *tsk) { write_lock_irq(&tasklist_lock); - __exit_sighand(tsk); + rcu_read_lock(); + if (tsk->sighand != NULL) { + struct sighand_struct *sighand = tsk->sighand; + spin_lock(&sighand->siglock); + __exit_sighand(tsk); + spin_unlock(&sighand->siglock); + } + rcu_read_unlock(); write_unlock_irq(&tasklist_lock); } @@ -351,6 +358,7 @@ void __exit_signal(struct task_struct *t BUG(); if (!atomic_read(&sig->count)) BUG(); + rcu_read_lock(); spin_lock(&sighand->siglock); posix_cpu_timers_exit(tsk); if (atomic_dec_and_test(&sig->count)) { @@ -358,6 +366,7 @@ void __exit_signal(struct task_struct *t if (tsk == sig->curr_target) sig->curr_target = next_thread(tsk); tsk->signal = NULL; + __exit_sighand(tsk); spin_unlock(&sighand->siglock); flush_sigqueue(&sig->shared_pending); } else { @@ -389,9 +398,11 @@ void __exit_signal(struct task_struct *t sig->nvcsw += tsk->nvcsw; sig->nivcsw += tsk->nivcsw; sig->sched_time += tsk->sched_time; + __exit_sighand(tsk); spin_unlock(&sighand->siglock); sig = NULL; /* Marker for below. */ } + rcu_read_unlock(); clear_tsk_thread_flag(tsk,TIF_SIGPENDING); flush_sigqueue(&tsk->pending); if (sig) { @@ -1094,13 +1105,24 @@ void zap_other_threads(struct task_struc int group_send_sig_info(int sig, struct siginfo *info, struct task_struct *p) { unsigned long flags; + struct sighand_struct *sp; int ret; +retry: ret = check_kill_permission(sig, info, p); - if (!ret && sig && p->sighand) { - spin_lock_irqsave(&p->sighand->siglock, flags); + if (!ret && sig && (sp = p->sighand)) { + if (!get_task_struct_rcu(p)) { + return -ESRCH; + } + spin_lock_irqsave(&sp->siglock, flags); + if (p->sighand != sp) { + spin_unlock_irqrestore(&sp->siglock, flags); + put_task_struct(p); + goto retry; + } ret = __group_send_sig_info(sig, info, p); - spin_unlock_irqrestore(&p->sighand->siglock, flags); + spin_unlock_irqrestore(&sp->siglock, flags); + put_task_struct(p); } return ret; @@ -1147,12 +1169,12 @@ kill_proc_info(int sig, struct siginfo * int error; struct task_struct *p; - read_lock(&tasklist_lock); + rcu_read_lock(); p = find_task_by_pid(pid); error = -ESRCH; if (p) error = group_send_sig_info(sig, info, p); - read_unlock(&tasklist_lock); + rcu_read_unlock(); return error; } @@ -1364,16 +1386,47 @@ send_sigqueue(int sig, struct sigqueue * { unsigned long flags; int ret = 0; + struct sighand_struct *sh = p->sighand; BUG_ON(!(q->flags & SIGQUEUE_PREALLOC)); - read_lock(&tasklist_lock); + + /* + * The rcu based delayed sighand destroy makes it possible to + * run this without tasklist lock held. The task struct itself + * cannot go away as create_timer did get_task_struct(). + * + * We return -1, when the task is marked exiting, so + * posix_timer_event can redirect it to the group leader + * + */ + rcu_read_lock(); if (unlikely(p->flags & PF_EXITING)) { ret = -1; goto out_err; } - spin_lock_irqsave(&p->sighand->siglock, flags); + spin_lock_irqsave(&sh->siglock, flags); + + /* + * We do the check here again to handle the following scenario: + * + * CPU 0 CPU 1 + * send_sigqueue + * check PF_EXITING + * interrupt exit code running + * __exit_signal + * lock sighand->siglock + * unlock sighand->siglock + * lock sh->siglock + * add(tsk->pending) flush_sigqueue(tsk->pending) + * + */ + + if (unlikely(p->flags & PF_EXITING)) { + ret = -1; + goto out; + } if (unlikely(!list_empty(&q->list))) { /* @@ -1397,9 +1450,9 @@ send_sigqueue(int sig, struct sigqueue * signal_wake_up(p, sig == SIGKILL); out: - spin_unlock_irqrestore(&p->sighand->siglock, flags); + spin_unlock_irqrestore(&sh->siglock, flags); out_err: - read_unlock(&tasklist_lock); + rcu_read_unlock(); return ret; } @@ -1411,7 +1464,16 @@ send_group_sigqueue(int sig, struct sigq int ret = 0; BUG_ON(!(q->flags & SIGQUEUE_PREALLOC)); - read_lock(&tasklist_lock); + + while(!read_trylock(&tasklist_lock)) { + if (!p->sighand) + return -1; + cpu_relax(); + } + if (unlikely(!p->sighand)) { + ret = -1; + goto out_err; + } spin_lock_irqsave(&p->sighand->siglock, flags); handle_stop_signal(sig, p); @@ -1444,8 +1506,9 @@ send_group_sigqueue(int sig, struct sigq __group_complete_signal(sig, p); out: spin_unlock_irqrestore(&p->sighand->siglock, flags); +out_err: read_unlock(&tasklist_lock); - return(ret); + return ret; } /* _