From: Ravikiran G Thirumalai Change the atomic_t sockets_allocated member of struct proto to a per-cpu counter. Signed-off-by: Pravin B. Shelar Signed-off-by: Ravikiran Thirumalai Signed-off-by: Shai Fultheim Signed-off-by: Andrew Morton --- include/net/sock.h | 8 +++++++- include/net/tcp.h | 2 +- net/core/sock.c | 21 +++++++++++++++++++-- net/core/stream.c | 2 +- net/ipv4/proc.c | 2 +- net/ipv4/tcp.c | 8 +++++++- net/ipv4/tcp_ipv4.c | 5 ++--- net/ipv6/tcp_ipv6.c | 4 ++-- 8 files changed, 40 insertions(+), 12 deletions(-) diff -puN include/net/sock.h~per-cpufy-net-proto-structures-sockets_allocated include/net/sock.h --- devel/include/net/sock.h~per-cpufy-net-proto-structures-sockets_allocated 2006-05-11 15:18:52.000000000 -0700 +++ devel-akpm/include/net/sock.h 2006-05-11 15:18:52.000000000 -0700 @@ -551,7 +551,7 @@ struct proto { /* Memory pressure */ void (*enter_memory_pressure)(void); struct percpu_counter *memory_allocated; /* Current allocated memory. */ - atomic_t *sockets_allocated; /* Current number of sockets. */ + int *sockets_allocated; /* Current number of sockets (percpu counter). */ /* * Pressure flag: try to collapse. @@ -587,6 +587,12 @@ struct proto { } stats[NR_CPUS]; }; +extern int read_sockets_allocated(struct proto *prot); +extern void mod_sockets_allocated(int *sockets_allocated, int count); + +#define inc_sockets_allocated(c) mod_sockets_allocated(c, 1) +#define dec_sockets_allocated(c) mod_sockets_allocated(c, -1) + extern int proto_register(struct proto *prot, int alloc_slab); extern void proto_unregister(struct proto *prot); diff -puN include/net/tcp.h~per-cpufy-net-proto-structures-sockets_allocated include/net/tcp.h --- devel/include/net/tcp.h~per-cpufy-net-proto-structures-sockets_allocated 2006-05-11 15:18:52.000000000 -0700 +++ devel-akpm/include/net/tcp.h 2006-05-11 15:18:52.000000000 -0700 @@ -226,7 +226,7 @@ extern int sysctl_tcp_base_mss; extern int sysctl_tcp_workaround_signed_windows; extern struct percpu_counter tcp_memory_allocated; -extern atomic_t tcp_sockets_allocated; +extern int *tcp_sockets_allocated; extern int tcp_memory_pressure; /* diff -puN net/core/sock.c~per-cpufy-net-proto-structures-sockets_allocated net/core/sock.c --- devel/net/core/sock.c~per-cpufy-net-proto-structures-sockets_allocated 2006-05-11 15:18:52.000000000 -0700 +++ devel-akpm/net/core/sock.c 2006-05-11 15:18:52.000000000 -0700 @@ -879,7 +879,7 @@ struct sock *sk_clone(const struct sock newsk->sk_sleep = NULL; if (newsk->sk_prot->sockets_allocated) - atomic_inc(newsk->sk_prot->sockets_allocated); + inc_sockets_allocated(newsk->sk_prot->sockets_allocated); } out: return newsk; @@ -1587,6 +1587,23 @@ void sk_common_release(struct sock *sk) EXPORT_SYMBOL(sk_common_release); +int read_sockets_allocated(struct proto *prot) +{ + int total = 0; + int cpu; + for_each_possible_cpu(cpu) + total += *per_cpu_ptr(prot->sockets_allocated, cpu); + return total; +} +EXPORT_SYMBOL(read_sockets_allocated); + +void mod_sockets_allocated(int *sockets_allocated, int count) +{ + (*per_cpu_ptr(sockets_allocated, get_cpu())) += count; + put_cpu(); +} +EXPORT_SYMBOL(mod_sockets_allocated); + static DEFINE_RWLOCK(proto_list_lock); static LIST_HEAD(proto_list); @@ -1756,7 +1773,7 @@ static void proto_seq_printf(struct seq_ "%2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c %2c\n", proto->name, proto->obj_size, - proto->sockets_allocated != NULL ? atomic_read(proto->sockets_allocated) : -1, + proto->sockets_allocated != NULL ? read_sockets_allocated(proto) : -1, proto->memory_allocated != NULL ? percpu_counter_read_positive(proto->memory_allocated) : -1, proto->memory_pressure != NULL ? *proto->memory_pressure ? "yes" : "no" : "NI", diff -puN net/core/stream.c~per-cpufy-net-proto-structures-sockets_allocated net/core/stream.c --- devel/net/core/stream.c~per-cpufy-net-proto-structures-sockets_allocated 2006-05-11 15:18:52.000000000 -0700 +++ devel-akpm/net/core/stream.c 2006-05-11 15:18:52.000000000 -0700 @@ -243,7 +243,7 @@ int sk_stream_mem_schedule(struct sock * return 1; if (!*sk->sk_prot->memory_pressure || - sk->sk_prot->sysctl_mem[2] > atomic_read(sk->sk_prot->sockets_allocated) * + sk->sk_prot->sysctl_mem[2] > read_sockets_allocated(sk->sk_prot) * sk_stream_pages(sk->sk_wmem_queued + atomic_read(&sk->sk_rmem_alloc) + sk->sk_forward_alloc)) diff -puN net/ipv4/proc.c~per-cpufy-net-proto-structures-sockets_allocated net/ipv4/proc.c --- devel/net/ipv4/proc.c~per-cpufy-net-proto-structures-sockets_allocated 2006-05-11 15:18:52.000000000 -0700 +++ devel-akpm/net/ipv4/proc.c 2006-05-11 15:18:52.000000000 -0700 @@ -63,7 +63,7 @@ static int sockstat_seq_show(struct seq_ socket_seq_show(seq); seq_printf(seq, "TCP: inuse %d orphan %d tw %d alloc %d mem %lu\n", fold_prot_inuse(&tcp_prot), atomic_read(&tcp_orphan_count), - tcp_death_row.tw_count, atomic_read(&tcp_sockets_allocated), + tcp_death_row.tw_count, read_sockets_allocated(&tcp_prot), percpu_counter_read_positive(&tcp_memory_allocated)); seq_printf(seq, "UDP: inuse %d\n", fold_prot_inuse(&udp_prot)); seq_printf(seq, "RAW: inuse %d\n", fold_prot_inuse(&raw_prot)); diff -puN net/ipv4/tcp.c~per-cpufy-net-proto-structures-sockets_allocated net/ipv4/tcp.c --- devel/net/ipv4/tcp.c~per-cpufy-net-proto-structures-sockets_allocated 2006-05-11 15:18:52.000000000 -0700 +++ devel-akpm/net/ipv4/tcp.c 2006-05-11 15:18:52.000000000 -0700 @@ -285,7 +285,7 @@ EXPORT_SYMBOL(sysctl_tcp_rmem); EXPORT_SYMBOL(sysctl_tcp_wmem); struct percpu_counter tcp_memory_allocated; /* Current allocated memory. */ -atomic_t tcp_sockets_allocated; /* Current number of TCP sockets. */ +int *tcp_sockets_allocated; /* Current number of TCP sockets. */ EXPORT_SYMBOL(tcp_memory_allocated); EXPORT_SYMBOL(tcp_sockets_allocated); @@ -2101,6 +2101,12 @@ void __init tcp_init(void) if (!tcp_hashinfo.bind_bucket_cachep) panic("tcp_init: Cannot alloc tcp_bind_bucket cache."); + tcp_sockets_allocated = alloc_percpu(int); + if (!tcp_sockets_allocated) + panic("tcp_init: Cannot alloc tcp_sockets_allocated"); + + tcp_prot.sockets_allocated = tcp_sockets_allocated; + /* Size and allocate the main established and bind bucket * hash tables. * diff -puN net/ipv4/tcp_ipv4.c~per-cpufy-net-proto-structures-sockets_allocated net/ipv4/tcp_ipv4.c --- devel/net/ipv4/tcp_ipv4.c~per-cpufy-net-proto-structures-sockets_allocated 2006-05-11 15:18:52.000000000 -0700 +++ devel-akpm/net/ipv4/tcp_ipv4.c 2006-05-11 15:18:52.000000000 -0700 @@ -1277,7 +1277,7 @@ static int tcp_v4_init_sock(struct sock sk->sk_sndbuf = sysctl_tcp_wmem[1]; sk->sk_rcvbuf = sysctl_tcp_rmem[1]; - atomic_inc(&tcp_sockets_allocated); + inc_sockets_allocated(tcp_sockets_allocated); return 0; } @@ -1311,7 +1311,7 @@ int tcp_v4_destroy_sock(struct sock *sk) sk->sk_sndmsg_page = NULL; } - atomic_dec(&tcp_sockets_allocated); + dec_sockets_allocated(tcp_sockets_allocated); return 0; } @@ -1819,7 +1819,6 @@ struct proto tcp_prot = { .unhash = tcp_unhash, .get_port = tcp_v4_get_port, .enter_memory_pressure = tcp_enter_memory_pressure, - .sockets_allocated = &tcp_sockets_allocated, .orphan_count = &tcp_orphan_count, .memory_allocated = &tcp_memory_allocated, .memory_pressure = &tcp_memory_pressure, diff -puN net/ipv6/tcp_ipv6.c~per-cpufy-net-proto-structures-sockets_allocated net/ipv6/tcp_ipv6.c --- devel/net/ipv6/tcp_ipv6.c~per-cpufy-net-proto-structures-sockets_allocated 2006-05-11 15:18:52.000000000 -0700 +++ devel-akpm/net/ipv6/tcp_ipv6.c 2006-05-11 15:18:52.000000000 -0700 @@ -1379,7 +1379,7 @@ static int tcp_v6_init_sock(struct sock sk->sk_sndbuf = sysctl_tcp_wmem[1]; sk->sk_rcvbuf = sysctl_tcp_rmem[1]; - atomic_inc(&tcp_sockets_allocated); + inc_sockets_allocated(tcp_sockets_allocated); return 0; } @@ -1577,7 +1577,6 @@ struct proto tcpv6_prot = { .unhash = tcp_unhash, .get_port = tcp_v6_get_port, .enter_memory_pressure = tcp_enter_memory_pressure, - .sockets_allocated = &tcp_sockets_allocated, .memory_allocated = &tcp_memory_allocated, .memory_pressure = &tcp_memory_pressure, .orphan_count = &tcp_orphan_count, @@ -1613,6 +1612,7 @@ static struct inet_protosw tcpv6_protosw void __init tcpv6_init(void) { + tcpv6_prot.sockets_allocated = tcp_sockets_allocated; /* register inet6 protocol */ if (inet6_add_protocol(&tcpv6_protocol, IPPROTO_TCP) < 0) printk(KERN_ERR "tcpv6_init: Could not register protocol\n"); _