From: Ravikiran G Thirumalai Use alloc_percpu to allocate per-CPU memory for the proto->inuse field. The inuse field is currently per-CPU as in NR_CPUS * cacheline size -- a big bloat on arches with large cachelines. Also marks some frequently used protos read mostly. Signed-off-by: Pravin B. Shelar Signed-off-by: Ravikiran Thirumalai Signed-off-by: Shai Fultheim Signed-off-by: Andrew Morton --- include/net/sock.h | 9 +++------ net/core/sock.c | 16 +++++++++++++++- net/ipv4/proc.c | 2 +- net/ipv4/raw.c | 2 +- net/ipv4/tcp_ipv4.c | 2 +- net/ipv4/udp.c | 2 +- net/ipv6/proc.c | 2 +- 7 files changed, 23 insertions(+), 12 deletions(-) diff -puN include/net/sock.h~per-cpufy-net-proto-structures-protoinuse include/net/sock.h --- devel/include/net/sock.h~per-cpufy-net-proto-structures-protoinuse 2006-04-22 01:39:34.000000000 -0700 +++ devel-akpm/include/net/sock.h 2006-04-22 01:39:34.000000000 -0700 @@ -582,10 +582,7 @@ struct proto { #ifdef SOCK_REFCNT_DEBUG atomic_t socks; #endif - struct { - int inuse; - u8 __pad[SMP_CACHE_BYTES - sizeof(int)]; - } stats[NR_CPUS]; + int *inuse; }; extern int read_sockets_allocated(struct proto *prot); @@ -625,12 +622,12 @@ static inline void sk_refcnt_debug_relea /* Called with local bh disabled */ static __inline__ void sock_prot_inc_use(struct proto *prot) { - prot->stats[smp_processor_id()].inuse++; + (*per_cpu_ptr(prot->inuse, smp_processor_id())) += 1; } static __inline__ void sock_prot_dec_use(struct proto *prot) { - prot->stats[smp_processor_id()].inuse--; + (*per_cpu_ptr(prot->inuse, smp_processor_id())) -= 1; } /* With per-bucket locks this operation is not-atomic, so that diff -puN net/core/sock.c~per-cpufy-net-proto-structures-protoinuse net/core/sock.c --- devel/net/core/sock.c~per-cpufy-net-proto-structures-protoinuse 2006-04-22 01:39:34.000000000 -0700 +++ devel-akpm/net/core/sock.c 2006-04-22 01:39:34.000000000 -0700 @@ -1610,7 +1610,7 @@ static LIST_HEAD(proto_list); int proto_register(struct proto *prot, int alloc_slab) { char *request_sock_slab_name = NULL; - char *timewait_sock_slab_name; + char *timewait_sock_slab_name = NULL; int rc = -ENOBUFS; if (alloc_slab) { @@ -1661,12 +1661,21 @@ int proto_register(struct proto *prot, i } } + prot->inuse = alloc_percpu(int); + if (prot->inuse == NULL) { + if (alloc_slab) + goto out_free_timewait_sock_slab_name_cache; + else + goto out; + } write_lock(&proto_list_lock); list_add(&prot->node, &proto_list); write_unlock(&proto_list_lock); rc = 0; out: return rc; +out_free_timewait_sock_slab_name_cache: + kmem_cache_destroy(prot->twsk_prot->twsk_slab); out_free_timewait_sock_slab_name: kfree(timewait_sock_slab_name); out_free_request_sock_slab: @@ -1690,6 +1699,11 @@ void proto_unregister(struct proto *prot list_del(&prot->node); write_unlock(&proto_list_lock); + if (prot->inuse != NULL) { + free_percpu(prot->inuse); + prot->inuse = NULL; + } + if (prot->slab != NULL) { kmem_cache_destroy(prot->slab); prot->slab = NULL; diff -puN net/ipv4/proc.c~per-cpufy-net-proto-structures-protoinuse net/ipv4/proc.c --- devel/net/ipv4/proc.c~per-cpufy-net-proto-structures-protoinuse 2006-04-22 01:39:34.000000000 -0700 +++ devel-akpm/net/ipv4/proc.c 2006-04-22 01:39:34.000000000 -0700 @@ -50,7 +50,7 @@ static int fold_prot_inuse(struct proto int cpu; for_each_possible_cpu(cpu) - res += proto->stats[cpu].inuse; + res += (*per_cpu_ptr(proto->inuse, cpu)); return res; } diff -puN net/ipv4/raw.c~per-cpufy-net-proto-structures-protoinuse net/ipv4/raw.c --- devel/net/ipv4/raw.c~per-cpufy-net-proto-structures-protoinuse 2006-04-22 01:39:34.000000000 -0700 +++ devel-akpm/net/ipv4/raw.c 2006-04-22 01:39:34.000000000 -0700 @@ -748,7 +748,7 @@ static int raw_ioctl(struct sock *sk, in } } -struct proto raw_prot = { +struct proto raw_prot __read_mostly = { .name = "RAW", .owner = THIS_MODULE, .close = raw_close, diff -puN net/ipv4/tcp_ipv4.c~per-cpufy-net-proto-structures-protoinuse net/ipv4/tcp_ipv4.c --- devel/net/ipv4/tcp_ipv4.c~per-cpufy-net-proto-structures-protoinuse 2006-04-22 01:39:34.000000000 -0700 +++ devel-akpm/net/ipv4/tcp_ipv4.c 2006-04-22 01:39:34.000000000 -0700 @@ -1799,7 +1799,7 @@ void tcp4_proc_exit(void) } #endif /* CONFIG_PROC_FS */ -struct proto tcp_prot = { +struct proto tcp_prot __read_mostly = { .name = "TCP", .owner = THIS_MODULE, .close = tcp_close, diff -puN net/ipv4/udp.c~per-cpufy-net-proto-structures-protoinuse net/ipv4/udp.c --- devel/net/ipv4/udp.c~per-cpufy-net-proto-structures-protoinuse 2006-04-22 01:39:34.000000000 -0700 +++ devel-akpm/net/ipv4/udp.c 2006-04-22 01:39:34.000000000 -0700 @@ -1369,7 +1369,7 @@ unsigned int udp_poll(struct file *file, } -struct proto udp_prot = { +struct proto udp_prot __read_mostly = { .name = "UDP", .owner = THIS_MODULE, .close = udp_close, diff -puN net/ipv6/proc.c~per-cpufy-net-proto-structures-protoinuse net/ipv6/proc.c --- devel/net/ipv6/proc.c~per-cpufy-net-proto-structures-protoinuse 2006-04-22 01:39:34.000000000 -0700 +++ devel-akpm/net/ipv6/proc.c 2006-04-22 01:39:34.000000000 -0700 @@ -39,7 +39,7 @@ static int fold_prot_inuse(struct proto int cpu; for_each_possible_cpu(cpu) - res += proto->stats[cpu].inuse; + res += (*per_cpu_ptr(proto->inuse, cpu)); return res; } _