FIXED_ADDR_PERCPU: Allow placing per cpu variables at a fixed address Add a new loader definition that can be used to choose the address of the per cpu area freely. All variable accesses will be moved by the linker to refer to the area of __per_cpu_start to __per_cpu_end However, the actual data will be loaded to __load_per_cpu_start to __load_per_cpu_end The per data must be moved to processor 0's cpu area before it can be used. If the offsets come from processor 0's addresses then the cpu_alloc functionality can be used to determine the address of the other processors variables. The cpu_offset arrays are no longer needed. A segment override or a per cpu virtual mapping can then also be used to avoid the need to add the current processors offset which will lead to a very effective way of accessing per cpu variables. IA64 already provides functionality like that in its ld script. Extract that logic. Signed-off-by: Christoph Lameter --- arch/ia64/kernel/vmlinux.lds.S | 12 ++---------- arch/ia64/mm/contig.c | 2 +- arch/ia64/mm/discontig.c | 4 ++-- include/asm-generic/sections.h | 1 + include/asm-generic/vmlinux.lds.h | 21 ++++++++++++++++++++- include/asm-ia64/percpu.h | 2 +- include/asm-ia64/sections.h | 1 - init/main.c | 3 ++- 8 files changed, 29 insertions(+), 17 deletions(-) Index: linux-2.6/arch/ia64/kernel/vmlinux.lds.S =================================================================== --- linux-2.6.orig/arch/ia64/kernel/vmlinux.lds.S 2007-11-17 16:46:26.304534847 -0800 +++ linux-2.6/arch/ia64/kernel/vmlinux.lds.S 2007-11-17 17:55:09.848773471 -0800 @@ -208,16 +208,8 @@ SECTIONS /* Per-cpu data: */ percpu : { } :percpu - . = ALIGN(PERCPU_PAGE_SIZE); - __phys_per_cpu_start = .; - .data.percpu PERCPU_ADDR : AT(__phys_per_cpu_start - LOAD_OFFSET) - { - __per_cpu_start = .; - *(.data.percpu) - *(.data.percpu.shared_aligned) - __per_cpu_end = .; - } - . = __phys_per_cpu_start + PERCPU_PAGE_SIZE; /* ensure percpu data fits + FIXED_ADDR_PERCPU(PERCPU_ADDR, PERCPU_PAGE_SIZE) + . = __load_per_cpu_start + PERCPU_PAGE_SIZE; /* ensure percpu data fits * into percpu page size */ Index: linux-2.6/include/asm-generic/vmlinux.lds.h =================================================================== --- linux-2.6.orig/include/asm-generic/vmlinux.lds.h 2007-11-17 17:54:34.380772664 -0800 +++ linux-2.6/include/asm-generic/vmlinux.lds.h 2007-11-17 17:57:31.909522666 -0800 @@ -257,10 +257,29 @@ #define PERCPU(align) \ . = ALIGN(align); \ + __load_per_cpu_start = .; \ __per_cpu_start = .; \ .data.percpu : AT(ADDR(.data.percpu) - LOAD_OFFSET) { \ *(.data.percpu.first) \ *(.data.percpu) \ *(.data.percpu.shared_aligned) \ } \ - __per_cpu_end = .; + __per_cpu_end = .; \ + __load_per_cpu_end = .; + +#define FIXED_ADDR_PERCPU(vaddr, align) \ + percpu : { } :percpu \ + . = ALIGN(align); \ + __load_per_cpu_start = .; \ + .data.percpu vaddr : AT(__load_per_cpu_start - LOAD_OFFSET) { \ + __per_cpu_start = .; \ + *(.data.percpu.first) \ + *(.data.percpu) \ + *(.data.percpu.shared_aligned) \ + __per_cpu_end = .; \ + } \ + . = __load_per_cpu_start + __per_cpu_end - __per_cpu_start; \ + __load_per_cpu_end = .; \ + data : { } :data \ + . = ALIGN(align); \ + Index: linux-2.6/include/asm-generic/sections.h =================================================================== --- linux-2.6.orig/include/asm-generic/sections.h 2007-11-17 16:46:26.352534784 -0800 +++ linux-2.6/include/asm-generic/sections.h 2007-11-17 17:55:09.864772607 -0800 @@ -12,6 +12,7 @@ extern char _sextratext[] __attribute__( extern char _eextratext[] __attribute__((weak)); extern char _end[]; extern char __per_cpu_start[], __per_cpu_end[]; +extern char __load_per_cpu_start[], __load_per_cpu_end[]; extern char __kprobes_text_start[], __kprobes_text_end[]; extern char __initdata_begin[], __initdata_end[]; extern char __start_rodata[], __end_rodata[]; Index: linux-2.6/include/asm-ia64/sections.h =================================================================== --- linux-2.6.orig/include/asm-ia64/sections.h 2007-11-17 16:46:26.356534456 -0800 +++ linux-2.6/include/asm-ia64/sections.h 2007-11-17 17:55:09.901149489 -0800 @@ -8,7 +8,6 @@ #include -extern char __per_cpu_start[], __per_cpu_end[], __phys_per_cpu_start[]; extern char __start___vtop_patchlist[], __end___vtop_patchlist[]; extern char __start___mckinley_e9_bundles[], __end___mckinley_e9_bundles[]; extern char __start___phys_stack_reg_patchlist[], __end___phys_stack_reg_patchlist[]; Index: linux-2.6/arch/ia64/mm/contig.c =================================================================== --- linux-2.6.orig/arch/ia64/mm/contig.c 2007-11-17 16:46:26.324534364 -0800 +++ linux-2.6/arch/ia64/mm/contig.c 2007-11-17 17:55:09.944804864 -0800 @@ -167,7 +167,7 @@ per_cpu_init (void) if (first_time) { first_time=0; for (cpu = 0; cpu < NR_CPUS; cpu++) { - memcpy(cpu_data, __phys_per_cpu_start, __per_cpu_end - __per_cpu_start); + memcpy(cpu_data, __load_per_cpu_start, __per_cpu_end - __per_cpu_start); __per_cpu_offset[cpu] = (char *) cpu_data - __per_cpu_start; cpu_data += PERCPU_PAGE_SIZE; per_cpu(local_per_cpu_offset, cpu) = __per_cpu_offset[cpu]; Index: linux-2.6/arch/ia64/mm/discontig.c =================================================================== --- linux-2.6.orig/arch/ia64/mm/discontig.c 2007-11-17 16:46:26.336534352 -0800 +++ linux-2.6/arch/ia64/mm/discontig.c 2007-11-17 17:55:09.961022464 -0800 @@ -144,7 +144,7 @@ static void *per_cpu_node_setup(void *cp for (cpu = 0; cpu < NR_CPUS; cpu++) { if (node == node_cpuid[cpu].nid) { - memcpy(__va(cpu_data), __phys_per_cpu_start, + memcpy(__va(cpu_data), __load_per_cpu_start, __per_cpu_end - __per_cpu_start); __per_cpu_offset[cpu] = (char*)__va(cpu_data) - __per_cpu_start; @@ -354,7 +354,7 @@ static void __init initialize_pernode_da struct cpuinfo_ia64 *cpu0_cpu_info; cpu = 0; node = node_cpuid[cpu].nid; - cpu0_cpu_info = (struct cpuinfo_ia64 *)(__phys_per_cpu_start + + cpu0_cpu_info = (struct cpuinfo_ia64 *)(__load_per_cpu_start + ((char *)&per_cpu__cpu_info - __per_cpu_start)); cpu0_cpu_info->node_data = mem_data[node].node_data; } Index: linux-2.6/include/asm-ia64/percpu.h =================================================================== --- linux-2.6.orig/include/asm-ia64/percpu.h 2007-11-17 16:46:26.364534278 -0800 +++ linux-2.6/include/asm-ia64/percpu.h 2007-11-17 17:55:09.992772951 -0800 @@ -64,7 +64,7 @@ extern void *per_cpu_init(void); #define per_cpu(var, cpu) (*((void)(cpu), &per_cpu__##var)) #define __get_cpu_var(var) per_cpu__##var #define __raw_get_cpu_var(var) per_cpu__##var -#define per_cpu_init() (__phys_per_cpu_start) +#define per_cpu_init() (__load_per_cpu_start) #endif /* SMP */ Index: linux-2.6/init/main.c =================================================================== --- linux-2.6.orig/init/main.c 2007-11-17 17:57:51.620772633 -0800 +++ linux-2.6/init/main.c 2007-11-17 17:58:36.869272679 -0800 @@ -380,7 +380,8 @@ static void __init setup_per_cpu_areas(v for_each_possible_cpu(i) { __per_cpu_offset[i] = ptr - __per_cpu_start; - memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start); + memcpy(ptr, __load_per_cpu_start, + __per_cpu_end - __per_cpu_start); ptr += size; } }