--- arch/x86/kernel/setup64.c | 3 +- arch/x86/kernel/smpboot_64.c | 24 +++++++++--------- arch/x86/mm/init_64.c | 10 +++++++ include/asm-generic/percpu.h | 6 +--- include/linux/cpu_alloc.h | 6 ++++ init/main.c | 21 +++++++--------- mm/cpu_alloc.c | 56 +++++++++++++++++++++++++++++-------------- 7 files changed, 81 insertions(+), 45 deletions(-) Index: linux-2.6/arch/x86/kernel/setup64.c =================================================================== --- linux-2.6.orig/arch/x86/kernel/setup64.c 2007-11-13 11:53:13.250383116 -0800 +++ linux-2.6/arch/x86/kernel/setup64.c 2007-11-13 11:53:18.537382583 -0800 @@ -28,9 +28,10 @@ struct boot_params __initdata boot_param cpumask_t cpu_initialized __cpuinitdata = CPU_MASK_NONE; +/* only used for code that must also run during early boot */ struct x8664_pda *_cpu_pda[NR_CPUS] __read_mostly; EXPORT_SYMBOL(_cpu_pda); -struct x8664_pda boot_cpu_pda[NR_CPUS] __cacheline_aligned; +struct x8664_pda __initdata boot_cpu_pda[NR_CPUS]; struct desc_ptr idt_descr = { 256 * 16 - 1, (unsigned long) idt_table }; Index: linux-2.6/arch/x86/kernel/smpboot_64.c =================================================================== --- linux-2.6.orig/arch/x86/kernel/smpboot_64.c 2007-11-13 11:53:13.258382827 -0800 +++ linux-2.6/arch/x86/kernel/smpboot_64.c 2007-11-13 11:53:18.537382583 -0800 @@ -49,6 +49,7 @@ #include #include #include +#include #include #include @@ -558,18 +559,17 @@ static int __cpuinit do_boot_cpu(int cpu /* Allocate node local memory for AP pdas */ if (cpu_pda(cpu) == &boot_cpu_pda[cpu]) { - struct x8664_pda *newpda, *pda; - int node = cpu_to_node(cpu); - pda = cpu_pda(cpu); - newpda = kmalloc_node(sizeof (struct x8664_pda), GFP_ATOMIC, - node); - if (newpda) { - memcpy(newpda, pda, sizeof (struct x8664_pda)); - cpu_pda(cpu) = newpda; - } else - printk(KERN_ERR - "Could not allocate node local PDA for CPU %d on node %d\n", - cpu, node); + /* + * Per CPU area was initialized during page allocator init + * and we have a reservation at the start of the cpu area + * for the pda area. + * + * After this point we can access the pda area without + * indirection through the cpu_pda(). + */ + memcpy(CPU_PTR((void *)CPU_AREA_BASE, cpu), cpu_pda(cpu), + sizeof (struct x8664_pda)); + cpu_pda(cpu) = CPU_PTR((void *)CPU_AREA_BASE, cpu); } alternatives_smp_switch(1); Index: linux-2.6/arch/x86/mm/init_64.c =================================================================== --- linux-2.6.orig/arch/x86/mm/init_64.c 2007-11-13 11:53:13.266384017 -0800 +++ linux-2.6/arch/x86/mm/init_64.c 2007-11-13 11:53:18.537382583 -0800 @@ -536,6 +536,16 @@ void __init mem_init(void) reservedpages = end_pfn - totalram_pages - absent_pages_in_range(0, end_pfn); + /* + * Move pda area of cpu zero. After this point we can use + * the generic CPU_xxx accessors for pda data. + */ + cpu_pda(0) = CPU_ALLOC(struct x8664_pda, GFP_KERNEL | __GFP_ZERO); + if (!cpu_pda(0)) + panic("Cannot allocate cpu memory\n"); + + pda_init(0); + after_bootmem = 1; codesize = (unsigned long) &_etext - (unsigned long) &_text; Index: linux-2.6/include/asm-generic/percpu.h =================================================================== --- linux-2.6.orig/include/asm-generic/percpu.h 2007-11-13 11:53:13.274383193 -0800 +++ linux-2.6/include/asm-generic/percpu.h 2007-11-13 11:53:18.537382583 -0800 @@ -6,9 +6,7 @@ #define __GENERIC_PER_CPU #ifdef CONFIG_SMP -extern unsigned long __per_cpu_offset[NR_CPUS]; - -#define per_cpu_offset(x) (__per_cpu_offset[x]) +#define per_cpu_offset(x) CPU_OFFSET(x) /* Separate out the type, so (int[3], foo) works. */ #define DEFINE_PER_CPU(type, name) \ @@ -22,7 +20,7 @@ extern unsigned long __per_cpu_offset[NR /* var is in discarded region: offset to particular copy we want */ #define per_cpu(var, cpu) (*({ \ extern int simple_identifier_##var(void); \ - RELOC_HIDE(&per_cpu__##var, __per_cpu_offset[cpu]); })) + RELOC_HIDE(&per_cpu__##var, CPU_OFFSET(cpu)); })) #define __get_cpu_var(var) per_cpu(var, smp_processor_id()) #define __raw_get_cpu_var(var) per_cpu(var, raw_smp_processor_id()) Index: linux-2.6/include/linux/cpu_alloc.h =================================================================== --- linux-2.6.orig/include/linux/cpu_alloc.h 2007-11-13 11:53:13.282382919 -0800 +++ linux-2.6/include/linux/cpu_alloc.h 2007-11-13 11:53:18.537382583 -0800 @@ -52,4 +52,10 @@ void *cpu_alloc(unsigned long size, gfp_t gfp, unsigned long align); void cpu_free(void *cpu_pointer, unsigned long size); +/* + * Early boot allocator for per_cpu variables and special per cpu areas. + * Allocations are not tracked and cannot be freed. + */ +void *boot_cpu_alloc(unsigned long size); + #endif /* _LINUX_CPU_ALLOC_H_ */ Index: linux-2.6/init/main.c =================================================================== --- linux-2.6.orig/init/main.c 2007-11-13 11:53:16.620391885 -0800 +++ linux-2.6/init/main.c 2007-11-13 11:58:13.121882646 -0800 @@ -370,19 +370,18 @@ EXPORT_SYMBOL(__per_cpu_offset); static void __init setup_per_cpu_areas(void) { - unsigned long size, i; char *ptr; - unsigned long nr_possible_cpus = num_possible_cpus(); - /* Copy section for each CPU (we discard the original) */ - size = ALIGN(PERCPU_ENOUGH_ROOM, PAGE_SIZE); - ptr = alloc_bootmem_pages(size * nr_possible_cpus); - - for_each_possible_cpu(i) { - __per_cpu_offset[i] = ptr - __per_cpu_start; - memcpy(ptr, __per_cpu_start, __per_cpu_end - __per_cpu_start); - ptr += size; - } + ptr = init_cpu_alloc(ALIGN(PERCPU_ENOUGH_ROOM, PAGE_SIZE)); + + if (!ptr) + printk("Cannot allocate per cpu areas\n"); + + BUG_ON(ptr != CPU_PTR(__per_cpu_start, 0)); + + for_each_possible_cpu(i) + memcpy(CPU_PTR(ptr, cpu), __per_cpu_start, + __per_cpu_end - __per_cpu_start); } #endif /* !__GENERIC_PER_CPU */ Index: linux-2.6/mm/cpu_alloc.c =================================================================== --- linux-2.6.orig/mm/cpu_alloc.c 2007-11-13 11:53:16.621882825 -0800 +++ linux-2.6/mm/cpu_alloc.c 2007-11-13 11:53:20.929882588 -0800 @@ -33,6 +33,14 @@ #define UNITS_PER_BLOCK (ALLOC_SIZE / UNIT_SIZE) /* + * How many units are needed for an object of a given size + */ +static int size_to_units(unsigned long size) +{ + return DIV_ROUND_UP(size, UNIT_SIZE); +} + +/* * Lock to protect the bitmap and the meta data for the cpu allocator. */ static DEFINE_SPINLOCK(cpu_alloc_map_lock); @@ -60,18 +68,22 @@ static int cpu_alloc_map_order = -1; /* static unsigned long active_blocks; /* Number of block allocated on each cpu */ static unsigned long units_free; /* Number of available units */ static unsigned long units_total; /* Total units that are managed */ - +static unsigned long units_reserved; /* Units reserved by boot allocations */ /* * Allocate a block of memory to be used to provide cpu area memory * or to extend the bitmap for the cpu map. */ void *cpu_area_alloc_block(unsigned long size, gfp_t flags, int node) { - struct page *page = alloc_pages_node(node, + if (slab_is_available()) { + struct page *page = alloc_pages_node(node, flags, get_order(size)); - if (page) - return page_address(page); - return NULL; + if (page) + return page_address(page); + return NULL; + } else + return __alloc_bootmem_node(NODE_DATA(node), size, size, + __pa(MAX_DMA_ADDRESS)); } pte_t *cpu_area_pte_populate(pmd_t *pmd, unsigned long addr, @@ -192,7 +204,7 @@ static int expand_cpu_area(gfp_t flags) /* * Determine the size of the bit map needed */ - bits = (blocks + 1) * UNITS_PER_BLOCK; + bits = (blocks + 1) * UNITS_PER_BLOCK - units_reserved; map_order = get_order(DIV_ROUND_UP(bits, 8)); start = cpu_area + \ (blocks << (PAGE_SHIFT + CONFIG_CPU_AREA_ALLOC_ORDER)); @@ -249,6 +261,15 @@ out: return err; } +void * __init boot_cpu_alloc(unsigned long size) +{ + unsigned long x = units_reserved; + + units_reserved += size_to_units(size); + while (units_reserved > units_total) + expand_cpu_area(0); + return cpu_area + x * UNIT_SIZE; +} #else /* @@ -269,17 +290,18 @@ static inline int expand_cpu_area(gfp_t { return -ENOSYS; } -#endif - -static int first_free; /* First known free unit */ -/* - * How many units are needed for an object of a given size - */ -static int size_to_units(unsigned long size) +void * __init boot_cpu_alloc(unsigned long size) { - return DIV_ROUND_UP(size, UNIT_SIZE); + unsigned long x = units_reserved; + + units_reserved += size_to_units(size); + BUG_ON(units_reserved > units_total); + return cpu_area + x * UNIT_SIZE; } +#endif + +static int first_free; /* First known free unit */ /* * Mark an object as used in the cpu_alloc_map @@ -365,7 +387,7 @@ restart: spin_unlock_irqrestore(&cpu_alloc_map_lock, flags); - ptr = cpu_area + start * UNIT_SIZE; + ptr = cpu_area + (start + units_reserved) * UNIT_SIZE; if (gfpflags & __GFP_ZERO) { int cpu; @@ -393,8 +415,8 @@ void cpu_free(void *start, unsigned long u8 *p = start; unsigned long flags; - BUG_ON(p < cpu_area); - index = (p - cpu_area) / UNIT_SIZE; + BUG_ON(p < (cpu_area + units_reserved * UNIT_SIZE)); + index = (p - cpu_area) / UNIT_SIZE - units_reserved; BUG_ON(!test_bit(index, cpu_alloc_map) || index >= units_total);