--- include/asm-x86/pgtable_64.h | 2 + include/linux/cpu.h | 28 +++++++++++++++ include/linux/slub_def.h | 15 -------- include/linux/types.h | 2 + mm/cpualloc.c | 80 +++++++++++++++++++++++++++++++++++++++++++ mm/slub.c | 9 ++++ 6 files changed, 122 insertions(+), 14 deletions(-) Index: linux-2.6/include/linux/cpu.h =================================================================== --- linux-2.6.orig/include/linux/cpu.h 2007-10-29 22:02:35.000000000 -0700 +++ linux-2.6/include/linux/cpu.h 2007-10-30 13:55:38.000000000 -0700 @@ -139,4 +139,32 @@ static inline int disable_nonboot_cpus(v static inline void enable_nonboot_cpus(void) {} #endif /* !CONFIG_PM_SLEEP_SMP */ +/* + * Manage pool of memory that is bound to a cpu and that changes + * if the cpu changes + */ +typedef cpu_area_t unsigned long; + +/* + * Allocate a cpu area and initialize the per cpu area for each cpu with + * the given data (or zero it if data == NULL) + */ +extern cpu_area_t cpu_alloc(unsigned long size, const void *data) + +/* + * Release a cpu area + */ +extern void cpu_free(cpu_area_t cpu_area, unsigned long size); + +/* + * Convert a cpu area pointer to a pointer that is independent + * from the cpu. This may be necessary to remotely read and + * manipulate cpu areas. + */ +extern void *cpu_addr_to_cpu(cpu_area_t, int cpu); +extern void *cpu_addr_to_local(cpu_area_t, int cpu) + +#define cpu_addr_to_local(x) (void *)x +#define cpu_addr_to_cpu(x, cpu) cpu_table[cpu] + (x - base) + #endif /* _LINUX_CPU_H_ */ Index: linux-2.6/include/linux/slub_def.h =================================================================== --- linux-2.6.orig/include/linux/slub_def.h 2007-10-29 22:02:40.000000000 -0700 +++ linux-2.6/include/linux/slub_def.h 2007-10-29 22:06:47.000000000 -0700 @@ -11,15 +11,6 @@ #include #include -struct kmem_cache_cpu { - void **freelist; - struct page *page; - int node; - unsigned int offset; - unsigned int objsize; - unsigned int objects; -}; - struct kmem_cache_node { spinlock_t list_lock; /* Protect partial list and nr_partial */ unsigned long nr_partial; @@ -34,6 +25,7 @@ struct kmem_cache_node { * Slab cache management. */ struct kmem_cache { + struct kmem_cache_cpu * cpu_slab cpu_area; /* Used for retriving partial slabs etc */ unsigned long flags; int size; /* The size of an object including meta data */ @@ -63,11 +55,6 @@ struct kmem_cache { int defrag_ratio; struct kmem_cache_node *node[MAX_NUMNODES]; #endif -#ifdef CONFIG_SMP - struct kmem_cache_cpu *cpu_slab[NR_CPUS]; -#else - struct kmem_cache_cpu cpu_slab; -#endif }; /* Index: linux-2.6/include/asm-x86/pgtable_64.h =================================================================== --- linux-2.6.orig/include/asm-x86/pgtable_64.h 2007-10-30 13:42:22.000000000 -0700 +++ linux-2.6/include/asm-x86/pgtable_64.h 2007-10-30 13:44:21.000000000 -0700 @@ -138,6 +138,8 @@ static inline pte_t ptep_get_and_clear_f #define VMALLOC_START _AC(0xffffc20000000000, UL) #define VMALLOC_END _AC(0xffffe1ffffffffff, UL) #define VMEMMAP_START _AC(0xffffe20000000000, UL) +#define PER_CPU_START __AC(0xfffff20000000000, UL) +#define PER_NODE_START __AC(0xfffff20000200000, UL) #define MODULES_VADDR _AC(0xffffffff88000000, UL) #define MODULES_END _AC(0xfffffffffff00000, UL) #define MODULES_LEN (MODULES_END - MODULES_VADDR) Index: linux-2.6/include/linux/types.h =================================================================== --- linux-2.6.orig/include/linux/types.h 2007-10-29 22:04:17.000000000 -0700 +++ linux-2.6/include/linux/types.h 2007-10-29 22:04:36.000000000 -0700 @@ -204,4 +204,6 @@ struct ustat { char f_fpack[6]; }; +#define cpu_area __thread + #endif /* _LINUX_TYPES_H */ Index: linux-2.6/mm/cpualloc.c =================================================================== --- /dev/null 1970-01-01 00:00:00.000000000 +0000 +++ linux-2.6/mm/cpualloc.c 2007-10-30 10:55:22.000000000 -0700 @@ -0,0 +1,80 @@ +/* + * Per cpu allocator + */ + +typedef unsigned long long per_cpu_unit; + +#define PER_CPU_UNITS sizeof(unsigned long long); +#define MAXIMUM_UNITS_PER_CPU 16384 + +typedef unsigned long per_cpu_t; + +static DEFINE_PER_CPU(per_cpu_unit, cpu_area)[MAXIMUM_UNITS_PER_CPU]; +static DECLARE_SPINLOCK(cpu_area_lock); +static DECLARE_BITMAP(cpu_alloc_map, MAXIMUM_UNITS_PER_CPU); + +static inline int size_to_units(unsigned long size) +{ + return (size + sizeof(per_cpu_unit) - 1) / sizeof(per_cpu_unit) +} + +void *cpu_address(per_cpu_t cpup, int cpu) +{ + return per_cpu(per_cpu_unit + cpuup, cpu) +} + +static inline void set_bits(unsigned long *bits, int start, int length) +{ + int i; + + for (i = start; i < start + length) + __set_bit(bits, i); +} + +static inline void clear_bits(unsigned long *bits, int start, int length) +{ + int i; + + for (i = start; i < start + length) + __clear_bit(bits, i); +} + +per_cpu_t cpu_alloc(unsigned long size, void *data) +{ + int start = 0; + int units = size_to_units(size); + int end; + + spin_lock(&cpu_area_lock); + do { + start = find_next_zero_bit(cpu_alloc_map, MAXIMUM_UNITS_PER_CPU, start); + if (start == MAXIMUM_UNITS_PER_CPU) + return NULL; + + end = find_next_bit(cpu_alloc_map, MAXIMUM_UNITS_PER_CPU, start); + if (end - start >= units) + break; + start = end; + } while (1); + + set_bits(cpu_alloc_map, start, units); + spin_unlock(&cpu_area_lock); + + if (data) { + for_each_online_cpu(cpu) + memcpy(cpu_address(cpup, cpu), data, size); + } + return &per_cpu(cpu_area[start], 0) - per_cpu_offset(0); +} + +void cpu_free(per_cpu_t per_cpu_offset, unsigned long size) +{ + int start = ((per_cpu_offset + per_cpu_offset(0)) - + &per_cpu(cpu_area, 0)) / sizeof(per_cpu_unit); + int units = size_to_units(size); + + spin_lock(&cpu_area_lock); + clear_bits(cpu_alloc_map, start, units); + spin_unlock(&cpu_area_lock); +} + Index: linux-2.6/mm/slub.c =================================================================== --- linux-2.6.orig/mm/slub.c 2007-10-29 22:03:35.000000000 -0700 +++ linux-2.6/mm/slub.c 2007-10-29 22:03:52.000000000 -0700 @@ -181,6 +181,15 @@ #define cache_line_size() L1_CACHE_BYTES #endif +struct kmem_cache_cpu { + void **freelist; + struct page *page; + int node; + unsigned int offset; + unsigned int objsize; + unsigned int objects; +}; + static int kmem_size = sizeof(struct kmem_cache); #ifdef CONFIG_SMP