--- include/linux/mm_types.h | 1 include/linux/slub_def.h | 23 ++++------- mm/slub.c | 98 +++++++++++++++++++++++++---------------------- 3 files changed, 63 insertions(+), 59 deletions(-) Index: linux-2.6/include/linux/mm_types.h =================================================================== --- linux-2.6.orig/include/linux/mm_types.h 2008-01-20 17:19:58.304115727 -0800 +++ linux-2.6/include/linux/mm_types.h 2008-01-20 17:19:59.564147028 -0800 @@ -60,6 +60,7 @@ struct page { * it points to anon_vma object: * see PAGE_MAPPING_ANON below. */ + int cpu; /* SLUB: Processor queue */ }; #if NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS spinlock_t ptl; Index: linux-2.6/include/linux/slub_def.h =================================================================== --- linux-2.6.orig/include/linux/slub_def.h 2008-01-20 17:19:58.280115133 -0800 +++ linux-2.6/include/linux/slub_def.h 2008-01-20 17:19:59.576147328 -0800 @@ -11,14 +11,6 @@ #include #include -struct kmem_cache_cpu { - void **freelist; - struct page *page; - int node; - unsigned int offset; - unsigned int objsize; -}; - struct kmem_cache_node { spinlock_t list_lock; /* Protect partial list and nr_partial */ unsigned long nr_partial; @@ -29,6 +21,15 @@ struct kmem_cache_node { #endif }; +struct kmem_cache_cpu { + void **freelist; + struct page *page; + int node; + unsigned int offset; + unsigned int objsize; + struct kmem_cache_node n; +}; + /* * Slab cache management. */ @@ -40,12 +41,6 @@ struct kmem_cache { int offset; /* Free pointer offset. */ int order; - /* - * Avoid an extra cache line for UP, SMP and for the node local to - * struct kmem_cache. - */ - struct kmem_cache_node local_node; - /* Allocation and freeing of slabs */ int objects; /* Number of objects in slab */ int refcount; /* Refcount for slab cache destroy */ Index: linux-2.6/mm/slub.c =================================================================== --- linux-2.6.orig/mm/slub.c 2008-01-20 17:19:58.316116027 -0800 +++ linux-2.6/mm/slub.c 2008-01-20 17:51:19.302810012 -0800 @@ -262,15 +262,6 @@ int slab_is_available(void) return slab_state >= UP; } -static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node) -{ -#ifdef CONFIG_NUMA - return s->node[node]; -#else - return &s->local_node; -#endif -} - static inline struct kmem_cache_cpu *get_cpu_slab(struct kmem_cache *s, int cpu) { #ifdef CONFIG_SMP @@ -280,6 +271,11 @@ static inline struct kmem_cache_cpu *get #endif } +static inline struct kmem_cache_node *get_slab_list(struct kmem_cache *s, int cpu) +{ + return &(get_cpu_slab(s, cpu)->n); +} + static inline int check_valid_pointer(struct kmem_cache *s, struct page *page, const void *object) { @@ -817,7 +813,7 @@ static void remove_full(struct kmem_cach if (!(s->flags & SLAB_STORE_USER)) return; - n = get_node(s, page_to_nid(page)); + n = get_slab_list(s, page->cpu); spin_lock(&n->list_lock); list_del(&page->lru); @@ -1092,10 +1088,11 @@ static struct page *new_slab(struct kmem if (!page) goto out; - n = get_node(s, page_to_nid(page)); + n = get_slab_list(s, smp_processor_id()); if (n) atomic_long_inc(&n->nr_slabs); page->slab = s; + page->cpu = smp_processor_id(); page->flags |= 1 << PG_slab; if (s->flags & (SLAB_DEBUG_FREE | SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER | SLAB_TRACE)) @@ -1165,7 +1162,7 @@ static void free_slab(struct kmem_cache static void discard_slab(struct kmem_cache *s, struct page *page) { - struct kmem_cache_node *n = get_node(s, page_to_nid(page)); + struct kmem_cache_node *n = get_slab_list(s, page->cpu); atomic_long_dec(&n->nr_slabs); reset_page_mapcount(page); @@ -1212,7 +1209,7 @@ static void add_partial(struct kmem_cach static void remove_partial(struct kmem_cache *s, struct page *page) { - struct kmem_cache_node *n = get_node(s, page_to_nid(page)); + struct kmem_cache_node *n = get_slab_list(s, page->cpu); spin_lock(&n->list_lock); list_del(&page->lru); @@ -1318,9 +1315,9 @@ static struct page *get_any_partial(stru static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node) { struct page *page; - int searchnode = (node == -1) ? numa_node_id() : node; +// int searchnode = (node == -1) ? numa_node_id() : node; - page = get_partial_node(get_node(s, searchnode)); + page = get_partial_node(get_slab_list(s, smp_processor_id())); if (page || (flags & __GFP_THISNODE)) return page; @@ -1336,7 +1333,7 @@ static struct page *get_partial(struct k */ static void unfreeze_slab(struct kmem_cache *s, struct page *page, int tail) { - struct kmem_cache_node *n = get_node(s, page_to_nid(page)); + struct kmem_cache_node *n = get_slab_list(s, page->cpu); ClearSlabFrozen(page); if (page->inuse) { @@ -1615,7 +1612,8 @@ checks_ok: * then add it. */ if (unlikely(!prior)) - add_partial(get_node(s, page_to_nid(page)), page, 1); + add_partial(get_slab_list(s, page->cpu), page, + page->cpu != smp_processor_id()); out_unlock: slab_unlock(page); @@ -2089,7 +2087,11 @@ static void free_kmem_cache_nodes(struct static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags) { - init_kmem_cache_node(&s->local_node); + int cpu; + + for_each_online_cpu(cpu) + init_kmem_cache_node(get_slab_list(s, cpu)); + return 1; } #endif @@ -2218,12 +2220,13 @@ static int kmem_cache_open(struct kmem_c #ifdef CONFIG_NUMA s->remote_node_defrag_ratio = 100; #endif - if (!init_kmem_cache_nodes(s, gfpflags & ~SLUB_DMA)) + if (!alloc_kmem_cache_cpus(s, gfpflags & ~SLUB_DMA)) goto error; - if (alloc_kmem_cache_cpus(s, gfpflags & ~SLUB_DMA)) + if (init_kmem_cache_nodes(s, gfpflags & ~SLUB_DMA)) return 1; - free_kmem_cache_nodes(s); + + free_kmem_cache_cpus(s); error: if (flags & SLAB_PANIC) panic("Cannot create slab %s size=%lu realsize=%u " @@ -2301,19 +2304,19 @@ static int free_list(struct kmem_cache * */ static inline int kmem_cache_close(struct kmem_cache *s) { - int node; + int cpu; flush_all(s); /* Attempt to free all objects */ - free_kmem_cache_cpus(s); - for_each_node_state(node, N_NORMAL_MEMORY) { - struct kmem_cache_node *n = get_node(s, node); + for_each_online_cpu(cpu) { + struct kmem_cache_node *n = get_slab_list(s, cpu); n->nr_partial -= free_list(s, n, &n->partial); if (atomic_long_read(&n->nr_slabs)) return 1; } + free_kmem_cache_cpus(s); free_kmem_cache_nodes(s); return 0; } @@ -2639,7 +2642,7 @@ static unsigned long count_partial(struc */ int kmem_cache_shrink(struct kmem_cache *s) { - int node; + int cpu; int i; struct kmem_cache_node *n; struct page *page; @@ -2652,8 +2655,8 @@ int kmem_cache_shrink(struct kmem_cache return -ENOMEM; flush_all(s); - for_each_node_state(node, N_NORMAL_MEMORY) { - n = get_node(s, node); + for_each_online_cpu(cpu) { + n = get_slab_list(s, cpu); if (!n->nr_partial) continue; @@ -2716,6 +2719,7 @@ static int slab_mem_going_offline_callba static void slab_mem_offline_callback(void *arg) { +#if 0 struct kmem_cache_node *n; struct kmem_cache *s; struct memory_notify *marg = arg; @@ -2747,15 +2751,17 @@ static void slab_mem_offline_callback(vo } } up_read(&slub_lock); +#endif } static int slab_mem_going_online_callback(void *arg) { + int ret = 0; +#if 0 struct kmem_cache_node *n; struct kmem_cache *s; struct memory_notify *marg = arg; int nid = marg->status_change_nid; - int ret = 0; /* * If the node's memory is already available, then kmem_cache_node is @@ -2786,6 +2792,7 @@ static int slab_mem_going_online_callbac } out: up_read(&slub_lock); +#endif return ret; } @@ -3034,9 +3041,11 @@ static int __cpuinit slab_cpuup_callback case CPU_UP_PREPARE_FROZEN: init_alloc_cpu_cpu(cpu); down_read(&slub_lock); - list_for_each_entry(s, &slab_caches, list) + list_for_each_entry(s, &slab_caches, list) { s->cpu_slab[cpu] = alloc_kmem_cache_cpu(s, cpu, GFP_KERNEL); + init_kmem_cache_node(get_slab_list(s, cpu)); + } up_read(&slub_lock); break; @@ -3183,7 +3192,7 @@ out: static long validate_slab_cache(struct kmem_cache *s) { - int node; + int cpu; unsigned long count = 0; unsigned long *map = kmalloc(BITS_TO_LONGS(s->objects) * sizeof(unsigned long), GFP_KERNEL); @@ -3192,8 +3201,8 @@ static long validate_slab_cache(struct k return -ENOMEM; flush_all(s); - for_each_node_state(node, N_NORMAL_MEMORY) { - struct kmem_cache_node *n = get_node(s, node); + for_each_online_cpu(cpu) { + struct kmem_cache_node *n = get_slab_list(s, cpu); count += validate_slab_node(s, n, map); } @@ -3403,7 +3412,7 @@ static int list_locations(struct kmem_ca int n = 0; unsigned long i; struct loc_track t = { 0, 0, NULL }; - int node; + int cpu; if (!alloc_loc_track(&t, PAGE_SIZE / sizeof(struct location), GFP_TEMPORARY)) @@ -3412,8 +3421,8 @@ static int list_locations(struct kmem_ca /* Push back cpu slabs */ flush_all(s); - for_each_node_state(node, N_NORMAL_MEMORY) { - struct kmem_cache_node *n = get_node(s, node); + for_each_online_cpu(cpu) { + struct kmem_cache_node *n = get_slab_list(s, cpu); unsigned long flags; struct page *page; @@ -3498,7 +3507,6 @@ static unsigned long slab_objects(struct { unsigned long total = 0; int cpu; - int node; int x; unsigned long *nodes; unsigned long *per_cpu; @@ -3533,8 +3541,9 @@ static unsigned long slab_objects(struct } } - for_each_node_state(node, N_NORMAL_MEMORY) { - struct kmem_cache_node *n = get_node(s, node); + for_each_online_cpu(cpu) { + struct kmem_cache_node *n = get_slab_list(s, cpu); + int node = cpu_to_node(cpu); if (flags & SO_PARTIAL) { if (flags & SO_OBJECTS) @@ -3572,7 +3581,6 @@ static unsigned long slab_objects(struct static int any_slab_objects(struct kmem_cache *s) { - int node; int cpu; for_each_possible_cpu(cpu) { @@ -3582,8 +3590,8 @@ static int any_slab_objects(struct kmem_ return 1; } - for_each_online_node(node) { - struct kmem_cache_node *n = get_node(s, node); + for_each_online_cpu(cpu) { + struct kmem_cache_node *n = get_slab_list(s, cpu); if (!n) continue; @@ -4193,12 +4201,12 @@ static int s_show(struct seq_file *m, vo unsigned long nr_inuse = 0; unsigned long nr_objs; struct kmem_cache *s; - int node; + int cpu; s = list_entry(p, struct kmem_cache, list); - for_each_online_node(node) { - struct kmem_cache_node *n = get_node(s, node); + for_each_online_cpu(cpu) { + struct kmem_cache_node *n = get_slab_list(s, cpu); if (!n) continue;