Add a per cpu special area Signed-off-by: Christoph Lameter Index: linux-2.6.21-rc1/include/linux/slub_def.h =================================================================== --- linux-2.6.21-rc1.orig/include/linux/slub_def.h 2007-02-25 16:20:02.000000000 -0800 +++ linux-2.6.21-rc1/include/linux/slub_def.h 2007-02-25 16:36:27.000000000 -0800 @@ -10,11 +10,16 @@ #include #include +struct kmem_cache_cpu { + struct page *page; +}; + struct kmem_cache_node { spinlock_t list_lock; /* Protect partial list and nr_partial */ unsigned int nr_partial; atomic_long_t nr_slabs; struct list_head partial; + struct kmem_cache_cpu local_cpu; }; /* @@ -27,6 +32,7 @@ struct kmem_cache { int size; /* Total size of an object */ int objects; /* Number of objects in slab */ struct kmem_cache_node local_node; + struct kmem_cache_node *node[MAX_NUMNODES]; atomic_t refcount; /* Refcount for destroy */ void (*ctor)(void *, struct kmem_cache *, unsigned long); void (*dtor)(void *, struct kmem_cache *, unsigned long); @@ -41,8 +47,7 @@ struct kmem_cache { atomic_t cpu_slabs; /* if >0 then flusher is scheduled */ struct delayed_work flush; #endif - struct kmem_cache_node *node[MAX_NUMNODES]; - struct page *cpu_slab[NR_CPUS]; + struct kmem_cache_cpu *cpu[NR_CPUS]; }; /* Index: linux-2.6.21-rc1/mm/slub.c =================================================================== --- linux-2.6.21-rc1.orig/mm/slub.c 2007-02-25 16:22:03.000000000 -0800 +++ linux-2.6.21-rc1/mm/slub.c 2007-02-25 17:01:14.000000000 -0800 @@ -248,6 +248,14 @@ struct kmem_cache_node *get_node(struct #endif } +struct kmem_cache_cpu *get_cpu_slab(struct kmem_cache *s, int cpu) +{ +#ifdef CONFIG_SMP + return s->cpu[cpu]; +#else + return &s->local_node.local_cpu; +#endif +} /* * Management of partially allocated slabs */ @@ -527,12 +535,13 @@ static void __always_inline putback_slab * Remove the cpu slab */ static void __always_inline deactivate_slab(struct kmem_cache *s, - struct page *page, int cpu) + struct kmem_cache_cpu *c) { - s->cpu_slab[cpu] = NULL; + struct page *page = c->page; + + c->page = NULL; ClearPageActive(page); ClearPageReferenced(page); - putback_slab(s, page); } @@ -542,11 +551,11 @@ static void __always_inline deactivate_s */ static void __flush_cpu_slab(struct kmem_cache *s, int cpu) { - struct page *page = s->cpu_slab[cpu]; + struct kmem_cache_cpu *c = get_cpu_slab(s, cpu); - if (likely(page)) { - slab_lock(page); - deactivate_slab(s, page, cpu); + if (likely(c->page)) { + slab_lock(c->page); + deactivate_slab(s, c); } } @@ -565,18 +574,17 @@ static void flush_cpu_slab(void *d) static void check_flush_cpu_slab(void *d) { struct kmem_cache *s = d; - int cpu = smp_processor_id(); - struct page *page = s->cpu_slab[cpu]; + struct kmem_cache_cpu *c = get_cpu_slab(s, smp_processor_id()); - if (!page) + if (!c->page) return; - if (PageReferenced(page)) { - ClearPageReferenced(page); + if (PageReferenced(c->page)) { + ClearPageReferenced(c->page); atomic_inc(&s->cpu_slabs); } else { - slab_lock(page); - deactivate_slab(s, page, cpu); + slab_lock(c->page); + deactivate_slab(s, c); } } @@ -625,66 +633,61 @@ static __always_inline void *__slab_allo void **object; void *next_object; unsigned long flags; - int cpu; + struct kmem_cache_cpu *c; local_irq_save(flags); - cpu = smp_processor_id(); - page = s->cpu_slab[cpu]; - if (!page) + c = get_cpu_slab(s, smp_processor_id()); + if (!c->page) goto new_slab; - slab_lock(page); - check_free_chain(s, page); - if (unlikely(!page->freelist)) + slab_lock(c->page); + check_free_chain(s, c->page); + if (unlikely(!c->page->freelist)) goto another_slab; - if (unlikely(node != -1 && page_to_nid(page) != node)) + if (unlikely(node != -1 && page_to_nid(c->page) != node)) goto another_slab; redo: - page->inuse++; - object = page->freelist; - page->freelist = next_object = object[page->offset]; - SetPageReferenced(page); - slab_unlock(page); + c->page->inuse++; + object = c->page->freelist; + c->page->freelist = next_object = object[c->page->offset]; + SetPageReferenced(c->page); + slab_unlock(c->page); local_irq_restore(flags); return object; another_slab: - deactivate_slab(s, page, cpu); + deactivate_slab(s, c); new_slab: page = get_partial(s, gfpflags, node); - if (page) - goto gotpage; - - page = new_slab(s, gfpflags, node); if (!page) { - local_irq_restore(flags); - return NULL; - } + page = new_slab(s, gfpflags, node); + if (!page) { + local_irq_restore(flags); + return NULL; + } - /* - * There is no point in putting single object slabs - * on a partial list. - */ - if (unlikely(s->objects == 1)) { - local_irq_restore(flags); - return page_address(page); + /* + * There is no point in putting single object slabs + * on a partial list. + */ + if (unlikely(s->objects == 1)) { + local_irq_restore(flags); + return page_address(page); + } + slab_lock(page); } - slab_lock(page); - -gotpage: - if (s->cpu_slab[cpu]) { + if (c->page) { slab_unlock(page); discard_slab(s, page); - page = s->cpu_slab[cpu]; - slab_lock(page); + slab_lock(c->page); } else - s->cpu_slab[cpu] = page; + c->page = page; - SetPageActive(page); - check_free_chain(s, page); + SetPageActive(c->page); + check_free_chain(s, c->page); #ifdef CONFIG_SMP if (keventd_up() && !atomic_read(&s->cpu_slabs)) { @@ -980,6 +983,80 @@ int init_kmem_cache_nodes(struct kmem_ca return 1; } +void free_kmem_cache_cpus(struct kmem_cache *s) +{ +#ifdef CONFIG_NUMA + int cpu; + + for_each_online_node(cpu) { + struct kmem_cache_cpu *c = s->cpu[cpu]; + int kmem_cache_node *n = s->node[cpu_to_node(cpu)]; + + if (c && c != &n->local_cpu) + kfree(c); + s->cpu[cpu] = NULL; + } +#endif +} + +static void init_kmem_cache_cpu(struct kmem_cache_cpu *c) +{ + c->page = NULL; +} + +int init_kmem_cache_cpus(struct kmem_cache *s, gfp_t gfpflags) +{ +#ifdef CONFIG_SMP + int cpu; + + for_each_online_cpu(cpu) { + struct kmem_cache_cpu *c; + int node = cpu_to_node(cpu); + struct kmem_cache_node *n = get_node(s, node); + + if (cpu == first_cpu(node_to_cpumask(node))) + c = &n->local_cpu; + else + if (slab_state == DOWN) { + /* + * No kmalloc_node yet so do it by hand. + * We know that this is the first slab on the + * node for this slabcache. There are no concurrent + * accesses possible. Which simplifies things. + */ + unsigned long flags; + struct page *page; + + local_irq_save(flags); + page = new_slab(s, gfpflags, node); + + BUG_ON(!page); + c = page->freelist; + page->freelist = *(void **)page->freelist; + page->inuse++; + local_irq_restore(flags); + } else + c = kmalloc_node(sizeof(struct kmem_cache_cpu), + gfpflags, node); + + if (!c) { + free_kmem_cache_nodes(s); + return 0; + } + + s->cpu[cpu] = c; + init_kmem_cache_cpu(c); + + if (slab_state == DOWN) + atomic_long_inc(&n->nr_slabs); + } +#else + init_kmem_cache_cpu(&s->local_node.local_cpu); +#endif + return 1; +} + + int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags, const char *name, size_t size, size_t align, unsigned long flags, @@ -1035,9 +1112,13 @@ int kmem_cache_open(struct kmem_cache *s atomic_set(&s->cpu_slabs, 0); INIT_DELAYED_WORK(&s->flush, flusher); #endif + if (init_kmem_cache_nodes(s, gfpflags)) { - register_slab(s); - return 1; + if (init_kmem_cache_cpus(s, gfpflags)) { + register_slab(s); + return 1; + } else + free_kmem_cache_nodes(s); } error: if (flags & SLAB_PANIC) @@ -1133,6 +1214,7 @@ int kmem_cache_close(struct kmem_cache * if (atomic_long_read(&n->nr_slabs)) return 1; } + free_kmem_cache_cpus(s); free_kmem_cache_nodes(s); unregister_slab(s); return 0; @@ -1165,10 +1247,10 @@ static unsigned long slab_objects(struct int cpu; int node; unsigned long flags; - struct page *page; for_each_online_node(node) { struct kmem_cache_node *n = get_node(s, node); + struct page *page; nr_slabs += atomic_read(&n->nr_slabs); nr_partial_slabs += n->nr_partial; @@ -1183,11 +1265,12 @@ static unsigned long slab_objects(struct } for_each_possible_cpu(cpu) { - page = s->cpu_slab[cpu]; - if (page) { + struct kmem_cache_cpu *c = get_cpu_slab(s, cpu); + + if (c->page) { nr_cpu_slabs++; - in_cpu_slabs += page->inuse; - nodes[page_to_nid(page)]++; + in_cpu_slabs += c->page->inuse; + nodes[page_to_nid(c->page)]++; } }