Index: linux-2.6.20-mm2/include/linux/slub_def.h =================================================================== --- linux-2.6.20-mm2.orig/include/linux/slub_def.h 2007-02-21 21:40:01.000000000 -0800 +++ linux-2.6.20-mm2/include/linux/slub_def.h 2007-02-21 21:41:02.000000000 -0800 @@ -18,7 +18,7 @@ struct list_head partial; unsigned long nr_partial; int offset; /* Free pointer offset. */ - struct page *active[NR_CPUS]; + struct page *cpu_slab[NR_CPUS]; atomic_long_t nr_slabs[MAX_NUMNODES]; unsigned int order; unsigned long flags; @@ -35,7 +35,7 @@ struct list_head list; /* List of slabs */ #ifdef CONFIG_SMP struct mutex flushing; - atomic_t active_cpus; /* if >0 then flusher is scheduled */ + atomic_t cpu_slabs; /* if >0 then flusher is scheduled */ struct delayed_work flush; #endif }; Index: linux-2.6.20-mm2/mm/slub.c =================================================================== --- linux-2.6.20-mm2.orig/mm/slub.c 2007-02-21 21:40:57.000000000 -0800 +++ linux-2.6.20-mm2/mm/slub.c 2007-02-21 21:56:19.000000000 -0800 @@ -2,8 +2,8 @@ * Slub a slab allocator that limits cache line use instead of queuing * objects in per cpu and per node queues. * - * The allocator synchronizes using slab based locks and only - * uses a centralized list lock to manage a pool of partial slabs. + * The allocator synchronizes using per slab locks and only + * uses a centralized lock to manage a pool of partial slabs. * * (C) 2007 Silicon Graphics Inc., Christoph Lameter */ @@ -20,12 +20,12 @@ /* * Overloading of page flags that are otherwise used for LRU management. * - * PageActive The page is used as an active cache. Allocations - * may be performed from the slab. The page is not + * PageActive The slab is used as an cpu cache. Allocations + * may be performed from the slab. The slab is not * on a partial list. * - * PageReferenced The active slab was used recently. This is used - * to push back active slabs if they are unused + * PageReferenced The per cpu slab was used recently. This is used + * to push back per cpu slabs if they are unused * for a longer time period. * * PagePrivate Only a single object exists per slab. Objects are not @@ -98,9 +98,9 @@ * 2. slab->list_lock * * SLUB assigns one slab for allocation to each processor. - * Allocations only occur from these slabs called active slabs. + * Allocations only occur from these slabs called cpu slabs. * - * If a cpu slab is active then a workqueue thread checks every 10 + * If a cpu slab exists then a workqueue thread checks every 10 * seconds if the cpu slab is still in use. The cpu slab is pushed back * to the list if inactive [only needed for SMP]. * @@ -364,7 +364,7 @@ /* * Determine if a certain object on a page is on the freelist and - * therefore free. Must hold the slab lock for active slabs to + * therefore free. Must hold the slab lock for cpu slabs to * guarantee that the chains are consistent. */ static int on_freelist(struct kmem_cache *s, struct page *page, void *search) @@ -483,12 +483,12 @@ } /* - * Remove the currently active slab + * Remove the cpu slab */ static void __always_inline deactivate_slab(struct kmem_cache *s, struct page *page, int cpu) { - s->active[cpu] = NULL; + s->cpu_slab[cpu] = NULL; ClearPageActive(page); ClearPageReferenced(page); @@ -496,38 +496,43 @@ } /* - * Flush active slab + * Flush cpu slab * Called from IPI handler with interrupts disabled. */ -static void flush_active(void *d) +static void __flush_cpu_slab(struct kmem_cache *s, int cpu) { - struct kmem_cache *s = d; - int cpu = smp_processor_id(); - struct page *page = s->active[cpu]; + struct page *page = s->cpu_slab[cpu]; - page = s->active[cpu]; if (likely(page)) { slab_lock(page); deactivate_slab(s, page, cpu); } } +static void flush_cpu_slab(void *d) +{ + struct kmem_cache *s = d; + int cpu = smp_processor_id(); + + __flush_cpu_slab(s, cpu); +} + #ifdef CONFIG_SMP /* - * Called from IPI during flushing to check and flush active slabs. + * Called from IPI during flushing to check and flush cpu slabs. */ -void check_flush_active(void *d) +void check_flush_cpu_slab(void *d) { struct kmem_cache *s = d; int cpu = smp_processor_id(); - struct page *page = s->active[cpu]; + struct page *page = s->cpu_slab[cpu]; if (!page) return; if (PageReferenced(page)) { ClearPageReferenced(page); - atomic_inc(&s->active_cpus); + atomic_inc(&s->cpu_slabs); } else { slab_lock(page); deactivate_slab(s, page, cpu); @@ -544,20 +549,20 @@ if (!mutex_trylock(&s->flushing)) return; - atomic_set(&s->active_cpus, num_online_cpus()); - on_each_cpu(check_flush_active, s, 1, 1); - if (atomic_read(&s->active_cpus)) + atomic_set(&s->cpu_slabs, num_online_cpus()); + on_each_cpu(check_flush_cpu_slab, s, 1, 1); + if (atomic_read(&s->cpu_slabs)) schedule_delayed_work(&s->flush, 2 * HZ); mutex_unlock(&s->flushing); } static void drain_all(struct kmem_cache *s) { - if (atomic_read(&s->active_cpus)) { + if (atomic_read(&s->cpu_slabs)) { mutex_lock(&s->flushing); cancel_delayed_work(&s->flush); - atomic_set(&s->active_cpus, 0); - on_each_cpu(flush_active, s, 1, 1); + atomic_set(&s->cpu_slabs, 0); + on_each_cpu(flush_cpu_slab, s, 1, 1); mutex_unlock(&s->flushing); } } @@ -567,7 +572,7 @@ unsigned long flags; local_irq_save(flags); - flush_active(s); + flush_cpu_slab(s); local_irq_restore(flags); } #endif @@ -583,7 +588,7 @@ local_irq_save(flags); cpu = smp_processor_id(); - page = s->active[cpu]; + page = s->cpu_slab[cpu]; if (!page) goto new_slab; @@ -619,7 +624,7 @@ /* * There is no point in putting single object slabs - * on an active list. + * on a partial list. */ if (unlikely(s->objects == 1)) { local_irq_restore(flags); @@ -629,20 +634,20 @@ slab_lock(page); gotpage: - if (s->active[cpu]) { + if (s->cpu_slab[cpu]) { slab_unlock(page); discard_slab(s, page); - page = s->active[cpu]; + page = s->cpu_slab[cpu]; slab_lock(page); } else - s->active[cpu] = page; + s->cpu_slab[cpu] = page; SetPageActive(page); check_free_chain(s, page); #ifdef CONFIG_SMP - if (keventd_up() && !atomic_read(&s->active_cpus)) { - atomic_inc(&s->active_cpus); + if (keventd_up() && !atomic_read(&s->cpu_slabs)) { + atomic_inc(&s->cpu_slabs); schedule_delayed_work(&s->flush, 2 * HZ); } #endif @@ -871,11 +876,11 @@ atomic_set(&s->refcount, 1); spin_lock_init(&s->list_lock); for_each_possible_cpu(cpu) - s->active[cpu] = NULL; + s->cpu_slab[cpu] = NULL; INIT_LIST_HEAD(&s->partial); #ifdef CONFIG_SMP mutex_init(&s->flushing); - atomic_set(&s->active_cpus, 0); + atomic_set(&s->cpu_slabs, 0); INIT_DELAYED_WORK(&s->flush, flusher); #endif s->name = name; @@ -1018,7 +1023,7 @@ } /* - * Shrinking drops the active per cpu slabs and also reaps all empty + * Shrinking drops the per cpu slabs and also reaps all empty * slabs off the partial list. Returns the number of slabs freed. * * The move_object function will be called for each objects in partially @@ -1141,13 +1146,13 @@ } static unsigned long slab_objects(struct kmem_cache *s, - unsigned long *p_total, unsigned long *p_active, + unsigned long *p_total, unsigned long *p_cpu_slabs, unsigned long *p_partial, unsigned long *nodes) { - int partial = count_objects(s, &s->partial, nodes); + int in_partial_slabs = count_objects(s, &s->partial, nodes); int nr_slabs = 0; - int active = 0; /* Active slabs */ - int nr_active = 0; /* Objects in active slabs */ + int cpu_slabs = 0; + int nr_in_cpu_slabs = 0; int cpu; int node; @@ -1155,11 +1160,11 @@ nr_slabs += nodes[node] = atomic_read(&s->nr_slabs[node]); for_each_possible_cpu(cpu) { - struct page *page = s->active[cpu]; + struct page *page = s->cpu_slab[cpu]; if (page) { - nr_active++; - active += page->inuse; + cpu_slabs++; + nr_in_cpu_slabs += page->inuse; nodes[page_to_nid(page)]++; } } @@ -1167,14 +1172,14 @@ if (p_partial) *p_partial = s->nr_partial; - if (p_active) - *p_active = nr_active; + if (p_cpu_slabs) + *p_cpu_slabs = cpu_slabs; if (p_total) *p_total = nr_slabs; - return partial + active + - (nr_slabs - s->nr_partial - nr_active) * s->objects; + return in_partial_slabs + nr_in_cpu_slabs + + (nr_slabs - s->nr_partial - cpu_slabs) * s->objects; } /******************************************************************** @@ -1184,8 +1189,9 @@ struct kmem_cache kmalloc_caches[KMALLOC_NR_CACHES] __cacheline_aligned; EXPORT_SYMBOL(kmalloc_caches); +#ifdef CONFIG_ZONE_DMA static struct kmem_cache *kmalloc_caches_dma[KMALLOC_NR_CACHES]; - +#endif static int __init setup_slab_min_order(char *str) { get_option (&str, &slab_min_order); @@ -1444,9 +1450,9 @@ * Output format version, so at least we can change it * without _too_ many complaints. */ - seq_puts(m, "slabinfo - version: 3.0\n"); + seq_puts(m, "slubinfo - version: 1.0\n"); seq_puts(m, "# name " - " // "); + " // "); #ifdef CONFIG_NUMA seq_puts(m, " "); #endif @@ -1498,7 +1504,7 @@ { struct kmem_cache *s = p; unsigned long total_slabs; - unsigned long active_slabs; + unsigned long cpu_slabs; unsigned long partial_slabs; unsigned long objects; unsigned char options[13]; @@ -1506,7 +1512,7 @@ char *x; unsigned long nodes[nr_node_ids]; - objects = slab_objects(s, &total_slabs, &active_slabs, + objects = slab_objects(s, &total_slabs, &cpu_slabs, &partial_slabs, nodes); if (s->ctor) *d++ = 'C'; @@ -1536,7 +1542,7 @@ *d = 0; x = kasprintf(GFP_KERNEL, "%lu/%lu/%lu", total_slabs, partial_slabs, - active_slabs); + cpu_slabs); seq_printf(m, "%-21s %6lu %1d %7u %12s %5s", s->name, objects, s->order, s->size, x, options); @@ -1559,12 +1565,6 @@ #ifdef CONFIG_SMP -void drain_all_slabs(struct kmem_cache *s, int cpu) -{ - /* Punt for now. Simple drain all cpus */ - drain_all(s); -} - /* * Use the cpu notifier to insure that the thresholds are recalculated * when necessary. @@ -1577,7 +1577,7 @@ switch (action) { case CPU_UP_CANCELED: case CPU_DEAD: - for_all_slabs(drain_all_slabs, cpu); + for_all_slabs(__flush_cpu_slab, cpu); break; default: break; @@ -1604,7 +1604,7 @@ /***************************************************************** * Generic reaper used to support the page allocator - * (the active slabs are reaped by a per processor workqueue). + * (the cpu slabs are reaped by a per processor workqueue). * * Maybe move this to the page allocator? ****************************************************************/