slab: simplify bootstrap It now becomes possible to simplify bootstrap since we can now operate slab caches without per cpu object caches and without shared object caches and also without alien caches, Throw out all the mind bending stuff and simplify bootstrap. Signed-off-by: Christoph Lameter Index: linux-2.6.18-rc3/mm/slab.c =================================================================== --- linux-2.6.18-rc3.orig/mm/slab.c 2006-08-03 23:06:54.507212113 -0700 +++ linux-2.6.18-rc3/mm/slab.c 2006-08-03 23:09:54.778287962 -0700 @@ -280,16 +280,6 @@ struct array_cache { #define ALIEN_LIMIT 12 /* - * bootstrap: The caches do not work without cpuarrays anymore, but the - * cpuarrays are allocated from the generic caches... - */ -#define BOOT_CPUCACHE_ENTRIES 1 -struct arraycache_init { - struct array_cache cache; - void *entries[BOOT_CPUCACHE_ENTRIES]; -}; - -/* * The slab lists for all objects. */ struct kmem_list3 { @@ -348,7 +338,6 @@ static __always_inline int index_of(cons static int slab_early_init = 1; -#define INDEX_AC index_of(sizeof(struct arraycache_init)) #define INDEX_L3 index_of(sizeof(struct kmem_list3)) static void kmem_list3_init(struct kmem_list3 *parent) @@ -659,15 +648,10 @@ static struct cache_names __initdata cac #undef CACHE }; -static struct arraycache_init initarray_cache __initdata = - { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; -static struct arraycache_init initarray_generic = - { {0, BOOT_CPUCACHE_ENTRIES, 1, 0} }; - /* internal cache of cache description objs */ static struct kmem_cache cache_cache = { .batchcount = 1, - .limit = BOOT_CPUCACHE_ENTRIES, + .limit = 0, .shared = 1, .buffer_size = sizeof(struct kmem_cache), .name = "kmem_cache", @@ -720,22 +704,11 @@ static struct list_head cache_chain; atomic_t slab_reclaim_pages; /* - * chicken and egg problem: delay the per-cpu array allocation - * until the general caches are up. - */ -static enum { - NONE, - PARTIAL_AC, - PARTIAL_L3, - FULL -} g_cpucache_up; - -/* * used by boot code to determine if it can use slab based allocator */ int slab_is_available(void) { - return g_cpucache_up == FULL; + return slab_is_available == 0; } static DEFINE_PER_CPU(struct work_struct, reap_work); @@ -755,7 +728,7 @@ static inline struct kmem_cache *__find_ * kmem_cache_create(), or __kmalloc(), before * the generic caches are initialized. */ - BUG_ON(malloc_sizes[INDEX_AC].cs_cachep == NULL); + BUG_ON(malloc_sizes[INDEX_L3].cs_cachep == NULL); #endif while (size > csizep->cs_size) csizep++; @@ -922,6 +895,9 @@ static struct array_cache *alloc_arrayca int memsize = sizeof(void *) * entries + sizeof(struct array_cache); struct array_cache *nc = NULL; + if (!entries) + return NULL; + nc = kmalloc_node(memsize, GFP_KERNEL, node); if (nc) { nc->avail = 0; @@ -1310,12 +1286,11 @@ void __init kmem_cache_init(void) struct cache_names *names; int i; int order; + int node; + struct kmem_cache *cache; - for (i = 0; i < NUM_INIT_LISTS; i++) { + for (i = 0; i < NUM_INIT_LISTS; i++) kmem_list3_init(&initkmem_list3[i]); - if (i < MAX_NUMNODES) - cache_cache.nodelists[i] = NULL; - } /* * Fragmentation resistance on low memory - only use bigger @@ -1348,7 +1323,6 @@ void __init kmem_cache_init(void) INIT_LIST_HEAD(&cache_chain); list_add(&cache_cache.next, &cache_chain); cache_cache.colour_off = cache_line_size(); - cache_cache.array[smp_processor_id()] = &initarray_cache.cache; cache_cache.nodelists[numa_node_id()] = &initkmem_list3[CACHE_CACHE]; cache_cache.buffer_size = ALIGN(cache_cache.buffer_size, @@ -1370,25 +1344,20 @@ void __init kmem_cache_init(void) sizes = malloc_sizes; names = cache_names; - /* - * Initialize the caches that provide memory for the array cache and the - * kmem_list3 structures first. Without this, further allocations will - * bug. - */ - - sizes[INDEX_AC].cs_cachep = kmem_cache_create(names[INDEX_AC].name, - sizes[INDEX_AC].cs_size, - ARCH_KMALLOC_MINALIGN, - ARCH_KMALLOC_FLAGS|SLAB_PANIC, - NULL, NULL); - if (INDEX_AC != INDEX_L3) { - sizes[INDEX_L3].cs_cachep = + cache = sizes[INDEX_L3].cs_cachep = kmem_cache_create(names[INDEX_L3].name, sizes[INDEX_L3].cs_size, ARCH_KMALLOC_MINALIGN, ARCH_KMALLOC_FLAGS|SLAB_PANIC, NULL, NULL); + + /* Early init. We have to setup the nodelists ourselves */ + for_each_online_node(node) { + cache->nodelists[node] = &initkmem_list3[INDEX_L3 + node]; + cache->nodelists[node]->next_reap = jiffies + + REAPTIMEOUT_LIST3 + + ((unsigned long)cache) % REAPTIMEOUT_LIST3; } slab_early_init = 0; @@ -1408,6 +1377,7 @@ void __init kmem_cache_init(void) ARCH_KMALLOC_FLAGS|SLAB_PANIC, NULL, NULL); } + init_lock_keys(sizes); sizes->cs_dmacachep = kmem_cache_create(names->name_dma, @@ -1419,71 +1389,30 @@ void __init kmem_cache_init(void) sizes++; names++; } - /* 4) Replace the bootstrap head arrays */ - { - struct array_cache *ptr; - - ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL); - - local_irq_disable(); - BUG_ON(cpu_cache_get(&cache_cache) != &initarray_cache.cache); - memcpy(ptr, cpu_cache_get(&cache_cache), - sizeof(struct arraycache_init)); - /* - * Do not assume that spinlocks can be initialized via memcpy: - */ - spin_lock_init(&ptr->lock); - - cache_cache.array[smp_processor_id()] = ptr; - local_irq_enable(); - - ptr = kmalloc(sizeof(struct arraycache_init), GFP_KERNEL); - - local_irq_disable(); - BUG_ON(cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep) - != &initarray_generic.cache); - memcpy(ptr, cpu_cache_get(malloc_sizes[INDEX_AC].cs_cachep), - sizeof(struct arraycache_init)); - /* - * Do not assume that spinlocks can be initialized via memcpy: - */ - spin_lock_init(&ptr->lock); - - malloc_sizes[INDEX_AC].cs_cachep->array[smp_processor_id()] = - ptr; - local_irq_enable(); - } /* 5) Replace the bootstrap kmem_list3's */ - { - int node; - /* Replace the static kmem_list3 structures for the boot cpu */ - init_list(&cache_cache, &initkmem_list3[CACHE_CACHE], + /* Replace the static kmem_list3 structures for the boot cpu */ + init_list(&cache_cache, &initkmem_list3[CACHE_CACHE], numa_node_id()); - for_each_online_node(node) { - init_list(malloc_sizes[INDEX_AC].cs_cachep, - &initkmem_list3[SIZE_AC + node], node); - - if (INDEX_AC != INDEX_L3) { - init_list(malloc_sizes[INDEX_L3].cs_cachep, - &initkmem_list3[SIZE_L3 + node], - node); - } - } + for_each_online_node(node) { + init_list(malloc_sizes[INDEX_L3].cs_cachep, + &initkmem_list3[SIZE_L3 + node], + node); } /* 6) resize the head arrays to their final sizes */ { struct kmem_cache *cachep; mutex_lock(&cache_chain_mutex); - list_for_each_entry(cachep, &cache_chain, next) + list_for_each_entry(cachep, &cache_chain, next) { + cachep->flags |= SLAB_CPUCACHE; if (enable_cpucache(cachep)) BUG(); + } mutex_unlock(&cache_chain_mutex); } /* Done! */ - g_cpucache_up = FULL; /* * Register a cpu startup notifier callback that initializes @@ -1805,22 +1734,6 @@ static void slab_destroy(struct kmem_cac } } -/* - * For setting up all the kmem_list3s for cache whose buffer_size is same as - * size of kmem_list3. - */ -static void set_up_list3s(struct kmem_cache *cachep, int index) -{ - int node; - - for_each_online_node(node) { - cachep->nodelists[node] = &initkmem_list3[index + node]; - cachep->nodelists[node]->next_reap = jiffies + - REAPTIMEOUT_LIST3 + - ((unsigned long)cachep) % REAPTIMEOUT_LIST3; - } -} - static void __kmem_cache_destroy(struct kmem_cache *cachep) { int i; @@ -1912,60 +1825,6 @@ static size_t calculate_slab_order(struc return left_over; } -static int setup_cpu_cache(struct kmem_cache *cachep) -{ - if (g_cpucache_up == FULL) - return enable_cpucache(cachep); - - if (g_cpucache_up == NONE) { - /* - * Note: the first kmem_cache_create must create the cache - * that's used by kmalloc(24), otherwise the creation of - * further caches will BUG(). - */ - cachep->array[smp_processor_id()] = &initarray_generic.cache; - - /* - * If the cache that's used by kmalloc(sizeof(kmem_list3)) is - * the first cache, then we need to set up all its list3s, - * otherwise the creation of further caches will BUG(). - */ - set_up_list3s(cachep, SIZE_AC); - if (INDEX_AC == INDEX_L3) - g_cpucache_up = PARTIAL_L3; - else - g_cpucache_up = PARTIAL_AC; - } else { - cachep->array[smp_processor_id()] = - kmalloc(sizeof(struct arraycache_init), GFP_KERNEL); - - if (g_cpucache_up == PARTIAL_AC) { - set_up_list3s(cachep, SIZE_L3); - g_cpucache_up = PARTIAL_L3; - } else { - int node; - for_each_online_node(node) { - cachep->nodelists[node] = - kmalloc_node(sizeof(struct kmem_list3), - GFP_KERNEL, node); - BUG_ON(!cachep->nodelists[node]); - kmem_list3_init(cachep->nodelists[node]); - } - } - } - cachep->nodelists[numa_node_id()]->next_reap = - jiffies + REAPTIMEOUT_LIST3 + - ((unsigned long)cachep) % REAPTIMEOUT_LIST3; - - cpu_cache_get(cachep)->avail = 0; - cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES; - cpu_cache_get(cachep)->batchcount = 1; - cpu_cache_get(cachep)->touched = 0; - cachep->batchcount = 1; - cachep->limit = BOOT_CPUCACHE_ENTRIES; - return 0; -} - /** * kmem_cache_create - Create a cache. * @name: A string which is used in /proc/slabinfo to identify this cache. @@ -2214,12 +2073,22 @@ kmem_cache_create (const char *name, siz cachep->dtor = dtor; cachep->name = name; - if (!setup_cpu_cache(cachep)) { - __kmem_cache_destroy(cachep); - cachep = NULL; - goto oops; + if (flags & SLAB_CPUCACHE) { + if (!enable_cpu_cache(cachep)) { + __kmem_cache_destroy(cachep); + cachep = NULL; + goto oops; + } + } else { + cachep->batchcount = 0; + cachep->limit = 0; + /* + * If its not too early in bootstrap then we can setup the + * lists. + */ + if (sizes->cs_cachep[INDEX_L3]) + alloc_kmemlist(cachep); } - /* cache setup completed, link it into the list */ list_add(&cachep->next, &cache_chain); oops: @@ -3516,11 +3385,14 @@ static int alloc_kmemlist(struct kmem_ca for_each_online_node(node) { - new_shared = alloc_arraycache(node, + if (cachep->flags & SLAB_CPUCACHE) { + new_shared = alloc_arraycache(node, cachep->shared*cachep->batchcount, 0xbaadf00d); - if (!new_shared) - goto fail; + if (!new_shared) + goto fail; + else + new_shared = NULL; l3 = cachep->nodelists[node]; if (l3) { @@ -3553,8 +3425,8 @@ static int alloc_kmemlist(struct kmem_ca l3->free_limit = (1 + nr_cpus_node(node)) * cachep->batchcount + cachep->num; cachep->nodelists[node] = l3; - alloc_alien_cache(l3, node); - + if (cachep->flags & SLAB_CPUCACHE) + alloc_alien_cache(l3, node); } return 0;