Index: linux-2.6/mm/slab.c =================================================================== --- linux-2.6.orig/mm/slab.c 2009-10-24 14:41:57.000000000 -0500 +++ linux-2.6/mm/slab.c 2009-10-24 14:42:45.000000000 -0500 @@ -1416,7 +1416,7 @@ INIT_LIST_HEAD(&cache_chain); list_add(&cache_cache.next, &cache_chain); cache_cache.colour_off = cache_line_size(); - cache_cache.cpu_array = alloc_percpu_arraycache(BOOT_CPUCACHE_ENTRIES, GFP_KERNEL); + cache_cache.cpu_array = alloc_cpu_arraycache(BOOT_CPUCACHE_ENTRIES, GFP_KERNEL); init_arraycache(this_cpu_ptr(cache_cache.cpu_array), BOOT_CPUCACHE_ENTRIES, 1); cache_cache.nodelists[node] = &initkmem_list3[CACHE_CACHE + node]; @@ -1984,7 +1984,7 @@ * that's used by kmalloc(24), otherwise the creation of * further caches will BUG(). */ - cachep->cpu_array = alloc_percpu_arraycache(BOOT_CPUCACHE_ENTRIES, + cachep->cpu_array = alloc_cpu_arraycache(BOOT_CPUCACHE_ENTRIES, GFP_KERNEL); /* @@ -1998,7 +1998,7 @@ else g_cpucache_up = PARTIAL_AC; } else { - cachep->cpu_array = alloc_percpu_arraycache(BOOT_CPUCACHE_ENTRIES, + cachep->cpu_array = alloc_cpu_arraycache(BOOT_CPUCACHE_ENTRIES, GFP_KERNEL); if (g_cpucache_up == PARTIAL_AC) { @@ -2019,10 +2019,10 @@ jiffies + REAPTIMEOUT_LIST3 + ((unsigned long)cachep) % REAPTIMEOUT_LIST3; - cpu_cache_get(cachep)->avail = 0; - cpu_cache_get(cachep)->limit = BOOT_CPUCACHE_ENTRIES; - cpu_cache_get(cachep)->batchcount = 1; - cpu_cache_get(cachep)->touched = 0; + __this_cpu_write(cachep->array->avail, 0); + __this_cpu_write(cachep->array->limit, BOOT_CPUCACHE_ENTRIES); + __this_cpu_write(cachep->array->batchcount, 1); + __this_cpu_write(cachep->array->touched, 0);; cachep->batchcount = 1; cachep->limit = BOOT_CPUCACHE_ENTRIES; return 0; @@ -2370,15 +2370,14 @@ static void do_drain(void *arg) { struct kmem_cache *cachep = arg; - struct array_cache *ac; int node = numa_node_id(); check_irq_off(); - ac = cpu_cache_get(cachep); spin_lock(&cachep->nodelists[node]->list_lock); - free_block(cachep, ac->entry, ac->avail, node); + free_block(cachep, this_cpu_ptr(&cachep->array->entry), + __this_cpu_read(cachep->array->avail), node); spin_unlock(&cachep->nodelists[node]->list_lock); - ac->avail = 0; + __this_cpu_write(cachep->array->avail, 0); } static void drain_cpu_caches(struct kmem_cache *cachep) @@ -3065,15 +3064,14 @@ static inline void *____cache_alloc(struct kmem_cache *cachep, gfp_t flags) { void *objp; - struct array_cache *ac; check_irq_off(); - ac = cpu_cache_get(cachep); - if (likely(ac->avail)) { + if (likely(__this_cpu_read(cachep->array->avail))) { STATS_INC_ALLOCHIT(cachep); - ac->touched = 1; - objp = ac->entry[--ac->avail]; + __this_cpu_write(cachep->array->touched, 1); + __this_cpu_dec(cachep->array->avail); + objp = __this_cpu_read(cachep->array->entry[__this_cpu_read(cachep->array->avail)]); } else { STATS_INC_ALLOCMISS(cachep); objp = cache_alloc_refill(cachep, flags); @@ -3083,7 +3081,8 @@ * per-CPU caches is leaked, we need to make sure kmemleak doesn't * treat the array pointers as a reference to the object. */ - kmemleak_erase(&ac->entry[ac->avail]); + kmemleak_erase(this_cpu_ptr(cachep->array->entry + + __this_cpu_read(cachep->array->avail))); return objp; } @@ -3484,7 +3483,7 @@ */ static inline void __cache_free(struct kmem_cache *cachep, void *objp) { - struct array_cache *ac = cpu_cache_get(cachep); + struct array_cache *ac = cachep->array; check_irq_off(); kmemleak_free_recursive(objp, cachep->flags); @@ -3502,15 +3501,14 @@ if (nr_online_nodes > 1 && cache_free_alien(cachep, objp)) return; - if (likely(ac->avail < ac->limit)) { + if (likely(__this_cpu_read(ac->avail) < __this_cpu_read(ac->limit))) STATS_INC_FREEHIT(cachep); - ac->entry[ac->avail++] = objp; - return; - } else { + else { STATS_INC_FREEMISS(cachep); - cache_flusharray(cachep, ac); - ac->entry[ac->avail++] = objp; + cache_flusharray(cachep, __this_cpu_ptr(ac)); } + __this_cpu_write(ac->entry[__this_cpu_read(ac->avail)], objp); + __this_cpu_inc(ac->avail); } /** @@ -3857,7 +3855,7 @@ struct array_cache *old; int i; - new = alloc_percpu_arraycache(limit, gfp); + new = alloc_cpu_arraycache(limit, gfp); if (!new) return -ENOMEM;