Index: linux-2.6.21-rc4-mm1/include/linux/mm_types.h =================================================================== --- linux-2.6.21-rc4-mm1.orig/include/linux/mm_types.h 2007-03-22 21:57:24.000000000 -0700 +++ linux-2.6.21-rc4-mm1/include/linux/mm_types.h 2007-03-22 21:57:48.000000000 -0700 @@ -51,7 +51,7 @@ struct page { #endif struct { /* SLUB uses */ struct page *first_page; /* Compound pages */ - struct kmem_cache *slab; /* Pointer to slab */ + void *slab; /* Pointer to slab */ }; }; union { Index: linux-2.6.21-rc4-mm1/include/linux/slub_def.h =================================================================== --- linux-2.6.21-rc4-mm1.orig/include/linux/slub_def.h 2007-03-22 20:51:46.000000000 -0700 +++ linux-2.6.21-rc4-mm1/include/linux/slub_def.h 2007-03-22 21:54:33.000000000 -0700 @@ -8,76 +8,10 @@ */ #include #include -#include -#include - -struct kmem_cache_node { - spinlock_t list_lock; /* Protect partial list and nr_partial */ - unsigned long nr_partial; - atomic_long_t nr_slabs; - struct list_head partial; -}; - -/* - * Slab cache management. - */ -struct kmem_cache { - /* Used for retriving partial slabs etc */ - unsigned long flags; - int size; /* The size of an object including meta data */ - int objsize; /* The size of an object without meta data */ - int offset; /* Free pointer offset. */ - atomic_t cpu_slabs; /* != 0 -> flusher scheduled. */ - - /* - * Avoid an extra cache line for UP, SMP and for the node local to - * struct kmem_cache. - */ - struct kmem_cache_node local_node; - - /* Allocation and freeing of slabs */ - unsigned int order; - int objects; /* Number of objects in slab */ - int refcount; /* Refcount for slab cache destroy */ - void (*ctor)(void *, struct kmem_cache *, unsigned long); - void (*dtor)(void *, struct kmem_cache *, unsigned long); - int inuse; /* Offset to metadata */ - int align; /* Alignment */ - const char *name; /* Name (only for display!) */ - struct list_head list; /* List of slab caches */ - struct kobject kobj; /* For sysfs */ - -#ifdef CONFIG_SMP - struct delayed_work flush; - struct mutex flushing; -#endif -#ifdef CONFIG_NUMA - struct kmem_cache_node *node[MAX_NUMNODES]; -#endif - struct page *cpu_slab[NR_CPUS]; -}; /* * Kmalloc subsystem. */ -#define KMALLOC_SHIFT_LOW 3 - -#define KMALLOC_SHIFT_HIGH 18 - -#if L1_CACHE_BYTES <= 64 -#define KMALLOC_EXTRAS 2 -#define KMALLOC_EXTRA -#else -#define KMALLOC_EXTRAS 0 -#endif - -#define KMALLOC_NR_CACHES (KMALLOC_SHIFT_HIGH - KMALLOC_SHIFT_LOW \ - + 1 + KMALLOC_EXTRAS) -/* - * We keep the general caches in an array of slab caches that are used for - * 2^x bytes of allocations. - */ -extern struct kmem_cache kmalloc_caches[KMALLOC_NR_CACHES]; /* * Sorry that the following has to be that ugly but some versions of GCC @@ -85,12 +19,10 @@ extern struct kmem_cache kmalloc_caches[ */ static inline int kmalloc_index(int size) { -#ifdef KMALLOC_EXTRA if (size > 64 && size <= 96) - return KMALLOC_SHIFT_HIGH + 1; + return 1; if (size > 128 && size <= 192) - return KMALLOC_SHIFT_HIGH + 2; -#endif + return 2; if (size < 8) return 3; if (size < 16) return 4; if (size < 32) return 5; @@ -107,13 +39,19 @@ static inline int kmalloc_index(int size if (size < 64 * 1024) return 16; if (size < 128 * 1024) return 17; if (size < 256 * 1024) return 18; - + if (size < 512 * 1024) return 19; + if (size < 1 * 1024 * 1024) return 20; + if (size < 2 * 1024 * 1024) return 21; + if (size < 4 * 1024 * 1024) return 22; + if (size < 8 * 1024 * 1024) return 23; + if (size < 16 * 1024 * 1024) return 24; + if (size < 32 * 1024 * 1024) return 25; return -1; /* * What we really wanted to do and cannot do because of compiler issues is: * int i; - * for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) + * for (i = 3; i <= 25; i++) * if (size <= (1 << i)) * return i; */ @@ -127,7 +65,7 @@ static inline int kmalloc_index(int size */ static inline struct kmem_cache *kmalloc_slab(size_t size) { - int index = kmalloc_index(size) - KMALLOC_SHIFT_LOW; + long index = kmalloc_index(size); if (index < 0) { /* @@ -137,7 +75,7 @@ static inline struct kmem_cache *kmalloc extern void __kmalloc_size_too_large(void); __kmalloc_size_too_large(); } - return &kmalloc_caches[index]; + return (struct kmem_cache *)index; } #ifdef CONFIG_ZONE_DMA Index: linux-2.6.21-rc4-mm1/mm/slub.c =================================================================== --- linux-2.6.21-rc4-mm1.orig/mm/slub.c 2007-03-22 20:52:20.000000000 -0700 +++ linux-2.6.21-rc4-mm1/mm/slub.c 2007-03-22 22:10:16.000000000 -0700 @@ -84,7 +84,67 @@ /* Internal SLUB flags */ #define __OBJECT_POISON 0x80000000 /* Poison object */ -static int kmem_size = sizeof(struct kmem_cache); +struct slab_node { + spinlock_t list_lock; /* Protect partial list and nr_partial */ + unsigned long nr_partial; + atomic_long_t nr_slabs; + struct list_head partial; +}; + +/* + * Slab cache management. + */ +struct slab_cache { + /* Used for retriving partial slabs etc */ + unsigned long flags; + int size; /* The size of an object including meta data */ + int objsize; /* The size of an object without meta data */ + int offset; /* Free pointer offset. */ + long slab_id; /* Nr for per cpu slabs */ + atomic_t cpu_slabs; /* != 0 -> flusher scheduled. */ + + /* + * Avoid an extra cache line for UP, SMP and for the node local to + * struct kmem_cache. + */ + struct slab_node local_node; + + /* Allocation and freeing of slabs */ + unsigned int order; + int objects; /* Number of objects in slab */ + int refcount; /* Refcount for slab cache destroy */ + void (*ctor)(void *, struct kmem_cache *, unsigned long); + void (*dtor)(void *, struct kmem_cache *, unsigned long); + int inuse; /* Offset to metadata */ + int align; /* Alignment */ + const char *name; /* Name (only for display!) */ + struct list_head list; /* List of slab caches */ + struct kobject kobj; /* For sysfs */ + +#ifdef CONFIG_SMP + struct delayed_work flush; + struct mutex flushing; +#endif +#ifdef CONFIG_NUMA + struct slab_node *node[MAX_NUMNODES]; +#endif +}; + +static int kmem_size = sizeof(struct slab_cache); + +#define NR_SLABS 256 +DEFINE_PER_CPU(struct page *, cpu_slab)[NR_SLABS]; + +static struct slab_cache *slab[NR_SLABS]; + +#define RESERVED_SLABS 25 +/* + * Reserved slabs + * 0 = BAD slab. Failure if accessed. + * 1 = 64 - 128 + * 2 = 196 - 256 + * 3 ... RESERVED_SLABS <= 2^SLAB_ID + */ #ifdef CONFIG_SMP static struct notifier_block slab_notifier; @@ -106,20 +166,20 @@ static DECLARE_RWSEM(slub_lock); LIST_HEAD(slab_caches); #ifdef CONFIG_SYSFS -static int sysfs_slab_add(struct kmem_cache *); -static int sysfs_slab_alias(struct kmem_cache *, const char *); -static void sysfs_slab_remove(struct kmem_cache *); +static int sysfs_slab_add(struct slab_cache *); +static int sysfs_slab_alias(struct slab_cache *, const char *); +static void sysfs_slab_remove(struct slab_cache *); #else -static int sysfs_slab_add(struct kmem_cache *) { return 0; } -static int sysfs_slab_alias(struct kmem_cache *, const char *) { return 0; } -static void sysfs_slab_remove(struct kmem_cache *) {} +static int sysfs_slab_add(struct slab_cache *) { return 0; } +static int sysfs_slab_alias(struct slab_cache *, const char *) { return 0; } +static void sysfs_slab_remove(struct slab_cache *) {} #endif /******************************************************************** * Core slab cache functions *******************************************************************/ -struct kmem_cache_node *get_node(struct kmem_cache *s, int node) +struct slab_node *get_node(struct slab_cache *s, int node) { #ifdef CONFIG_NUMA return s->node[node]; @@ -172,12 +232,12 @@ static void print_section(char *text, u8 * The offset can also be obtained from the page. In that * case it is in the cacheline that we already need to touch. */ -static void *get_freepointer(struct kmem_cache *s, void *object) +static void *get_freepointer(struct slab_cache *s, void *object) { return *(void **)(object + s->offset); } -static void set_freepointer(struct kmem_cache *s, void *object, void *fp) +static void set_freepointer(struct slab_cache *s, void *object, void *fp) { *(void **)(object + s->offset) = fp; } @@ -192,7 +252,7 @@ struct track { unsigned long when; /* When did the operation occur */ }; -struct track *get_track(struct kmem_cache *s, void *object, int alloc) +struct track *get_track(struct slab_cache *s, void *object, int alloc) { struct track *p; @@ -204,7 +264,7 @@ struct track *get_track(struct kmem_cach return p + alloc; } -static void set_track(struct kmem_cache *s, void *object, +static void set_track(struct slab_cache *s, void *object, int alloc, void *addr) { struct track *p; @@ -227,7 +287,7 @@ static void set_track(struct kmem_cache #define set_tracking(__s, __o, __a) set_track(__s, __o, __a, \ __builtin_return_address(0)) -static void init_tracking(struct kmem_cache *s, void *object) +static void init_tracking(struct slab_cache *s, void *object) { if (s->flags & SLAB_STORE_USER) { set_track(s, object, 0, NULL); @@ -261,7 +321,7 @@ static void print_track(const char *s, s printk(" jiffies since=%lu cpu=%u pid=%d\n", jiffies - t->when, t->cpu, t->pid); } -static void print_trailer(struct kmem_cache *s, u8 *p) +static void print_trailer(struct slab_cache *s, u8 *p) { unsigned int off; /* Offset of last byte */ @@ -289,7 +349,7 @@ static void print_trailer(struct kmem_ca print_section("Filler", p + off, s->size - off); } -static void object_err(struct kmem_cache *s, struct page *page, +static void object_err(struct slab_cache *s, struct page *page, u8 *object, char *reason) { u8 *addr = page_address(page); @@ -305,7 +365,7 @@ static void object_err(struct kmem_cache dump_stack(); } -static void init_object(struct kmem_cache *s, void *object, int active) +static void init_object(struct slab_cache *s, void *object, int active) { u8 *p = object; @@ -335,7 +395,7 @@ static int check_bytes(u8 *start, unsign } -static int check_valid_pointer(struct kmem_cache *s, struct page *page, +static int check_valid_pointer(struct slab_cache *s, struct page *page, void *object) { void *base; @@ -382,7 +442,7 @@ static int check_valid_pointer(struct km * merged slabcaches. */ -static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p) +static int check_pad_bytes(struct slab_cache *s, struct page *page, u8 *p) { unsigned long off = s->inuse; /* The end of info */ @@ -404,7 +464,7 @@ static int check_pad_bytes(struct kmem_c return 0; } -static int slab_pad_check(struct kmem_cache *s, struct page *page) +static int slab_pad_check(struct slab_cache *s, struct page *page) { u8 *p; int length, remainder; @@ -427,7 +487,7 @@ static int slab_pad_check(struct kmem_ca return 1; } -static int check_object(struct kmem_cache *s, struct page *page, +static int check_object(struct slab_cache *s, struct page *page, void *object, int active) { u8 *p = object; @@ -484,7 +544,7 @@ static int check_object(struct kmem_cach return 1; } -static int check_slab(struct kmem_cache *s, struct page *page) +static int check_slab(struct slab_cache *s, struct page *page) { if (!PageSlab(page)) { printk(KERN_CRIT "SLUB: %s Not a valid slab page @0x%p flags=%lx" @@ -519,7 +579,7 @@ static int check_slab(struct kmem_cache * therefore free. Must hold the slab lock for cpu slabs to * guarantee that the chains are consistent. */ -static int on_freelist(struct kmem_cache *s, struct page *page, void *search) +static int on_freelist(struct slab_cache *s, struct page *page, void *search) { int nr = 0; void *fp = page->freelist; @@ -563,7 +623,7 @@ static int on_freelist(struct kmem_cache return 0; } -static int alloc_object_checks(struct kmem_cache *s, struct page *page, +static int alloc_object_checks(struct slab_cache *s, struct page *page, void *object) { if (!check_slab(s, page)) @@ -603,7 +663,7 @@ bad: return 0; } -static int free_object_checks(struct kmem_cache *s, struct page *page, void *object) +static int free_object_checks(struct slab_cache *s, struct page *page, void *object) { if (!check_slab(s, page)) { goto fail; @@ -634,11 +694,13 @@ static int free_object_checks(struct kme printk(KERN_CRIT "slab_free : no slab(NULL) for object 0x%p.\n", object); - else - printk(KERN_CRIT "slab_free %s(%d): object at 0x%p" + else { + struct slab_cache *o = page->slab; + printk(KERN_CRIT "slab_free %s(%d): object at 0x%p" " belongs to slab %s(%d)\n", s->name, s->size, object, - page->slab->name, page->slab->size); + o->name, o->size); + } goto fail; } if (s->flags & SLAB_TRACE) { @@ -658,7 +720,7 @@ fail: /* * Slab allocation and freeing */ -static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node) +static struct page *allocate_slab(struct slab_cache *s, gfp_t flags, int node) { struct page * page; int pages = 1 << s->order; @@ -685,7 +747,7 @@ static struct page *allocate_slab(struct return page; } -static void setup_object(struct kmem_cache *s, struct page *page, +static void setup_object(struct slab_cache *s, struct page *page, void *object) { if (PageError(page)) { @@ -699,14 +761,14 @@ static void setup_object(struct kmem_cac if (!(s->flags & __GFP_WAIT)) mode |= SLAB_CTOR_ATOMIC; - s->ctor(object, s, mode); + s->ctor(object, (void *)s->slab_id, mode); } } -static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node) +static struct page *new_slab(struct slab_cache *s, gfp_t flags, int node) { struct page *page; - struct kmem_cache_node *n; + struct slab_node *n; BUG_ON(flags & ~(GFP_DMA | GFP_LEVEL_MASK | __GFP_NO_GROW)); if (flags & __GFP_NO_GROW) @@ -757,8 +819,9 @@ out: return page; } -static void __free_slab(struct kmem_cache *s, struct page *page) +static void __free_slab(struct page *page) { + struct slab_cache *s = page->slab; int pages = 1 << s->order; if (unlikely(PageError(page) || s->dtor)) { @@ -769,7 +832,7 @@ static void __free_slab(struct kmem_cach slab_pad_check(s, page); for (p = start; p <= end - s->size; p += s->size) { if (s->dtor) - s->dtor(p, s, 0); + s->dtor(p, (void *)s->slab_id, 0); check_object(s, page, p, 0); } } @@ -788,11 +851,13 @@ static void rcu_free_slab(struct rcu_hea struct page *page; page = container_of((struct list_head *)h, struct page, lru); - __free_slab(page->slab, page); + __free_slab(page); } -static void free_slab(struct kmem_cache *s, struct page *page) +static void free_slab(struct page *page) { + struct slab_cache *s = page->slab; + if (unlikely(s->flags & SLAB_DESTROY_BY_RCU)) { /* * RCU free overloads the RCU head over the LRU @@ -801,18 +866,18 @@ static void free_slab(struct kmem_cache call_rcu(head, rcu_free_slab); } else - __free_slab(s, page); + __free_slab(page); } -static void discard_slab(struct kmem_cache *s, struct page *page) +static void discard_slab(struct page *page) { - struct kmem_cache_node *n = get_node(s, page_to_nid(page)); + struct slab_node *n = get_node(page->slab, page_to_nid(page)); atomic_long_dec(&n->nr_slabs); reset_page_mapcount(page); page->flags &= ~(1 << PG_slab | 1 << PG_error); - free_slab(s, page); + free_slab(page); } /* @@ -844,9 +909,9 @@ static __always_inline int slab_trylock( /* * Management of partially allocated slabs */ -static void __always_inline add_partial(struct kmem_cache *s, struct page *page) +static void __always_inline add_partial(struct page *page) { - struct kmem_cache_node *n = get_node(s, page_to_nid(page)); + struct slab_node *n = get_node(page->slab, page_to_nid(page)); spin_lock(&n->list_lock); n->nr_partial++; @@ -854,10 +919,9 @@ static void __always_inline add_partial( spin_unlock(&n->list_lock); } -static void __always_inline remove_partial(struct kmem_cache *s, - struct page *page) +static void __always_inline remove_partial(struct page *page) { - struct kmem_cache_node *n = get_node(s, page_to_nid(page)); + struct slab_node *n = get_node(page->slab, page_to_nid(page)); spin_lock(&n->list_lock); list_del(&page->lru); @@ -870,7 +934,7 @@ static void __always_inline remove_parti * * Must hold list_lock */ -static __always_inline int lock_and_del_slab(struct kmem_cache_node *n, +static __always_inline int lock_and_del_slab(struct slab_node *n, struct page *page) { if (slab_trylock(page)) { @@ -884,7 +948,7 @@ static __always_inline int lock_and_del_ /* * Try to get a partial slab from a specific node */ -static struct page *get_partial_node(struct kmem_cache_node *n) +static struct page *get_partial_node(struct slab_node *n) { struct page *page; @@ -910,7 +974,7 @@ out: * Get a page from somewhere. Search in increasing NUMA * distances. */ -static struct page *get_any_partial(struct kmem_cache *s, gfp_t flags) +static struct page *get_any_partial(struct slab_cache *s, gfp_t flags) { #ifdef CONFIG_NUMA struct zonelist *zonelist = &NODE_DATA(slab_node(current->mempolicy)) @@ -919,7 +983,7 @@ static struct page *get_any_partial(stru struct page *page; for (z = zonelist->zones; *z; z++) { - struct kmem_cache_node *n; + struct slab_node *n; n = get_node(s, zone_to_nid(*z)); @@ -937,7 +1001,7 @@ static struct page *get_any_partial(stru /* * Get a partial page, lock it and return it. */ -static struct page *get_partial(struct kmem_cache *s, gfp_t flags, int node) +static struct page *get_partial(struct slab_cache *s, gfp_t flags, int node) { struct page *page; int searchnode = (node == -1) ? numa_node_id() : node; @@ -956,55 +1020,53 @@ static struct page *get_partial(struct k * * On exit the slab lock will have been dropped. */ -static void __always_inline putback_slab(struct kmem_cache *s, struct page *page) +static void __always_inline putback_slab(struct slab_cache *s, struct page *page) { if (page->inuse) { if (page->inuse < s->objects) - add_partial(s, page); + add_partial(page); slab_unlock(page); } else { slab_unlock(page); - discard_slab(s, page); + discard_slab(page); } } /* * Remove the cpu slab */ -static void __always_inline deactivate_slab(struct kmem_cache *s, - struct page *page, int cpu) +static void __always_inline deactivate_slab(int slab_id, struct page *page) { - s->cpu_slab[cpu] = NULL; + __get_cpu_var(cpu_slab)[slab_id] = NULL; ClearPageActive(page); ClearPageReferenced(page); - putback_slab(s, page); + putback_slab(page->slab, page); } -static void flush_slab(struct kmem_cache *s, struct page *page, int cpu) +static void flush_slab(int slab_id, struct page *page) { slab_lock(page); - deactivate_slab(s, page, cpu); + deactivate_slab(slab_id, page); } /* * Flush cpu slab. * Called from IPI handler with interrupts disabled. */ -static void __flush_cpu_slab(struct kmem_cache *s, int cpu) +static void __flush_cpu_slab(int slab_id) { - struct page *page = s->cpu_slab[cpu]; + struct page *page = __get_cpu_var(cpu_slab)[slab_id]; if (likely(page)) - flush_slab(s, page, cpu); + flush_slab(slab_id, page); } static void flush_cpu_slab(void *d) { - struct kmem_cache *s = d; - int cpu = smp_processor_id(); + int slab_id = (long)d; - __flush_cpu_slab(s, cpu); + __flush_cpu_slab(slab_id); } #ifdef CONFIG_SMP @@ -1013,14 +1075,14 @@ static void flush_cpu_slab(void *d) */ static void check_flush_cpu_slab(void *private) { - struct kmem_cache *s = private; - int cpu = smp_processor_id(); - struct page *page = s->cpu_slab[cpu]; + int slab_id = (long)private; + struct page *page = __get_cpu_var(cpu_slab)[slab_id]; + struct slab_cache *s = page->slab; if (page) { if (!TestClearPageReferenced(page)) return; - flush_slab(s, page, cpu); + flush_slab(slab_id, page); } atomic_dec(&s->cpu_slabs); } @@ -1030,19 +1092,19 @@ static void check_flush_cpu_slab(void *p */ static void flusher(struct work_struct *w) { - struct kmem_cache *s = container_of(w, struct kmem_cache, flush.work); + struct slab_cache *s = container_of(w, struct slab_cache, flush.work); if (!mutex_trylock(&s->flushing)) return; atomic_set(&s->cpu_slabs, num_online_cpus()); - on_each_cpu(check_flush_cpu_slab, s, 1, 1); + on_each_cpu(check_flush_cpu_slab, (void *)s->slab_id, 1, 1); if (atomic_read(&s->cpu_slabs)) schedule_delayed_work(&s->flush, 30 * HZ); mutex_unlock(&s->flushing); } -static void flush_all(struct kmem_cache *s) +static void flush_all(struct slab_cache *s) { if (atomic_read(&s->cpu_slabs)) { mutex_lock(&s->flushing); @@ -1053,12 +1115,12 @@ static void flush_all(struct kmem_cache } } #else -static void flush_all(struct kmem_cache *s) +static void flush_all(struct slab_cache *s) { unsigned long flags; local_irq_save(flags); - flush_cpu_slab(s); + flush_cpu_slab((void *)s->slab_id); local_irq_restore(flags); } #endif @@ -1071,22 +1133,23 @@ static void flush_all(struct kmem_cache * 2. The first cacheline of the object to be allocated. * * The only cache lines that are read (apart from code) is the - * per cpu array in the kmem_cache struct. + * per cpu array in the slab_cache struct. * * Fastpath is not possible if we need to get a new slab or have * debugging enabled (which means all slabs are marked with PageError) */ -static __always_inline void *slab_alloc(struct kmem_cache *s, +static __always_inline void *slab_alloc(struct kmem_cache *k, gfp_t gfpflags, int node) { struct page *page; void **object; unsigned long flags; - int cpu; + int slab_id = (long)k; + struct slab_cache *s; + local_irq_save(flags); - cpu = smp_processor_id(); - page = s->cpu_slab[cpu]; + page = __get_cpu_var(cpu_slab)[slab_id]; if (!page) goto new_slab; @@ -1098,10 +1161,10 @@ redo: goto another_slab; object = page->freelist; if (unlikely(PageError(page))) { - if (!alloc_object_checks(s, page, object)) + if (!alloc_object_checks(page->slab, page, object)) goto another_slab; - if (s->flags & SLAB_STORE_USER) - set_tracking(s, object, 0); + if (((struct slab_cache *)(page->slab))->flags & SLAB_STORE_USER) + set_tracking(page->slab, object, 0); } page->inuse++; page->freelist = object[page->offset]; @@ -1111,11 +1174,13 @@ redo: return object; another_slab: - deactivate_slab(s, page, cpu); + deactivate_slab(slab_id, page); new_slab: + s = slab[slab_id]; page = get_partial(s, gfpflags, node); if (unlikely(!page)) { + struct page *npage; page = new_slab(s, gfpflags, node); if (!page) { @@ -1128,29 +1193,30 @@ new_slab: return page_address(page); } - if (s->cpu_slab[cpu]) { + npage = __get_cpu_var(cpu_slab)[slab_id]; + if (npage) { /* * Someone else populated the cpu_slab while * we enabled interrupts. The page may not * be on the required node. */ if (node == -1 || - page_to_nid(s->cpu_slab[cpu]) == node) { + page_to_nid(npage) == node) { /* * Current cpuslab is acceptable and we * want the current one since its cache hot */ - discard_slab(s, page); - page = s->cpu_slab[cpu]; + discard_slab(page); + page = npage; slab_lock(page); goto redo; } - flush_slab(s, s->cpu_slab[cpu], cpu); + flush_slab(s->slab_id, npage); } slab_lock(page); } - s->cpu_slab[cpu] = page; + __get_cpu_var(cpu_slab)[slab_id] = page; SetPageActive(page); #ifdef CONFIG_SMP @@ -1162,16 +1228,16 @@ new_slab: goto redo; } -void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags) +void *kmem_cache_alloc(struct kmem_cache *k, gfp_t gfpflags) { - return slab_alloc(s, gfpflags, -1); + return slab_alloc(k, gfpflags, -1); } EXPORT_SYMBOL(kmem_cache_alloc); #ifdef CONFIG_NUMA -void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node) +void *kmem_cache_alloc_node(struct kmem_cache *k, gfp_t gfpflags, int node) { - return slab_alloc(s, gfpflags, node); + return slab_alloc(k, gfpflags, node); } EXPORT_SYMBOL(kmem_cache_alloc_node); #endif @@ -1182,21 +1248,22 @@ EXPORT_SYMBOL(kmem_cache_alloc_node); * * No special cachelines need to be read */ -static void slab_free(struct kmem_cache *s, struct page *page, void *x) +static void slab_free(struct kmem_cache *k, struct page *page, void *x) { void *prior; void **object = (void *)x; unsigned long flags; + int slab_id = (long)k; local_irq_save(flags); - if (unlikely(PageError(page)) && s->objects == 1) + if (unlikely(PageError(page)) && page->slab->objects == 1) goto single_object_slab; slab_lock(page); if (unlikely(PageError(page)) && - !free_object_checks(s, page, x)) + !free_object_checks(slab[slab_id], page, x)) goto out_unlock; prior = object[page->offset] = page->freelist; @@ -1211,17 +1278,17 @@ static void slab_free(struct kmem_cache * The slab was full before. It will have one free * object now. So move to the partial list. */ - add_partial(s, page); + add_partial(page); goto out_unlock; } /* * All object have been freed. */ - remove_partial(s, page); + remove_partial(page); slab_unlock(page); single_object_slab: - discard_slab(s, page); + discard_slab(page); local_irq_restore(flags); return; @@ -1230,7 +1297,7 @@ out_unlock: local_irq_restore(flags); } -void kmem_cache_free(struct kmem_cache *s, void *x) +void kmem_cache_free(struct kmem_cache *k, void *x) { struct page * page; @@ -1240,9 +1307,9 @@ void kmem_cache_free(struct kmem_cache * page = page->first_page; - if (unlikely(PageError(page) && (s->flags & SLAB_STORE_USER))) - set_tracking(s, x, 1); - slab_free(s, page, x); + if (unlikely(PageError(page) && (page->slab->flags & SLAB_STORE_USER))) + set_tracking(page->slab, x, 1); + slab_free(k, page, x); } EXPORT_SYMBOL(kmem_cache_free); @@ -1359,7 +1426,7 @@ static unsigned long calculate_alignment return ALIGN(align, sizeof(void *)); } -static void free_kmem_cache_nodes(struct kmem_cache *s) +static void free_kmem_cache_nodes(struct slab_cache *s) { #ifdef CONFIG_NUMA int node; @@ -1373,15 +1440,15 @@ static void free_kmem_cache_nodes(struct #endif } -static void init_kmem_cache_node(struct kmem_cache_node *n) +static void init_kmem_cache_node(struct slab_node *n) { - memset(n, 0, sizeof(struct kmem_cache_node)); + memset(n, 0, sizeof(struct slab_node)); atomic_long_set(&n->nr_slabs, 0); spin_lock_init(&n->list_lock); INIT_LIST_HEAD(&n->partial); } -static int init_kmem_cache_nodes(struct kmem_cache *s, gfp_t gfpflags) +static int init_kmem_cache_nodes(struct slab_cache *s, gfp_t gfpflags) { #ifdef CONFIG_NUMA int node; @@ -1438,7 +1505,7 @@ static int init_kmem_cache_nodes(struct return 1; } -int calculate_sizes(struct kmem_cache *s) +int calculate_sizes(struct slab_cache *s) { int tentative_size; unsigned long flags = s->flags; @@ -1508,7 +1575,7 @@ int calculate_sizes(struct kmem_cache *s } -static int kmem_cache_open(struct kmem_cache *s, gfp_t gfpflags, +static int kmem_cache_open(struct slab_cache *s, gfp_t gfpflags, const char *name, size_t size, size_t align, unsigned long flags, void (*ctor)(void *, struct kmem_cache *, unsigned long), @@ -1558,10 +1625,12 @@ EXPORT_SYMBOL(kmem_cache_open); /* * Check if a given pointer is valid */ -int kmem_ptr_validate(struct kmem_cache *s, const void *object) +int kmem_ptr_validate(struct kmem_cache *k, const void *object) { struct page * page; void *addr; + int slab_id = (long)k; + struct slab_cache *s = slab[slab_id]; page = get_object_page(object); @@ -1574,7 +1643,7 @@ int kmem_ptr_validate(struct kmem_cache /* Out of bounds */ return 0; - if ((object - addr) & s->size) + if ((object - addr) % s->size) /* Improperly aligned */ return 0; @@ -1591,19 +1660,23 @@ EXPORT_SYMBOL(kmem_ptr_validate); /* * Determine the size of a slab object */ -unsigned int kmem_cache_size(struct kmem_cache *s) +unsigned int kmem_cache_size(struct kmem_cache *k) { + struct slab_cache *s = slab[(long)k]; + return s->objsize; } EXPORT_SYMBOL(kmem_cache_size); const char *kmem_cache_name(struct kmem_cache *s) { + struct slab_cache *s = slab[(long)k]; + return s->name; } EXPORT_SYMBOL(kmem_cache_name); -static int free_list(struct kmem_cache *s, struct kmem_cache_node *n, +static int free_list(struct slab_node *n, struct list_head *list) { int slabs_inuse = 0; @@ -1614,7 +1687,7 @@ static int free_list(struct kmem_cache * list_for_each_entry_safe(page, h, list, lru) if (!page->inuse) { list_del(&page->lru); - discard_slab(s, page); + discard_slab(page); } else slabs_inuse++; spin_unlock_irqrestore(&n->list_lock, flags); @@ -1625,9 +1698,10 @@ static int free_list(struct kmem_cache * * Release all resources used by slab cache * (if possible...) */ -static int kmem_cache_close(struct kmem_cache *s) +static int kmem_cache_close(struct kmem_cache *k) { int node; + struct slab_cache *s = slab[(long)k]; flush_all(s); @@ -1635,7 +1709,7 @@ static int kmem_cache_close(struct kmem_ for_each_online_node(node) { struct kmem_cache_node *n = get_node(s, node); - free_list(s, n, &n->partial); + free_list(n, &n->partial); if (atomic_long_read(&n->nr_slabs)) return 1; } @@ -1648,8 +1722,9 @@ EXPORT_SYMBOL(kmem_cache_close); * Close a cache and release the kmem_cache structure * (must be used for caches created using kmem_cache_create) */ -void kmem_cache_destroy(struct kmem_cache *s) +void kmem_cache_destroy(struct kmem_cache *k) { + struct slab_cache *s = slab[(long)k]; down_write(&slub_lock); if (s->refcount) s->refcount--; @@ -1663,7 +1738,7 @@ void kmem_cache_destroy(struct kmem_cach } EXPORT_SYMBOL(kmem_cache_destroy); -static unsigned long slab_objects(struct kmem_cache *s, +static unsigned long slab_objects(struct slab_cache *s, unsigned long *p_total, unsigned long *p_cpu_slabs, unsigned long *p_partial, unsigned long *nodes) { @@ -1678,7 +1753,7 @@ static unsigned long slab_objects(struct struct page *page; for_each_online_node(node) { - struct kmem_cache_node *n = get_node(s, node); + struct slab_node *n = get_node(s, node); nr_slabs += atomic_read(&n->nr_slabs); nr_partial_slabs += n->nr_partial; @@ -1694,7 +1769,7 @@ static unsigned long slab_objects(struct } for_each_possible_cpu(cpu) { - page = s->cpu_slab[cpu]; + page = __get_per_cpu(cpu_slab, cpu)[s->slab_id]; if (page) { nr_cpu_slabs++; in_cpu_slabs += page->inuse; @@ -1720,13 +1795,6 @@ static unsigned long slab_objects(struct * Kmalloc subsystem *******************************************************************/ -struct kmem_cache kmalloc_caches[KMALLOC_NR_CACHES] __cacheline_aligned; -EXPORT_SYMBOL(kmalloc_caches); - -#ifdef CONFIG_ZONE_DMA -static struct kmem_cache *kmalloc_caches_dma[KMALLOC_NR_CACHES]; -#endif - static int __init setup_slub_min_order(char *str) { get_option (&str, &slub_min_order); @@ -1790,7 +1858,7 @@ static int __init setup_slub_debug(char __setup("slub_debug", setup_slub_debug); -static struct kmem_cache *create_kmalloc_cache(struct kmem_cache *s, +static struct slab_cache *create_kmalloc_cache(struct slab_cache *s, const char *name, int size, gfp_t gfp_flags) { unsigned int flags = 0; @@ -1816,12 +1884,12 @@ panic: static struct kmem_cache *get_slab(size_t size, gfp_t flags) { - int index = kmalloc_index(size) - KMALLOC_SHIFT_LOW; + int index = kmalloc_index(size); /* SLAB allows allocations with zero size. So warn on those */ WARN_ON(size == 0); /* Allocation too large? */ - BUG_ON(index < 0); + BUG_ON(index < 1); #ifdef CONFIG_ZONE_DMA if ((flags & SLUB_DMA)) { @@ -1860,7 +1928,7 @@ static struct kmem_cache *get_slab(size_ return s; } #endif - return &kmalloc_caches[index]; + return (struct kmem_cache *)index; } void *__kmalloc(size_t size, gfp_t flags) @@ -1963,7 +2031,7 @@ void *krealloc(const void *p, size_t new /* * If new size fits in the current cache, bail out. */ - if (likely(page->slab == new_cache)) + if (likely((struct kmem_cache *)(page->slab->slabid) == new_cache)) return (void *)p; ret = kmalloc(new_size, flags); @@ -2033,8 +2101,8 @@ void __init kmem_cache_init(void) register_cpu_notifier(&slab_notifier); #endif if (nr_cpu_ids) /* Remove when nr_cpu_ids was fixed ! */ - kmem_size = offsetof(struct kmem_cache, cpu_slab) - + nr_cpu_ids * sizeof(struct page *); + kmem_size = offsetof(struct slab_cache, node) + + nr_node_ids * sizeof(struct slab_node *); printk(KERN_INFO "SLUB V6: General Slabs=%ld, HW alignment=%d, " "Processors=%d, Nodes=%d\n", @@ -2048,7 +2116,7 @@ void __init kmem_cache_init(void) /* * Find a mergeable slab cache */ -static struct kmem_cache *find_mergeable(size_t size, +static struct slab_cache *find_mergeable(size_t size, size_t align, unsigned long flags, void (*ctor)(void *, struct kmem_cache *, unsigned long), void (*dtor)(void *, struct kmem_cache *, unsigned long)) @@ -2101,7 +2169,7 @@ struct kmem_cache *kmem_cache_create(con void (*ctor)(void *, struct kmem_cache *, unsigned long), void (*dtor)(void *, struct kmem_cache *, unsigned long)) { - struct kmem_cache *s; + struct slab_cache *s; down_write(&slub_lock); s = find_mergeable(size, align, flags, dtor, ctor); @@ -2135,30 +2203,30 @@ err: if (flags & SLAB_PANIC) panic("Cannot create slabcache %s\n", name); else - s = NULL; - return s; + return NULL; + return (struct kmem_cache *)(s->slabid); } EXPORT_SYMBOL(kmem_cache_create); -void *kmem_cache_zalloc(struct kmem_cache *s, gfp_t flags) +void *kmem_cache_zalloc(struct kmem_cache *k, gfp_t flags) { void *x; - x = kmem_cache_alloc(s, flags); + x = kmem_cache_alloc(k, flags); if (x) - memset(x, 0, s->objsize); + memset(x, 0, slab[(long)k]->objsize); return x; } EXPORT_SYMBOL(kmem_cache_zalloc); #ifdef CONFIG_SMP -static void for_all_slabs(void (*func)(struct kmem_cache *, int), int cpu) +static void for_all_slabs(void (*func)(struct slab_cache *, int), int cpu) { struct list_head *h; down_read(&slub_lock); list_for_each(h, &slab_caches) { - struct kmem_cache *s = + struct slab_cache *s = container_of(h, struct kmem_cache, list); func(s, cpu); @@ -2195,9 +2263,9 @@ static struct notifier_block __cpuinitda * Compatiblility definitions **************************************************************/ -int kmem_cache_shrink(struct kmem_cache *s) +int kmem_cache_shrink(struct kmem_cache *k) { - flush_all(s); + flush_all(slab[(long)k]); return 0; } EXPORT_SYMBOL(kmem_cache_shrink); @@ -2297,7 +2365,7 @@ void *__kmalloc_track_caller(size_t size struct kmem_cache *s = get_slab(size, gfpflags); void *object = kmem_cache_alloc(s, gfpflags); - if (object && (s->flags & SLAB_STORE_USER)) + if (object && (slab[(long)s)]->flags & SLAB_STORE_USER)) set_track(s, object, 0, caller); return object; } @@ -2308,7 +2376,7 @@ void *__kmalloc_node_track_caller(size_t struct kmem_cache *s = get_slab(size, gfpflags); void *object = kmem_cache_alloc_node(s, gfpflags, node); - if (object && (s->flags & SLAB_STORE_USER)) + if (object && (slab[(long)s]->flags & SLAB_STORE_USER)) set_track(s, object, 0, caller); return object; } @@ -2316,12 +2384,12 @@ void *__kmalloc_node_track_caller(size_t #ifdef CONFIG_SYSFS #define to_slab_attr(n) container_of(n, struct slab_attribute, attr) -#define to_slab(n) container_of(n, struct kmem_cache, kobj); +#define to_slab(n) container_of(n, struct slab_cache, kobj); struct slab_attribute { struct attribute attr; - ssize_t (*show)(struct kmem_cache *s, char *buf); - ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count); + ssize_t (*show)(struct slab_cache *s, char *buf); + ssize_t (*store)(struct slab_cache *s, const char *x, size_t count); }; #define SLAB_ATTR_RO(_name) \ @@ -2332,37 +2400,37 @@ struct slab_attribute { __ATTR(_name, 0644, _name##_show, _name##_store) -static ssize_t slab_size_show(struct kmem_cache *s, char *buf) +static ssize_t slab_size_show(struct slab_cache *s, char *buf) { return sprintf(buf, "%d\n", s->size); } SLAB_ATTR_RO(slab_size); -static ssize_t align_show(struct kmem_cache *s, char *buf) +static ssize_t align_show(struct slab_cache *s, char *buf) { return sprintf(buf, "%d\n", s->align); } SLAB_ATTR_RO(align); -static ssize_t object_size_show(struct kmem_cache *s, char *buf) +static ssize_t object_size_show(struct slab_cache *s, char *buf) { return sprintf(buf, "%d\n", s->objsize); } SLAB_ATTR_RO(object_size); -static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf) +static ssize_t objs_per_slab_show(struct slab_cache *s, char *buf) { return sprintf(buf, "%d\n", s->objects); } SLAB_ATTR_RO(objs_per_slab); -static ssize_t order_show(struct kmem_cache *s, char *buf) +static ssize_t order_show(struct slab_cache *s, char *buf) { return sprintf(buf, "%d\n", s->order); } SLAB_ATTR_RO(order); -static ssize_t ctor_show(struct kmem_cache *s, char *buf) +static ssize_t ctor_show(struct slab_cache *s, char *buf) { if (s->ctor) return sprint_symbol(buf, (unsigned long)s->ctor); @@ -2370,7 +2438,7 @@ static ssize_t ctor_show(struct kmem_cac } SLAB_ATTR_RO(ctor); -static ssize_t dtor_show(struct kmem_cache *s, char *buf) +static ssize_t dtor_show(struct slab_cache *s, char *buf) { if (s->dtor) return sprint_symbol(buf, (unsigned long)s->dtor); @@ -2378,13 +2446,13 @@ static ssize_t dtor_show(struct kmem_cac } SLAB_ATTR_RO(dtor); -static ssize_t aliases_show(struct kmem_cache *s, char *buf) +static ssize_t aliases_show(struct slab_cache *s, char *buf) { return sprintf(buf, "%d\n", s->refcount - 1); } SLAB_ATTR_RO(aliases); -static ssize_t slabs_show(struct kmem_cache *s, char *buf) +static ssize_t slabs_show(struct slab_cache *s, char *buf) { unsigned long x; @@ -2393,7 +2461,7 @@ static ssize_t slabs_show(struct kmem_ca } SLAB_ATTR_RO(slabs); -static ssize_t partial_show(struct kmem_cache *s, char *buf) +static ssize_t partial_show(struct slab_cache *s, char *buf) { unsigned long x; @@ -2402,7 +2470,7 @@ static ssize_t partial_show(struct kmem_ } SLAB_ATTR_RO(partial); -static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf) +static ssize_t cpu_slabs_show(struct slab_cache *s, char *buf) { unsigned long x; @@ -2411,7 +2479,7 @@ static ssize_t cpu_slabs_show(struct kme } SLAB_ATTR_RO(cpu_slabs); -static ssize_t objects_show(struct kmem_cache *s, char *buf) +static ssize_t objects_show(struct slab_cache *s, char *buf) { unsigned long x; @@ -2420,12 +2488,12 @@ static ssize_t objects_show(struct kmem_ } SLAB_ATTR_RO(objects); -static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf) +static ssize_t sanity_checks_show(struct slab_cache *s, char *buf) { return sprintf(buf, "%d\n", !!(s->flags & SLAB_DEBUG_FREE)); } -static ssize_t sanity_checks_store(struct kmem_cache *s, const char *buf, size_t length) +static ssize_t sanity_checks_store(struct slab_cache *s, const char *buf, size_t length) { s->flags &= ~SLAB_DEBUG_FREE; if (buf[0] == '1') @@ -2434,12 +2502,12 @@ static ssize_t sanity_checks_store(struc } SLAB_ATTR(sanity_checks); -static ssize_t trace_show(struct kmem_cache *s, char *buf) +static ssize_t trace_show(struct slab_cache *s, char *buf) { return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE)); } -static ssize_t trace_store(struct kmem_cache *s, const char *buf, size_t length) +static ssize_t trace_store(struct slab_cache *s, const char *buf, size_t length) { s->flags &= ~SLAB_TRACE; printk("_trace_store = %s\n", buf); @@ -2449,12 +2517,12 @@ static ssize_t trace_store(struct kmem_c } SLAB_ATTR(trace); -static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf) +static ssize_t reclaim_account_show(struct slab_cache *s, char *buf) { return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT)); } -static ssize_t reclaim_account_store(struct kmem_cache *s, const char *buf, size_t length) +static ssize_t reclaim_account_store(struct slab_cache *s, const char *buf, size_t length) { s->flags &= ~SLAB_RECLAIM_ACCOUNT; if (buf[0] == '1') @@ -2463,7 +2531,7 @@ static ssize_t reclaim_account_store(str } SLAB_ATTR(reclaim_account); -static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf) +static ssize_t hwcache_align_show(struct slab_cache *s, char *buf) { return sprintf(buf, "%d\n", !!(s->flags & (SLAB_HWCACHE_ALIGN|SLAB_MUST_HWCACHE_ALIGN))); @@ -2471,25 +2539,25 @@ static ssize_t hwcache_align_show(struct SLAB_ATTR_RO(hwcache_align); #ifdef CONFIG_ZONE_DMA -static ssize_t cache_dma_show(struct kmem_cache *s, char *buf) +static ssize_t cache_dma_show(struct slab_cache *s, char *buf) { return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA)); } SLAB_ATTR_RO(cache_dma); #endif -static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf) +static ssize_t destroy_by_rcu_show(struct slab_cache *s, char *buf) { return sprintf(buf, "%d\n", !!(s->flags & SLAB_DESTROY_BY_RCU)); } SLAB_ATTR_RO(destroy_by_rcu); -static ssize_t red_zone_show(struct kmem_cache *s, char *buf) +static ssize_t red_zone_show(struct slab_cache *s, char *buf) { return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE)); } -static ssize_t red_zone_store(struct kmem_cache *s, const char *buf, size_t length) +static ssize_t red_zone_store(struct slab_cache *s, const char *buf, size_t length) { if (slab_objects(s, NULL, NULL, NULL, NULL)) return -EBUSY; @@ -2502,12 +2570,12 @@ static ssize_t red_zone_store(struct kme } SLAB_ATTR(red_zone); -static ssize_t poison_show(struct kmem_cache *s, char *buf) +static ssize_t poison_show(struct slab_cache *s, char *buf) { return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON)); } -static ssize_t poison_store(struct kmem_cache *s, const char *buf, size_t length) +static ssize_t poison_store(struct slab_cache *s, const char *buf, size_t length) { if (slab_objects(s, NULL, NULL, NULL, NULL)) return -EBUSY; @@ -2520,12 +2588,12 @@ static ssize_t poison_store(struct kmem_ } SLAB_ATTR(poison); -static ssize_t store_user_show(struct kmem_cache *s, char *buf) +static ssize_t store_user_show(struct slab_cache *s, char *buf) { return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER)); } -static ssize_t store_user_store(struct kmem_cache *s, const char *buf, size_t length) +static ssize_t store_user_store(struct slab_cache *s, const char *buf, size_t length) { if (slab_objects(s, NULL, NULL, NULL, NULL)) return -EBUSY; @@ -2574,7 +2642,7 @@ static ssize_t slab_attr_show(struct kob char *buf) { struct slab_attribute *attribute; - struct kmem_cache *s; + struct slab_cache *s; int err; attribute = to_slab_attr(attr); @@ -2593,7 +2661,7 @@ static ssize_t slab_attr_store(struct ko const char *buf, size_t len) { struct slab_attribute *attribute; - struct kmem_cache *s; + struct slab_cache *s; int err; attribute = to_slab_attr(attr); @@ -2631,7 +2699,7 @@ static struct kset_uevent_ops slab_ueven decl_subsys(slab, &slab_ktype, &slab_uevent_ops); -static int sysfs_slab_add(struct kmem_cache *s) +static int sysfs_slab_add(struct slab_cache *s) { int err; @@ -2653,7 +2721,7 @@ static int sysfs_slab_add(struct kmem_ca return 0; } -static void sysfs_slab_remove(struct kmem_cache *s) +static void sysfs_slab_remove(struct slab_cache *s) { kobject_uevent(&s->kobj, KOBJ_REMOVE); @@ -2667,14 +2735,14 @@ static void sysfs_slab_remove(struct kme * available lest we loose that information. */ struct saved_alias { - struct kmem_cache *s; + struct slab_cache *s; const char *name; struct saved_alias *next; }; struct saved_alias *alias_list; -int sysfs_slab_alias(struct kmem_cache *s, const char *name) +int sysfs_slab_alias(struct slab_cache *s, const char *name) { struct saved_alias *al;