--- include/linux/mm_types.h | 5 - mm/slub.c | 134 ++++++++++++++++++++--------------------------- 2 files changed, 60 insertions(+), 79 deletions(-) Index: linux-2.6/include/linux/mm_types.h =================================================================== --- linux-2.6.orig/include/linux/mm_types.h 2007-10-17 13:58:09.000000000 -0700 +++ linux-2.6/include/linux/mm_types.h 2007-10-17 13:59:27.000000000 -0700 @@ -37,13 +37,10 @@ struct page { unsigned long flags; /* Atomic flags, some possibly * updated asynchronously */ atomic_t _count; /* Usage count, see below. */ - union { - atomic_t _mapcount; /* Count of ptes mapped in mms, + atomic_t _mapcount; /* Count of ptes mapped in mms, * to show when page is mapped * & limit reverse map searches. */ - unsigned int inuse; /* SLUB: Nr of objects */ - }; union { struct { unsigned long private; /* Mapping-private opaque data: Index: linux-2.6/mm/slub.c =================================================================== --- linux-2.6.orig/mm/slub.c 2007-10-17 13:58:09.000000000 -0700 +++ linux-2.6/mm/slub.c 2007-10-17 16:16:50.000000000 -0700 @@ -435,8 +435,8 @@ static void print_tracking(struct kmem_c static void print_page_info(struct page *page) { - printk(KERN_ERR "INFO: Slab 0x%p used=%u fp=0x%p flags=0x%04lx\n", - page, page->inuse, page->freelist, page->flags); + printk(KERN_ERR "INFO: Slab 0x%p fp=0x%p flags=0x%04lx\n", + page, page->freelist, page->flags); } @@ -720,17 +720,45 @@ static int check_object(struct kmem_cach return 1; } +/* + * Determine the number of objects in use in a slab. + * + * This is a slow path function which uses get_free_pointer! + * + * Function is used in a racy way. The result may not be accurate. + * We terminate when a pointer is encountered that is outside of + * the slab or the number of objects is zero. + * + * The function is useful for statistics and debugging. + */ +static int count_inuse(struct kmem_cache *s, struct page *page) +{ + int n = s->objects; + void **p; + + for (p = page->freelist; + p && check_valid_pointer(s, page, p) && n; + p = get_freepointer(s, p)) + n--; + return n; +} + static int check_slab(struct kmem_cache *s, struct page *page) { + int inuse; + VM_BUG_ON(!irqs_disabled()); if (!PageSlab(page)) { slab_err(s, page, "Not a valid slab page"); return 0; } - if (page->inuse > s->objects) { + + inuse = count_inuse(s, page); + + if (inuse > s->objects) { slab_err(s, page, "inuse %u > max %u", - s->name, page->inuse, s->objects); + s->name, inuse, s->objects); return 0; } /* Slab_pad_check fixes things up after itself */ @@ -747,6 +775,7 @@ static int on_freelist(struct kmem_cache int nr = 0; void *fp = page->freelist; void *object = NULL; + int inuse; while (fp && nr <= s->objects) { if (fp == search) @@ -760,7 +789,6 @@ static int on_freelist(struct kmem_cache } else { slab_err(s, page, "Freepointer corrupt"); page->freelist = NULL; - page->inuse = s->objects; slab_fix(s, "Freelist cleared"); return 0; } @@ -771,10 +799,11 @@ static int on_freelist(struct kmem_cache nr++; } - if (page->inuse != s->objects - nr) { + inuse = count_inuse(s, page); + + if (inuse != s->objects - nr) { slab_err(s, page, "Wrong object count. Counter is %d but " - "counted were %d", page->inuse, s->objects - nr); - page->inuse = s->objects - nr; + "counted were %d", inuse, s->objects - nr); slab_fix(s, "Object count adjusted."); } return search == NULL; @@ -786,7 +815,7 @@ static void trace(struct kmem_cache *s, printk(KERN_INFO "TRACE %s %s 0x%p inuse=%d fp=0x%p\n", s->name, alloc ? "alloc" : "free", - object, page->inuse, + object, count_inuse(s, page), page->freelist); if (!alloc) @@ -864,7 +893,6 @@ bad: * as used avoids touching the remaining objects. */ slab_fix(s, "Marking all objects used"); - page->inuse = s->objects; page->freelist = NULL; } return 0; @@ -1115,7 +1143,6 @@ static struct page *new_slab(struct kmem set_freepointer(s, last, NULL); page->freelist = start; - page->inuse = 0; out: return page; } @@ -1208,17 +1235,6 @@ static void add_partial(struct kmem_cach spin_unlock(&n->list_lock); } -static void remove_partial(struct kmem_cache *s, - struct page *page) -{ - struct kmem_cache_node *n = get_node(s, page_to_nid(page)); - - spin_lock(&n->list_lock); - list_del(&page->lru); - n->nr_partial--; - spin_unlock(&n->list_lock); -} - /* * Lock slab and remove from the partial list. * @@ -1337,31 +1353,17 @@ static void unfreeze_slab(struct kmem_ca struct kmem_cache_node *n = get_node(s, page_to_nid(page)); ClearSlabFrozen(page); - if (page->inuse) { - if (page->freelist) - add_partial(n, page, tail); - else if (SlabDebug(page) && (s->flags & SLAB_STORE_USER)) - add_full(n, page); - slab_unlock(page); - - } else { - if (n->nr_partial < MIN_PARTIAL) { - /* - * Adding an empty slab to the partial slabs in order - * to avoid page allocator overhead. This slab needs - * to come after the other slabs with objects in - * order to fill them up. That way the size of the - * partial list stays small. kmem_cache_shrink can - * reclaim empty slabs from the partial list. - */ - add_partial(n, page, 1); - slab_unlock(page); - } else { - slab_unlock(page); - discard_slab(s, page); - } - } + /* + * Unfreezing of slabs that have all objects free will result in + * them being put onto the partial list. We cannot check in an + * efficient way if all objects in a slab have been freed. + */ + if (page->freelist) + add_partial(n, page, tail); + else if (SlabDebug(page) && (s->flags & SLAB_STORE_USER)) + add_full(n, page); + slab_unlock(page); } /* @@ -1388,7 +1390,6 @@ static void deactivate_slab(struct kmem_ /* And put onto the regular freelist */ object[c->offset] = page->freelist; page->freelist = object; - page->inuse--; } c->page = NULL; unfreeze_slab(s, page, tail); @@ -1483,7 +1484,6 @@ load_freelist: object = c->page->freelist; c->freelist = object[c->offset]; - c->page->inuse = s->objects; c->page->freelist = NULL; c->node = page_to_nid(c->page); slab_unlock(c->page); @@ -1540,7 +1540,6 @@ debug: if (!alloc_debug_processing(s, c->page, object, addr)) goto another_slab; - c->page->inuse++; c->page->freelist = object[c->offset]; c->node = -1; slab_unlock(c->page); @@ -1617,18 +1616,13 @@ static void __slab_free(struct kmem_cach checks_ok: prior = object[offset] = page->freelist; page->freelist = object; - page->inuse--; if (unlikely(SlabFrozen(page))) goto out_unlock; - if (unlikely(!page->inuse)) - goto slab_empty; - /* - * Objects left in the slab. If it - * was not on the partial list before - * then add it. + * The slab is now guaranteed to have free objects. + * We may have to move it to the partial list. */ if (unlikely(!prior)) add_partial(get_node(s, page_to_nid(page)), page, 0); @@ -1637,17 +1631,6 @@ out_unlock: slab_unlock(page); return; -slab_empty: - if (prior) - /* - * Slab still on the partial list. - */ - remove_partial(s, page); - - slab_unlock(page); - discard_slab(s, page); - return; - debug: if (!free_debug_processing(s, page, x, addr)) goto out_unlock; @@ -2031,7 +2014,6 @@ static struct kmem_cache_node *early_kme n = page->freelist; BUG_ON(!n); page->freelist = get_freepointer(kmalloc_caches, n); - page->inuse++; kmalloc_caches->node[node] = n; #ifdef CONFIG_SLUB_DEBUG init_object(kmalloc_caches, n, 1); @@ -2295,7 +2277,7 @@ static int free_list(struct kmem_cache * spin_lock_irqsave(&n->list_lock, flags); list_for_each_entry_safe(page, h, list, lru) - if (!page->inuse) { + if (!count_inuse(s, page)) { list_del(&page->lru); discard_slab(s, page); } else @@ -2619,7 +2601,7 @@ void kfree(const void *x) } EXPORT_SYMBOL(kfree); -static unsigned long count_partial(struct kmem_cache_node *n) +static unsigned long count_partial(struct kmem_cache *s, struct kmem_cache_node *n) { unsigned long flags; unsigned long x = 0; @@ -2627,7 +2609,7 @@ static unsigned long count_partial(struc spin_lock_irqsave(&n->list_lock, flags); list_for_each_entry(page, &n->partial, lru) - x += page->inuse; + x += count_inuse(s, page); spin_unlock_irqrestore(&n->list_lock, flags); return x; } @@ -2675,7 +2657,9 @@ int kmem_cache_shrink(struct kmem_cache * list_lock. page->inuse here is the upper limit. */ list_for_each_entry_safe(page, t, &n->partial, lru) { - if (!page->inuse && slab_trylock(page)) { + int inuse = count_inuse(s, page); + + if (!inuse && slab_trylock(page)) { /* * Must hold slab lock here because slab_free * may have freed the last object and be @@ -2687,7 +2671,7 @@ int kmem_cache_shrink(struct kmem_cache discard_slab(s, page); } else { list_move(&page->lru, - slabs_by_inuse + page->inuse); + slabs_by_inuse + inuse); } } @@ -3410,7 +3394,7 @@ static unsigned long slab_objects(struct int x = 0; if (flags & SO_OBJECTS) - x = page->inuse; + x = count_inuse(s, page); else x = 1; total += x; @@ -3425,7 +3409,7 @@ static unsigned long slab_objects(struct if (flags & SO_PARTIAL) { if (flags & SO_OBJECTS) - x = count_partial(n); + x = count_partial(s, n); else x = n->nr_partial; total += x;