SLUB: Use mapping for the address of the slab's memory

Bring back the use of the mapping field by SLUB. We now have to call a
relatively expense function page_address in the half hot path of
__slab_alloc. Use of the field allows to avoid that function and allows
various scanning functions (in the future also slab defrag) avoid calling
page_address().

Signed-off-by: Christoph Lameter <clameter@sgi.com>

---
 include/linux/mm.h       |    4 ++--
 include/linux/mm_types.h |    5 ++++-
 mm/slub.c                |   21 ++++++++++-----------
 3 files changed, 16 insertions(+), 14 deletions(-)

Index: linux-2.6/include/linux/mm_types.h
===================================================================
--- linux-2.6.orig/include/linux/mm_types.h	2007-10-22 09:14:20.000000000 -0700
+++ linux-2.6/include/linux/mm_types.h	2007-10-22 10:20:21.000000000 -0700
@@ -64,7 +64,10 @@
 #if NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS
 	    spinlock_t ptl;
 #endif
-	    struct kmem_cache *slab;	/* SLUB: Pointer to slab */
+	    struct {
+	 	   struct kmem_cache *slab;	/* SLUB: Pointer to slab */
+		   void *end;			/* SLUB: end marker */
+	    };
 	    struct page *first_page;	/* Compound tail pages */
 	};
 	union {
Index: linux-2.6/mm/slub.c
===================================================================
--- linux-2.6.orig/mm/slub.c	2007-10-22 09:14:22.000000000 -0700
+++ linux-2.6/mm/slub.c	2007-10-22 10:47:27.000000000 -0700
@@ -280,19 +280,14 @@
  * The end pointer in a slab is special. It points to the first object in the
  * slab but has bit 0 set to mark it.
  */
-static inline void **make_end(void *addr)
+static inline int is_end(void *addr)
 {
-	return addr + 1;
+	return (unsigned long)addr & 1;
 }
 
-static inline int is_end(const void *p)
+void *slab_address(struct page *page)
 {
-	return (unsigned long)p & 1;
-}
-
-static inline void **end(struct page *page)
-{
-	return make_end(page_address(page));
+	return page->end - 1;
 }
 
 static inline int check_valid_pointer(struct kmem_cache *s,
@@ -300,10 +295,10 @@
 {
 	void *base;
 
-	if (is_end(object))
+	if (object == page->end)
 		return 1;
 
-	base = page_address(page);
+	base = slab_address(page);
 	if (object < base || object >= base + s->objects * s->size ||
 		(object - base) % s->size) {
 		return 0;
@@ -336,7 +331,8 @@
 
 /* Scan freelist */
 #define for_each_free_object(__p, __s, __free) \
-	for (__p = (__free); !is_end(__p); __p = get_freepointer((__s), __p))
+	for (__p = (__free); (__p) != page->end; __p = get_freepointer((__s),\
+		__p))
 
 /* Determine object index from a given position */
 static inline int slab_index(void *p, struct kmem_cache *s, void *addr)
@@ -488,7 +484,7 @@
 static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
 {
 	unsigned int off;	/* Offset of last byte */
-	u8 *addr = page_address(page);
+	u8 *addr = slab_address(page);
 
 	print_tracking(s, p);
 
@@ -666,7 +662,7 @@
 	if (!(s->flags & SLAB_POISON))
 		return 1;
 
-	start = page_address(page);
+	start = slab_address(page);
 	end = start + (PAGE_SIZE << s->order);
 	length = s->objects * s->size;
 	remainder = end - (start + length);
@@ -733,7 +729,7 @@
 		 * of the free objects in this slab. May cause
 		 * another error because the object count is now wrong.
 		 */
-		set_freepointer(s, p, end(page));
+		set_freepointer(s, p, page->end);
 		return 0;
 	}
 	return 1;
@@ -767,18 +763,18 @@
 	void *fp = page->freelist;
 	void *object = NULL;
 
-	while (!is_end(fp) && nr <= s->objects) {
+	while (fp != page->end && nr <= s->objects) {
 		if (fp == search)
 			return 1;
 		if (!check_valid_pointer(s, page, fp)) {
 			if (object) {
 				object_err(s, page, object,
 					"Freechain corrupt");
-				set_freepointer(s, object, end(page));
+				set_freepointer(s, object, page->end);
 				break;
 			} else {
 				slab_err(s, page, "Freepointer corrupt");
-				page->freelist = end(page);
+				page->freelist = page->end;
 				page->inuse = s->objects;
 				slab_fix(s, "Freelist cleared");
 				return 0;
@@ -888,7 +884,7 @@
 		 */
 		slab_fix(s, "Marking all objects used");
 		page->inuse = s->objects;
-		page->freelist = end(page);
+		page->freelist = page->end;
 	}
 	return 0;
 }
@@ -1102,7 +1098,6 @@
 	struct page *page;
 	struct kmem_cache_node *n;
 	void *start;
-	void *end;
 	void *last;
 	void *p;
 
@@ -1123,7 +1118,7 @@
 		SetSlabDebug(page);
 
 	start = page_address(page);
-	end = start + s->objects * s->size;
+	page->end = start + 1;
 
 	if (unlikely(s->flags & SLAB_POISON))
 		memset(start, POISON_INUSE, PAGE_SIZE << s->order);
@@ -1135,7 +1130,7 @@
 		last = p;
 	}
 	setup_object(s, page, last);
-	set_freepointer(s, last, make_end(start));
+	set_freepointer(s, last, page->end);
 
 	page->freelist = start;
 	page->inuse = 0;
@@ -1151,7 +1146,7 @@
 		void *p;
 
 		slab_pad_check(s, page);
-		for_each_object(p, s, page_address(page))
+		for_each_object(p, s, slab_address(page))
 			check_object(s, page, p, 0);
 		ClearSlabDebug(page);
 	}
@@ -1161,6 +1156,7 @@
 		NR_SLAB_RECLAIMABLE : NR_SLAB_UNRECLAIMABLE,
 		- pages);
 
+	page->mapping = NULL;
 	__free_pages(page, s->order);
 }
 
@@ -1362,7 +1358,7 @@
 	ClearSlabFrozen(page);
 	if (page->inuse) {
 
-		if (!is_end(page->freelist))
+		if (page->freelist != page->end)
 			add_partial(s, page, tail);
 		else
 			add_full(s, page);
@@ -1400,7 +1396,7 @@
 	 * because both freelists are empty. So this is unlikely
 	 * to occur.
 	 */
-	while (unlikely(!is_end(c->freelist))) {
+	while (unlikely(c->freelist != page->end)) {
 		void **object;
 
 		tail = 0;	/* Hot objects. Put the slab first */
@@ -1500,7 +1496,7 @@
 		goto another_slab;
 load_freelist:
 	object = c->page->freelist;
-	if (unlikely(is_end(object)))
+	if (unlikely(object == c->page->end))
 		goto another_slab;
 	if (unlikely(SlabDebug(c->page)))
 		goto debug;
@@ -1508,7 +1504,7 @@
 	object = c->page->freelist;
 	c->freelist = object[c->offset];
 	c->page->inuse = s->objects;
-	c->page->freelist = end(c->page);
+	c->page->freelist = c->page->end;
 	c->node = page_to_nid(c->page);
 unlock_out:
 	slab_unlock(c->page);
@@ -1592,7 +1588,7 @@
 
 	local_irq_save(flags);
 	c = get_cpu_slab(s, smp_processor_id());
-	if (unlikely(is_end(c->freelist) || !node_match(c, node))) {
+	if (unlikely((is_end(c->freelist)) || !node_match(c, node))) {
 
 		object = __slab_alloc(s, gfpflags, node, addr, c);
 		if (unlikely(!object)) {
@@ -1659,7 +1655,7 @@
 	 * was not on the partial list before
 	 * then add it.
 	 */
-	if (unlikely(is_end(prior)))
+	if (unlikely(prior == page->end))
 		add_partial(s, page, 0);
 
 out_unlock:
@@ -1667,7 +1663,7 @@
 	return;
 
 slab_empty:
-	if (!is_end(prior))
+	if (prior != page->end)
 		/*
 		 * Slab still on the partial list.
 		 */
@@ -1887,7 +1883,7 @@
 			struct kmem_cache_cpu *c)
 {
 	c->page = NULL;
-	c->freelist = make_end(NULL);
+	c->freelist = (void *)1;
 	c->node = 0;
 	c->offset = s->offset / sizeof(void *);
 	c->objsize = s->objsize;
@@ -3007,7 +3003,7 @@
 						unsigned long *map)
 {
 	void *p;
-	void *addr = page_address(page);
+	void *addr = slab_address(page);
 
 	if (!check_slab(s, page) ||
 			!on_freelist(s, page, NULL))
@@ -3287,7 +3283,7 @@
 static void process_slab(struct loc_track *t, struct kmem_cache *s,
 		struct page *page, enum track_item alloc)
 {
-	void *addr = page_address(page);
+	void *addr = slab_address(page);
 	DECLARE_BITMAP(map, s->objects);
 	void *p;