[PATCH] Use separate page bit for tail
We cannot use PG_reclaim for compound pages if we put them onto the LRU.
Make them use a separate page flag.
Add compound handling to pagevec_free.
Signed-off-by: Christoph Lameter <clameter@sgi.com>
---
include/linux/mm.h | 17 ++++++++++++++++-
include/linux/page-flags.h | 43 ++++++++++++-------------------------------
mm/internal.h | 2 +-
mm/page_alloc.c | 23 ++++++++++++++---------
mm/swap.c | 8 +++++++-
5 files changed, 50 insertions(+), 43 deletions(-)
Index: linux-2.6.21-rc7-mm2/include/linux/page-flags.h
===================================================================
--- linux-2.6.21-rc7-mm2.orig/include/linux/page-flags.h 2007-04-26 22:04:19.000000000 -0700
+++ linux-2.6.21-rc7-mm2/include/linux/page-flags.h 2007-04-26 22:06:02.000000000 -0700
@@ -83,7 +83,8 @@
#define PG_private 11 /* If pagecache, has fs-private data */
#define PG_writeback 12 /* Page is under writeback */
-#define PG_compound 14 /* Part of a compound page */
+#define PG_head 13 /* Page is head of a compound page */
+#define PG_tail 14 /* Page is tail of a compound page */
#define PG_swapcache 15 /* Swap page: swp_entry_t in private */
#define PG_mappedtodisk 16 /* Has blocks allocated on-disk */
@@ -95,6 +96,7 @@
#define PG_readahead 21 /* Reminder to do read-ahead */
+
/* PG_owner_priv_1 users should have descriptive aliases */
#define PG_checked PG_owner_priv_1 /* Used by some filesystems */
@@ -218,37 +220,16 @@ static inline void SetPageUptodate(struc
#define ClearPageLazyFree(page) clear_bit(PG_lazyfree, &(page)->flags)
#define __ClearPageLazyFree(page) __clear_bit(PG_lazyfree, &(page)->flags)
-#define PageCompound(page) test_bit(PG_compound, &(page)->flags)
-#define __SetPageCompound(page) __set_bit(PG_compound, &(page)->flags)
-#define __ClearPageCompound(page) __clear_bit(PG_compound, &(page)->flags)
-
-/*
- * PG_reclaim is used in combination with PG_compound to mark the
- * head and tail of a compound page
- *
- * PG_compound & PG_reclaim => Tail page
- * PG_compound & ~PG_reclaim => Head page
- */
-
-#define PG_head_tail_mask ((1L << PG_compound) | (1L << PG_reclaim))
-
-#define PageTail(page) ((page->flags & PG_head_tail_mask) \
- == PG_head_tail_mask)
-
-static inline void __SetPageTail(struct page *page)
-{
- page->flags |= PG_head_tail_mask;
-}
-
-static inline void __ClearPageTail(struct page *page)
-{
- page->flags &= ~PG_head_tail_mask;
-}
+#define PageHead(page) test_bit(PG_head, &(page)->flags)
+#define __SetPageHead(page) __set_bit(PG_head, &(page)->flags)
+#define __ClearPageHead(page) __clear_bit(PG_head, &(page)->flags)
+
+#define PageTail(page) test_bit(PG_tail, &(page->flags))
+#define __SetPageTail(page) __set_bit(PG_tail, &(page)->flags)
+#define __ClearPageTail(page) __clear_bit(PG_tail, &(page)->flags)
-#define PageHead(page) ((page->flags & PG_head_tail_mask) \
- == (1L << PG_compound))
-#define __SetPageHead(page) __SetPageCompound(page)
-#define __ClearPageHead(page) __ClearPageCompound(page)
+#define PageCompound(page) ((page)->flags & \
+ ((1L << PG_head) | (1L << PG_tail)))
#ifdef CONFIG_SWAP
#define PageSwapCache(page) test_bit(PG_swapcache, &(page)->flags)
Index: linux-2.6.21-rc7-mm2/mm/page_alloc.c
===================================================================
--- linux-2.6.21-rc7-mm2.orig/mm/page_alloc.c 2007-04-26 22:04:20.000000000 -0700
+++ linux-2.6.21-rc7-mm2/mm/page_alloc.c 2007-04-27 15:30:12.000000000 -0700
@@ -444,7 +444,7 @@ static inline void __free_one_page(struc
int order_size = 1 << order;
int migratetype = get_pageblock_migratetype(page);
- if (unlikely(PageCompound(page)))
+ if (unlikely(PageHead(page)))
destroy_compound_page(page, order);
page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1);
@@ -485,18 +485,14 @@ static inline int free_pages_check(struc
1 << PG_private |
1 << PG_locked |
1 << PG_active |
+ 1 << PG_reclaim |
+ 1 << PG_tail |
1 << PG_slab |
1 << PG_swapcache |
1 << PG_writeback |
1 << PG_reserved |
1 << PG_buddy ))))
bad_page(page);
- /*
- * PageReclaim == PageTail. It is only an error
- * for PageReclaim to be set if PageCompound is clear.
- */
- if (unlikely(!PageCompound(page) && PageReclaim(page)))
- bad_page(page);
if (PageDirty(page))
__ClearPageDirty(page);
if (PageLazyFree(page))
@@ -1779,8 +1775,17 @@ void __pagevec_free(struct pagevec *pvec
{
int i = pagevec_count(pvec);
- while (--i >= 0)
- free_hot_cold_page(pvec->pages[i], pvec->cold);
+ while (--i >= 0) {
+ struct page *page = pvec->pages[i];
+
+ if (PageHead(page)) {
+ compound_page_dtor *dtor;
+
+ dtor = get_compound_page_dtor(page);
+ (*dtor)(page);
+ } else
+ free_hot_cold_page(page, pvec->cold);
+ }
}
fastcall void __free_pages(struct page *page, unsigned int order)
Index: linux-2.6.21-rc7-mm2/include/linux/mm.h
===================================================================
--- linux-2.6.21-rc7-mm2.orig/include/linux/mm.h 2007-04-26 22:04:19.000000000 -0700
+++ linux-2.6.21-rc7-mm2/include/linux/mm.h 2007-04-27 15:33:04.000000000 -0700
@@ -295,7 +295,7 @@ static inline int put_page_testzero(stru
*/
static inline int get_page_unless_zero(struct page *page)
{
- VM_BUG_ON(PageCompound(page));
+ VM_BUG_ON(PageTail(page));
return atomic_inc_not_zero(&page->_count);
}
@@ -368,6 +368,21 @@ static inline void set_compound_order(st
page[1].lru.prev = (void *)order;
}
+static inline int compound_pages(struct page *page)
+{
+ return 1 << compound_order(page);
+}
+
+static inline int compound_shift(struct page *page)
+{
+ return PAGE_SHIFT + compound_order(page);
+}
+
+static inline int compound_size(struct page *page)
+{
+ return PAGE_SIZE << compound_order(page);
+}
+
/*
* Multiple processes may "see" the same page. E.g. for untouched
* mappings of /dev/null, all processes see the same page full of
Index: linux-2.6.21-rc7-mm2/mm/internal.h
===================================================================
--- linux-2.6.21-rc7-mm2.orig/mm/internal.h 2007-04-26 22:04:20.000000000 -0700
+++ linux-2.6.21-rc7-mm2/mm/internal.h 2007-04-26 22:06:02.000000000 -0700
@@ -24,7 +24,7 @@ static inline void set_page_count(struct
*/
static inline void set_page_refcounted(struct page *page)
{
- VM_BUG_ON(PageCompound(page) && PageTail(page));
+ VM_BUG_ON(PageTail(page));
VM_BUG_ON(atomic_read(&page->_count));
set_page_count(page, 1);
}
Index: linux-2.6.21-rc7-mm2/mm/swap.c
===================================================================
--- linux-2.6.21-rc7-mm2.orig/mm/swap.c 2007-04-27 15:32:53.000000000 -0700
+++ linux-2.6.21-rc7-mm2/mm/swap.c 2007-04-27 15:33:30.000000000 -0700
@@ -307,7 +307,13 @@ void release_pages(struct page **pages,
for (i = 0; i < nr; i++) {
struct page *page = pages[i];
- if (unlikely(PageCompound(page))) {
+ /*
+ * If we have a tail page on the LRU then we need to
+ * decrement the page count of the head page. There
+ * is no further need to do anything since tail pages
+ * cannot be on the LRU.
+ */
+ if (unlikely(PageTail(page))) {
if (zone) {
spin_unlock_irq(&zone->lru_lock);
zone = NULL;