[PATCH] Use separate page bit for tail We cannot use PG_reclaim for compound pages if we put them onto the LRU. Make them use a separate page flag. Add compound handling to pagevec_free. Signed-off-by: Christoph Lameter --- include/linux/mm.h | 17 ++++++++++++++++- include/linux/page-flags.h | 43 ++++++++++++------------------------------- mm/internal.h | 2 +- mm/page_alloc.c | 23 ++++++++++++++--------- mm/swap.c | 8 +++++++- 5 files changed, 50 insertions(+), 43 deletions(-) Index: linux-2.6.21-rc7-mm2/include/linux/page-flags.h =================================================================== --- linux-2.6.21-rc7-mm2.orig/include/linux/page-flags.h 2007-04-26 22:04:19.000000000 -0700 +++ linux-2.6.21-rc7-mm2/include/linux/page-flags.h 2007-04-26 22:06:02.000000000 -0700 @@ -83,7 +83,8 @@ #define PG_private 11 /* If pagecache, has fs-private data */ #define PG_writeback 12 /* Page is under writeback */ -#define PG_compound 14 /* Part of a compound page */ +#define PG_head 13 /* Page is head of a compound page */ +#define PG_tail 14 /* Page is tail of a compound page */ #define PG_swapcache 15 /* Swap page: swp_entry_t in private */ #define PG_mappedtodisk 16 /* Has blocks allocated on-disk */ @@ -95,6 +96,7 @@ #define PG_readahead 21 /* Reminder to do read-ahead */ + /* PG_owner_priv_1 users should have descriptive aliases */ #define PG_checked PG_owner_priv_1 /* Used by some filesystems */ @@ -218,37 +220,16 @@ static inline void SetPageUptodate(struc #define ClearPageLazyFree(page) clear_bit(PG_lazyfree, &(page)->flags) #define __ClearPageLazyFree(page) __clear_bit(PG_lazyfree, &(page)->flags) -#define PageCompound(page) test_bit(PG_compound, &(page)->flags) -#define __SetPageCompound(page) __set_bit(PG_compound, &(page)->flags) -#define __ClearPageCompound(page) __clear_bit(PG_compound, &(page)->flags) - -/* - * PG_reclaim is used in combination with PG_compound to mark the - * head and tail of a compound page - * - * PG_compound & PG_reclaim => Tail page - * PG_compound & ~PG_reclaim => Head page - */ - -#define PG_head_tail_mask ((1L << PG_compound) | (1L << PG_reclaim)) - -#define PageTail(page) ((page->flags & PG_head_tail_mask) \ - == PG_head_tail_mask) - -static inline void __SetPageTail(struct page *page) -{ - page->flags |= PG_head_tail_mask; -} - -static inline void __ClearPageTail(struct page *page) -{ - page->flags &= ~PG_head_tail_mask; -} +#define PageHead(page) test_bit(PG_head, &(page)->flags) +#define __SetPageHead(page) __set_bit(PG_head, &(page)->flags) +#define __ClearPageHead(page) __clear_bit(PG_head, &(page)->flags) + +#define PageTail(page) test_bit(PG_tail, &(page->flags)) +#define __SetPageTail(page) __set_bit(PG_tail, &(page)->flags) +#define __ClearPageTail(page) __clear_bit(PG_tail, &(page)->flags) -#define PageHead(page) ((page->flags & PG_head_tail_mask) \ - == (1L << PG_compound)) -#define __SetPageHead(page) __SetPageCompound(page) -#define __ClearPageHead(page) __ClearPageCompound(page) +#define PageCompound(page) ((page)->flags & \ + ((1L << PG_head) | (1L << PG_tail))) #ifdef CONFIG_SWAP #define PageSwapCache(page) test_bit(PG_swapcache, &(page)->flags) Index: linux-2.6.21-rc7-mm2/mm/page_alloc.c =================================================================== --- linux-2.6.21-rc7-mm2.orig/mm/page_alloc.c 2007-04-26 22:04:20.000000000 -0700 +++ linux-2.6.21-rc7-mm2/mm/page_alloc.c 2007-04-27 15:30:12.000000000 -0700 @@ -444,7 +444,7 @@ static inline void __free_one_page(struc int order_size = 1 << order; int migratetype = get_pageblock_migratetype(page); - if (unlikely(PageCompound(page))) + if (unlikely(PageHead(page))) destroy_compound_page(page, order); page_idx = page_to_pfn(page) & ((1 << MAX_ORDER) - 1); @@ -485,18 +485,14 @@ static inline int free_pages_check(struc 1 << PG_private | 1 << PG_locked | 1 << PG_active | + 1 << PG_reclaim | + 1 << PG_tail | 1 << PG_slab | 1 << PG_swapcache | 1 << PG_writeback | 1 << PG_reserved | 1 << PG_buddy )))) bad_page(page); - /* - * PageReclaim == PageTail. It is only an error - * for PageReclaim to be set if PageCompound is clear. - */ - if (unlikely(!PageCompound(page) && PageReclaim(page))) - bad_page(page); if (PageDirty(page)) __ClearPageDirty(page); if (PageLazyFree(page)) @@ -1779,8 +1775,17 @@ void __pagevec_free(struct pagevec *pvec { int i = pagevec_count(pvec); - while (--i >= 0) - free_hot_cold_page(pvec->pages[i], pvec->cold); + while (--i >= 0) { + struct page *page = pvec->pages[i]; + + if (PageHead(page)) { + compound_page_dtor *dtor; + + dtor = get_compound_page_dtor(page); + (*dtor)(page); + } else + free_hot_cold_page(page, pvec->cold); + } } fastcall void __free_pages(struct page *page, unsigned int order) Index: linux-2.6.21-rc7-mm2/include/linux/mm.h =================================================================== --- linux-2.6.21-rc7-mm2.orig/include/linux/mm.h 2007-04-26 22:04:19.000000000 -0700 +++ linux-2.6.21-rc7-mm2/include/linux/mm.h 2007-04-27 15:33:04.000000000 -0700 @@ -295,7 +295,7 @@ static inline int put_page_testzero(stru */ static inline int get_page_unless_zero(struct page *page) { - VM_BUG_ON(PageCompound(page)); + VM_BUG_ON(PageTail(page)); return atomic_inc_not_zero(&page->_count); } @@ -368,6 +368,21 @@ static inline void set_compound_order(st page[1].lru.prev = (void *)order; } +static inline int compound_pages(struct page *page) +{ + return 1 << compound_order(page); +} + +static inline int compound_shift(struct page *page) +{ + return PAGE_SHIFT + compound_order(page); +} + +static inline int compound_size(struct page *page) +{ + return PAGE_SIZE << compound_order(page); +} + /* * Multiple processes may "see" the same page. E.g. for untouched * mappings of /dev/null, all processes see the same page full of Index: linux-2.6.21-rc7-mm2/mm/internal.h =================================================================== --- linux-2.6.21-rc7-mm2.orig/mm/internal.h 2007-04-26 22:04:20.000000000 -0700 +++ linux-2.6.21-rc7-mm2/mm/internal.h 2007-04-26 22:06:02.000000000 -0700 @@ -24,7 +24,7 @@ static inline void set_page_count(struct */ static inline void set_page_refcounted(struct page *page) { - VM_BUG_ON(PageCompound(page) && PageTail(page)); + VM_BUG_ON(PageTail(page)); VM_BUG_ON(atomic_read(&page->_count)); set_page_count(page, 1); } Index: linux-2.6.21-rc7-mm2/mm/swap.c =================================================================== --- linux-2.6.21-rc7-mm2.orig/mm/swap.c 2007-04-27 15:32:53.000000000 -0700 +++ linux-2.6.21-rc7-mm2/mm/swap.c 2007-04-27 15:33:30.000000000 -0700 @@ -307,7 +307,13 @@ void release_pages(struct page **pages, for (i = 0; i < nr; i++) { struct page *page = pages[i]; - if (unlikely(PageCompound(page))) { + /* + * If we have a tail page on the LRU then we need to + * decrement the page count of the head page. There + * is no further need to do anything since tail pages + * cannot be on the LRU. + */ + if (unlikely(PageTail(page))) { if (zone) { spin_unlock_irq(&zone->lru_lock); zone = NULL;