Index: linux-2.6.20-mm2/include/linux/mm_types.h =================================================================== --- linux-2.6.20-mm2.orig/include/linux/mm_types.h 2007-02-20 15:42:26.000000000 -0800 +++ linux-2.6.20-mm2/include/linux/mm_types.h 2007-02-20 16:29:34.000000000 -0800 @@ -16,8 +16,24 @@ struct address_space; * who is mapping it. */ struct page { - unsigned long flags; /* Atomic flags, some possibly + u16 flags; /* Atomic flags, some possibly * updated asynchronously */ + u8 zone_flags; /* Flags updates under lru_lock */ +#ifdef CONFIG_64BIT + u8 flags64; + u16 node; + u16 section; +#else +/* + * 32bit platform and therefore only 8 bits available and two ways to use + * them. We can deduce the node from the section so use section if we need it. + */ +#ifdef CONFIG_SPARSEMEM + u8 section; +#else + u8 node; +#endif +#endif atomic_t _count; /* Usage count, see below. */ atomic_t _mapcount; /* Count of ptes mapped in mms, * to show when page is mapped Index: linux-2.6.20-mm2/include/linux/page-flags.h =================================================================== --- linux-2.6.20-mm2.orig/include/linux/page-flags.h 2007-02-20 15:37:11.000000000 -0800 +++ linux-2.6.20-mm2/include/linux/page-flags.h 2007-02-20 16:30:41.000000000 -0800 @@ -8,6 +8,29 @@ #include /* + * Describes the bits used in page->zone_flags. + */ +enum zone_page_flags { + /* Lower bits are used for zoneid information */ + ZONE_BIT1, + ZONE_BIT2, + /* Higer bits are used for page flags under zone->lru lock */ + PG_lru, + PG_active, + PG_mlocked, + PG_swapcache, + PG_spare1, + PG_spare2, + NR_ZONE_PAGE_FLAGS +}; + +/* 64 bit only flags */ +enum page_flags64 { + PG_uncached, + NR_FLAGS64 +}; + +/* * Various page->flags bits: * * PG_reserved is set for special pages, which can never be swapped out. Some @@ -72,38 +95,25 @@ #define PG_uptodate 3 #define PG_dirty 4 -#define PG_lru 5 -#define PG_active 6 -#define PG_slab 7 /* slab debug (Suparna wants this) */ - -#define PG_checked 8 /* kill me in 2.5.. */ -#define PG_arch_1 9 -#define PG_reserved 10 -#define PG_private 11 /* If pagecache, has fs-private data */ - -#define PG_writeback 12 /* Page is under writeback */ -#define PG_nosave 13 /* Used for system suspend/resume */ -#define PG_compound 14 /* Part of a compound page */ -#define PG_swapcache 15 /* Swap page: swp_entry_t in private */ +#define PG_slab 5 /* slab debug (Suparna wants this) */ +#define PG_checked 6 /* kill me in 2.5.. */ +#define PG_arch_1 7 + +#define PG_reserved 8 +#define PG_private 9 /* If pagecache, has fs-private data */ +#define PG_writeback 10 /* Page is under writeback */ +#define PG_compound 11 /* Part of a compound page */ + +#define PG_mappedtodisk 12 /* Has blocks allocated on-disk */ +#define PG_reclaim 13 /* To be reclaimed asap */ +#define PG_buddy 14 /* Page is free, on buddy lists */ +#define PG_readahead 15 /* Reminder to do read-ahead */ -#define PG_mappedtodisk 16 /* Has blocks allocated on-disk */ -#define PG_reclaim 17 /* To be reclaimed asap */ -#define PG_nosave_free 18 /* Used for system suspend/resume */ -#define PG_buddy 19 /* Page is free, on buddy lists */ - -#define PG_mlocked 20 /* Page is mlocked */ -#define PG_readahead 21 /* Reminder to do read-ahead */ - -#if (BITS_PER_LONG > 32) /* - * 64-bit-only flags build down from bit 31 - * - * 32 bit -------------------------------| FIELDS | FLAGS | - * 64 bit | FIELDS | ?????? FLAGS | - * 63 32 0 + * These will be replaced by a bit array */ -#define PG_uncached 31 /* Page has been mapped as uncached */ -#endif +#define PG_nosave_free -1 /* Used for system suspend/resume */ +#define PG_nosave -1 /* Used for system suspend/resume */ /* * Manipulation of page state flags @@ -147,15 +157,15 @@ static inline void SetPageUptodate(struc #define __ClearPageDirty(page) __clear_bit(PG_dirty, &(page)->flags) #define TestClearPageDirty(page) test_and_clear_bit(PG_dirty, &(page)->flags) -#define PageLRU(page) test_bit(PG_lru, &(page)->flags) -#define SetPageLRU(page) set_bit(PG_lru, &(page)->flags) -#define ClearPageLRU(page) clear_bit(PG_lru, &(page)->flags) -#define __ClearPageLRU(page) __clear_bit(PG_lru, &(page)->flags) - -#define PageActive(page) test_bit(PG_active, &(page)->flags) -#define SetPageActive(page) set_bit(PG_active, &(page)->flags) -#define ClearPageActive(page) clear_bit(PG_active, &(page)->flags) -#define __ClearPageActive(page) __clear_bit(PG_active, &(page)->flags) +#define PageLRU(page) test_bit(PG_lru, &(page)->zone_flags) +#define SetPageLRU(page) __set_bit(PG_lru, &(page)->zone_flags) +#define ClearPageLRU(page) __clear_bit(PG_lru, &(page)->zone_flags) +#define __ClearPageLRU(page) __clear_bit(PG_lru, &(page)->zone_flags) + +#define PageActive(page) test_bit(PG_active, &(page)->zone_flags) +#define SetPageActive(page) __set_bit(PG_active, &(page)->zone_flags) +#define ClearPageActive(page) __clear_bit(PG_active, &(page)->zone_flags) +#define __ClearPageActive(page) __clear_bit(PG_active, &(page)->zone_flags) #define PageSlab(page) test_bit(PG_slab, &(page)->flags) #define __SetPageSlab(page) __set_bit(PG_slab, &(page)->flags) @@ -220,16 +230,18 @@ static inline void SetPageUptodate(struc #define __ClearPageCompound(page) __clear_bit(PG_compound, &(page)->flags) #ifdef CONFIG_SWAP -#define PageSwapCache(page) test_bit(PG_swapcache, &(page)->flags) -#define SetPageSwapCache(page) set_bit(PG_swapcache, &(page)->flags) -#define ClearPageSwapCache(page) clear_bit(PG_swapcache, &(page)->flags) +#define PageSwapCache(page) test_bit(PG_swapcache, &(page)->zone_flags) +#define SetPageSwapCache(page) __set_bit(PG_swapcache, &(page)->zone_flags) +#define ClearPageSwapCache(page) __clear_bit(PG_swapcache, &(page)->zone_flags) #else #define PageSwapCache(page) 0 #endif -#define PageUncached(page) test_bit(PG_uncached, &(page)->flags) -#define SetPageUncached(page) set_bit(PG_uncached, &(page)->flags) -#define ClearPageUncached(page) clear_bit(PG_uncached, &(page)->flags) +#ifdef CONFIG_64BIT +#define PageUncached(page) test_bit(PG_uncached, &(page)->flags64) +#define SetPageUncached(page) set_bit(PG_uncached, &(page)->flags64) +#define ClearPageUncached(page) clear_bit(PG_uncached, &(page)->flags64) +#endif /* * PageMlocked set means that the page was taken off the LRU because @@ -237,10 +249,10 @@ static inline void SetPageUptodate(struc * page is put back onto the LRU. PageMlocked is only modified * under the zone->lru_lock like PageLRU. */ -#define PageMlocked(page) test_bit(PG_mlocked, &(page)->flags) -#define SetPageMlocked(page) set_bit(PG_mlocked, &(page)->flags) -#define ClearPageMlocked(page) clear_bit(PG_mlocked, &(page)->flags) -#define __ClearPageMlocked(page) __clear_bit(PG_mlocked, &(page)->flags) +#define PageMlocked(page) test_bit(PG_mlocked, &(page)->zone_flags) +#define SetPageMlocked(page) __set_bit(PG_mlocked, &(page)->zone_flags) +#define ClearPageMlocked(page) __clear_bit(PG_mlocked, &(page)->zone_flags) +#define __ClearPageMlocked(page) __clear_bit(PG_mlocked, &(page)->zone_flags) #define PageReadahead(page) test_bit(PG_readahead, &(page)->flags) #define SetPageReadahead(page) set_bit(PG_readahead, &(page)->flags) Index: linux-2.6.20-mm2/mm/hugetlb.c =================================================================== --- linux-2.6.20-mm2.orig/mm/hugetlb.c 2007-02-20 16:15:22.000000000 -0800 +++ linux-2.6.20-mm2/mm/hugetlb.c 2007-02-20 16:15:57.000000000 -0800 @@ -180,7 +180,7 @@ static void update_and_free_page(struct nr_huge_pages_node[page_to_nid(page)]--; for (i = 0; i < (HPAGE_SIZE / PAGE_SIZE); i++) { page[i].flags &= ~(1 << PG_locked | 1 << PG_error | 1 << PG_referenced | - 1 << PG_dirty | 1 << PG_active | 1 << PG_reserved | + 1 << PG_dirty | 1 << PG_reserved | 1 << PG_private | 1<< PG_writeback); } page[1].lru.next = NULL; Index: linux-2.6.20-mm2/mm/page_alloc.c =================================================================== --- linux-2.6.20-mm2.orig/mm/page_alloc.c 2007-02-20 15:45:54.000000000 -0800 +++ linux-2.6.20-mm2/mm/page_alloc.c 2007-02-20 16:09:34.000000000 -0800 @@ -209,17 +209,14 @@ static void bad_page(struct page *page) } printk("\n"); } - page->flags &= ~(1 << PG_lru | - 1 << PG_private | + page->flags &= ~(1 << PG_private | 1 << PG_locked | - 1 << PG_active | 1 << PG_dirty | 1 << PG_reclaim | 1 << PG_slab | - 1 << PG_swapcache | 1 << PG_writeback | - 1 << PG_mlocked | 1 << PG_buddy ); + page->zone_flags &= ~ZONEID_MASK; set_page_count(page, 0); reset_page_mapcount(page); page->mapping = NULL; @@ -441,28 +438,26 @@ static inline void __free_one_page(struc static inline int free_pages_check(struct page *page) { + if (PageMlocked(page)) { + /* Page is unused so no need to take the lru lock */ + __ClearPageMlocked(page); + dec_zone_page_state(page, NR_MLOCK); + } if (unlikely(page_mapcount(page) | (page->mapping != NULL) | (page_count(page) != 0) | + ((page->zone_flags & ~ZONEID_MASK) != 0) | (page->flags & ( - 1 << PG_lru | 1 << PG_private | 1 << PG_locked | - 1 << PG_active | 1 << PG_reclaim | 1 << PG_slab | - 1 << PG_swapcache | 1 << PG_writeback | 1 << PG_reserved | 1 << PG_buddy )))) bad_page(page); if (PageDirty(page)) __ClearPageDirty(page); - if (PageMlocked(page)) { - /* Page is unused so no need to take the lru lock */ - __ClearPageMlocked(page); - dec_zone_page_state(page, NR_MLOCK); - } /* * For now, we report if PG_reserved was found set, but do not * clear it, and do not free the page. But we shall soon need @@ -598,18 +593,15 @@ static int prep_new_page(struct page *pa if (unlikely(page_mapcount(page) | (page->mapping != NULL) | (page_count(page) != 0) | + ((page->zone_flags & ~ZONEID_MASK) != 0) | (page->flags & ( - 1 << PG_lru | 1 << PG_private | 1 << PG_locked | - 1 << PG_active | 1 << PG_dirty | 1 << PG_reclaim | 1 << PG_slab | - 1 << PG_swapcache | 1 << PG_writeback | 1 << PG_reserved | - 1 << PG_mlocked | 1 << PG_buddy )))) bad_page(page); Index: linux-2.6.20-mm2/mm/readahead.c =================================================================== --- linux-2.6.20-mm2.orig/mm/readahead.c 2007-02-20 16:14:44.000000000 -0800 +++ linux-2.6.20-mm2/mm/readahead.c 2007-02-20 16:19:25.000000000 -0800 @@ -1188,28 +1188,18 @@ state_based_readahead(struct address_spa * a lower estimation of the true thrashing-threshold. */ -#if PG_active < PG_referenced -# error unexpected page flags order -#endif - -#define PAGE_REFCNT_0 0 -#define PAGE_REFCNT_1 (1 << PG_referenced) -#define PAGE_REFCNT_2 (1 << PG_active) -#define PAGE_REFCNT_3 ((1 << PG_active) | (1 << PG_referenced)) -#define PAGE_REFCNT_MASK PAGE_REFCNT_3 - /* * STATUS REFERENCE COUNT TYPE * __ 0 fresh - * _R PAGE_REFCNT_1 stale - * A_ PAGE_REFCNT_2 disturbed once - * AR PAGE_REFCNT_3 disturbed twice + * _R 1 stale + * A_ 2 disturbed once + * AR 3 disturbed twice * * A/R: Active / Referenced */ static inline unsigned long page_refcnt(struct page *page) { - return page->flags & PAGE_REFCNT_MASK; + return PageReferenced(page) + 2 * PageActive(page); } /* @@ -1256,7 +1246,7 @@ static int __count_cache_hit(struct addr for (i = 0; i < 8;) { struct page *page = radix_tree_lookup(&mapping->page_tree, begin + size * ((i++ * CACHE_HIT_HASH_KEY) & 7) / 8); - if (inactive_page_refcnt(page) >= PAGE_REFCNT_1 && ++count >= 2) + if (inactive_page_refcnt(page) >= 1 && ++count >= 2) break; }