Index: linux-2.6.16-rc1-mm3/mm/page_alloc.c =================================================================== --- linux-2.6.16-rc1-mm3.orig/mm/page_alloc.c 2006-01-25 17:03:17.000000000 -0800 +++ linux-2.6.16-rc1-mm3/mm/page_alloc.c 2006-01-25 18:29:02.000000000 -0800 @@ -231,8 +231,10 @@ static inline unsigned long page_order(s return page_private(page); } -static inline void set_page_order(struct page *page, int order) { - set_page_private(page, order); +#define AREA_SHIFT 16 + +static inline void set_page_order(struct page *page, int order, int area) { + set_page_private(page, order + (area << AREA_SHIFT)); __SetPagePrivate(page); } @@ -283,7 +285,7 @@ __find_combined_index(unsigned long page * for recording page's order, we use page_private(page) and PG_private. * */ -static inline int page_is_buddy(struct page *page, int order) +static inline int page_is_buddy(struct page *page, int order, int area) { #ifdef CONFIG_HOLES_IN_ZONE if (!pfn_valid(page_to_pfn(page))) @@ -291,7 +293,7 @@ static inline int page_is_buddy(struct p #endif if (PagePrivate(page) && - (page_order(page) == order) && + (page_order(page) == order + (area << AREA_SHIFT)) && page_count(page) == 0) return 1; return 0; @@ -322,7 +324,7 @@ static inline int page_is_buddy(struct p */ static inline void __free_one_page(struct page *page, - struct zone *zone, unsigned int order) + struct zone *zone, unsigned int order, int area) { unsigned long page_idx; int order_size = 1 << order; @@ -338,25 +340,25 @@ static inline void __free_one_page(struc zone->free_pages += order_size; while (order < MAX_ORDER-1) { unsigned long combined_idx; - struct free_area *area; + struct free_area *free_area; struct page *buddy; buddy = __page_find_buddy(page, page_idx, order); - if (!page_is_buddy(buddy, order)) + if (!page_is_buddy(buddy, order, area)) break; /* Move the buddy up one level. */ list_del(&buddy->lru); - area = zone->free_area + order; - area->nr_free--; + free_area = zone->free_area[area] + order; + free_area->nr_free--; rmv_page_order(buddy); combined_idx = __find_combined_index(page_idx, order); page = page + (combined_idx - page_idx); page_idx = combined_idx; order++; } - set_page_order(page, order); - list_add(&page->lru, &zone->free_area[order].free_list); - zone->free_area[order].nr_free++; + set_page_order(page, order, area); + list_add(&page->lru, &zone->free_area[area][order].free_list); + zone->free_area[area][order].nr_free++; } static inline int free_pages_check(struct page *page) @@ -397,7 +399,7 @@ static inline int free_pages_check(struc * pinned" detection logic. */ static void free_pages_bulk(struct zone *zone, int count, - struct list_head *list, int order) + struct list_head *list, int order, int area) { spin_lock(&zone->lock); zone->all_unreclaimable = 0; @@ -409,19 +411,19 @@ static void free_pages_bulk(struct zone page = list_entry(list->prev, struct page, lru); /* have to delete it as __free_one_page list manipulates */ list_del(&page->lru); - __free_one_page(page, zone, order); + __free_one_page(page, zone, order, area); } spin_unlock(&zone->lock); } -static void free_one_page(struct zone *zone, struct page *page, int order) +static void free_one_page(struct zone *zone, struct page *page, int order, int area) { LIST_HEAD(list); list_add(&page->lru, &list); - free_pages_bulk(zone, 1, &list, order); + free_pages_bulk(zone, 1, &list, order, area); } -static void __free_pages_ok(struct page *page, unsigned int order) +static void __free_pages_ok(struct page *page, unsigned int order, int area) { unsigned long flags; int i; @@ -445,7 +447,7 @@ static void __free_pages_ok(struct page kernel_map_pages(page, 1 << order, 0); local_irq_save(flags); __mod_page_state(pgfree, 1 << order); - free_one_page(page_zone(page), page, order); + free_one_page(page_zone(page), page, order, area); local_irq_restore(flags); } @@ -493,18 +495,18 @@ void fastcall __init __free_pages_bootme * -- wli */ static inline void expand(struct zone *zone, struct page *page, - int low, int high, struct free_area *area) + int low, int high, struct free_area *free_area, int area) { unsigned long size = 1 << high; while (high > low) { - area--; + free_area--; high--; size >>= 1; BUG_ON(bad_range(zone, &page[size])); - list_add(&page[size].lru, &area->free_list); - area->nr_free++; - set_page_order(&page[size], high); + list_add(&page[size].lru, &free_area->free_list); + free_area->nr_free++; + set_page_order(&page[size], high, area); } } @@ -549,23 +551,23 @@ static int prep_new_page(struct page *pa * Do the hard work of removing an element from the buddy allocator. * Call me with the zone->lock already held. */ -static struct page *__rmqueue(struct zone *zone, unsigned int order) +static struct page *__rmqueue(struct zone *zone, unsigned int order, int area) { - struct free_area * area; + struct free_area * free_area; unsigned int current_order; struct page *page; for (current_order = order; current_order < MAX_ORDER; ++current_order) { - area = zone->free_area + current_order; - if (list_empty(&area->free_list)) + free_area = zone->free_area[area] + current_order; + if (list_empty(&free_area->free_list)) continue; - page = list_entry(area->free_list.next, struct page, lru); + page = list_entry(free_area->free_list.next, struct page, lru); list_del(&page->lru); rmv_page_order(page); - area->nr_free--; + free_area->nr_free--; zone->free_pages -= 1UL << order; - expand(zone, page, order, current_order, area); + expand(zone, page, order, current_order, free_area, area); return page; } @@ -578,13 +580,13 @@ static struct page *__rmqueue(struct zon * Returns the number of new pages which were placed at *list. */ static int rmqueue_bulk(struct zone *zone, unsigned int order, - unsigned long count, struct list_head *list) + unsigned long count, struct list_head *list, int area) { int i; spin_lock(&zone->lock); for (i = 0; i < count; ++i) { - struct page *page = __rmqueue(zone, order); + struct page *page = __rmqueue(zone, order, area); if (unlikely(page == NULL)) break; list_add_tail(&page->lru, list); @@ -598,7 +600,7 @@ static int rmqueue_bulk(struct zone *zon void drain_remote_pages(void) { struct zone *zone; - int i; + int i, area; unsigned long flags; local_irq_save(flags); @@ -614,7 +616,8 @@ void drain_remote_pages(void) struct per_cpu_pages *pcp; pcp = &pset->pcp[i]; - free_pages_bulk(zone, pcp->count, &pcp->list, 0); + for (area = 0; area < NR_FREE_AREAS; area++) + free_pages_bulk(zone, pcp->count, &pcp->list, 0, area); pcp->count = 0; } } @@ -627,7 +630,7 @@ static void __drain_pages(unsigned int c { unsigned long flags; struct zone *zone; - int i; + int i, area; for_each_zone(zone) { struct per_cpu_pageset *pset; @@ -638,7 +641,8 @@ static void __drain_pages(unsigned int c pcp = &pset->pcp[i]; local_irq_save(flags); - free_pages_bulk(zone, pcp->count, &pcp->list, 0); + for (area = 0; area < NR_FREE_AREAS; area++) + free_pages_bulk(zone, pcp->count, &pcp->list, 0, area); pcp->count = 0; local_irq_restore(flags); } @@ -651,7 +655,7 @@ static void __drain_pages(unsigned int c void mark_free_pages(struct zone *zone) { unsigned long zone_pfn, flags; - int order; + int order, area; struct list_head *curr; if (!zone->spanned_pages) @@ -661,15 +665,16 @@ void mark_free_pages(struct zone *zone) for (zone_pfn = 0; zone_pfn < zone->spanned_pages; ++zone_pfn) ClearPageNosaveFree(pfn_to_page(zone_pfn + zone->zone_start_pfn)); - for (order = MAX_ORDER - 1; order >= 0; --order) - list_for_each(curr, &zone->free_area[order].free_list) { - unsigned long start_pfn, i; + for (area = 0; area < NR_FREE_AREAS; area++) + for (order = MAX_ORDER - 1; order >= 0; --order) + list_for_each(curr, &zone->free_area[area][order].free_list) { + unsigned long start_pfn, i; - start_pfn = page_to_pfn(list_entry(curr, struct page, lru)); + start_pfn = page_to_pfn(list_entry(curr, struct page, lru)); - for (i=0; i < (1<lock, flags); } @@ -731,7 +736,7 @@ static void fastcall free_hot_cold_page( list_add(&page->lru, &pcp->list); pcp->count++; if (pcp->count >= pcp->high) { - free_pages_bulk(zone, pcp->batch, &pcp->list, 0); + free_pages_bulk(zone, pcp->batch, &pcp->list, 0, FREE_AREA_DEFAULT); pcp->count -= pcp->batch; } local_irq_restore(flags); @@ -801,7 +806,7 @@ again: local_irq_save(flags); if (!pcp->count) { pcp->count += rmqueue_bulk(zone, 0, - pcp->batch, &pcp->list); + pcp->batch, &pcp->list, FREE_AREA_DEFAULT); if (unlikely(!pcp->count)) goto failed; } @@ -810,7 +815,7 @@ again: pcp->count--; } else { spin_lock_irqsave(&zone->lock, flags); - page = __rmqueue(zone, order); + page = __rmqueue(zone, order, FREE_AREA_DEFAULT); spin_unlock(&zone->lock); if (!page) goto failed; @@ -853,9 +858,9 @@ failed: int zone_watermark_ok(struct zone *z, int order, unsigned long mark, int classzone_idx, int alloc_flags) { - /* free_pages my go negative - that's OK */ + /* free_pages may go negative - that's OK */ long min = mark, free_pages = z->free_pages - (1 << order) + 1; - int o; + int o, area; if (alloc_flags & ALLOC_HIGH) min -= min / 2; @@ -865,8 +870,9 @@ int zone_watermark_ok(struct zone *z, in if (free_pages <= min + z->lowmem_reserve[classzone_idx]) return 0; for (o = 0; o < order; o++) { - /* At the next order, this order's pages become unavailable */ - free_pages -= z->free_area[o].nr_free << o; + for (area = 0; area > NR_FREE_AREAS; area++) + /* At the next order, this order's pages become unavailable */ + free_pages -= z->free_area[area][o].nr_free << o; /* Require fewer higher order pages to be free */ min >>= 1; @@ -1180,7 +1186,7 @@ fastcall void __free_pages(struct page * if (order == 0) free_hot_page(page); else - __free_pages_ok(page, order); + __free_pages_ok(page, order, 0); #ifdef CONFIG_PAGE_OWNER page->order = -1; #endif @@ -1452,7 +1458,7 @@ void si_meminfo_node(struct sysinfo *val #define K(x) ((x) << (PAGE_SHIFT-10)) -char *temperature_descr[] = { "cold", "hot" }; +const char *temperature_descr[] = { "cold", "hot" }; /* * Show free area list (used inside shift_scroll-lock stuff) @@ -1545,7 +1551,7 @@ void show_free_areas(void) } for_each_zone(zone) { - unsigned long nr, flags, order, total = 0; + unsigned long nr, flags, order, area, total = 0; show_node(zone); printk("%s: ", zone->name); @@ -1556,7 +1562,8 @@ void show_free_areas(void) spin_lock_irqsave(&zone->lock, flags); for (order = 0; order < MAX_ORDER; order++) { - nr = zone->free_area[order].nr_free; + for (area = 0; area < NR_FREE_AREAS; area++) + nr = zone->free_area[area][order].nr_free; total += nr << order; printk("%lu*%lukB ", nr, K(1UL) << order); } @@ -1868,10 +1875,13 @@ void zone_init_free_lists(struct pglist_ unsigned long size) { int order; - for (order = 0; order < MAX_ORDER ; order++) { - INIT_LIST_HEAD(&zone->free_area[order].free_list); - zone->free_area[order].nr_free = 0; - } + int area; + + for (area = 0; area < NR_FREE_AREAS; area++) + for (order = 0; order < MAX_ORDER ; order++) { + INIT_LIST_HEAD(&zone->free_area[area][order].free_list); + zone->free_area[area][order].nr_free = 0; + } } #define ZONETABLE_INDEX(x, zone_nr) ((x << ZONES_SHIFT) | zone_nr) @@ -1931,13 +1941,13 @@ inline void setup_pageset(struct per_cpu memset(p, 0, sizeof(*p)); - pcp = &p->pcp[0]; /* hot */ + pcp = &p->pcp[PER_CPU_HOT]; pcp->count = 0; pcp->high = 6 * batch; pcp->batch = max(1UL, 1 * batch); INIT_LIST_HEAD(&pcp->list); - pcp = &p->pcp[1]; /* cold*/ + pcp = &p->pcp[PER_CPU_COLD]; pcp->count = 0; pcp->high = 2 * batch; pcp->batch = max(1UL, batch/2); @@ -2270,7 +2280,7 @@ static int frag_show(struct seq_file *m, struct zone *zone; struct zone *node_zones = pgdat->node_zones; unsigned long flags; - int order; + int order, area; for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) { if (!populated_zone(zone)) @@ -2279,7 +2289,8 @@ static int frag_show(struct seq_file *m, spin_lock_irqsave(&zone->lock, flags); seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name); for (order = 0; order < MAX_ORDER; ++order) - seq_printf(m, "%6lu ", zone->free_area[order].nr_free); + for (area = 0; area < NR_FREE_AREAS; area++) + seq_printf(m, "%6lu ", zone->free_area[area][order].nr_free); spin_unlock_irqrestore(&zone->lock, flags); seq_putc(m, '\n'); } Index: linux-2.6.16-rc1-mm3/include/linux/mmzone.h =================================================================== --- linux-2.6.16-rc1-mm3.orig/include/linux/mmzone.h 2006-01-25 16:22:01.000000000 -0800 +++ linux-2.6.16-rc1-mm3/include/linux/mmzone.h 2006-01-25 17:15:18.000000000 -0800 @@ -24,6 +24,15 @@ #define NR_PER_CPU_PAGES 2 +/* Types of per cpu pages */ +#define PER_CPU_HOT 0 +#define PER_CPU_COLD 1 + +#define NR_FREE_AREAS 1 + +/* Types of free areas */ +#define FREE_AREA_DEFAULT 0 + struct free_area { struct list_head free_list; unsigned long nr_free; @@ -54,7 +63,7 @@ struct per_cpu_pages { }; struct per_cpu_pageset { - struct per_cpu_pages pcp[NR_PER_CPU_PAGES]; /* 0: hot. 1: cold */ + struct per_cpu_pages pcp[NR_PER_CPU_PAGES]; #ifdef CONFIG_NUMA unsigned long numa_hit; /* allocated in intended node */ unsigned long numa_miss; /* allocated in non intended node */ @@ -146,7 +155,7 @@ struct zone { /* see spanned/present_pages for more description */ seqlock_t span_seqlock; #endif - struct free_area free_area[MAX_ORDER]; + struct free_area free_area[NR_FREE_AREAS][MAX_ORDER]; ZONE_PADDING(_pad1_)