--- fs/proc/task_mmu.c | 11 +++++++---- include/linux/mm_types.h | 7 ++++++- include/linux/sched.h | 9 ++++++--- kernel/fork.c | 9 ++++++--- mm/fremap.c | 2 +- mm/memory.c | 26 +++++++++++++------------- mm/rmap.c | 10 +++++----- mm/swapfile.c | 2 +- 8 files changed, 45 insertions(+), 31 deletions(-) Index: linux-2.6/kernel/fork.c =================================================================== --- linux-2.6.orig/kernel/fork.c 2009-11-04 13:14:19.000000000 -0600 +++ linux-2.6/kernel/fork.c 2009-11-04 13:47:19.000000000 -0600 @@ -457,7 +457,7 @@ static struct mm_struct * mm_init(struct for_each_possible_cpu(cpu) { struct mm_counter *m; - memset(m, sizeof(struct mm_counter), 0); + memset(m, sizeof(struct mm_counter_node) * nr_node_ids, 0); } spin_lock_init(&mm->page_table_lock); mm->free_area_cache = TASK_UNMAPPED_BASE; @@ -485,7 +485,9 @@ struct mm_struct * mm_alloc(void) mm = allocate_mm(); if (mm) { memset(mm, 0, sizeof(*mm)); - mm->rss = alloc_percpu(struct mm_counter); + mm->rss = __alloc_percpu( + nr_node_ids * sizeof(struct mm_counter_node), + __alignof__(struct mm_counter_node)); if (mm->rss) mm = mm_init(mm, current); else { @@ -643,7 +645,8 @@ struct mm_struct *dup_mm(struct task_str goto fail_nomem; memcpy(mm, oldmm, sizeof(*mm)); - mm->rss = alloc_percpu(struct mm_counter); + mm->rss = __alloc_percpu(nr_node_ids * sizeof(struct mm_counter_node), + __alignof__(struct mm_counter_node)); if (!mm->rss) goto fail_nomem; Index: linux-2.6/fs/proc/task_mmu.c =================================================================== --- linux-2.6.orig/fs/proc/task_mmu.c 2009-11-04 13:13:42.000000000 -0600 +++ linux-2.6/fs/proc/task_mmu.c 2009-11-04 13:38:33.000000000 -0600 @@ -65,15 +65,18 @@ unsigned long task_vsize(struct mm_struc int task_statm(struct mm_struct *mm, int *shared, int *text, int *data, int *resident) { - int cpu; + int cpu, node; int anon_rss = 0; int file_rss = 0; for_each_possible_cpu(cpu) { - struct mm_counter *c = per_cpu_ptr(mm->rss, cpu); + for_each_online_node(node) { + struct mm_counter_node *c = + &per_cpu_ptr(mm->rss, cpu)->n[node]; - anon_rss += c->anon; - file_rss += c->file; + anon_rss += c->anon; + file_rss += c->file; + } } *shared = file_rss; *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK)) Index: linux-2.6/include/linux/mm_types.h =================================================================== --- linux-2.6.orig/include/linux/mm_types.h 2009-11-04 13:13:42.000000000 -0600 +++ linux-2.6/include/linux/mm_types.h 2009-11-04 13:46:52.000000000 -0600 @@ -14,6 +14,7 @@ #include #include #include +#include #ifndef AT_VECTOR_SIZE_ARCH #define AT_VECTOR_SIZE_ARCH 0 @@ -24,11 +25,15 @@ struct address_space; #define USE_SPLIT_PTLOCKS (NR_CPUS >= CONFIG_SPLIT_PTLOCK_CPUS) -struct mm_counter { +struct mm_counter_node { long file; long anon; }; +struct mm_counter { + struct mm_counter_node n[MAX_NUMNODES]; +}; + /* * Each physical page in the system has a struct page associated with * it to keep track of whatever it is we are using the page for at the Index: linux-2.6/include/linux/sched.h =================================================================== --- linux-2.6.orig/include/linux/sched.h 2009-11-04 13:13:42.000000000 -0600 +++ linux-2.6/include/linux/sched.h 2009-11-04 13:38:33.000000000 -0600 @@ -387,13 +387,16 @@ extern void arch_unmap_area_topdown(stru static inline unsigned long get_mm_rss(struct mm_struct *mm) { - int cpu; + int cpu, node; unsigned long r = 0; for_each_possible_cpu(cpu) { - struct mm_counter *c = per_cpu_ptr(mm->rss, cpu); + for_each_online_node(node) { + struct mm_counter_node *c = + &per_cpu_ptr(mm->rss, cpu)->n[node]; - r = c->file + c->anon; + r = c->file + c->anon; + } } return r; Index: linux-2.6/mm/fremap.c =================================================================== --- linux-2.6.orig/mm/fremap.c 2009-11-04 13:13:42.000000000 -0600 +++ linux-2.6/mm/fremap.c 2009-11-04 13:38:33.000000000 -0600 @@ -40,7 +40,7 @@ static void zap_pte(struct mm_struct *mm page_remove_rmap(page); page_cache_release(page); update_hiwater_rss(mm); - __this_cpu_dec(mm->rss->file); + __this_cpu_dec(mm->rss->n[page_to_nid(page)].file); } } else { if (!pte_file(pte)) Index: linux-2.6/mm/memory.c =================================================================== --- linux-2.6.orig/mm/memory.c 2009-11-04 13:38:31.000000000 -0600 +++ linux-2.6/mm/memory.c 2009-11-04 13:38:33.000000000 -0600 @@ -623,9 +623,9 @@ copy_one_pte(struct mm_struct *dst_mm, s get_page(page); page_dup_rmap(page); if (PageAnon(page)) - __this_cpu_inc(dst_mm->rss->anon); + __this_cpu_inc(dst_mm->rss->n[page_to_nid(page)].anon); else - __this_cpu_inc(dst_mm->rss->file); + __this_cpu_inc(dst_mm->rss->n[page_to_nid(page)].file); } out_set_pte: @@ -840,14 +840,14 @@ static unsigned long zap_pte_range(struc set_pte_at(mm, addr, pte, pgoff_to_pte(page->index)); if (PageAnon(page)) - __this_cpu_dec(mm->rss->anon); + __this_cpu_dec(mm->rss->n[page_to_nid(page)].anon); else { if (pte_dirty(ptent)) set_page_dirty(page); if (pte_young(ptent) && likely(!VM_SequentialReadHint(vma))) mark_page_accessed(page); - __this_cpu_dec(mm->rss->file); + __this_cpu_dec(mm->rss->n[page_to_nid(page)].file); } page_remove_rmap(page); if (unlikely(page_mapcount(page) < 0)) @@ -1501,7 +1501,7 @@ static int insert_page(struct vm_area_st /* Ok, finally just insert the thing.. */ get_page(page); - __this_cpu_inc(mm->rss->file); + __this_cpu_inc(mm->rss->n[page_to_nid(page)].file); page_add_file_rmap(page); set_pte_at(mm, addr, pte, mk_pte(page, prot)); @@ -2137,11 +2137,11 @@ gotten: if (likely(pte_same(*page_table, orig_pte))) { if (old_page) { if (!PageAnon(old_page)) { - __this_cpu_dec(mm->rss->file); - __this_cpu_inc(mm->rss->anon); + __this_cpu_dec(mm->rss->n[page_to_nid(old_page)].file); + __this_cpu_inc(mm->rss->n[page_to_nid(new_page)].anon); } } else - __this_cpu_inc(mm->rss->anon); + __this_cpu_inc(mm->rss->n[page_to_nid(new_page)].anon); flush_cache_page(vma, address, pte_pfn(orig_pte)); entry = mk_pte(new_page, vma->vm_page_prot); entry = maybe_mkwrite(pte_mkdirty(entry), vma); @@ -2250,7 +2250,7 @@ unwritable_page: } /* - * Helper functions for unmap_mapping_range(). + * Helper functions for unmap_mapping_range()/. * * __ Notes on dropping i_mmap_lock to reduce latency while unmapping __ * @@ -2568,7 +2568,7 @@ static int do_swap_page(struct mm_struct * discarded at swap_free(). */ - __this_cpu_inc(mm->rss->anon); + __this_cpu_inc(mm->rss->n[page_to_nid(page)].anon); pte = mk_pte(page, vma->vm_page_prot); if ((flags & FAULT_FLAG_WRITE) && reuse_swap_page(page)) { pte = maybe_mkwrite(pte_mkdirty(pte), vma); @@ -2652,7 +2652,7 @@ static int do_anonymous_page(struct mm_s if (!pte_none(*page_table)) goto release; - __this_cpu_inc(mm->rss->anon); + __this_cpu_inc(mm->rss->n[page_to_nid(page)].anon); page_add_new_anon_rmap(page, vma, address); setpte: set_pte_at(mm, address, page_table, entry); @@ -2806,10 +2806,10 @@ static int __do_fault(struct mm_struct * if (flags & FAULT_FLAG_WRITE) entry = maybe_mkwrite(pte_mkdirty(entry), vma); if (anon) { - __this_cpu_inc(mm->rss->anon); + __this_cpu_inc(mm->rss->n[page_to_nid(page)].anon); page_add_new_anon_rmap(page, vma, address); } else { - __this_cpu_inc(mm->rss->file); + __this_cpu_inc(mm->rss->n[page_to_nid(page)].file); page_add_file_rmap(page); if (flags & FAULT_FLAG_WRITE) { dirty_page = page; Index: linux-2.6/mm/swapfile.c =================================================================== --- linux-2.6.orig/mm/swapfile.c 2009-11-04 13:13:42.000000000 -0600 +++ linux-2.6/mm/swapfile.c 2009-11-04 13:38:33.000000000 -0600 @@ -831,7 +831,7 @@ static int unuse_pte(struct vm_area_stru goto out; } - __this_cpu_inc(vma->vm_mm->rss->anon); + __this_cpu_inc(vma->vm_mm->rss->n[page_to_nid(page)].anon); get_page(page); set_pte_at(vma->vm_mm, addr, pte, pte_mkold(mk_pte(page, vma->vm_page_prot))); Index: linux-2.6/mm/rmap.c =================================================================== --- linux-2.6.orig/mm/rmap.c 2009-11-04 13:13:42.000000000 -0600 +++ linux-2.6/mm/rmap.c 2009-11-04 13:38:33.000000000 -0600 @@ -809,9 +809,9 @@ static int try_to_unmap_one(struct page if (PageHWPoison(page) && !(flags & TTU_IGNORE_HWPOISON)) { if (PageAnon(page)) - __this_cpu_dec(mm->rss->anon); + __this_cpu_dec(mm->rss->n[page_to_nid(page)].anon); else - __this_cpu_dec(mm->rss->file); + __this_cpu_dec(mm->rss->n[page_to_nid(page)].file); set_pte_at(mm, address, pte, swp_entry_to_pte(make_hwpoison_entry(page))); } else if (PageAnon(page)) { @@ -829,7 +829,7 @@ static int try_to_unmap_one(struct page list_add(&mm->mmlist, &init_mm.mmlist); spin_unlock(&mmlist_lock); } - __this_cpu_dec(mm->rss->anon); + __this_cpu_dec(mm->rss->n[page_to_nid(page)].anon); } else if (PAGE_MIGRATION) { /* * Store the pfn of the page in a special migration @@ -847,7 +847,7 @@ static int try_to_unmap_one(struct page entry = make_migration_entry(page, pte_write(pteval)); set_pte_at(mm, address, pte, swp_entry_to_pte(entry)); } else - __this_cpu_dec(mm->rss->file); + __this_cpu_dec(mm->rss->n[page_to_nid(page)].file); page_remove_rmap(page); @@ -967,7 +967,7 @@ static int try_to_unmap_cluster(unsigned page_remove_rmap(page); page_cache_release(page); - __this_cpu_dec(mm->rss->file); + __this_cpu_dec(mm->rss->n[page_to_nid(page)].file); (*mapcount)--; } pte_unmap_unlock(pte - 1, ptl);