--- include/linux/mmu_notifier.h | 11 +++++++++-- mm/fremap.c | 3 ++- mm/hugetlb.c | 3 ++- mm/memory.c | 16 ++++++++++------ mm/mmu_notifier.c | 9 ++++----- 5 files changed, 27 insertions(+), 15 deletions(-) Index: linux-2.6/mm/mmu_notifier.c =================================================================== --- linux-2.6.orig/mm/mmu_notifier.c 2008-01-30 17:58:48.000000000 -0800 +++ linux-2.6/mm/mmu_notifier.c 2008-01-30 18:00:26.000000000 -0800 @@ -13,23 +13,22 @@ #include #include +/* + * No synchronization. This function can only be called when only a single + * process remains that performs teardown. + */ void mmu_notifier_release(struct mm_struct *mm) { struct mmu_notifier *mn; struct hlist_node *n, *t; if (unlikely(!hlist_empty(&mm->mmu_notifier.head))) { - down_write(&mm->mmap_sem); - rcu_read_lock(); hlist_for_each_entry_safe_rcu(mn, n, t, &mm->mmu_notifier.head, hlist) { hlist_del_rcu(&mn->hlist); if (mn->ops->release) mn->ops->release(mn, mm); } - rcu_read_unlock(); - up_write(&mm->mmap_sem); - synchronize_rcu(); } } Index: linux-2.6/include/linux/mmu_notifier.h =================================================================== --- linux-2.6.orig/include/linux/mmu_notifier.h 2008-01-30 17:58:48.000000000 -0800 +++ linux-2.6/include/linux/mmu_notifier.h 2008-01-30 18:00:26.000000000 -0800 @@ -67,15 +67,22 @@ struct mmu_notifier_ops { int dummy); /* + * invalidate_range_begin() and invalidate_range_end() are paired. + * + * invalidate_range_begin must clear all references in the range + * and stop the establishment of new references. + * + * invalidate_range_end() reenables the establishment of references. + * * lock indicates that the function is called under spinlock. */ void (*invalidate_range_begin)(struct mmu_notifier *mn, struct mm_struct *mm, + unsigned long start, unsigned long end, int lock); void (*invalidate_range_end)(struct mmu_notifier *mn, - struct mm_struct *mm, - unsigned long start, unsigned long end); + struct mm_struct *mm); }; struct mmu_rmap_notifier_ops; Index: linux-2.6/mm/fremap.c =================================================================== --- linux-2.6.orig/mm/fremap.c 2008-01-30 17:58:48.000000000 -0800 +++ linux-2.6/mm/fremap.c 2008-01-30 18:00:26.000000000 -0800 @@ -212,8 +212,9 @@ asmlinkage long sys_remap_file_pages(uns spin_unlock(&mapping->i_mmap_lock); } + mmu_notifier(invalidate_range_start, mm, start, start + size, 0); err = populate_range(mm, vma, start, size, pgoff); - mmu_notifier(invalidate_range, mm, start, start + size, 0); + mmu_notifier(invalidate_range_end, mm); if (!err && !(flags & MAP_NONBLOCK)) { if (unlikely(has_write_lock)) { downgrade_write(&mm->mmap_sem); Index: linux-2.6/mm/hugetlb.c =================================================================== --- linux-2.6.orig/mm/hugetlb.c 2008-01-30 17:58:48.000000000 -0800 +++ linux-2.6/mm/hugetlb.c 2008-01-30 18:00:26.000000000 -0800 @@ -744,6 +744,7 @@ void __unmap_hugepage_range(struct vm_ar BUG_ON(start & ~HPAGE_MASK); BUG_ON(end & ~HPAGE_MASK); + mmu_notifier(invalidate_range_start, mm, start, end, 1); spin_lock(&mm->page_table_lock); for (address = start; address < end; address += HPAGE_SIZE) { ptep = huge_pte_offset(mm, address); @@ -764,7 +765,7 @@ void __unmap_hugepage_range(struct vm_ar } spin_unlock(&mm->page_table_lock); flush_tlb_range(vma, start, end); - mmu_notifier(invalidate_range, mm, start, end, 1); + mmu_notifier(invalidate_range_end, mm); list_for_each_entry_safe(page, tmp, &page_list, lru) { list_del(&page->lru); put_page(page); Index: linux-2.6/mm/memory.c =================================================================== --- linux-2.6.orig/mm/memory.c 2008-01-30 17:58:48.000000000 -0800 +++ linux-2.6/mm/memory.c 2008-01-30 18:00:51.000000000 -0800 @@ -888,11 +888,12 @@ unsigned long zap_page_range(struct vm_a lru_add_drain(); tlb = tlb_gather_mmu(mm, 0); update_hiwater_rss(mm); + mmu_notifier(invalidate_range_start, mm, address, end, + (details ? (details->i_mmap_lock != NULL) : 0)); end = unmap_vmas(&tlb, vma, address, end, &nr_accounted, details); if (tlb) tlb_finish_mmu(tlb, address, end); - mmu_notifier(invalidate_range, mm, address, end, - (details ? (details->i_mmap_lock != NULL) : 0)); + mmu_notifier(invalidate_range_end, mm); return end; } @@ -1355,6 +1356,7 @@ int remap_pfn_range(struct vm_area_struc pfn -= addr >> PAGE_SHIFT; pgd = pgd_offset(mm, addr); flush_cache_range(vma, addr, end); + mmu_notifier(invalidate_range_start, mm, start, end, 0); do { next = pgd_addr_end(addr, end); err = remap_pud_range(mm, pgd, addr, next, @@ -1362,7 +1364,7 @@ int remap_pfn_range(struct vm_area_struc if (err) break; } while (pgd++, addr = next, addr != end); - mmu_notifier(invalidate_range, mm, start, end, 0); + mmu_notifier(invalidate_range_end, mm); return err; } EXPORT_SYMBOL(remap_pfn_range); @@ -1450,6 +1452,7 @@ int apply_to_page_range(struct mm_struct int err; BUG_ON(addr >= end); + mmu_notifier(invalidate_range_start, mm, start, end, 0); pgd = pgd_offset(mm, addr); do { next = pgd_addr_end(addr, end); @@ -1457,7 +1460,7 @@ int apply_to_page_range(struct mm_struct if (err) break; } while (pgd++, addr = next, addr != end); - mmu_notifier(invalidate_range, mm, start, end, 0); + mmu_notifier(invalidate_range_end, mm); return err; } EXPORT_SYMBOL_GPL(apply_to_page_range); @@ -1635,6 +1638,8 @@ gotten: goto oom; cow_user_page(new_page, old_page, address, vma); + mmu_notifier(invalidate_range_start, mm, address, + address + PAGE_SIZE - 1, 0); /* * Re-check the pte - we dropped the lock */ @@ -1673,8 +1678,7 @@ gotten: page_cache_release(old_page); unlock: pte_unmap_unlock(page_table, ptl); - mmu_notifier(invalidate_range, mm, address, - address + PAGE_SIZE - 1, 0); + mmu_notifier(invalidate_range_end, mm); if (dirty_page) { if (vma->vm_file) file_update_time(vma->vm_file);