From: Christoph Lameter This implements the use of migration entries to preserve ptes of file backed pages during migration. Processes can therefore be migrated back and forth without loosing their connection to pagecache pages. Note that we implement the migration entries only for linear mappings. Nonlinear mappings still require the unmapping of the ptes for migration. And another writepage() ugliness shows up. writepage() can drop the page lock. Therefore we have to remove migration ptes before calling writepages() in order to avoid having migration entries point to unlocked pages. Signed-off-by: Christoph Lameter Signed-off-by: Andrew Morton --- include/linux/swap.h | 15 ---- mm/migrate.c | 127 ++++++++++++++++++++++++++++++++--------- mm/rmap.c | 11 +++ mm/vmscan.c | 14 ++++ 4 files changed, 124 insertions(+), 43 deletions(-) diff -puN mm/migrate.c~more-page-migration-use-migration-entries-for-file-pages mm/migrate.c --- 25/mm/migrate.c~more-page-migration-use-migration-entries-for-file-pages Fri May 26 16:00:23 2006 +++ 25-akpm/mm/migrate.c Fri May 26 16:02:28 2006 @@ -24,6 +24,7 @@ #include #include #include +#include #include "internal.h" @@ -123,7 +124,7 @@ static inline int is_swap_pte(pte_t pte) /* * Restore a potential migration pte to a working pte entry */ -static void remove_migration_pte(struct vm_area_struct *vma, unsigned long addr, +static void remove_migration_pte(struct vm_area_struct *vma, struct page *old, struct page *new) { struct mm_struct *mm = vma->vm_mm; @@ -133,6 +134,10 @@ static void remove_migration_pte(struct pmd_t *pmd; pte_t *ptep, pte; spinlock_t *ptl; + unsigned long addr = page_address_in_vma(new, vma); + + if (addr == -EFAULT) + return; pgd = pgd_offset(mm, addr); if (!pgd_present(*pgd)) @@ -169,19 +174,47 @@ static void remove_migration_pte(struct if (is_write_migration_entry(entry)) pte = pte_mkwrite(pte); set_pte_at(mm, addr, ptep, pte); - page_add_anon_rmap(new, vma, addr); + + if (PageAnon(new)) + page_add_anon_rmap(new, vma, addr); + else + page_add_file_rmap(new); + + /* No need to invalidate - it was non-present before */ + update_mmu_cache(vma, addr, pte); + lazy_mmu_prot_update(pte); + out: pte_unmap_unlock(ptep, ptl); } /* - * Get rid of all migration entries and replace them by - * references to the indicated page. - * + * Note that remove_file_migration_ptes will only work on regular mappings, + * Nonlinear mappings do not use migration entries. + */ +static void remove_file_migration_ptes(struct page *old, struct page *new) +{ + struct vm_area_struct *vma; + struct address_space *mapping = page_mapping(new); + struct prio_tree_iter iter; + pgoff_t pgoff = new->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT); + + if (!mapping) + return; + + spin_lock(&mapping->i_mmap_lock); + + vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff) + remove_migration_pte(vma, old, new); + + spin_unlock(&mapping->i_mmap_lock); +} + +/* * Must hold mmap_sem lock on at least one of the vmas containing * the page so that the anon_vma cannot vanish. */ -static void remove_migration_ptes(struct page *old, struct page *new) +static void remove_anon_migration_ptes(struct page *old, struct page *new) { struct anon_vma *anon_vma; struct vm_area_struct *vma; @@ -199,13 +232,24 @@ static void remove_migration_ptes(struct spin_lock(&anon_vma->lock); list_for_each_entry(vma, &anon_vma->head, anon_vma_node) - remove_migration_pte(vma, page_address_in_vma(new, vma), - old, new); + remove_migration_pte(vma, old, new); spin_unlock(&anon_vma->lock); } /* + * Get rid of all migration entries and replace them by + * references to the indicated page. + */ +static void remove_migration_ptes(struct page *old, struct page *new) +{ + if (PageAnon(new)) + remove_anon_migration_ptes(old, new); + else + remove_file_migration_ptes(old, new); +} + +/* * Something used the pte of a page under migration. We need to * get to the page and wait until migration is finished. * When we return from this function the fault will be retried. @@ -424,30 +468,59 @@ int buffer_migrate_page(struct address_s } EXPORT_SYMBOL(buffer_migrate_page); -static int fallback_migrate_page(struct address_space *mapping, - struct page *newpage, struct page *page) +/* + * Writeback a page to clean the dirty state + */ +static int writeout(struct address_space *mapping, struct page *page) { + struct writeback_control wbc = { + .sync_mode = WB_SYNC_NONE, + .nr_to_write = 1, + .range_start = 0, + .range_end = LLONG_MAX, + .nonblocking = 1, + .for_reclaim = 1 + }; + int rc; + + if (!mapping->a_ops->writepage) + /* No write method for the address space */ + return -EINVAL; + + if (!clear_page_dirty_for_io(page)) + /* Someone else already triggered a write */ + return -EAGAIN; + /* - * Default handling if a filesystem does not provide - * a migration function. We can only migrate clean - * pages so try to write out any dirty pages first. + * A dirty page may imply that the underlying filesystem has + * the page on some queue. So the page must be clean for + * migration. Writeout may mean we loose the lock and the + * page state is no longer what we checked for earlier. + * At this point we know that the migration attempt cannot + * be successful. */ - if (PageDirty(page)) { - switch (pageout(page, mapping)) { - case PAGE_KEEP: - case PAGE_ACTIVATE: - return -EAGAIN; + remove_migration_ptes(page, page); - case PAGE_SUCCESS: - /* Relock since we lost the lock */ - lock_page(page); - /* Must retry since page state may have changed */ - return -EAGAIN; + rc = mapping->a_ops->writepage(page, &wbc); + if (rc < 0) + /* I/O Error writing */ + return -EIO; + + if (rc != AOP_WRITEPAGE_ACTIVATE) + /* unlocked. Relock */ + lock_page(page); - case PAGE_CLEAN: - ; /* try to migrate the page below */ - } - } + return -EAGAIN; +} + +/* + * Default handling if a filesystem does not provide a migration function. + */ +static int fallback_migrate_page(struct address_space *mapping, + struct page *newpage, struct page *page) +{ + if (PageDirty(page)) + return writeout(mapping, page); /* * Buffers may be managed in a filesystem specific way. diff -puN mm/rmap.c~more-page-migration-use-migration-entries-for-file-pages mm/rmap.c --- 25/mm/rmap.c~more-page-migration-use-migration-entries-for-file-pages Fri May 26 16:00:23 2006 +++ 25-akpm/mm/rmap.c Fri May 26 16:02:27 2006 @@ -596,6 +596,7 @@ static int try_to_unmap_one(struct page spin_unlock(&mmlist_lock); } dec_mm_counter(mm, anon_rss); +#ifdef CONFIG_MIGRATION } else { /* * Store the pfn of the page in a special migration @@ -604,12 +605,22 @@ static int try_to_unmap_one(struct page */ BUG_ON(!migration); entry = make_migration_entry(page, pte_write(pteval)); +#endif } set_pte_at(mm, address, pte, swp_entry_to_pte(entry)); BUG_ON(pte_file(*pte)); } else +#ifdef CONFIG_MIGRATION + if (migration) { + /* Establish migration entry for a file page */ + swp_entry_t entry; + entry = make_migration_entry(page, pte_write(pteval)); + set_pte_at(mm, address, pte, swp_entry_to_pte(entry)); + } else +#endif dec_mm_counter(mm, file_rss); + page_remove_rmap(page); page_cache_release(page); diff -puN include/linux/swap.h~more-page-migration-use-migration-entries-for-file-pages include/linux/swap.h --- 25/include/linux/swap.h~more-page-migration-use-migration-entries-for-file-pages Fri May 26 16:02:28 2006 +++ 25-akpm/include/linux/swap.h Fri May 26 16:02:28 2006 @@ -186,20 +186,6 @@ extern unsigned long shrink_all_memory(u extern int vm_swappiness; extern int remove_mapping(struct address_space *mapping, struct page *page); -/* possible outcome of pageout() */ -typedef enum { - /* failed to write page out, page is locked */ - PAGE_KEEP, - /* move page to the active list, page is locked */ - PAGE_ACTIVATE, - /* page has been sent to the disk successfully, page is unlocked */ - PAGE_SUCCESS, - /* page is clean and locked */ - PAGE_CLEAN, -} pageout_t; - -extern pageout_t pageout(struct page *page, struct address_space *mapping); - #ifdef CONFIG_NUMA extern int zone_reclaim_mode; extern int zone_reclaim_interval; @@ -261,7 +247,6 @@ extern int remove_exclusive_swap_page(st struct backing_dev_info; extern spinlock_t swap_lock; -extern int remove_vma_swap(struct vm_area_struct *vma, struct page *page); /* linux/mm/thrash.c */ extern struct mm_struct * swap_token_mm; diff -puN mm/vmscan.c~more-page-migration-use-migration-entries-for-file-pages mm/vmscan.c --- 25/mm/vmscan.c~more-page-migration-use-migration-entries-for-file-pages Fri May 26 16:02:28 2006 +++ 25-akpm/mm/vmscan.c Fri May 26 16:02:28 2006 @@ -291,11 +291,23 @@ static void handle_write_error(struct ad unlock_page(page); } +/* possible outcome of pageout() */ +typedef enum { + /* failed to write page out, page is locked */ + PAGE_KEEP, + /* move page to the active list, page is locked */ + PAGE_ACTIVATE, + /* page has been sent to the disk successfully, page is unlocked */ + PAGE_SUCCESS, + /* page is clean and locked */ + PAGE_CLEAN, +} pageout_t; + /* * pageout is called by shrink_page_list() for each dirty page. * Calls ->writepage(). */ -pageout_t pageout(struct page *page, struct address_space *mapping) +static pageout_t pageout(struct page *page, struct address_space *mapping) { /* * If the page is dirty, only perform writeback if that write _