AIO support for O_SYNC buffered writes, built over O_SYNC-speedup. It uses the tagged radix tree lookups to writeout just the pages pertaining to this request, and retries instead of blocking for writeback to complete on the same range. All the writeout is issued at the time of io submission, and there is a check to make sure that retries skip over straight to the wait_on_page_writeback_range. Limitations: Extending file writes or hole overwrites with O_SYNC may still block because we have yet to convert generic_osync_inode to be asynchronous. For non O_SYNC writes, writeout happens in the background and so typically appears async to the caller except for memory throttling and non-block aligned writes involving read-modify-write. Signed-off-by: Suparna Bhattacharya diff -urp -X dontdiff2 linux-2.6.10-rc1/include/linux/aio.h linux-2.6.10-rc1-new/include/linux/aio.h --- linux-2.6.19-rc4-root/include/linux/aio.h | 8 +++ linux-2.6.19-rc4-root/include/linux/fs.h | 12 ++++- linux-2.6.19-rc4-root/mm/filemap.c | 61 +++++++++++++++++++++--------- 3 files changed, 60 insertions(+), 21 deletions(-) diff -puN include/linux/aio.h~aio-fs-write include/linux/aio.h --- linux-2.6.19-rc4/include/linux/aio.h~aio-fs-write 2006-11-04 08:44:12.000000000 +0530 +++ linux-2.6.19-rc4-root/include/linux/aio.h 2006-11-04 08:44:13.000000000 +0530 @@ -34,21 +34,26 @@ struct kioctx; /* #define KIF_LOCKED 0 */ #define KIF_KICKED 1 #define KIF_CANCELLED 2 +#define KIF_SYNCED 3 #define kiocbTryLock(iocb) test_and_set_bit(KIF_LOCKED, &(iocb)->ki_flags) #define kiocbTryKick(iocb) test_and_set_bit(KIF_KICKED, &(iocb)->ki_flags) +#define kiocbTrySync(iocb) test_and_set_bit(KIF_SYNCED, &(iocb)->ki_flags) #define kiocbSetLocked(iocb) set_bit(KIF_LOCKED, &(iocb)->ki_flags) #define kiocbSetKicked(iocb) set_bit(KIF_KICKED, &(iocb)->ki_flags) #define kiocbSetCancelled(iocb) set_bit(KIF_CANCELLED, &(iocb)->ki_flags) +#define kiocbSetSynced(iocb) set_bit(KIF_SYNCED, &(iocb)->ki_flags) #define kiocbClearLocked(iocb) clear_bit(KIF_LOCKED, &(iocb)->ki_flags) #define kiocbClearKicked(iocb) clear_bit(KIF_KICKED, &(iocb)->ki_flags) #define kiocbClearCancelled(iocb) clear_bit(KIF_CANCELLED, &(iocb)->ki_flags) +#define kiocbClearSynced(iocb) clear_bit(KIF_SYNCED, &(iocb)->ki_flags) #define kiocbIsLocked(iocb) test_bit(KIF_LOCKED, &(iocb)->ki_flags) #define kiocbIsKicked(iocb) test_bit(KIF_KICKED, &(iocb)->ki_flags) #define kiocbIsCancelled(iocb) test_bit(KIF_CANCELLED, &(iocb)->ki_flags) +#define kiocbIsSynced(iocb) test_bit(KIF_SYNCED, &(iocb)->ki_flags) /* is there a better place to document function pointer methods? */ /** @@ -237,7 +242,8 @@ do { \ } \ } while (0) -#define io_wait_to_kiocb(wait) container_of(wait, struct kiocb, ki_wait) +#define io_wait_to_kiocb(io_wait) container_of(container_of(io_wait, \ + struct wait_bit_queue, wait), struct kiocb, ki_wait) #define is_retried_kiocb(iocb) ((iocb)->ki_retried > 1) #include diff -puN mm/filemap.c~aio-fs-write mm/filemap.c --- linux-2.6.19-rc4/mm/filemap.c~aio-fs-write 2006-11-04 08:44:12.000000000 +0530 +++ linux-2.6.19-rc4-root/mm/filemap.c 2006-11-04 09:30:00.000000000 +0530 @@ -239,10 +239,11 @@ EXPORT_SYMBOL(filemap_flush); * @end: ending page index * * Wait for writeback to complete against pages indexed by start->end - * inclusive + * inclusive. In AIO context, this may queue an async notification + * and retry callback and return, instead of blocking the caller. */ -int wait_on_page_writeback_range(struct address_space *mapping, - pgoff_t start, pgoff_t end) +int __wait_on_page_writeback_range(struct address_space *mapping, + pgoff_t start, pgoff_t end, wait_queue_t *wait) { struct pagevec pvec; int nr_pages; @@ -254,20 +255,20 @@ int wait_on_page_writeback_range(struct pagevec_init(&pvec, 0); index = start; - while ((index <= end) && + while (!ret && (index <= end) && (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, PAGECACHE_TAG_WRITEBACK, min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) { unsigned i; - for (i = 0; i < nr_pages; i++) { + for (i = 0; !ret && (i < nr_pages); i++) { struct page *page = pvec.pages[i]; /* until radix tree lookup accepts end_index */ if (page->index > end) continue; - wait_on_page_writeback(page); + ret = __wait_on_page_writeback(page, wait); if (PageError(page)) ret = -EIO; } @@ -303,18 +304,27 @@ int sync_page_range(struct inode *inode, { pgoff_t start = pos >> PAGE_CACHE_SHIFT; pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT; - int ret; + int ret = 0; if (!mapping_cap_writeback_dirty(mapping) || !count) return 0; + if (in_aio()) { + /* Already issued writeouts for this iocb ? */ + if (kiocbTrySync(io_wait_to_kiocb(current->io_wait))) + goto do_wait; /* just need to check if done */ + } ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1); - if (ret == 0) { + + if (ret >= 0) { mutex_lock(&inode->i_mutex); ret = generic_osync_inode(inode, mapping, OSYNC_METADATA); mutex_unlock(&inode->i_mutex); } - if (ret == 0) - ret = wait_on_page_writeback_range(mapping, start, end); +do_wait: + if (ret >= 0) { + ret = __wait_on_page_writeback_range(mapping, start, end, + current->io_wait); + } return ret; } EXPORT_SYMBOL(sync_page_range); @@ -335,15 +345,23 @@ int sync_page_range_nolock(struct inode { pgoff_t start = pos >> PAGE_CACHE_SHIFT; pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT; - int ret; + int ret = 0; if (!mapping_cap_writeback_dirty(mapping) || !count) return 0; + if (in_aio()) { + /* Already issued writeouts for this iocb ? */ + if (kiocbTrySync(io_wait_to_kiocb(current->io_wait))) + goto do_wait; /* just need to check if done */ + } ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1); - if (ret == 0) + if (ret >= 0) ret = generic_osync_inode(inode, mapping, OSYNC_METADATA); - if (ret == 0) - ret = wait_on_page_writeback_range(mapping, start, end); +do_wait: + if (ret >= 0) { + ret = __wait_on_page_writeback_range(mapping, start, end, + current->io_wait); + } return ret; } EXPORT_SYMBOL(sync_page_range_nolock); @@ -2219,7 +2237,7 @@ zero_length_segment: */ if (likely(status >= 0)) { if (unlikely((file->f_flags & O_SYNC) || IS_SYNC(inode))) { - if (!a_ops->writepage || !is_sync_kiocb(iocb)) + if (!a_ops->writepage) status = generic_osync_inode(inode, mapping, OSYNC_METADATA|OSYNC_DATA); } @@ -2271,7 +2289,10 @@ __generic_file_aio_write_nolock(struct k ocount -= iv->iov_len; /* This segment is no good */ break; } - + if (!is_sync_kiocb(iocb) && kiocbIsSynced(iocb)) { + /* nothing to transfer, may just need to sync data */ + return ocount; + } count = ocount; pos = *ppos; @@ -2371,8 +2392,10 @@ ssize_t generic_file_aio_write_nolock(st ssize_t err; err = sync_page_range_nolock(inode, mapping, pos, ret); - if (err < 0) + if (err < 0) { ret = err; + iocb->ki_pos = pos; + } } return ret; } @@ -2397,8 +2420,10 @@ ssize_t generic_file_aio_write(struct ki ssize_t err; err = sync_page_range(inode, mapping, pos, ret); - if (err < 0) + if (err < 0) { ret = err; + iocb->ki_pos = pos; + } } return ret; } diff -puN include/linux/fs.h~aio-fs-write include/linux/fs.h --- linux-2.6.19-rc4/include/linux/fs.h~aio-fs-write 2006-11-04 08:44:13.000000000 +0530 +++ linux-2.6.19-rc4-root/include/linux/fs.h 2006-11-04 08:44:13.000000000 +0530 @@ -1623,8 +1623,16 @@ extern int filemap_fdatawait(struct addr extern int filemap_write_and_wait(struct address_space *mapping); extern int filemap_write_and_wait_range(struct address_space *mapping, loff_t lstart, loff_t lend); -extern int wait_on_page_writeback_range(struct address_space *mapping, - pgoff_t start, pgoff_t end); +extern int __wait_on_page_writeback_range(struct address_space *mapping, + pgoff_t start, pgoff_t end, wait_queue_t *wait); + +static inline int wait_on_page_writeback_range(struct address_space *mapping, + pgoff_t start, pgoff_t end) +{ + return __wait_on_page_writeback_range(mapping, start, end, + ¤t->__wait.wait); +} + extern int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start, loff_t end, int sync_mode); _