From 218abc854504fa387349f8659edc6c1047c09b31 Mon Sep 17 00:00:00 2001 From: Mike Christie Date: Wed, 31 Oct 2007 15:41:56 -0500 Subject: [PATCH 09/13] split bioset_add_pages for sg mmap use The pages for the mmap operation are reused for each sg request, so they cannot be freed when the bio is like with the copy or map operations. sg.c does open(/dev/sg) mmap() write(/dev/sg) read(/dev/sg) Then can do write(/dev/sg) read(/dev/sg) multiple times. close(/dev/sg) This patch breaks up the bio page allocation from the page addition, so that a mmap helper can alloc pages then reuse them for another requests. The next patch contains the mmap helper and then the last patch will have sg use all the helpers. Signed-off-by: Mike Christie --- block/ll_rw_blk.c | 6 +- fs/bio.c | 115 +++++++++++++++++++++++++++++++++++---------------- include/linux/bio.h | 5 +- 3 files changed, 84 insertions(+), 42 deletions(-) diff --git a/block/ll_rw_blk.c b/block/ll_rw_blk.c index 6d94567..23474b9 100644 --- a/block/ll_rw_blk.c +++ b/block/ll_rw_blk.c @@ -2368,7 +2368,7 @@ static void __blk_rq_destroy_buffer(struct bio *bio) if (bio_flagged(bio, BIO_USER_MAPPED)) bio_unmap_user(bio); else - bioset_free_pages(bio); + bio_put(bio); } void blk_rq_destroy_buffer(struct bio *bio) @@ -2416,8 +2416,8 @@ static int __blk_rq_setup_buffer(struct bio_set *bs, struct request *rq, bio = bio_map_user(q, bs, (unsigned long)ubuf, map_len, reading, gfp_mask); } else - bio = bioset_add_pages(q, bs, len, reading, - gfp_mask); + bio = bioset_setup_pages(q, bs, len, reading, + gfp_mask); if (IS_ERR(bio)) return PTR_ERR(bio); diff --git a/fs/bio.c b/fs/bio.c index 4e7a6e5..c42ebcf 100644 --- a/fs/bio.c +++ b/fs/bio.c @@ -220,21 +220,8 @@ static struct bio_map_data *bio_alloc_map_data(int nr_segs, gfp_t gfp_mask) return NULL; } -static void bio_bmd_destructor(struct bio *bio) +void bioset_free_pages(struct bio_map_data *bmd) { - struct bio_map_data *bmd = bio->bi_private; - struct bio_set *bs; - - if (!bmd) - return; - bs = bmd->bs; - bio_free_map_data(bmd); - bio_free(bio, bs); -} - -void bioset_free_pages(struct bio *bio) -{ - struct bio_map_data *bmd = bio->bi_private; struct bio_set *bs = bmd->bs; int i; @@ -242,49 +229,90 @@ void bioset_free_pages(struct bio *bio) if (bmd->iovecs[i].page) mempool_free(bmd->iovecs[i].page, bs->page_pool); } - bio_put(bio); + bio_free_map_data(bmd); } -struct bio *bioset_add_pages(struct request_queue *q, struct bio_set *bs, - unsigned int len, int write_to_vm, gfp_t gfp_mask) +struct bio_map_data *bioset_alloc_pages(struct request_queue *q, + struct bio_set *bs, unsigned int len, + gfp_t gfp_mask) { int nr_pages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; struct bio_map_data *bmd; struct page *page; - struct bio *bio; int i = 0, ret; bmd = bio_alloc_map_data(nr_pages, gfp_mask); if (!bmd) return ERR_PTR(-ENOMEM); + bmd->bs = bs; + if (!bmd->bs) + bmd->bs = blk_bio_set; - ret = -ENOMEM; - if (!bs) - bs = blk_bio_set; - bio = bio_alloc_bioset(gfp_mask, nr_pages, bs); + ret = 0; + while (len) { + page = mempool_alloc(bmd->bs->page_pool, + q->bounce_gfp | gfp_mask); + if (!page) { + ret = -ENOMEM; + goto fail; + } + bmd->nr_vecs++; + bmd->iovecs[i].page = page; + + len -= min_t(unsigned int, + (1 << bmd->bs->page_pool_order) << PAGE_SHIFT, len); + i++; + } + return bmd; +fail: + bioset_free_pages(bmd); + return ERR_PTR(ret); +} + +static void bio_bmd_destructor(struct bio *bio) +{ + struct bio_map_data *bmd = bio->bi_private; + struct bio_set *bs; + + if (!bmd) + return; + bs = bmd->bs; + bioset_free_pages(bmd); + bio_free(bio, bs); +} + +struct bio *bioset_add_pages(struct request_queue *q, struct bio_map_data *bmd, + unsigned int len, int write_to_vm, gfp_t gfp_mask) +{ + int nr_pages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT; + struct page *page; + struct bio *bio; + int i = 0, ret; + + bio = bio_alloc_bioset(gfp_mask, nr_pages, bmd->bs); if (!bio) - goto out_bmd; + return ERR_PTR(-ENOMEM); bio->bi_rw |= (!write_to_vm << BIO_RW); - bio->bi_destructor = bio_bmd_destructor; bio->bi_private = bmd; - bmd->bs = bs; + bio->bi_destructor = bio_bmd_destructor; ret = 0; while (len) { unsigned add_len; - page = mempool_alloc(bs->page_pool, q->bounce_gfp | gfp_mask); + page = bmd->iovecs[i].page; if (!page) { - ret = -ENOMEM; - bioset_free_pages(bio); - goto fail; + ret = -EINVAL; + printk(KERN_ERR "Invalid bio map data. Not enough " + "pages allocated to handle req of len %d\n", + len); + goto free_bio; } - bmd->nr_vecs++; - bmd->iovecs[i].page = page; bmd->iovecs[i].len = 0; add_len = min_t(unsigned int, - (1 << bs->page_pool_order) << PAGE_SHIFT, len); + (1 << bmd->bs->page_pool_order) << PAGE_SHIFT, + len); while (add_len) { unsigned int added, bytes = PAGE_SIZE; @@ -300,15 +328,30 @@ struct bio *bioset_add_pages(struct request_queue *q, struct bio_set *bs, } i++; } - return bio; -out_bmd: - bio_free_map_data(bmd); -fail: +free_bio: + bio_free(bio, bmd->bs); return ERR_PTR(ret); } +struct bio *bioset_setup_pages(struct request_queue *q, struct bio_set *bs, + unsigned int len, int write_to_vm, + gfp_t gfp_mask) +{ + struct bio_map_data *bmd; + struct bio *bio; + + bmd = bioset_alloc_pages(q, bs, len, gfp_mask); + if (IS_ERR(bmd)) + return ERR_PTR(-ENOMEM); + + bio = bioset_add_pages(q, bmd, len, write_to_vm, gfp_mask); + if (IS_ERR(bio)) + bioset_free_pages(bmd); + return bio; +} + int bio_copy_user_iov(struct bio *head, struct sg_iovec *iov, int iov_count) { unsigned int iov_len = 0; diff --git a/include/linux/bio.h b/include/linux/bio.h index 8300e4b..c7e81a1 100644 --- a/include/linux/bio.h +++ b/include/linux/bio.h @@ -330,9 +330,8 @@ extern struct bio *bio_map_kern(struct request_queue *, struct bio_set *, extern void bio_set_pages_dirty(struct bio *bio); extern void bio_check_pages_dirty(struct bio *bio); extern void bio_release_pages(struct bio *bio); -extern void bioset_free_pages(struct bio *); -extern struct bio *bioset_add_pages(struct request_queue *, - struct bio_set *, unsigned int, int, gfp_t); +extern struct bio *bioset_setup_pages(struct request_queue *, struct bio_set *, + unsigned int, int, gfp_t); void zero_fill_bio(struct bio *bio); #ifdef CONFIG_HIGHMEM -- 1.5.4.1