blk-mq: introduce blk_mq_delay_kick_requeue_list()

[karo-tx-linux.git] / block / blk-lib.c

diff --git a/block/blk-lib.c b/block/blk-lib.c
index 9e29dc35169560a223ae7eea9dffc80da15fc7f9..083e56f72308e7ebed8721ba300eb59c22f953cf 100644 (file)
--- a/block/blk-lib.c
+++ b/block/blk-lib.c
@@ -9,33 +9,46 @@
 
 #include "blk.h"
 
-static struct bio *next_bio(struct bio *bio, int rw, unsigned int nr_pages,
+static struct bio *next_bio(struct bio *bio, unsigned int nr_pages,
                gfp_t gfp)
 {
        struct bio *new = bio_alloc(gfp, nr_pages);
 
        if (bio) {
                bio_chain(bio, new);
-               submit_bio(rw, bio);
+               submit_bio(bio);
        }
 
        return new;
 }
 
 int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
-               sector_t nr_sects, gfp_t gfp_mask, int type, struct bio **biop)
+               sector_t nr_sects, gfp_t gfp_mask, int flags,
+               struct bio **biop)
 {
        struct request_queue *q = bdev_get_queue(bdev);
        struct bio *bio = *biop;
        unsigned int granularity;
+       enum req_op op;
        int alignment;
 
        if (!q)
                return -ENXIO;
-       if (!blk_queue_discard(q))
-               return -EOPNOTSUPP;
-       if ((type & REQ_SECURE) && !blk_queue_secdiscard(q))
-               return -EOPNOTSUPP;
+
+       if (flags & BLKDEV_DISCARD_SECURE) {
+               if (flags & BLKDEV_DISCARD_ZERO)
+                       return -EOPNOTSUPP;
+               if (!blk_queue_secure_erase(q))
+                       return -EOPNOTSUPP;
+               op = REQ_OP_SECURE_ERASE;
+       } else {
+               if (!blk_queue_discard(q))
+                       return -EOPNOTSUPP;
+               if ((flags & BLKDEV_DISCARD_ZERO) &&
+                   !q->limits.discard_zeroes_data)
+                       return -EOPNOTSUPP;
+               op = REQ_OP_DISCARD;
+       }
 
        /* Zero-sector (unknown) and one-sector granularities are the same.  */
        granularity = max(q->limits.discard_granularity >> 9, 1U);
@@ -62,9 +75,10 @@ int __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
                        req_sects = end_sect - sector;
                }
 
-               bio = next_bio(bio, type, 1, gfp_mask);
+               bio = next_bio(bio, 1, gfp_mask);
                bio->bi_iter.bi_sector = sector;
                bio->bi_bdev = bdev;
+               bio_set_op_attrs(bio, op, 0);
 
                bio->bi_iter.bi_size = req_sects << 9;
                nr_sects -= req_sects;
@@ -98,20 +112,16 @@ EXPORT_SYMBOL(__blkdev_issue_discard);
 int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
                sector_t nr_sects, gfp_t gfp_mask, unsigned long flags)
 {
-       int type = REQ_WRITE | REQ_DISCARD;
        struct bio *bio = NULL;
        struct blk_plug plug;
        int ret;
 
-       if (flags & BLKDEV_DISCARD_SECURE)
-               type |= REQ_SECURE;
-
        blk_start_plug(&plug);
-       ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, type,
+       ret = __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, flags,
                        &bio);
        if (!ret && bio) {
-               ret = submit_bio_wait(type, bio);
-               if (ret == -EOPNOTSUPP)
+               ret = submit_bio_wait(bio);
+               if (ret == -EOPNOTSUPP && !(flags & BLKDEV_DISCARD_ZERO))
                        ret = 0;
                bio_put(bio);
        }
@@ -148,13 +158,14 @@ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
        max_write_same_sectors = UINT_MAX >> 9;
 
        while (nr_sects) {
-               bio = next_bio(bio, REQ_WRITE | REQ_WRITE_SAME, 1, gfp_mask);
+               bio = next_bio(bio, 1, gfp_mask);
                bio->bi_iter.bi_sector = sector;
                bio->bi_bdev = bdev;
                bio->bi_vcnt = 1;
                bio->bi_io_vec->bv_page = page;
                bio->bi_io_vec->bv_offset = 0;
                bio->bi_io_vec->bv_len = bdev_logical_block_size(bdev);
+               bio_set_op_attrs(bio, REQ_OP_WRITE_SAME, 0);
 
                if (nr_sects > max_write_same_sectors) {
                        bio->bi_iter.bi_size = max_write_same_sectors << 9;
@@ -167,10 +178,10 @@ int blkdev_issue_write_same(struct block_device *bdev, sector_t sector,
        }
 
        if (bio) {
-               ret = submit_bio_wait(REQ_WRITE | REQ_WRITE_SAME, bio);
+               ret = submit_bio_wait(bio);
                bio_put(bio);
        }
-       return ret != -EOPNOTSUPP ? ret : 0;
+       return ret;
 }
 EXPORT_SYMBOL(blkdev_issue_write_same);
 
@@ -193,11 +204,11 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
        unsigned int sz;
 
        while (nr_sects != 0) {
-               bio = next_bio(bio, WRITE,
-                               min(nr_sects, (sector_t)BIO_MAX_PAGES),
+               bio = next_bio(bio, min(nr_sects, (sector_t)BIO_MAX_PAGES),
                                gfp_mask);
                bio->bi_iter.bi_sector = sector;
                bio->bi_bdev   = bdev;
+               bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
 
                while (nr_sects != 0) {
                        sz = min((sector_t) PAGE_SIZE >> 9 , nr_sects);
@@ -210,7 +221,7 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
        }
 
        if (bio) {
-               ret = submit_bio_wait(WRITE, bio);
+               ret = submit_bio_wait(bio);
                bio_put(bio);
                return ret;
        }
@@ -241,11 +252,11 @@ static int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
 int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
                         sector_t nr_sects, gfp_t gfp_mask, bool discard)
 {
-       struct request_queue *q = bdev_get_queue(bdev);
-
-       if (discard && blk_queue_discard(q) && q->limits.discard_zeroes_data &&
-           blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, 0) == 0)
-               return 0;
+       if (discard) {
+               if (!blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask,
+                               BLKDEV_DISCARD_ZERO))
+                       return 0;
+       }
 
        if (bdev_write_same(bdev) &&
            blkdev_issue_write_same(bdev, sector, nr_sects, gfp_mask,