fs/btrfs/scrub.c

   1 /*
   2  * Copyright (C) 2011 STRATO.  All rights reserved.
   3  *
   4  * This program is free software; you can redistribute it and/or
   5  * modify it under the terms of the GNU General Public
   6  * License v2 as published by the Free Software Foundation.
   7  *
   8  * This program is distributed in the hope that it will be useful,
   9  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  10  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  11  * General Public License for more details.
  12  *
  13  * You should have received a copy of the GNU General Public
  14  * License along with this program; if not, write to the
  15  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  16  * Boston, MA 021110-1307, USA.
  17  */
  18
  19 #include <linux/sched.h>
  20 #include <linux/pagemap.h>
  21 #include <linux/writeback.h>
  22 #include <linux/blkdev.h>
  23 #include <linux/rbtree.h>
  24 #include <linux/slab.h>
  25 #include <linux/workqueue.h>
  26 #include "ctree.h"
  27 #include "volumes.h"
  28 #include "disk-io.h"
  29 #include "ordered-data.h"
  30
  31 /*
  32  * This is only the first step towards a full-features scrub. It reads all
  33  * extent and super block and verifies the checksums. In case a bad checksum
  34  * is found or the extent cannot be read, good data will be written back if
  35  * any can be found.
  36  *
  37  * Future enhancements:
  38  *  - To enhance the performance, better read-ahead strategies for the
  39  *    extent-tree can be employed.
  40  *  - In case an unrepairable extent is encountered, track which files are
  41  *    affected and report them
  42  *  - In case of a read error on files with nodatasum, map the file and read
  43  *    the extent to trigger a writeback of the good copy
  44  *  - track and record media errors, throw out bad devices
  45  *  - add a mode to also read unallocated space
  46  *  - make the prefetch cancellable
  47  */
  48
  49 struct scrub_bio;
  50 struct scrub_page;
  51 struct scrub_dev;
  52 static void scrub_bio_end_io(struct bio *bio, int err);
  53 static void scrub_checksum(struct btrfs_work *work);
  54 static int scrub_checksum_data(struct scrub_dev *sdev,
  55                                struct scrub_page *spag, void *buffer);
  56 static int scrub_checksum_tree_block(struct scrub_dev *sdev,
  57                                      struct scrub_page *spag, u64 logical,
  58                                      void *buffer);
  59 static int scrub_checksum_super(struct scrub_bio *sbio, void *buffer);
  60 static int scrub_fixup_check(struct scrub_bio *sbio, int ix);
  61 static void scrub_fixup_end_io(struct bio *bio, int err);
  62 static int scrub_fixup_io(int rw, struct block_device *bdev, sector_t sector,
  63                           struct page *page);
  64 static void scrub_fixup(struct scrub_bio *sbio, int ix);
  65
  66 #define SCRUB_PAGES_PER_BIO     16      /* 64k per bio */
  67 #define SCRUB_BIOS_PER_DEV      16      /* 1 MB per device in flight */
  68
  69 struct scrub_page {
  70         u64                     flags;  /* extent flags */
  71         u64                     generation;
  72         u64                     mirror_num;
  73         int                     have_csum;
  74         u8                      csum[BTRFS_CSUM_SIZE];
  75 };
  76
  77 struct scrub_bio {
  78         int                     index;
  79         struct scrub_dev        *sdev;
  80         struct bio              *bio;
  81         int                     err;
  82         u64                     logical;
  83         u64                     physical;
  84         struct scrub_page       spag[SCRUB_PAGES_PER_BIO];
  85         u64                     count;
  86         int                     next_free;
  87         struct btrfs_work       work;
  88 };
  89
  90 struct scrub_dev {
  91         struct scrub_bio        *bios[SCRUB_BIOS_PER_DEV];
  92         struct btrfs_device     *dev;
  93         int                     first_free;
  94         int                     curr;
  95         atomic_t                in_flight;
  96         spinlock_t              list_lock;
  97         wait_queue_head_t       list_wait;
  98         u16                     csum_size;
  99         struct list_head        csum_list;
 100         atomic_t                cancel_req;
 101         int                     readonly;
 102         /*
 103          * statistics
 104          */
 105         struct btrfs_scrub_progress stat;
 106         spinlock_t              stat_lock;
 107 };
 108
 109 static void scrub_free_csums(struct scrub_dev *sdev)
 110 {
 111         while (!list_empty(&sdev->csum_list)) {
 112                 struct btrfs_ordered_sum *sum;
 113                 sum = list_first_entry(&sdev->csum_list,
 114                                        struct btrfs_ordered_sum, list);
 115                 list_del(&sum->list);
 116                 kfree(sum);
 117         }
 118 }
 119
 120 static noinline_for_stack void scrub_free_dev(struct scrub_dev *sdev)
 121 {
 122         int i;
 123         int j;
 124         struct page *last_page;
 125
 126         if (!sdev)
 127                 return;
 128
 129         for (i = 0; i < SCRUB_BIOS_PER_DEV; ++i) {
 130                 struct scrub_bio *sbio = sdev->bios[i];
 131                 struct bio *bio;
 132
 133                 if (!sbio)
 134                         break;
 135
 136                 bio = sbio->bio;
 137                 if (bio) {
 138                         last_page = NULL;
 139                         for (j = 0; j < bio->bi_vcnt; ++j) {
 140                                 if (bio->bi_io_vec[j].bv_page == last_page)
 141                                         continue;
 142                                 last_page = bio->bi_io_vec[j].bv_page;
 143                                 __free_page(last_page);
 144                         }
 145                         bio_put(bio);
 146                 }
 147                 kfree(sbio);
 148         }
 149
 150         scrub_free_csums(sdev);
 151         kfree(sdev);
 152 }
 153
 154 static noinline_for_stack
 155 struct scrub_dev *scrub_setup_dev(struct btrfs_device *dev)
 156 {
 157         struct scrub_dev *sdev;
 158         int             i;
 159         int             j;
 160         int             ret;
 161         struct btrfs_fs_info *fs_info = dev->dev_root->fs_info;
 162
 163         sdev = kzalloc(sizeof(*sdev), GFP_NOFS);
 164         if (!sdev)
 165                 goto nomem;
 166         sdev->dev = dev;
 167         for (i = 0; i < SCRUB_BIOS_PER_DEV; ++i) {
 168                 struct bio *bio;
 169                 struct scrub_bio *sbio;
 170
 171                 sbio = kzalloc(sizeof(*sbio), GFP_NOFS);
 172                 if (!sbio)
 173                         goto nomem;
 174                 sdev->bios[i] = sbio;
 175
 176                 bio = bio_kmalloc(GFP_NOFS, SCRUB_PAGES_PER_BIO);
 177                 if (!bio)
 178                         goto nomem;
 179
 180                 sbio->index = i;
 181                 sbio->sdev = sdev;
 182                 sbio->bio = bio;
 183                 sbio->count = 0;
 184                 sbio->work.func = scrub_checksum;
 185                 bio->bi_private = sdev->bios[i];
 186                 bio->bi_end_io = scrub_bio_end_io;
 187                 bio->bi_sector = 0;
 188                 bio->bi_bdev = dev->bdev;
 189                 bio->bi_size = 0;
 190
 191                 for (j = 0; j < SCRUB_PAGES_PER_BIO; ++j) {
 192                         struct page *page;
 193                         page = alloc_page(GFP_NOFS);
 194                         if (!page)
 195                                 goto nomem;
 196
 197                         ret = bio_add_page(bio, page, PAGE_SIZE, 0);
 198                         if (!ret)
 199                                 goto nomem;
 200                 }
 201                 WARN_ON(bio->bi_vcnt != SCRUB_PAGES_PER_BIO);
 202
 203                 if (i != SCRUB_BIOS_PER_DEV-1)
 204                         sdev->bios[i]->next_free = i + 1;
 205                  else
 206                         sdev->bios[i]->next_free = -1;
 207         }
 208         sdev->first_free = 0;
 209         sdev->curr = -1;
 210         atomic_set(&sdev->in_flight, 0);
 211         atomic_set(&sdev->cancel_req, 0);
 212         sdev->csum_size = btrfs_super_csum_size(&fs_info->super_copy);
 213         INIT_LIST_HEAD(&sdev->csum_list);
 214
 215         spin_lock_init(&sdev->list_lock);
 216         spin_lock_init(&sdev->stat_lock);
 217         init_waitqueue_head(&sdev->list_wait);
 218         return sdev;
 219
 220 nomem:
 221         scrub_free_dev(sdev);
 222         return ERR_PTR(-ENOMEM);
 223 }
 224
 225 /*
 226  * scrub_recheck_error gets called when either verification of the page
 227  * failed or the bio failed to read, e.g. with EIO. In the latter case,
 228  * recheck_error gets called for every page in the bio, even though only
 229  * one may be bad
 230  */
 231 static void scrub_recheck_error(struct scrub_bio *sbio, int ix)
 232 {
 233         if (sbio->err) {
 234                 if (scrub_fixup_io(READ, sbio->sdev->dev->bdev,
 235                                    (sbio->physical + ix * PAGE_SIZE) >> 9,
 236                                    sbio->bio->bi_io_vec[ix].bv_page) == 0) {
 237                         if (scrub_fixup_check(sbio, ix) == 0)
 238                                 return;
 239                 }
 240         }
 241
 242         scrub_fixup(sbio, ix);
 243 }
 244
 245 static int scrub_fixup_check(struct scrub_bio *sbio, int ix)
 246 {
 247         int ret = 1;
 248         struct page *page;
 249         void *buffer;
 250         u64 flags = sbio->spag[ix].flags;
 251
 252         page = sbio->bio->bi_io_vec[ix].bv_page;
 253         buffer = kmap_atomic(page, KM_USER0);
 254         if (flags & BTRFS_EXTENT_FLAG_DATA) {
 255                 ret = scrub_checksum_data(sbio->sdev,
 256                                           sbio->spag + ix, buffer);
 257         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
 258                 ret = scrub_checksum_tree_block(sbio->sdev,
 259                                                 sbio->spag + ix,
 260                                                 sbio->logical + ix * PAGE_SIZE,
 261                                                 buffer);
 262         } else {
 263                 WARN_ON(1);
 264         }
 265         kunmap_atomic(buffer, KM_USER0);
 266
 267         return ret;
 268 }
 269
 270 static void scrub_fixup_end_io(struct bio *bio, int err)
 271 {
 272         complete((struct completion *)bio->bi_private);
 273 }
 274
 275 static void scrub_fixup(struct scrub_bio *sbio, int ix)
 276 {
 277         struct scrub_dev *sdev = sbio->sdev;
 278         struct btrfs_fs_info *fs_info = sdev->dev->dev_root->fs_info;
 279         struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
 280         struct btrfs_multi_bio *multi = NULL;
 281         u64 logical = sbio->logical + ix * PAGE_SIZE;
 282         u64 length;
 283         int i;
 284         int ret;
 285         DECLARE_COMPLETION_ONSTACK(complete);
 286
 287         if ((sbio->spag[ix].flags & BTRFS_EXTENT_FLAG_DATA) &&
 288             (sbio->spag[ix].have_csum == 0)) {
 289                 /*
 290                  * nodatasum, don't try to fix anything
 291                  * FIXME: we can do better, open the inode and trigger a
 292                  * writeback
 293                  */
 294                 goto uncorrectable;
 295         }
 296
 297         length = PAGE_SIZE;
 298         ret = btrfs_map_block(map_tree, REQ_WRITE, logical, &length,
 299                               &multi, 0);
 300         if (ret || !multi || length < PAGE_SIZE) {
 301                 printk(KERN_ERR
 302                        "scrub_fixup: btrfs_map_block failed us for %llu\n",
 303                        (unsigned long long)logical);
 304                 WARN_ON(1);
 305                 return;
 306         }
 307
 308         if (multi->num_stripes == 1)
 309                 /* there aren't any replicas */
 310                 goto uncorrectable;
 311
 312         /*
 313          * first find a good copy
 314          */
 315         for (i = 0; i < multi->num_stripes; ++i) {
 316                 if (i == sbio->spag[ix].mirror_num)
 317                         continue;
 318
 319                 if (scrub_fixup_io(READ, multi->stripes[i].dev->bdev,
 320                                    multi->stripes[i].physical >> 9,
 321                                    sbio->bio->bi_io_vec[ix].bv_page)) {
 322                         /* I/O-error, this is not a good copy */
 323                         continue;
 324                 }
 325
 326                 if (scrub_fixup_check(sbio, ix) == 0)
 327                         break;
 328         }
 329         if (i == multi->num_stripes)
 330                 goto uncorrectable;
 331
 332         if (!sdev->readonly) {
 333                 /*
 334                  * bi_io_vec[ix].bv_page now contains good data, write it back
 335                  */
 336                 if (scrub_fixup_io(WRITE, sdev->dev->bdev,
 337                                    (sbio->physical + ix * PAGE_SIZE) >> 9,
 338                                    sbio->bio->bi_io_vec[ix].bv_page)) {
 339                         /* I/O-error, writeback failed, give up */
 340                         goto uncorrectable;
 341                 }
 342         }
 343
 344         kfree(multi);
 345         spin_lock(&sdev->stat_lock);
 346         ++sdev->stat.corrected_errors;
 347         spin_unlock(&sdev->stat_lock);
 348
 349         if (printk_ratelimit())
 350                 printk(KERN_ERR "btrfs: fixed up at %llu\n",
 351                        (unsigned long long)logical);
 352         return;
 353
 354 uncorrectable:
 355         kfree(multi);
 356         spin_lock(&sdev->stat_lock);
 357         ++sdev->stat.uncorrectable_errors;
 358         spin_unlock(&sdev->stat_lock);
 359
 360         if (printk_ratelimit())
 361                 printk(KERN_ERR "btrfs: unable to fixup at %llu\n",
 362                          (unsigned long long)logical);
 363 }
 364
 365 static int scrub_fixup_io(int rw, struct block_device *bdev, sector_t sector,
 366                          struct page *page)
 367 {
 368         struct bio *bio = NULL;
 369         int ret;
 370         DECLARE_COMPLETION_ONSTACK(complete);
 371
 372         /* we are going to wait on this IO */
 373         rw |= REQ_SYNC | REQ_UNPLUG;
 374
 375         bio = bio_alloc(GFP_NOFS, 1);
 376         bio->bi_bdev = bdev;
 377         bio->bi_sector = sector;
 378         bio_add_page(bio, page, PAGE_SIZE, 0);
 379         bio->bi_end_io = scrub_fixup_end_io;
 380         bio->bi_private = &complete;
 381         submit_bio(rw, bio);
 382
 383         wait_for_completion(&complete);
 384
 385         ret = !test_bit(BIO_UPTODATE, &bio->bi_flags);
 386         bio_put(bio);
 387         return ret;
 388 }
 389
 390 static void scrub_bio_end_io(struct bio *bio, int err)
 391 {
 392         struct scrub_bio *sbio = bio->bi_private;
 393         struct scrub_dev *sdev = sbio->sdev;
 394         struct btrfs_fs_info *fs_info = sdev->dev->dev_root->fs_info;
 395
 396         sbio->err = err;
 397
 398         btrfs_queue_worker(&fs_info->scrub_workers, &sbio->work);
 399 }
 400
 401 static void scrub_checksum(struct btrfs_work *work)
 402 {
 403         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
 404         struct scrub_dev *sdev = sbio->sdev;
 405         struct page *page;
 406         void *buffer;
 407         int i;
 408         u64 flags;
 409         u64 logical;
 410         int ret;
 411
 412         if (sbio->err) {
 413                 for (i = 0; i < sbio->count; ++i)
 414                         scrub_recheck_error(sbio, i);
 415
 416                 sbio->bio->bi_flags &= ~(BIO_POOL_MASK - 1);
 417                 sbio->bio->bi_flags |= 1 << BIO_UPTODATE;
 418                 sbio->bio->bi_phys_segments = 0;
 419                 sbio->bio->bi_idx = 0;
 420
 421                 for (i = 0; i < sbio->count; i++) {
 422                         struct bio_vec *bi;
 423                         bi = &sbio->bio->bi_io_vec[i];
 424                         bi->bv_offset = 0;
 425                         bi->bv_len = PAGE_SIZE;
 426                 }
 427
 428                 spin_lock(&sdev->stat_lock);
 429                 ++sdev->stat.read_errors;
 430                 spin_unlock(&sdev->stat_lock);
 431                 goto out;
 432         }
 433         for (i = 0; i < sbio->count; ++i) {
 434                 page = sbio->bio->bi_io_vec[i].bv_page;
 435                 buffer = kmap_atomic(page, KM_USER0);
 436                 flags = sbio->spag[i].flags;
 437                 logical = sbio->logical + i * PAGE_SIZE;
 438                 ret = 0;
 439                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
 440                         ret = scrub_checksum_data(sdev, sbio->spag + i, buffer);
 441                 } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
 442                         ret = scrub_checksum_tree_block(sdev, sbio->spag + i,
 443                                                         logical, buffer);
 444                 } else if (flags & BTRFS_EXTENT_FLAG_SUPER) {
 445                         BUG_ON(i);
 446                         (void)scrub_checksum_super(sbio, buffer);
 447                 } else {
 448                         WARN_ON(1);
 449                 }
 450                 kunmap_atomic(buffer, KM_USER0);
 451                 if (ret)
 452                         scrub_recheck_error(sbio, i);
 453         }
 454
 455 out:
 456         spin_lock(&sdev->list_lock);
 457         sbio->next_free = sdev->first_free;
 458         sdev->first_free = sbio->index;
 459         spin_unlock(&sdev->list_lock);
 460         atomic_dec(&sdev->in_flight);
 461         wake_up(&sdev->list_wait);
 462 }
 463
 464 static int scrub_checksum_data(struct scrub_dev *sdev,
 465                                struct scrub_page *spag, void *buffer)
 466 {
 467         u8 csum[BTRFS_CSUM_SIZE];
 468         u32 crc = ~(u32)0;
 469         int fail = 0;
 470         struct btrfs_root *root = sdev->dev->dev_root;
 471
 472         if (!spag->have_csum)
 473                 return 0;
 474
 475         crc = btrfs_csum_data(root, buffer, crc, PAGE_SIZE);
 476         btrfs_csum_final(crc, csum);
 477         if (memcmp(csum, spag->csum, sdev->csum_size))
 478                 fail = 1;
 479
 480         spin_lock(&sdev->stat_lock);
 481         ++sdev->stat.data_extents_scrubbed;
 482         sdev->stat.data_bytes_scrubbed += PAGE_SIZE;
 483         if (fail)
 484                 ++sdev->stat.csum_errors;
 485         spin_unlock(&sdev->stat_lock);
 486
 487         return fail;
 488 }
 489
 490 static int scrub_checksum_tree_block(struct scrub_dev *sdev,
 491                                      struct scrub_page *spag, u64 logical,
 492                                      void *buffer)
 493 {
 494         struct btrfs_header *h;
 495         struct btrfs_root *root = sdev->dev->dev_root;
 496         struct btrfs_fs_info *fs_info = root->fs_info;
 497         u8 csum[BTRFS_CSUM_SIZE];
 498         u32 crc = ~(u32)0;
 499         int fail = 0;
 500         int crc_fail = 0;
 501
 502         /*
 503          * we don't use the getter functions here, as we
 504          * a) don't have an extent buffer and
 505          * b) the page is already kmapped
 506          */
 507         h = (struct btrfs_header *)buffer;
 508
 509         if (logical != le64_to_cpu(h->bytenr))
 510                 ++fail;
 511
 512         if (spag->generation != le64_to_cpu(h->generation))
 513                 ++fail;
 514
 515         if (memcmp(h->fsid, fs_info->fsid, BTRFS_UUID_SIZE))
 516                 ++fail;
 517
 518         if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
 519                    BTRFS_UUID_SIZE))
 520                 ++fail;
 521
 522         crc = btrfs_csum_data(root, buffer + BTRFS_CSUM_SIZE, crc,
 523                               PAGE_SIZE - BTRFS_CSUM_SIZE);
 524         btrfs_csum_final(crc, csum);
 525         if (memcmp(csum, h->csum, sdev->csum_size))
 526                 ++crc_fail;
 527
 528         spin_lock(&sdev->stat_lock);
 529         ++sdev->stat.tree_extents_scrubbed;
 530         sdev->stat.tree_bytes_scrubbed += PAGE_SIZE;
 531         if (crc_fail)
 532                 ++sdev->stat.csum_errors;
 533         if (fail)
 534                 ++sdev->stat.verify_errors;
 535         spin_unlock(&sdev->stat_lock);
 536
 537         return fail || crc_fail;
 538 }
 539
 540 static int scrub_checksum_super(struct scrub_bio *sbio, void *buffer)
 541 {
 542         struct btrfs_super_block *s;
 543         u64 logical;
 544         struct scrub_dev *sdev = sbio->sdev;
 545         struct btrfs_root *root = sdev->dev->dev_root;
 546         struct btrfs_fs_info *fs_info = root->fs_info;
 547         u8 csum[BTRFS_CSUM_SIZE];
 548         u32 crc = ~(u32)0;
 549         int fail = 0;
 550
 551         s = (struct btrfs_super_block *)buffer;
 552         logical = sbio->logical;
 553
 554         if (logical != le64_to_cpu(s->bytenr))
 555                 ++fail;
 556
 557         if (sbio->spag[0].generation != le64_to_cpu(s->generation))
 558                 ++fail;
 559
 560         if (memcmp(s->fsid, fs_info->fsid, BTRFS_UUID_SIZE))
 561                 ++fail;
 562
 563         crc = btrfs_csum_data(root, buffer + BTRFS_CSUM_SIZE, crc,
 564                               PAGE_SIZE - BTRFS_CSUM_SIZE);
 565         btrfs_csum_final(crc, csum);
 566         if (memcmp(csum, s->csum, sbio->sdev->csum_size))
 567                 ++fail;
 568
 569         if (fail) {
 570                 /*
 571                  * if we find an error in a super block, we just report it.
 572                  * They will get written with the next transaction commit
 573                  * anyway
 574                  */
 575                 spin_lock(&sdev->stat_lock);
 576                 ++sdev->stat.super_errors;
 577                 spin_unlock(&sdev->stat_lock);
 578         }
 579
 580         return fail;
 581 }
 582
 583 static int scrub_submit(struct scrub_dev *sdev)
 584 {
 585         struct scrub_bio *sbio;
 586
 587         if (sdev->curr == -1)
 588                 return 0;
 589
 590         sbio = sdev->bios[sdev->curr];
 591
 592         sbio->bio->bi_sector = sbio->physical >> 9;
 593         sbio->bio->bi_size = sbio->count * PAGE_SIZE;
 594         sbio->bio->bi_next = NULL;
 595         sbio->bio->bi_flags |= 1 << BIO_UPTODATE;
 596         sbio->bio->bi_comp_cpu = -1;
 597         sbio->bio->bi_bdev = sdev->dev->bdev;
 598         sbio->err = 0;
 599         sdev->curr = -1;
 600         atomic_inc(&sdev->in_flight);
 601
 602         submit_bio(0, sbio->bio);
 603
 604         return 0;
 605 }
 606
 607 static int scrub_page(struct scrub_dev *sdev, u64 logical, u64 len,
 608                       u64 physical, u64 flags, u64 gen, u64 mirror_num,
 609                       u8 *csum, int force)
 610 {
 611         struct scrub_bio *sbio;
 612
 613 again:
 614         /*
 615          * grab a fresh bio or wait for one to become available
 616          */
 617         while (sdev->curr == -1) {
 618                 spin_lock(&sdev->list_lock);
 619                 sdev->curr = sdev->first_free;
 620                 if (sdev->curr != -1) {
 621                         sdev->first_free = sdev->bios[sdev->curr]->next_free;
 622                         sdev->bios[sdev->curr]->next_free = -1;
 623                         sdev->bios[sdev->curr]->count = 0;
 624                         spin_unlock(&sdev->list_lock);
 625                 } else {
 626                         spin_unlock(&sdev->list_lock);
 627                         wait_event(sdev->list_wait, sdev->first_free != -1);
 628                 }
 629         }
 630         sbio = sdev->bios[sdev->curr];
 631         if (sbio->count == 0) {
 632                 sbio->physical = physical;
 633                 sbio->logical = logical;
 634         } else if (sbio->physical + sbio->count * PAGE_SIZE != physical) {
 635                 scrub_submit(sdev);
 636                 goto again;
 637         }
 638         sbio->spag[sbio->count].flags = flags;
 639         sbio->spag[sbio->count].generation = gen;
 640         sbio->spag[sbio->count].have_csum = 0;
 641         sbio->spag[sbio->count].mirror_num = mirror_num;
 642         if (csum) {
 643                 sbio->spag[sbio->count].have_csum = 1;
 644                 memcpy(sbio->spag[sbio->count].csum, csum, sdev->csum_size);
 645         }
 646         ++sbio->count;
 647         if (sbio->count == SCRUB_PAGES_PER_BIO || force)
 648                 scrub_submit(sdev);
 649
 650         return 0;
 651 }
 652
 653 static int scrub_find_csum(struct scrub_dev *sdev, u64 logical, u64 len,
 654                            u8 *csum)
 655 {
 656         struct btrfs_ordered_sum *sum = NULL;
 657         int ret = 0;
 658         unsigned long i;
 659         unsigned long num_sectors;
 660         u32 sectorsize = sdev->dev->dev_root->sectorsize;
 661
 662         while (!list_empty(&sdev->csum_list)) {
 663                 sum = list_first_entry(&sdev->csum_list,
 664                                        struct btrfs_ordered_sum, list);
 665                 if (sum->bytenr > logical)
 666                         return 0;
 667                 if (sum->bytenr + sum->len > logical)
 668                         break;
 669
 670                 ++sdev->stat.csum_discards;
 671                 list_del(&sum->list);
 672                 kfree(sum);
 673                 sum = NULL;
 674         }
 675         if (!sum)
 676                 return 0;
 677
 678         num_sectors = sum->len / sectorsize;
 679         for (i = 0; i < num_sectors; ++i) {
 680                 if (sum->sums[i].bytenr == logical) {
 681                         memcpy(csum, &sum->sums[i].sum, sdev->csum_size);
 682                         ret = 1;
 683                         break;
 684                 }
 685         }
 686         if (ret && i == num_sectors - 1) {
 687                 list_del(&sum->list);
 688                 kfree(sum);
 689         }
 690         return ret;
 691 }
 692
 693 /* scrub extent tries to collect up to 64 kB for each bio */
 694 static int scrub_extent(struct scrub_dev *sdev, u64 logical, u64 len,
 695                         u64 physical, u64 flags, u64 gen, u64 mirror_num)
 696 {
 697         int ret;
 698         u8 csum[BTRFS_CSUM_SIZE];
 699
 700         while (len) {
 701                 u64 l = min_t(u64, len, PAGE_SIZE);
 702                 int have_csum = 0;
 703
 704                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
 705                         /* push csums to sbio */
 706                         have_csum = scrub_find_csum(sdev, logical, l, csum);
 707                         if (have_csum == 0)
 708                                 ++sdev->stat.no_csum;
 709                 }
 710                 ret = scrub_page(sdev, logical, l, physical, flags, gen,
 711                                  mirror_num, have_csum ? csum : NULL, 0);
 712                 if (ret)
 713                         return ret;
 714                 len -= l;
 715                 logical += l;
 716                 physical += l;
 717         }
 718         return 0;
 719 }
 720
 721 static noinline_for_stack int scrub_stripe(struct scrub_dev *sdev,
 722         struct map_lookup *map, int num, u64 base, u64 length)
 723 {
 724         struct btrfs_path *path;
 725         struct btrfs_fs_info *fs_info = sdev->dev->dev_root->fs_info;
 726         struct btrfs_root *root = fs_info->extent_root;
 727         struct btrfs_root *csum_root = fs_info->csum_root;
 728         struct btrfs_extent_item *extent;
 729         u64 flags;
 730         int ret;
 731         int slot;
 732         int i;
 733         u64 nstripes;
 734         int start_stripe;
 735         struct extent_buffer *l;
 736         struct btrfs_key key;
 737         u64 physical;
 738         u64 logical;
 739         u64 generation;
 740         u64 mirror_num;
 741
 742         u64 increment = map->stripe_len;
 743         u64 offset;
 744
 745         nstripes = length;
 746         offset = 0;
 747         do_div(nstripes, map->stripe_len);
 748         if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
 749                 offset = map->stripe_len * num;
 750                 increment = map->stripe_len * map->num_stripes;
 751                 mirror_num = 0;
 752         } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
 753                 int factor = map->num_stripes / map->sub_stripes;
 754                 offset = map->stripe_len * (num / map->sub_stripes);
 755                 increment = map->stripe_len * factor;
 756                 mirror_num = num % map->sub_stripes;
 757         } else if (map->type & BTRFS_BLOCK_GROUP_RAID1) {
 758                 increment = map->stripe_len;
 759                 mirror_num = num % map->num_stripes;
 760         } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
 761                 increment = map->stripe_len;
 762                 mirror_num = num % map->num_stripes;
 763         } else {
 764                 increment = map->stripe_len;
 765                 mirror_num = 0;
 766         }
 767
 768         path = btrfs_alloc_path();
 769         if (!path)
 770                 return -ENOMEM;
 771
 772         path->reada = 2;
 773         path->search_commit_root = 1;
 774         path->skip_locking = 1;
 775
 776         /*
 777          * find all extents for each stripe and just read them to get
 778          * them into the page cache
 779          * FIXME: we can do better. build a more intelligent prefetching
 780          */
 781         logical = base + offset;
 782         physical = map->stripes[num].physical;
 783         ret = 0;
 784         for (i = 0; i < nstripes; ++i) {
 785                 key.objectid = logical;
 786                 key.type = BTRFS_EXTENT_ITEM_KEY;
 787                 key.offset = (u64)0;
 788
 789                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
 790                 if (ret < 0)
 791                         goto out;
 792
 793                 l = path->nodes[0];
 794                 slot = path->slots[0];
 795                 btrfs_item_key_to_cpu(l, &key, slot);
 796                 if (key.objectid != logical) {
 797                         ret = btrfs_previous_item(root, path, 0,
 798                                                   BTRFS_EXTENT_ITEM_KEY);
 799                         if (ret < 0)
 800                                 goto out;
 801                 }
 802
 803                 while (1) {
 804                         l = path->nodes[0];
 805                         slot = path->slots[0];
 806                         if (slot >= btrfs_header_nritems(l)) {
 807                                 ret = btrfs_next_leaf(root, path);
 808                                 if (ret == 0)
 809                                         continue;
 810                                 if (ret < 0)
 811                                         goto out;
 812
 813                                 break;
 814                         }
 815                         btrfs_item_key_to_cpu(l, &key, slot);
 816
 817                         if (key.objectid >= logical + map->stripe_len)
 818                                 break;
 819
 820                         path->slots[0]++;
 821                 }
 822                 btrfs_release_path(root, path);
 823                 logical += increment;
 824                 physical += map->stripe_len;
 825                 cond_resched();
 826         }
 827
 828         /*
 829          * collect all data csums for the stripe to avoid seeking during
 830          * the scrub. This might currently (crc32) end up to be about 1MB
 831          */
 832         start_stripe = 0;
 833 again:
 834         logical = base + offset + start_stripe * increment;
 835         for (i = start_stripe; i < nstripes; ++i) {
 836                 ret = btrfs_lookup_csums_range(csum_root, logical,
 837                                                logical + map->stripe_len - 1,
 838                                                &sdev->csum_list, 1);
 839                 if (ret)
 840                         goto out;
 841
 842                 logical += increment;
 843                 cond_resched();
 844         }
 845         /*
 846          * now find all extents for each stripe and scrub them
 847          */
 848         logical = base + offset + start_stripe * increment;
 849         physical = map->stripes[num].physical + start_stripe * map->stripe_len;
 850         ret = 0;
 851         for (i = start_stripe; i < nstripes; ++i) {
 852                 /*
 853                  * canceled?
 854                  */
 855                 if (atomic_read(&fs_info->scrub_cancel_req) ||
 856                     atomic_read(&sdev->cancel_req)) {
 857                         ret = -ECANCELED;
 858                         goto out;
 859                 }
 860                 /*
 861                  * check to see if we have to pause
 862                  */
 863                 if (atomic_read(&fs_info->scrub_pause_req)) {
 864                         /* push queued extents */
 865                         scrub_submit(sdev);
 866                         wait_event(sdev->list_wait,
 867                                    atomic_read(&sdev->in_flight) == 0);
 868                         atomic_inc(&fs_info->scrubs_paused);
 869                         wake_up(&fs_info->scrub_pause_wait);
 870                         mutex_lock(&fs_info->scrub_lock);
 871                         while (atomic_read(&fs_info->scrub_pause_req)) {
 872                                 mutex_unlock(&fs_info->scrub_lock);
 873                                 wait_event(fs_info->scrub_pause_wait,
 874                                    atomic_read(&fs_info->scrub_pause_req) == 0);
 875                                 mutex_lock(&fs_info->scrub_lock);
 876                         }
 877                         atomic_dec(&fs_info->scrubs_paused);
 878                         mutex_unlock(&fs_info->scrub_lock);
 879                         wake_up(&fs_info->scrub_pause_wait);
 880                         scrub_free_csums(sdev);
 881                         start_stripe = i;
 882                         goto again;
 883                 }
 884
 885                 key.objectid = logical;
 886                 key.type = BTRFS_EXTENT_ITEM_KEY;
 887                 key.offset = (u64)0;
 888
 889                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
 890                 if (ret < 0)
 891                         goto out;
 892
 893                 l = path->nodes[0];
 894                 slot = path->slots[0];
 895                 btrfs_item_key_to_cpu(l, &key, slot);
 896                 if (key.objectid != logical) {
 897                         ret = btrfs_previous_item(root, path, 0,
 898                                                   BTRFS_EXTENT_ITEM_KEY);
 899                         if (ret < 0)
 900                                 goto out;
 901                 }
 902
 903                 while (1) {
 904                         l = path->nodes[0];
 905                         slot = path->slots[0];
 906                         if (slot >= btrfs_header_nritems(l)) {
 907                                 ret = btrfs_next_leaf(root, path);
 908                                 if (ret == 0)
 909                                         continue;
 910                                 if (ret < 0)
 911                                         goto out;
 912
 913                                 break;
 914                         }
 915                         btrfs_item_key_to_cpu(l, &key, slot);
 916
 917                         if (key.objectid + key.offset <= logical)
 918                                 goto next;
 919
 920                         if (key.objectid >= logical + map->stripe_len)
 921                                 break;
 922
 923                         if (btrfs_key_type(&key) != BTRFS_EXTENT_ITEM_KEY)
 924                                 goto next;
 925
 926                         extent = btrfs_item_ptr(l, slot,
 927                                                 struct btrfs_extent_item);
 928                         flags = btrfs_extent_flags(l, extent);
 929                         generation = btrfs_extent_generation(l, extent);
 930
 931                         if (key.objectid < logical &&
 932                             (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)) {
 933                                 printk(KERN_ERR
 934                                        "btrfs scrub: tree block %llu spanning "
 935                                        "stripes, ignored. logical=%llu\n",
 936                                        (unsigned long long)key.objectid,
 937                                        (unsigned long long)logical);
 938                                 goto next;
 939                         }
 940
 941                         /*
 942                          * trim extent to this stripe
 943                          */
 944                         if (key.objectid < logical) {
 945                                 key.offset -= logical - key.objectid;
 946                                 key.objectid = logical;
 947                         }
 948                         if (key.objectid + key.offset >
 949                             logical + map->stripe_len) {
 950                                 key.offset = logical + map->stripe_len -
 951                                              key.objectid;
 952                         }
 953
 954                         ret = scrub_extent(sdev, key.objectid, key.offset,
 955                                            key.objectid - logical + physical,
 956                                            flags, generation, mirror_num);
 957                         if (ret)
 958                                 goto out;
 959
 960 next:
 961                         path->slots[0]++;
 962                 }
 963                 btrfs_release_path(root, path);
 964                 logical += increment;
 965                 physical += map->stripe_len;
 966                 spin_lock(&sdev->stat_lock);
 967                 sdev->stat.last_physical = physical;
 968                 spin_unlock(&sdev->stat_lock);
 969         }
 970         /* push queued extents */
 971         scrub_submit(sdev);
 972
 973 out:
 974         btrfs_free_path(path);
 975         return ret < 0 ? ret : 0;
 976 }
 977
 978 static noinline_for_stack int scrub_chunk(struct scrub_dev *sdev,
 979         u64 chunk_tree, u64 chunk_objectid, u64 chunk_offset, u64 length)
 980 {
 981         struct btrfs_mapping_tree *map_tree =
 982                 &sdev->dev->dev_root->fs_info->mapping_tree;
 983         struct map_lookup *map;
 984         struct extent_map *em;
 985         int i;
 986         int ret = -EINVAL;
 987
 988         read_lock(&map_tree->map_tree.lock);
 989         em = lookup_extent_mapping(&map_tree->map_tree, chunk_offset, 1);
 990         read_unlock(&map_tree->map_tree.lock);
 991
 992         if (!em)
 993                 return -EINVAL;
 994
 995         map = (struct map_lookup *)em->bdev;
 996         if (em->start != chunk_offset)
 997                 goto out;
 998
 999         if (em->len < length)
1000                 goto out;
1001
1002         for (i = 0; i < map->num_stripes; ++i) {
1003                 if (map->stripes[i].dev == sdev->dev) {
1004                         ret = scrub_stripe(sdev, map, i, chunk_offset, length);
1005                         if (ret)
1006                                 goto out;
1007                 }
1008         }
1009 out:
1010         free_extent_map(em);
1011
1012         return ret;
1013 }
1014
1015 static noinline_for_stack
1016 int scrub_enumerate_chunks(struct scrub_dev *sdev, u64 start, u64 end)
1017 {
1018         struct btrfs_dev_extent *dev_extent = NULL;
1019         struct btrfs_path *path;
1020         struct btrfs_root *root = sdev->dev->dev_root;
1021         struct btrfs_fs_info *fs_info = root->fs_info;
1022         u64 length;
1023         u64 chunk_tree;
1024         u64 chunk_objectid;
1025         u64 chunk_offset;
1026         int ret;
1027         int slot;
1028         struct extent_buffer *l;
1029         struct btrfs_key key;
1030         struct btrfs_key found_key;
1031         struct btrfs_block_group_cache *cache;
1032
1033         path = btrfs_alloc_path();
1034         if (!path)
1035                 return -ENOMEM;
1036
1037         path->reada = 2;
1038         path->search_commit_root = 1;
1039         path->skip_locking = 1;
1040
1041         key.objectid = sdev->dev->devid;
1042         key.offset = 0ull;
1043         key.type = BTRFS_DEV_EXTENT_KEY;
1044
1045
1046         while (1) {
1047                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1048                 if (ret < 0)
1049                         goto out;
1050                 ret = 0;
1051
1052                 l = path->nodes[0];
1053                 slot = path->slots[0];
1054
1055                 btrfs_item_key_to_cpu(l, &found_key, slot);
1056
1057                 if (found_key.objectid != sdev->dev->devid)
1058                         break;
1059
1060                 if (btrfs_key_type(&key) != BTRFS_DEV_EXTENT_KEY)
1061                         break;
1062
1063                 if (found_key.offset >= end)
1064                         break;
1065
1066                 if (found_key.offset < key.offset)
1067                         break;
1068
1069                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
1070                 length = btrfs_dev_extent_length(l, dev_extent);
1071
1072                 if (found_key.offset + length <= start) {
1073                         key.offset = found_key.offset + length;
1074                         btrfs_release_path(root, path);
1075                         continue;
1076                 }
1077
1078                 chunk_tree = btrfs_dev_extent_chunk_tree(l, dev_extent);
1079                 chunk_objectid = btrfs_dev_extent_chunk_objectid(l, dev_extent);
1080                 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
1081
1082                 /*
1083                  * get a reference on the corresponding block group to prevent
1084                  * the chunk from going away while we scrub it
1085                  */
1086                 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
1087                 if (!cache) {
1088                         ret = -ENOENT;
1089                         goto out;
1090                 }
1091                 ret = scrub_chunk(sdev, chunk_tree, chunk_objectid,
1092                                   chunk_offset, length);
1093                 btrfs_put_block_group(cache);
1094                 if (ret)
1095                         break;
1096
1097                 key.offset = found_key.offset + length;
1098                 btrfs_release_path(root, path);
1099         }
1100
1101 out:
1102         btrfs_free_path(path);
1103         return ret;
1104 }
1105
1106 static noinline_for_stack int scrub_supers(struct scrub_dev *sdev)
1107 {
1108         int     i;
1109         u64     bytenr;
1110         u64     gen;
1111         int     ret;
1112         struct btrfs_device *device = sdev->dev;
1113         struct btrfs_root *root = device->dev_root;
1114
1115         gen = root->fs_info->last_trans_committed;
1116
1117         for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
1118                 bytenr = btrfs_sb_offset(i);
1119                 if (bytenr + BTRFS_SUPER_INFO_SIZE >= device->total_bytes)
1120                         break;
1121
1122                 ret = scrub_page(sdev, bytenr, PAGE_SIZE, bytenr,
1123                                  BTRFS_EXTENT_FLAG_SUPER, gen, i, NULL, 1);
1124                 if (ret)
1125                         return ret;
1126         }
1127         wait_event(sdev->list_wait, atomic_read(&sdev->in_flight) == 0);
1128
1129         return 0;
1130 }
1131
1132 /*
1133  * get a reference count on fs_info->scrub_workers. start worker if necessary
1134  */
1135 static noinline_for_stack int scrub_workers_get(struct btrfs_root *root)
1136 {
1137         struct btrfs_fs_info *fs_info = root->fs_info;
1138
1139         mutex_lock(&fs_info->scrub_lock);
1140         if (fs_info->scrub_workers_refcnt == 0)
1141                 btrfs_start_workers(&fs_info->scrub_workers, 1);
1142         ++fs_info->scrub_workers_refcnt;
1143         mutex_unlock(&fs_info->scrub_lock);
1144
1145         return 0;
1146 }
1147
1148 static noinline_for_stack void scrub_workers_put(struct btrfs_root *root)
1149 {
1150         struct btrfs_fs_info *fs_info = root->fs_info;
1151
1152         mutex_lock(&fs_info->scrub_lock);
1153         if (--fs_info->scrub_workers_refcnt == 0)
1154                 btrfs_stop_workers(&fs_info->scrub_workers);
1155         WARN_ON(fs_info->scrub_workers_refcnt < 0);
1156         mutex_unlock(&fs_info->scrub_lock);
1157 }
1158
1159
1160 int btrfs_scrub_dev(struct btrfs_root *root, u64 devid, u64 start, u64 end,
1161                     struct btrfs_scrub_progress *progress, int readonly)
1162 {
1163         struct scrub_dev *sdev;
1164         struct btrfs_fs_info *fs_info = root->fs_info;
1165         int ret;
1166         struct btrfs_device *dev;
1167
1168         if (root->fs_info->closing)
1169                 return -EINVAL;
1170
1171         /*
1172          * check some assumptions
1173          */
1174         if (root->sectorsize != PAGE_SIZE ||
1175             root->sectorsize != root->leafsize ||
1176             root->sectorsize != root->nodesize) {
1177                 printk(KERN_ERR "btrfs_scrub: size assumptions fail\n");
1178                 return -EINVAL;
1179         }
1180
1181         ret = scrub_workers_get(root);
1182         if (ret)
1183                 return ret;
1184
1185         mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
1186         dev = btrfs_find_device(root, devid, NULL, NULL);
1187         if (!dev || dev->missing) {
1188                 mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
1189                 scrub_workers_put(root);
1190                 return -ENODEV;
1191         }
1192         mutex_lock(&fs_info->scrub_lock);
1193
1194         if (!dev->in_fs_metadata) {
1195                 mutex_unlock(&fs_info->scrub_lock);
1196                 mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
1197                 scrub_workers_put(root);
1198                 return -ENODEV;
1199         }
1200
1201         if (dev->scrub_device) {
1202                 mutex_unlock(&fs_info->scrub_lock);
1203                 mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
1204                 scrub_workers_put(root);
1205                 return -EINPROGRESS;
1206         }
1207         sdev = scrub_setup_dev(dev);
1208         if (IS_ERR(sdev)) {
1209                 mutex_unlock(&fs_info->scrub_lock);
1210                 mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
1211                 scrub_workers_put(root);
1212                 return PTR_ERR(sdev);
1213         }
1214         sdev->readonly = readonly;
1215         dev->scrub_device = sdev;
1216
1217         atomic_inc(&fs_info->scrubs_running);
1218         mutex_unlock(&fs_info->scrub_lock);
1219         mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
1220
1221         down_read(&fs_info->scrub_super_lock);
1222         ret = scrub_supers(sdev);
1223         up_read(&fs_info->scrub_super_lock);
1224
1225         if (!ret)
1226                 ret = scrub_enumerate_chunks(sdev, start, end);
1227
1228         wait_event(sdev->list_wait, atomic_read(&sdev->in_flight) == 0);
1229
1230         atomic_dec(&fs_info->scrubs_running);
1231         wake_up(&fs_info->scrub_pause_wait);
1232
1233         if (progress)
1234                 memcpy(progress, &sdev->stat, sizeof(*progress));
1235
1236         mutex_lock(&fs_info->scrub_lock);
1237         dev->scrub_device = NULL;
1238         mutex_unlock(&fs_info->scrub_lock);
1239
1240         scrub_free_dev(sdev);
1241         scrub_workers_put(root);
1242
1243         return ret;
1244 }
1245
1246 int btrfs_scrub_pause(struct btrfs_root *root)
1247 {
1248         struct btrfs_fs_info *fs_info = root->fs_info;
1249
1250         mutex_lock(&fs_info->scrub_lock);
1251         atomic_inc(&fs_info->scrub_pause_req);
1252         while (atomic_read(&fs_info->scrubs_paused) !=
1253                atomic_read(&fs_info->scrubs_running)) {
1254                 mutex_unlock(&fs_info->scrub_lock);
1255                 wait_event(fs_info->scrub_pause_wait,
1256                            atomic_read(&fs_info->scrubs_paused) ==
1257                            atomic_read(&fs_info->scrubs_running));
1258                 mutex_lock(&fs_info->scrub_lock);
1259         }
1260         mutex_unlock(&fs_info->scrub_lock);
1261
1262         return 0;
1263 }
1264
1265 int btrfs_scrub_continue(struct btrfs_root *root)
1266 {
1267         struct btrfs_fs_info *fs_info = root->fs_info;
1268
1269         atomic_dec(&fs_info->scrub_pause_req);
1270         wake_up(&fs_info->scrub_pause_wait);
1271         return 0;
1272 }
1273
1274 int btrfs_scrub_pause_super(struct btrfs_root *root)
1275 {
1276         down_write(&root->fs_info->scrub_super_lock);
1277         return 0;
1278 }
1279
1280 int btrfs_scrub_continue_super(struct btrfs_root *root)
1281 {
1282         up_write(&root->fs_info->scrub_super_lock);
1283         return 0;
1284 }
1285
1286 int btrfs_scrub_cancel(struct btrfs_root *root)
1287 {
1288         struct btrfs_fs_info *fs_info = root->fs_info;
1289
1290         mutex_lock(&fs_info->scrub_lock);
1291         if (!atomic_read(&fs_info->scrubs_running)) {
1292                 mutex_unlock(&fs_info->scrub_lock);
1293                 return -ENOTCONN;
1294         }
1295
1296         atomic_inc(&fs_info->scrub_cancel_req);
1297         while (atomic_read(&fs_info->scrubs_running)) {
1298                 mutex_unlock(&fs_info->scrub_lock);
1299                 wait_event(fs_info->scrub_pause_wait,
1300                            atomic_read(&fs_info->scrubs_running) == 0);
1301                 mutex_lock(&fs_info->scrub_lock);
1302         }
1303         atomic_dec(&fs_info->scrub_cancel_req);
1304         mutex_unlock(&fs_info->scrub_lock);
1305
1306         return 0;
1307 }
1308
1309 int btrfs_scrub_cancel_dev(struct btrfs_root *root, struct btrfs_device *dev)
1310 {
1311         struct btrfs_fs_info *fs_info = root->fs_info;
1312         struct scrub_dev *sdev;
1313
1314         mutex_lock(&fs_info->scrub_lock);
1315         sdev = dev->scrub_device;
1316         if (!sdev) {
1317                 mutex_unlock(&fs_info->scrub_lock);
1318                 return -ENOTCONN;
1319         }
1320         atomic_inc(&sdev->cancel_req);
1321         while (dev->scrub_device) {
1322                 mutex_unlock(&fs_info->scrub_lock);
1323                 wait_event(fs_info->scrub_pause_wait,
1324                            dev->scrub_device == NULL);
1325                 mutex_lock(&fs_info->scrub_lock);
1326         }
1327         mutex_unlock(&fs_info->scrub_lock);
1328
1329         return 0;
1330 }
1331 int btrfs_scrub_cancel_devid(struct btrfs_root *root, u64 devid)
1332 {
1333         struct btrfs_fs_info *fs_info = root->fs_info;
1334         struct btrfs_device *dev;
1335         int ret;
1336
1337         /*
1338          * we have to hold the device_list_mutex here so the device
1339          * does not go away in cancel_dev. FIXME: find a better solution
1340          */
1341         mutex_lock(&fs_info->fs_devices->device_list_mutex);
1342         dev = btrfs_find_device(root, devid, NULL, NULL);
1343         if (!dev) {
1344                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
1345                 return -ENODEV;
1346         }
1347         ret = btrfs_scrub_cancel_dev(root, dev);
1348         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
1349
1350         return ret;
1351 }
1352
1353 int btrfs_scrub_progress(struct btrfs_root *root, u64 devid,
1354                          struct btrfs_scrub_progress *progress)
1355 {
1356         struct btrfs_device *dev;
1357         struct scrub_dev *sdev = NULL;
1358
1359         mutex_lock(&root->fs_info->fs_devices->device_list_mutex);
1360         dev = btrfs_find_device(root, devid, NULL, NULL);
1361         if (dev)
1362                 sdev = dev->scrub_device;
1363         if (sdev)
1364                 memcpy(progress, &sdev->stat, sizeof(*progress));
1365         mutex_unlock(&root->fs_info->fs_devices->device_list_mutex);
1366
1367         return dev ? (sdev ? 0 : -ENOTCONN) : -ENODEV;
1368 }