drivers/md/dm-mpath.c

   1 /*
   2  * Copyright (C) 2003 Sistina Software Limited.
   3  * Copyright (C) 2004-2005 Red Hat, Inc. All rights reserved.
   4  *
   5  * This file is released under the GPL.
   6  */
   7
   8 #include <linux/device-mapper.h>
   9
  10 #include "dm-rq.h"
  11 #include "dm-bio-record.h"
  12 #include "dm-path-selector.h"
  13 #include "dm-uevent.h"
  14
  15 #include <linux/blkdev.h>
  16 #include <linux/ctype.h>
  17 #include <linux/init.h>
  18 #include <linux/mempool.h>
  19 #include <linux/module.h>
  20 #include <linux/pagemap.h>
  21 #include <linux/slab.h>
  22 #include <linux/time.h>
  23 #include <linux/workqueue.h>
  24 #include <linux/delay.h>
  25 #include <scsi/scsi_dh.h>
  26 #include <linux/atomic.h>
  27 #include <linux/blk-mq.h>
  28
  29 #define DM_MSG_PREFIX "multipath"
  30 #define DM_PG_INIT_DELAY_MSECS 2000
  31 #define DM_PG_INIT_DELAY_DEFAULT ((unsigned) -1)
  32
  33 /* Path properties */
  34 struct pgpath {
  35         struct list_head list;
  36
  37         struct priority_group *pg;      /* Owning PG */
  38         unsigned fail_count;            /* Cumulative failure count */
  39
  40         struct dm_path path;
  41         struct delayed_work activate_path;
  42
  43         bool is_active:1;               /* Path status */
  44 };
  45
  46 #define path_to_pgpath(__pgp) container_of((__pgp), struct pgpath, path)
  47
  48 /*
  49  * Paths are grouped into Priority Groups and numbered from 1 upwards.
  50  * Each has a path selector which controls which path gets used.
  51  */
  52 struct priority_group {
  53         struct list_head list;
  54
  55         struct multipath *m;            /* Owning multipath instance */
  56         struct path_selector ps;
  57
  58         unsigned pg_num;                /* Reference number */
  59         unsigned nr_pgpaths;            /* Number of paths in PG */
  60         struct list_head pgpaths;
  61
  62         bool bypassed:1;                /* Temporarily bypass this PG? */
  63 };
  64
  65 /* Multipath context */
  66 struct multipath {
  67         struct list_head list;
  68         struct dm_target *ti;
  69
  70         const char *hw_handler_name;
  71         char *hw_handler_params;
  72
  73         spinlock_t lock;
  74
  75         unsigned nr_priority_groups;
  76         struct list_head priority_groups;
  77
  78         wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
  79
  80         struct pgpath *current_pgpath;
  81         struct priority_group *current_pg;
  82         struct priority_group *next_pg; /* Switch to this PG if set */
  83
  84         unsigned long flags;            /* Multipath state flags */
  85
  86         unsigned pg_init_retries;       /* Number of times to retry pg_init */
  87         unsigned pg_init_delay_msecs;   /* Number of msecs before pg_init retry */
  88
  89         atomic_t nr_valid_paths;        /* Total number of usable paths */
  90         atomic_t pg_init_in_progress;   /* Only one pg_init allowed at once */
  91         atomic_t pg_init_count;         /* Number of times pg_init called */
  92
  93         unsigned queue_mode;
  94
  95         struct mutex work_mutex;
  96         struct work_struct trigger_event;
  97
  98         struct work_struct process_queued_bios;
  99         struct bio_list queued_bios;
 100 };
 101
 102 /*
 103  * Context information attached to each io we process.
 104  */
 105 struct dm_mpath_io {
 106         struct pgpath *pgpath;
 107         size_t nr_bytes;
 108 };
 109
 110 typedef int (*action_fn) (struct pgpath *pgpath);
 111
 112 static struct workqueue_struct *kmultipathd, *kmpath_handlerd;
 113 static void trigger_event(struct work_struct *work);
 114 static void activate_path(struct work_struct *work);
 115 static void process_queued_bios(struct work_struct *work);
 116
 117 /*-----------------------------------------------
 118  * Multipath state flags.
 119  *-----------------------------------------------*/
 120
 121 #define MPATHF_QUEUE_IO 0                       /* Must we queue all I/O? */
 122 #define MPATHF_QUEUE_IF_NO_PATH 1               /* Queue I/O if last path fails? */
 123 #define MPATHF_SAVED_QUEUE_IF_NO_PATH 2         /* Saved state during suspension */
 124 #define MPATHF_RETAIN_ATTACHED_HW_HANDLER 3     /* If there's already a hw_handler present, don't change it. */
 125 #define MPATHF_PG_INIT_DISABLED 4               /* pg_init is not currently allowed */
 126 #define MPATHF_PG_INIT_REQUIRED 5               /* pg_init needs calling? */
 127 #define MPATHF_PG_INIT_DELAY_RETRY 6            /* Delay pg_init retry? */
 128
 129 /*-----------------------------------------------
 130  * Allocation routines
 131  *-----------------------------------------------*/
 132
 133 static struct pgpath *alloc_pgpath(void)
 134 {
 135         struct pgpath *pgpath = kzalloc(sizeof(*pgpath), GFP_KERNEL);
 136
 137         if (pgpath) {
 138                 pgpath->is_active = true;
 139                 INIT_DELAYED_WORK(&pgpath->activate_path, activate_path);
 140         }
 141
 142         return pgpath;
 143 }
 144
 145 static void free_pgpath(struct pgpath *pgpath)
 146 {
 147         kfree(pgpath);
 148 }
 149
 150 static struct priority_group *alloc_priority_group(void)
 151 {
 152         struct priority_group *pg;
 153
 154         pg = kzalloc(sizeof(*pg), GFP_KERNEL);
 155
 156         if (pg)
 157                 INIT_LIST_HEAD(&pg->pgpaths);
 158
 159         return pg;
 160 }
 161
 162 static void free_pgpaths(struct list_head *pgpaths, struct dm_target *ti)
 163 {
 164         struct pgpath *pgpath, *tmp;
 165
 166         list_for_each_entry_safe(pgpath, tmp, pgpaths, list) {
 167                 list_del(&pgpath->list);
 168                 dm_put_device(ti, pgpath->path.dev);
 169                 free_pgpath(pgpath);
 170         }
 171 }
 172
 173 static void free_priority_group(struct priority_group *pg,
 174                                 struct dm_target *ti)
 175 {
 176         struct path_selector *ps = &pg->ps;
 177
 178         if (ps->type) {
 179                 ps->type->destroy(ps);
 180                 dm_put_path_selector(ps->type);
 181         }
 182
 183         free_pgpaths(&pg->pgpaths, ti);
 184         kfree(pg);
 185 }
 186
 187 static struct multipath *alloc_multipath(struct dm_target *ti)
 188 {
 189         struct multipath *m;
 190
 191         m = kzalloc(sizeof(*m), GFP_KERNEL);
 192         if (m) {
 193                 INIT_LIST_HEAD(&m->priority_groups);
 194                 spin_lock_init(&m->lock);
 195                 set_bit(MPATHF_QUEUE_IO, &m->flags);
 196                 atomic_set(&m->nr_valid_paths, 0);
 197                 atomic_set(&m->pg_init_in_progress, 0);
 198                 atomic_set(&m->pg_init_count, 0);
 199                 m->pg_init_delay_msecs = DM_PG_INIT_DELAY_DEFAULT;
 200                 INIT_WORK(&m->trigger_event, trigger_event);
 201                 init_waitqueue_head(&m->pg_init_wait);
 202                 mutex_init(&m->work_mutex);
 203
 204                 m->queue_mode = DM_TYPE_NONE;
 205
 206                 m->ti = ti;
 207                 ti->private = m;
 208         }
 209
 210         return m;
 211 }
 212
 213 static int alloc_multipath_stage2(struct dm_target *ti, struct multipath *m)
 214 {
 215         if (m->queue_mode == DM_TYPE_NONE) {
 216                 /*
 217                  * Default to request-based.
 218                  */
 219                 if (dm_use_blk_mq(dm_table_get_md(ti->table)))
 220                         m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
 221                 else
 222                         m->queue_mode = DM_TYPE_REQUEST_BASED;
 223         } else if (m->queue_mode == DM_TYPE_BIO_BASED) {
 224                 INIT_WORK(&m->process_queued_bios, process_queued_bios);
 225                 /*
 226                  * bio-based doesn't support any direct scsi_dh management;
 227                  * it just discovers if a scsi_dh is attached.
 228                  */
 229                 set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
 230         }
 231
 232         dm_table_set_type(ti->table, m->queue_mode);
 233
 234         return 0;
 235 }
 236
 237 static void free_multipath(struct multipath *m)
 238 {
 239         struct priority_group *pg, *tmp;
 240
 241         list_for_each_entry_safe(pg, tmp, &m->priority_groups, list) {
 242                 list_del(&pg->list);
 243                 free_priority_group(pg, m->ti);
 244         }
 245
 246         kfree(m->hw_handler_name);
 247         kfree(m->hw_handler_params);
 248         kfree(m);
 249 }
 250
 251 static struct dm_mpath_io *get_mpio(union map_info *info)
 252 {
 253         return info->ptr;
 254 }
 255
 256 static size_t multipath_per_bio_data_size(void)
 257 {
 258         return sizeof(struct dm_mpath_io) + sizeof(struct dm_bio_details);
 259 }
 260
 261 static struct dm_mpath_io *get_mpio_from_bio(struct bio *bio)
 262 {
 263         return dm_per_bio_data(bio, multipath_per_bio_data_size());
 264 }
 265
 266 static struct dm_bio_details *get_bio_details_from_bio(struct bio *bio)
 267 {
 268         /* dm_bio_details is immediately after the dm_mpath_io in bio's per-bio-data */
 269         struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
 270         void *bio_details = mpio + 1;
 271
 272         return bio_details;
 273 }
 274
 275 static void multipath_init_per_bio_data(struct bio *bio, struct dm_mpath_io **mpio_p,
 276                                         struct dm_bio_details **bio_details_p)
 277 {
 278         struct dm_mpath_io *mpio = get_mpio_from_bio(bio);
 279         struct dm_bio_details *bio_details = get_bio_details_from_bio(bio);
 280
 281         memset(mpio, 0, sizeof(*mpio));
 282         memset(bio_details, 0, sizeof(*bio_details));
 283         dm_bio_record(bio_details, bio);
 284
 285         if (mpio_p)
 286                 *mpio_p = mpio;
 287         if (bio_details_p)
 288                 *bio_details_p = bio_details;
 289 }
 290
 291 /*-----------------------------------------------
 292  * Path selection
 293  *-----------------------------------------------*/
 294
 295 static int __pg_init_all_paths(struct multipath *m)
 296 {
 297         struct pgpath *pgpath;
 298         unsigned long pg_init_delay = 0;
 299
 300         if (atomic_read(&m->pg_init_in_progress) || test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
 301                 return 0;
 302
 303         atomic_inc(&m->pg_init_count);
 304         clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
 305
 306         /* Check here to reset pg_init_required */
 307         if (!m->current_pg)
 308                 return 0;
 309
 310         if (test_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags))
 311                 pg_init_delay = msecs_to_jiffies(m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT ?
 312                                                  m->pg_init_delay_msecs : DM_PG_INIT_DELAY_MSECS);
 313         list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) {
 314                 /* Skip failed paths */
 315                 if (!pgpath->is_active)
 316                         continue;
 317                 if (queue_delayed_work(kmpath_handlerd, &pgpath->activate_path,
 318                                        pg_init_delay))
 319                         atomic_inc(&m->pg_init_in_progress);
 320         }
 321         return atomic_read(&m->pg_init_in_progress);
 322 }
 323
 324 static void pg_init_all_paths(struct multipath *m)
 325 {
 326         unsigned long flags;
 327
 328         spin_lock_irqsave(&m->lock, flags);
 329         __pg_init_all_paths(m);
 330         spin_unlock_irqrestore(&m->lock, flags);
 331 }
 332
 333 static void __switch_pg(struct multipath *m, struct priority_group *pg)
 334 {
 335         m->current_pg = pg;
 336
 337         /* Must we initialise the PG first, and queue I/O till it's ready? */
 338         if (m->hw_handler_name) {
 339                 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
 340                 set_bit(MPATHF_QUEUE_IO, &m->flags);
 341         } else {
 342                 clear_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
 343                 clear_bit(MPATHF_QUEUE_IO, &m->flags);
 344         }
 345
 346         atomic_set(&m->pg_init_count, 0);
 347 }
 348
 349 static struct pgpath *choose_path_in_pg(struct multipath *m,
 350                                         struct priority_group *pg,
 351                                         size_t nr_bytes)
 352 {
 353         unsigned long flags;
 354         struct dm_path *path;
 355         struct pgpath *pgpath;
 356
 357         path = pg->ps.type->select_path(&pg->ps, nr_bytes);
 358         if (!path)
 359                 return ERR_PTR(-ENXIO);
 360
 361         pgpath = path_to_pgpath(path);
 362
 363         if (unlikely(lockless_dereference(m->current_pg) != pg)) {
 364                 /* Only update current_pgpath if pg changed */
 365                 spin_lock_irqsave(&m->lock, flags);
 366                 m->current_pgpath = pgpath;
 367                 __switch_pg(m, pg);
 368                 spin_unlock_irqrestore(&m->lock, flags);
 369         }
 370
 371         return pgpath;
 372 }
 373
 374 static struct pgpath *choose_pgpath(struct multipath *m, size_t nr_bytes)
 375 {
 376         unsigned long flags;
 377         struct priority_group *pg;
 378         struct pgpath *pgpath;
 379         unsigned bypassed = 1;
 380
 381         if (!atomic_read(&m->nr_valid_paths)) {
 382                 clear_bit(MPATHF_QUEUE_IO, &m->flags);
 383                 goto failed;
 384         }
 385
 386         /* Were we instructed to switch PG? */
 387         if (lockless_dereference(m->next_pg)) {
 388                 spin_lock_irqsave(&m->lock, flags);
 389                 pg = m->next_pg;
 390                 if (!pg) {
 391                         spin_unlock_irqrestore(&m->lock, flags);
 392                         goto check_current_pg;
 393                 }
 394                 m->next_pg = NULL;
 395                 spin_unlock_irqrestore(&m->lock, flags);
 396                 pgpath = choose_path_in_pg(m, pg, nr_bytes);
 397                 if (!IS_ERR_OR_NULL(pgpath))
 398                         return pgpath;
 399         }
 400
 401         /* Don't change PG until it has no remaining paths */
 402 check_current_pg:
 403         pg = lockless_dereference(m->current_pg);
 404         if (pg) {
 405                 pgpath = choose_path_in_pg(m, pg, nr_bytes);
 406                 if (!IS_ERR_OR_NULL(pgpath))
 407                         return pgpath;
 408         }
 409
 410         /*
 411          * Loop through priority groups until we find a valid path.
 412          * First time we skip PGs marked 'bypassed'.
 413          * Second time we only try the ones we skipped, but set
 414          * pg_init_delay_retry so we do not hammer controllers.
 415          */
 416         do {
 417                 list_for_each_entry(pg, &m->priority_groups, list) {
 418                         if (pg->bypassed == !!bypassed)
 419                                 continue;
 420                         pgpath = choose_path_in_pg(m, pg, nr_bytes);
 421                         if (!IS_ERR_OR_NULL(pgpath)) {
 422                                 if (!bypassed)
 423                                         set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
 424                                 return pgpath;
 425                         }
 426                 }
 427         } while (bypassed--);
 428
 429 failed:
 430         spin_lock_irqsave(&m->lock, flags);
 431         m->current_pgpath = NULL;
 432         m->current_pg = NULL;
 433         spin_unlock_irqrestore(&m->lock, flags);
 434
 435         return NULL;
 436 }
 437
 438 /*
 439  * Check whether bios must be queued in the device-mapper core rather
 440  * than here in the target.
 441  *
 442  * If m->queue_if_no_path and m->saved_queue_if_no_path hold the
 443  * same value then we are not between multipath_presuspend()
 444  * and multipath_resume() calls and we have no need to check
 445  * for the DMF_NOFLUSH_SUSPENDING flag.
 446  */
 447 static bool __must_push_back(struct multipath *m)
 448 {
 449         return ((test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) !=
 450                  test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags)) &&
 451                 dm_noflush_suspending(m->ti));
 452 }
 453
 454 static bool must_push_back_rq(struct multipath *m)
 455 {
 456         bool r;
 457         unsigned long flags;
 458
 459         spin_lock_irqsave(&m->lock, flags);
 460         r = (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) ||
 461              __must_push_back(m));
 462         spin_unlock_irqrestore(&m->lock, flags);
 463
 464         return r;
 465 }
 466
 467 static bool must_push_back_bio(struct multipath *m)
 468 {
 469         bool r;
 470         unsigned long flags;
 471
 472         spin_lock_irqsave(&m->lock, flags);
 473         r = __must_push_back(m);
 474         spin_unlock_irqrestore(&m->lock, flags);
 475
 476         return r;
 477 }
 478
 479 /*
 480  * Map cloned requests (request-based multipath)
 481  */
 482 static int multipath_clone_and_map(struct dm_target *ti, struct request *rq,
 483                                    union map_info *map_context,
 484                                    struct request **__clone)
 485 {
 486         struct multipath *m = ti->private;
 487         int r = DM_MAPIO_REQUEUE;
 488         size_t nr_bytes = blk_rq_bytes(rq);
 489         struct pgpath *pgpath;
 490         struct block_device *bdev;
 491         struct dm_mpath_io *mpio = get_mpio(map_context);
 492         struct request *clone;
 493
 494         /* Do we need to select a new pgpath? */
 495         pgpath = lockless_dereference(m->current_pgpath);
 496         if (!pgpath || !test_bit(MPATHF_QUEUE_IO, &m->flags))
 497                 pgpath = choose_pgpath(m, nr_bytes);
 498
 499         if (!pgpath) {
 500                 if (must_push_back_rq(m))
 501                         return DM_MAPIO_DELAY_REQUEUE;
 502                 return -EIO;    /* Failed */
 503         } else if (test_bit(MPATHF_QUEUE_IO, &m->flags) ||
 504                    test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
 505                 pg_init_all_paths(m);
 506                 return r;
 507         }
 508
 509         memset(mpio, 0, sizeof(*mpio));
 510         mpio->pgpath = pgpath;
 511         mpio->nr_bytes = nr_bytes;
 512
 513         bdev = pgpath->path.dev->bdev;
 514
 515         clone = blk_get_request(bdev_get_queue(bdev),
 516                         rq->cmd_flags | REQ_NOMERGE,
 517                         GFP_ATOMIC);
 518         if (IS_ERR(clone)) {
 519                 /* EBUSY, ENODEV or EWOULDBLOCK: requeue */
 520                 return r;
 521         }
 522         clone->bio = clone->biotail = NULL;
 523         clone->rq_disk = bdev->bd_disk;
 524         clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
 525         *__clone = clone;
 526
 527         if (pgpath->pg->ps.type->start_io)
 528                 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
 529                                               &pgpath->path,
 530                                               nr_bytes);
 531         return DM_MAPIO_REMAPPED;
 532 }
 533
 534 static void multipath_release_clone(struct request *clone)
 535 {
 536         blk_put_request(clone);
 537 }
 538
 539 /*
 540  * Map cloned bios (bio-based multipath)
 541  */
 542 static int __multipath_map_bio(struct multipath *m, struct bio *bio, struct dm_mpath_io *mpio)
 543 {
 544         size_t nr_bytes = bio->bi_iter.bi_size;
 545         struct pgpath *pgpath;
 546         unsigned long flags;
 547         bool queue_io;
 548
 549         /* Do we need to select a new pgpath? */
 550         pgpath = lockless_dereference(m->current_pgpath);
 551         queue_io = test_bit(MPATHF_QUEUE_IO, &m->flags);
 552         if (!pgpath || !queue_io)
 553                 pgpath = choose_pgpath(m, nr_bytes);
 554
 555         if ((pgpath && queue_io) ||
 556             (!pgpath && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))) {
 557                 /* Queue for the daemon to resubmit */
 558                 spin_lock_irqsave(&m->lock, flags);
 559                 bio_list_add(&m->queued_bios, bio);
 560                 spin_unlock_irqrestore(&m->lock, flags);
 561                 /* PG_INIT_REQUIRED cannot be set without QUEUE_IO */
 562                 if (queue_io || test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
 563                         pg_init_all_paths(m);
 564                 else if (!queue_io)
 565                         queue_work(kmultipathd, &m->process_queued_bios);
 566                 return DM_MAPIO_SUBMITTED;
 567         }
 568
 569         if (!pgpath) {
 570                 if (!must_push_back_bio(m))
 571                         return -EIO;
 572                 return DM_MAPIO_REQUEUE;
 573         }
 574
 575         mpio->pgpath = pgpath;
 576         mpio->nr_bytes = nr_bytes;
 577
 578         bio->bi_error = 0;
 579         bio->bi_bdev = pgpath->path.dev->bdev;
 580         bio->bi_opf |= REQ_FAILFAST_TRANSPORT;
 581
 582         if (pgpath->pg->ps.type->start_io)
 583                 pgpath->pg->ps.type->start_io(&pgpath->pg->ps,
 584                                               &pgpath->path,
 585                                               nr_bytes);
 586         return DM_MAPIO_REMAPPED;
 587 }
 588
 589 static int multipath_map_bio(struct dm_target *ti, struct bio *bio)
 590 {
 591         struct multipath *m = ti->private;
 592         struct dm_mpath_io *mpio = NULL;
 593
 594         multipath_init_per_bio_data(bio, &mpio, NULL);
 595
 596         return __multipath_map_bio(m, bio, mpio);
 597 }
 598
 599 static void process_queued_io_list(struct multipath *m)
 600 {
 601         if (m->queue_mode == DM_TYPE_MQ_REQUEST_BASED)
 602                 dm_mq_kick_requeue_list(dm_table_get_md(m->ti->table));
 603         else if (m->queue_mode == DM_TYPE_BIO_BASED)
 604                 queue_work(kmultipathd, &m->process_queued_bios);
 605 }
 606
 607 static void process_queued_bios(struct work_struct *work)
 608 {
 609         int r;
 610         unsigned long flags;
 611         struct bio *bio;
 612         struct bio_list bios;
 613         struct blk_plug plug;
 614         struct multipath *m =
 615                 container_of(work, struct multipath, process_queued_bios);
 616
 617         bio_list_init(&bios);
 618
 619         spin_lock_irqsave(&m->lock, flags);
 620
 621         if (bio_list_empty(&m->queued_bios)) {
 622                 spin_unlock_irqrestore(&m->lock, flags);
 623                 return;
 624         }
 625
 626         bio_list_merge(&bios, &m->queued_bios);
 627         bio_list_init(&m->queued_bios);
 628
 629         spin_unlock_irqrestore(&m->lock, flags);
 630
 631         blk_start_plug(&plug);
 632         while ((bio = bio_list_pop(&bios))) {
 633                 r = __multipath_map_bio(m, bio, get_mpio_from_bio(bio));
 634                 if (r < 0 || r == DM_MAPIO_REQUEUE) {
 635                         bio->bi_error = r;
 636                         bio_endio(bio);
 637                 } else if (r == DM_MAPIO_REMAPPED)
 638                         generic_make_request(bio);
 639         }
 640         blk_finish_plug(&plug);
 641 }
 642
 643 /*
 644  * If we run out of usable paths, should we queue I/O or error it?
 645  */
 646 static int queue_if_no_path(struct multipath *m, bool queue_if_no_path,
 647                             bool save_old_value)
 648 {
 649         unsigned long flags;
 650
 651         spin_lock_irqsave(&m->lock, flags);
 652
 653         if (save_old_value) {
 654                 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
 655                         set_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
 656                 else
 657                         clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
 658         } else {
 659                 if (queue_if_no_path)
 660                         set_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
 661                 else
 662                         clear_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags);
 663         }
 664         if (queue_if_no_path)
 665                 set_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
 666         else
 667                 clear_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
 668
 669         spin_unlock_irqrestore(&m->lock, flags);
 670
 671         if (!queue_if_no_path) {
 672                 dm_table_run_md_queue_async(m->ti->table);
 673                 process_queued_io_list(m);
 674         }
 675
 676         return 0;
 677 }
 678
 679 /*
 680  * An event is triggered whenever a path is taken out of use.
 681  * Includes path failure and PG bypass.
 682  */
 683 static void trigger_event(struct work_struct *work)
 684 {
 685         struct multipath *m =
 686                 container_of(work, struct multipath, trigger_event);
 687
 688         dm_table_event(m->ti->table);
 689 }
 690
 691 /*-----------------------------------------------------------------
 692  * Constructor/argument parsing:
 693  * <#multipath feature args> [<arg>]*
 694  * <#hw_handler args> [hw_handler [<arg>]*]
 695  * <#priority groups>
 696  * <initial priority group>
 697  *     [<selector> <#selector args> [<arg>]*
 698  *      <#paths> <#per-path selector args>
 699  *         [<path> [<arg>]* ]+ ]+
 700  *---------------------------------------------------------------*/
 701 static int parse_path_selector(struct dm_arg_set *as, struct priority_group *pg,
 702                                struct dm_target *ti)
 703 {
 704         int r;
 705         struct path_selector_type *pst;
 706         unsigned ps_argc;
 707
 708         static struct dm_arg _args[] = {
 709                 {0, 1024, "invalid number of path selector args"},
 710         };
 711
 712         pst = dm_get_path_selector(dm_shift_arg(as));
 713         if (!pst) {
 714                 ti->error = "unknown path selector type";
 715                 return -EINVAL;
 716         }
 717
 718         r = dm_read_arg_group(_args, as, &ps_argc, &ti->error);
 719         if (r) {
 720                 dm_put_path_selector(pst);
 721                 return -EINVAL;
 722         }
 723
 724         r = pst->create(&pg->ps, ps_argc, as->argv);
 725         if (r) {
 726                 dm_put_path_selector(pst);
 727                 ti->error = "path selector constructor failed";
 728                 return r;
 729         }
 730
 731         pg->ps.type = pst;
 732         dm_consume_args(as, ps_argc);
 733
 734         return 0;
 735 }
 736
 737 static struct pgpath *parse_path(struct dm_arg_set *as, struct path_selector *ps,
 738                                struct dm_target *ti)
 739 {
 740         int r;
 741         struct pgpath *p;
 742         struct multipath *m = ti->private;
 743         struct request_queue *q = NULL;
 744         const char *attached_handler_name;
 745
 746         /* we need at least a path arg */
 747         if (as->argc < 1) {
 748                 ti->error = "no device given";
 749                 return ERR_PTR(-EINVAL);
 750         }
 751
 752         p = alloc_pgpath();
 753         if (!p)
 754                 return ERR_PTR(-ENOMEM);
 755
 756         r = dm_get_device(ti, dm_shift_arg(as), dm_table_get_mode(ti->table),
 757                           &p->path.dev);
 758         if (r) {
 759                 ti->error = "error getting device";
 760                 goto bad;
 761         }
 762
 763         if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) || m->hw_handler_name)
 764                 q = bdev_get_queue(p->path.dev->bdev);
 765
 766         if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags)) {
 767 retain:
 768                 attached_handler_name = scsi_dh_attached_handler_name(q, GFP_KERNEL);
 769                 if (attached_handler_name) {
 770                         /*
 771                          * Clear any hw_handler_params associated with a
 772                          * handler that isn't already attached.
 773                          */
 774                         if (m->hw_handler_name && strcmp(attached_handler_name, m->hw_handler_name)) {
 775                                 kfree(m->hw_handler_params);
 776                                 m->hw_handler_params = NULL;
 777                         }
 778
 779                         /*
 780                          * Reset hw_handler_name to match the attached handler
 781                          *
 782                          * NB. This modifies the table line to show the actual
 783                          * handler instead of the original table passed in.
 784                          */
 785                         kfree(m->hw_handler_name);
 786                         m->hw_handler_name = attached_handler_name;
 787                 }
 788         }
 789
 790         if (m->hw_handler_name) {
 791                 r = scsi_dh_attach(q, m->hw_handler_name);
 792                 if (r == -EBUSY) {
 793                         char b[BDEVNAME_SIZE];
 794
 795                         printk(KERN_INFO "dm-mpath: retaining handler on device %s\n",
 796                                 bdevname(p->path.dev->bdev, b));
 797                         goto retain;
 798                 }
 799                 if (r < 0) {
 800                         ti->error = "error attaching hardware handler";
 801                         dm_put_device(ti, p->path.dev);
 802                         goto bad;
 803                 }
 804
 805                 if (m->hw_handler_params) {
 806                         r = scsi_dh_set_params(q, m->hw_handler_params);
 807                         if (r < 0) {
 808                                 ti->error = "unable to set hardware "
 809                                                         "handler parameters";
 810                                 dm_put_device(ti, p->path.dev);
 811                                 goto bad;
 812                         }
 813                 }
 814         }
 815
 816         r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
 817         if (r) {
 818                 dm_put_device(ti, p->path.dev);
 819                 goto bad;
 820         }
 821
 822         return p;
 823
 824  bad:
 825         free_pgpath(p);
 826         return ERR_PTR(r);
 827 }
 828
 829 static struct priority_group *parse_priority_group(struct dm_arg_set *as,
 830                                                    struct multipath *m)
 831 {
 832         static struct dm_arg _args[] = {
 833                 {1, 1024, "invalid number of paths"},
 834                 {0, 1024, "invalid number of selector args"}
 835         };
 836
 837         int r;
 838         unsigned i, nr_selector_args, nr_args;
 839         struct priority_group *pg;
 840         struct dm_target *ti = m->ti;
 841
 842         if (as->argc < 2) {
 843                 as->argc = 0;
 844                 ti->error = "not enough priority group arguments";
 845                 return ERR_PTR(-EINVAL);
 846         }
 847
 848         pg = alloc_priority_group();
 849         if (!pg) {
 850                 ti->error = "couldn't allocate priority group";
 851                 return ERR_PTR(-ENOMEM);
 852         }
 853         pg->m = m;
 854
 855         r = parse_path_selector(as, pg, ti);
 856         if (r)
 857                 goto bad;
 858
 859         /*
 860          * read the paths
 861          */
 862         r = dm_read_arg(_args, as, &pg->nr_pgpaths, &ti->error);
 863         if (r)
 864                 goto bad;
 865
 866         r = dm_read_arg(_args + 1, as, &nr_selector_args, &ti->error);
 867         if (r)
 868                 goto bad;
 869
 870         nr_args = 1 + nr_selector_args;
 871         for (i = 0; i < pg->nr_pgpaths; i++) {
 872                 struct pgpath *pgpath;
 873                 struct dm_arg_set path_args;
 874
 875                 if (as->argc < nr_args) {
 876                         ti->error = "not enough path parameters";
 877                         r = -EINVAL;
 878                         goto bad;
 879                 }
 880
 881                 path_args.argc = nr_args;
 882                 path_args.argv = as->argv;
 883
 884                 pgpath = parse_path(&path_args, &pg->ps, ti);
 885                 if (IS_ERR(pgpath)) {
 886                         r = PTR_ERR(pgpath);
 887                         goto bad;
 888                 }
 889
 890                 pgpath->pg = pg;
 891                 list_add_tail(&pgpath->list, &pg->pgpaths);
 892                 dm_consume_args(as, nr_args);
 893         }
 894
 895         return pg;
 896
 897  bad:
 898         free_priority_group(pg, ti);
 899         return ERR_PTR(r);
 900 }
 901
 902 static int parse_hw_handler(struct dm_arg_set *as, struct multipath *m)
 903 {
 904         unsigned hw_argc;
 905         int ret;
 906         struct dm_target *ti = m->ti;
 907
 908         static struct dm_arg _args[] = {
 909                 {0, 1024, "invalid number of hardware handler args"},
 910         };
 911
 912         if (dm_read_arg_group(_args, as, &hw_argc, &ti->error))
 913                 return -EINVAL;
 914
 915         if (!hw_argc)
 916                 return 0;
 917
 918         if (m->queue_mode == DM_TYPE_BIO_BASED) {
 919                 dm_consume_args(as, hw_argc);
 920                 DMERR("bio-based multipath doesn't allow hardware handler args");
 921                 return 0;
 922         }
 923
 924         m->hw_handler_name = kstrdup(dm_shift_arg(as), GFP_KERNEL);
 925         if (!m->hw_handler_name)
 926                 return -EINVAL;
 927
 928         if (hw_argc > 1) {
 929                 char *p;
 930                 int i, j, len = 4;
 931
 932                 for (i = 0; i <= hw_argc - 2; i++)
 933                         len += strlen(as->argv[i]) + 1;
 934                 p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
 935                 if (!p) {
 936                         ti->error = "memory allocation failed";
 937                         ret = -ENOMEM;
 938                         goto fail;
 939                 }
 940                 j = sprintf(p, "%d", hw_argc - 1);
 941                 for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
 942                         j = sprintf(p, "%s", as->argv[i]);
 943         }
 944         dm_consume_args(as, hw_argc - 1);
 945
 946         return 0;
 947 fail:
 948         kfree(m->hw_handler_name);
 949         m->hw_handler_name = NULL;
 950         return ret;
 951 }
 952
 953 static int parse_features(struct dm_arg_set *as, struct multipath *m)
 954 {
 955         int r;
 956         unsigned argc;
 957         struct dm_target *ti = m->ti;
 958         const char *arg_name;
 959
 960         static struct dm_arg _args[] = {
 961                 {0, 8, "invalid number of feature args"},
 962                 {1, 50, "pg_init_retries must be between 1 and 50"},
 963                 {0, 60000, "pg_init_delay_msecs must be between 0 and 60000"},
 964         };
 965
 966         r = dm_read_arg_group(_args, as, &argc, &ti->error);
 967         if (r)
 968                 return -EINVAL;
 969
 970         if (!argc)
 971                 return 0;
 972
 973         do {
 974                 arg_name = dm_shift_arg(as);
 975                 argc--;
 976
 977                 if (!strcasecmp(arg_name, "queue_if_no_path")) {
 978                         r = queue_if_no_path(m, true, false);
 979                         continue;
 980                 }
 981
 982                 if (!strcasecmp(arg_name, "retain_attached_hw_handler")) {
 983                         set_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags);
 984                         continue;
 985                 }
 986
 987                 if (!strcasecmp(arg_name, "pg_init_retries") &&
 988                     (argc >= 1)) {
 989                         r = dm_read_arg(_args + 1, as, &m->pg_init_retries, &ti->error);
 990                         argc--;
 991                         continue;
 992                 }
 993
 994                 if (!strcasecmp(arg_name, "pg_init_delay_msecs") &&
 995                     (argc >= 1)) {
 996                         r = dm_read_arg(_args + 2, as, &m->pg_init_delay_msecs, &ti->error);
 997                         argc--;
 998                         continue;
 999                 }
1000
1001                 if (!strcasecmp(arg_name, "queue_mode") &&
1002                     (argc >= 1)) {
1003                         const char *queue_mode_name = dm_shift_arg(as);
1004
1005                         if (!strcasecmp(queue_mode_name, "bio"))
1006                                 m->queue_mode = DM_TYPE_BIO_BASED;
1007                         else if (!strcasecmp(queue_mode_name, "rq"))
1008                                 m->queue_mode = DM_TYPE_REQUEST_BASED;
1009                         else if (!strcasecmp(queue_mode_name, "mq"))
1010                                 m->queue_mode = DM_TYPE_MQ_REQUEST_BASED;
1011                         else {
1012                                 ti->error = "Unknown 'queue_mode' requested";
1013                                 r = -EINVAL;
1014                         }
1015                         argc--;
1016                         continue;
1017                 }
1018
1019                 ti->error = "Unrecognised multipath feature request";
1020                 r = -EINVAL;
1021         } while (argc && !r);
1022
1023         return r;
1024 }
1025
1026 static int multipath_ctr(struct dm_target *ti, unsigned argc, char **argv)
1027 {
1028         /* target arguments */
1029         static struct dm_arg _args[] = {
1030                 {0, 1024, "invalid number of priority groups"},
1031                 {0, 1024, "invalid initial priority group number"},
1032         };
1033
1034         int r;
1035         struct multipath *m;
1036         struct dm_arg_set as;
1037         unsigned pg_count = 0;
1038         unsigned next_pg_num;
1039
1040         as.argc = argc;
1041         as.argv = argv;
1042
1043         m = alloc_multipath(ti);
1044         if (!m) {
1045                 ti->error = "can't allocate multipath";
1046                 return -EINVAL;
1047         }
1048
1049         r = parse_features(&as, m);
1050         if (r)
1051                 goto bad;
1052
1053         r = alloc_multipath_stage2(ti, m);
1054         if (r)
1055                 goto bad;
1056
1057         r = parse_hw_handler(&as, m);
1058         if (r)
1059                 goto bad;
1060
1061         r = dm_read_arg(_args, &as, &m->nr_priority_groups, &ti->error);
1062         if (r)
1063                 goto bad;
1064
1065         r = dm_read_arg(_args + 1, &as, &next_pg_num, &ti->error);
1066         if (r)
1067                 goto bad;
1068
1069         if ((!m->nr_priority_groups && next_pg_num) ||
1070             (m->nr_priority_groups && !next_pg_num)) {
1071                 ti->error = "invalid initial priority group";
1072                 r = -EINVAL;
1073                 goto bad;
1074         }
1075
1076         /* parse the priority groups */
1077         while (as.argc) {
1078                 struct priority_group *pg;
1079                 unsigned nr_valid_paths = atomic_read(&m->nr_valid_paths);
1080
1081                 pg = parse_priority_group(&as, m);
1082                 if (IS_ERR(pg)) {
1083                         r = PTR_ERR(pg);
1084                         goto bad;
1085                 }
1086
1087                 nr_valid_paths += pg->nr_pgpaths;
1088                 atomic_set(&m->nr_valid_paths, nr_valid_paths);
1089
1090                 list_add_tail(&pg->list, &m->priority_groups);
1091                 pg_count++;
1092                 pg->pg_num = pg_count;
1093                 if (!--next_pg_num)
1094                         m->next_pg = pg;
1095         }
1096
1097         if (pg_count != m->nr_priority_groups) {
1098                 ti->error = "priority group count mismatch";
1099                 r = -EINVAL;
1100                 goto bad;
1101         }
1102
1103         ti->num_flush_bios = 1;
1104         ti->num_discard_bios = 1;
1105         ti->num_write_same_bios = 1;
1106         if (m->queue_mode == DM_TYPE_BIO_BASED)
1107                 ti->per_io_data_size = multipath_per_bio_data_size();
1108         else
1109                 ti->per_io_data_size = sizeof(struct dm_mpath_io);
1110
1111         return 0;
1112
1113  bad:
1114         free_multipath(m);
1115         return r;
1116 }
1117
1118 static void multipath_wait_for_pg_init_completion(struct multipath *m)
1119 {
1120         DEFINE_WAIT(wait);
1121
1122         while (1) {
1123                 prepare_to_wait(&m->pg_init_wait, &wait, TASK_UNINTERRUPTIBLE);
1124
1125                 if (!atomic_read(&m->pg_init_in_progress))
1126                         break;
1127
1128                 io_schedule();
1129         }
1130         finish_wait(&m->pg_init_wait, &wait);
1131 }
1132
1133 static void flush_multipath_work(struct multipath *m)
1134 {
1135         set_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1136         smp_mb__after_atomic();
1137
1138         flush_workqueue(kmpath_handlerd);
1139         multipath_wait_for_pg_init_completion(m);
1140         flush_workqueue(kmultipathd);
1141         flush_work(&m->trigger_event);
1142
1143         clear_bit(MPATHF_PG_INIT_DISABLED, &m->flags);
1144         smp_mb__after_atomic();
1145 }
1146
1147 static void multipath_dtr(struct dm_target *ti)
1148 {
1149         struct multipath *m = ti->private;
1150
1151         flush_multipath_work(m);
1152         free_multipath(m);
1153 }
1154
1155 /*
1156  * Take a path out of use.
1157  */
1158 static int fail_path(struct pgpath *pgpath)
1159 {
1160         unsigned long flags;
1161         struct multipath *m = pgpath->pg->m;
1162
1163         spin_lock_irqsave(&m->lock, flags);
1164
1165         if (!pgpath->is_active)
1166                 goto out;
1167
1168         DMWARN("Failing path %s.", pgpath->path.dev->name);
1169
1170         pgpath->pg->ps.type->fail_path(&pgpath->pg->ps, &pgpath->path);
1171         pgpath->is_active = false;
1172         pgpath->fail_count++;
1173
1174         atomic_dec(&m->nr_valid_paths);
1175
1176         if (pgpath == m->current_pgpath)
1177                 m->current_pgpath = NULL;
1178
1179         dm_path_uevent(DM_UEVENT_PATH_FAILED, m->ti,
1180                        pgpath->path.dev->name, atomic_read(&m->nr_valid_paths));
1181
1182         schedule_work(&m->trigger_event);
1183
1184 out:
1185         spin_unlock_irqrestore(&m->lock, flags);
1186
1187         return 0;
1188 }
1189
1190 /*
1191  * Reinstate a previously-failed path
1192  */
1193 static int reinstate_path(struct pgpath *pgpath)
1194 {
1195         int r = 0, run_queue = 0;
1196         unsigned long flags;
1197         struct multipath *m = pgpath->pg->m;
1198         unsigned nr_valid_paths;
1199
1200         spin_lock_irqsave(&m->lock, flags);
1201
1202         if (pgpath->is_active)
1203                 goto out;
1204
1205         DMWARN("Reinstating path %s.", pgpath->path.dev->name);
1206
1207         r = pgpath->pg->ps.type->reinstate_path(&pgpath->pg->ps, &pgpath->path);
1208         if (r)
1209                 goto out;
1210
1211         pgpath->is_active = true;
1212
1213         nr_valid_paths = atomic_inc_return(&m->nr_valid_paths);
1214         if (nr_valid_paths == 1) {
1215                 m->current_pgpath = NULL;
1216                 run_queue = 1;
1217         } else if (m->hw_handler_name && (m->current_pg == pgpath->pg)) {
1218                 if (queue_work(kmpath_handlerd, &pgpath->activate_path.work))
1219                         atomic_inc(&m->pg_init_in_progress);
1220         }
1221
1222         dm_path_uevent(DM_UEVENT_PATH_REINSTATED, m->ti,
1223                        pgpath->path.dev->name, nr_valid_paths);
1224
1225         schedule_work(&m->trigger_event);
1226
1227 out:
1228         spin_unlock_irqrestore(&m->lock, flags);
1229         if (run_queue) {
1230                 dm_table_run_md_queue_async(m->ti->table);
1231                 process_queued_io_list(m);
1232         }
1233
1234         return r;
1235 }
1236
1237 /*
1238  * Fail or reinstate all paths that match the provided struct dm_dev.
1239  */
1240 static int action_dev(struct multipath *m, struct dm_dev *dev,
1241                       action_fn action)
1242 {
1243         int r = -EINVAL;
1244         struct pgpath *pgpath;
1245         struct priority_group *pg;
1246
1247         list_for_each_entry(pg, &m->priority_groups, list) {
1248                 list_for_each_entry(pgpath, &pg->pgpaths, list) {
1249                         if (pgpath->path.dev == dev)
1250                                 r = action(pgpath);
1251                 }
1252         }
1253
1254         return r;
1255 }
1256
1257 /*
1258  * Temporarily try to avoid having to use the specified PG
1259  */
1260 static void bypass_pg(struct multipath *m, struct priority_group *pg,
1261                       bool bypassed)
1262 {
1263         unsigned long flags;
1264
1265         spin_lock_irqsave(&m->lock, flags);
1266
1267         pg->bypassed = bypassed;
1268         m->current_pgpath = NULL;
1269         m->current_pg = NULL;
1270
1271         spin_unlock_irqrestore(&m->lock, flags);
1272
1273         schedule_work(&m->trigger_event);
1274 }
1275
1276 /*
1277  * Switch to using the specified PG from the next I/O that gets mapped
1278  */
1279 static int switch_pg_num(struct multipath *m, const char *pgstr)
1280 {
1281         struct priority_group *pg;
1282         unsigned pgnum;
1283         unsigned long flags;
1284         char dummy;
1285
1286         if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
1287             !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
1288                 DMWARN("invalid PG number supplied to switch_pg_num");
1289                 return -EINVAL;
1290         }
1291
1292         spin_lock_irqsave(&m->lock, flags);
1293         list_for_each_entry(pg, &m->priority_groups, list) {
1294                 pg->bypassed = false;
1295                 if (--pgnum)
1296                         continue;
1297
1298                 m->current_pgpath = NULL;
1299                 m->current_pg = NULL;
1300                 m->next_pg = pg;
1301         }
1302         spin_unlock_irqrestore(&m->lock, flags);
1303
1304         schedule_work(&m->trigger_event);
1305         return 0;
1306 }
1307
1308 /*
1309  * Set/clear bypassed status of a PG.
1310  * PGs are numbered upwards from 1 in the order they were declared.
1311  */
1312 static int bypass_pg_num(struct multipath *m, const char *pgstr, bool bypassed)
1313 {
1314         struct priority_group *pg;
1315         unsigned pgnum;
1316         char dummy;
1317
1318         if (!pgstr || (sscanf(pgstr, "%u%c", &pgnum, &dummy) != 1) || !pgnum ||
1319             !m->nr_priority_groups || (pgnum > m->nr_priority_groups)) {
1320                 DMWARN("invalid PG number supplied to bypass_pg");
1321                 return -EINVAL;
1322         }
1323
1324         list_for_each_entry(pg, &m->priority_groups, list) {
1325                 if (!--pgnum)
1326                         break;
1327         }
1328
1329         bypass_pg(m, pg, bypassed);
1330         return 0;
1331 }
1332
1333 /*
1334  * Should we retry pg_init immediately?
1335  */
1336 static bool pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
1337 {
1338         unsigned long flags;
1339         bool limit_reached = false;
1340
1341         spin_lock_irqsave(&m->lock, flags);
1342
1343         if (atomic_read(&m->pg_init_count) <= m->pg_init_retries &&
1344             !test_bit(MPATHF_PG_INIT_DISABLED, &m->flags))
1345                 set_bit(MPATHF_PG_INIT_REQUIRED, &m->flags);
1346         else
1347                 limit_reached = true;
1348
1349         spin_unlock_irqrestore(&m->lock, flags);
1350
1351         return limit_reached;
1352 }
1353
1354 static void pg_init_done(void *data, int errors)
1355 {
1356         struct pgpath *pgpath = data;
1357         struct priority_group *pg = pgpath->pg;
1358         struct multipath *m = pg->m;
1359         unsigned long flags;
1360         bool delay_retry = false;
1361
1362         /* device or driver problems */
1363         switch (errors) {
1364         case SCSI_DH_OK:
1365                 break;
1366         case SCSI_DH_NOSYS:
1367                 if (!m->hw_handler_name) {
1368                         errors = 0;
1369                         break;
1370                 }
1371                 DMERR("Could not failover the device: Handler scsi_dh_%s "
1372                       "Error %d.", m->hw_handler_name, errors);
1373                 /*
1374                  * Fail path for now, so we do not ping pong
1375                  */
1376                 fail_path(pgpath);
1377                 break;
1378         case SCSI_DH_DEV_TEMP_BUSY:
1379                 /*
1380                  * Probably doing something like FW upgrade on the
1381                  * controller so try the other pg.
1382                  */
1383                 bypass_pg(m, pg, true);
1384                 break;
1385         case SCSI_DH_RETRY:
1386                 /* Wait before retrying. */
1387                 delay_retry = 1;
1388         case SCSI_DH_IMM_RETRY:
1389         case SCSI_DH_RES_TEMP_UNAVAIL:
1390                 if (pg_init_limit_reached(m, pgpath))
1391                         fail_path(pgpath);
1392                 errors = 0;
1393                 break;
1394         case SCSI_DH_DEV_OFFLINED:
1395         default:
1396                 /*
1397                  * We probably do not want to fail the path for a device
1398                  * error, but this is what the old dm did. In future
1399                  * patches we can do more advanced handling.
1400                  */
1401                 fail_path(pgpath);
1402         }
1403
1404         spin_lock_irqsave(&m->lock, flags);
1405         if (errors) {
1406                 if (pgpath == m->current_pgpath) {
1407                         DMERR("Could not failover device. Error %d.", errors);
1408                         m->current_pgpath = NULL;
1409                         m->current_pg = NULL;
1410                 }
1411         } else if (!test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
1412                 pg->bypassed = false;
1413
1414         if (atomic_dec_return(&m->pg_init_in_progress) > 0)
1415                 /* Activations of other paths are still on going */
1416                 goto out;
1417
1418         if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags)) {
1419                 if (delay_retry)
1420                         set_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
1421                 else
1422                         clear_bit(MPATHF_PG_INIT_DELAY_RETRY, &m->flags);
1423
1424                 if (__pg_init_all_paths(m))
1425                         goto out;
1426         }
1427         clear_bit(MPATHF_QUEUE_IO, &m->flags);
1428
1429         process_queued_io_list(m);
1430
1431         /*
1432          * Wake up any thread waiting to suspend.
1433          */
1434         wake_up(&m->pg_init_wait);
1435
1436 out:
1437         spin_unlock_irqrestore(&m->lock, flags);
1438 }
1439
1440 static void activate_path(struct work_struct *work)
1441 {
1442         struct pgpath *pgpath =
1443                 container_of(work, struct pgpath, activate_path.work);
1444         struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1445
1446         if (pgpath->is_active && !blk_queue_dying(q))
1447                 scsi_dh_activate(q, pg_init_done, pgpath);
1448         else
1449                 pg_init_done(pgpath, SCSI_DH_DEV_OFFLINED);
1450 }
1451
1452 static int noretry_error(int error)
1453 {
1454         switch (error) {
1455         case -EBADE:
1456                 /*
1457                  * EBADE signals an reservation conflict.
1458                  * We shouldn't fail the path here as we can communicate with
1459                  * the target.  We should failover to the next path, but in
1460                  * doing so we might be causing a ping-pong between paths.
1461                  * So just return the reservation conflict error.
1462                  */
1463         case -EOPNOTSUPP:
1464         case -EREMOTEIO:
1465         case -EILSEQ:
1466         case -ENODATA:
1467         case -ENOSPC:
1468                 return 1;
1469         }
1470
1471         /* Anything else could be a path failure, so should be retried */
1472         return 0;
1473 }
1474
1475 /*
1476  * end_io handling
1477  */
1478 static int do_end_io(struct multipath *m, struct request *clone,
1479                      int error, struct dm_mpath_io *mpio)
1480 {
1481         /*
1482          * We don't queue any clone request inside the multipath target
1483          * during end I/O handling, since those clone requests don't have
1484          * bio clones.  If we queue them inside the multipath target,
1485          * we need to make bio clones, that requires memory allocation.
1486          * (See drivers/md/dm-rq.c:end_clone_bio() about why the clone requests
1487          *  don't have bio clones.)
1488          * Instead of queueing the clone request here, we queue the original
1489          * request into dm core, which will remake a clone request and
1490          * clone bios for it and resubmit it later.
1491          */
1492         int r = DM_ENDIO_REQUEUE;
1493
1494         if (!error && !clone->errors)
1495                 return 0;       /* I/O complete */
1496
1497         if (noretry_error(error))
1498                 return error;
1499
1500         if (mpio->pgpath)
1501                 fail_path(mpio->pgpath);
1502
1503         if (!atomic_read(&m->nr_valid_paths)) {
1504                 if (!test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
1505                         if (!must_push_back_rq(m))
1506                                 r = -EIO;
1507                 }
1508         }
1509
1510         return r;
1511 }
1512
1513 static int multipath_end_io(struct dm_target *ti, struct request *clone,
1514                             int error, union map_info *map_context)
1515 {
1516         struct multipath *m = ti->private;
1517         struct dm_mpath_io *mpio = get_mpio(map_context);
1518         struct pgpath *pgpath;
1519         struct path_selector *ps;
1520         int r;
1521
1522         BUG_ON(!mpio);
1523
1524         r = do_end_io(m, clone, error, mpio);
1525         pgpath = mpio->pgpath;
1526         if (pgpath) {
1527                 ps = &pgpath->pg->ps;
1528                 if (ps->type->end_io)
1529                         ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1530         }
1531
1532         return r;
1533 }
1534
1535 static int do_end_io_bio(struct multipath *m, struct bio *clone,
1536                          int error, struct dm_mpath_io *mpio)
1537 {
1538         unsigned long flags;
1539
1540         if (!error)
1541                 return 0;       /* I/O complete */
1542
1543         if (noretry_error(error))
1544                 return error;
1545
1546         if (mpio->pgpath)
1547                 fail_path(mpio->pgpath);
1548
1549         if (!atomic_read(&m->nr_valid_paths)) {
1550                 if (!test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags)) {
1551                         if (!must_push_back_bio(m))
1552                                 return -EIO;
1553                         return DM_ENDIO_REQUEUE;
1554                 }
1555         }
1556
1557         /* Queue for the daemon to resubmit */
1558         dm_bio_restore(get_bio_details_from_bio(clone), clone);
1559
1560         spin_lock_irqsave(&m->lock, flags);
1561         bio_list_add(&m->queued_bios, clone);
1562         spin_unlock_irqrestore(&m->lock, flags);
1563         if (!test_bit(MPATHF_QUEUE_IO, &m->flags))
1564                 queue_work(kmultipathd, &m->process_queued_bios);
1565
1566         return DM_ENDIO_INCOMPLETE;
1567 }
1568
1569 static int multipath_end_io_bio(struct dm_target *ti, struct bio *clone, int error)
1570 {
1571         struct multipath *m = ti->private;
1572         struct dm_mpath_io *mpio = get_mpio_from_bio(clone);
1573         struct pgpath *pgpath;
1574         struct path_selector *ps;
1575         int r;
1576
1577         BUG_ON(!mpio);
1578
1579         r = do_end_io_bio(m, clone, error, mpio);
1580         pgpath = mpio->pgpath;
1581         if (pgpath) {
1582                 ps = &pgpath->pg->ps;
1583                 if (ps->type->end_io)
1584                         ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
1585         }
1586
1587         return r;
1588 }
1589
1590 /*
1591  * Suspend can't complete until all the I/O is processed so if
1592  * the last path fails we must error any remaining I/O.
1593  * Note that if the freeze_bdev fails while suspending, the
1594  * queue_if_no_path state is lost - userspace should reset it.
1595  */
1596 static void multipath_presuspend(struct dm_target *ti)
1597 {
1598         struct multipath *m = ti->private;
1599
1600         queue_if_no_path(m, false, true);
1601 }
1602
1603 static void multipath_postsuspend(struct dm_target *ti)
1604 {
1605         struct multipath *m = ti->private;
1606
1607         mutex_lock(&m->work_mutex);
1608         flush_multipath_work(m);
1609         mutex_unlock(&m->work_mutex);
1610 }
1611
1612 /*
1613  * Restore the queue_if_no_path setting.
1614  */
1615 static void multipath_resume(struct dm_target *ti)
1616 {
1617         struct multipath *m = ti->private;
1618         unsigned long flags;
1619
1620         spin_lock_irqsave(&m->lock, flags);
1621         if (test_bit(MPATHF_SAVED_QUEUE_IF_NO_PATH, &m->flags))
1622                 set_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
1623         else
1624                 clear_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags);
1625         spin_unlock_irqrestore(&m->lock, flags);
1626 }
1627
1628 /*
1629  * Info output has the following format:
1630  * num_multipath_feature_args [multipath_feature_args]*
1631  * num_handler_status_args [handler_status_args]*
1632  * num_groups init_group_number
1633  *            [A|D|E num_ps_status_args [ps_status_args]*
1634  *             num_paths num_selector_args
1635  *             [path_dev A|F fail_count [selector_args]* ]+ ]+
1636  *
1637  * Table output has the following format (identical to the constructor string):
1638  * num_feature_args [features_args]*
1639  * num_handler_args hw_handler [hw_handler_args]*
1640  * num_groups init_group_number
1641  *     [priority selector-name num_ps_args [ps_args]*
1642  *      num_paths num_selector_args [path_dev [selector_args]* ]+ ]+
1643  */
1644 static void multipath_status(struct dm_target *ti, status_type_t type,
1645                              unsigned status_flags, char *result, unsigned maxlen)
1646 {
1647         int sz = 0;
1648         unsigned long flags;
1649         struct multipath *m = ti->private;
1650         struct priority_group *pg;
1651         struct pgpath *p;
1652         unsigned pg_num;
1653         char state;
1654
1655         spin_lock_irqsave(&m->lock, flags);
1656
1657         /* Features */
1658         if (type == STATUSTYPE_INFO)
1659                 DMEMIT("2 %u %u ", test_bit(MPATHF_QUEUE_IO, &m->flags),
1660                        atomic_read(&m->pg_init_count));
1661         else {
1662                 DMEMIT("%u ", test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags) +
1663                               (m->pg_init_retries > 0) * 2 +
1664                               (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT) * 2 +
1665                               test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags) +
1666                               (m->queue_mode != DM_TYPE_REQUEST_BASED) * 2);
1667
1668                 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1669                         DMEMIT("queue_if_no_path ");
1670                 if (m->pg_init_retries)
1671                         DMEMIT("pg_init_retries %u ", m->pg_init_retries);
1672                 if (m->pg_init_delay_msecs != DM_PG_INIT_DELAY_DEFAULT)
1673                         DMEMIT("pg_init_delay_msecs %u ", m->pg_init_delay_msecs);
1674                 if (test_bit(MPATHF_RETAIN_ATTACHED_HW_HANDLER, &m->flags))
1675                         DMEMIT("retain_attached_hw_handler ");
1676                 if (m->queue_mode != DM_TYPE_REQUEST_BASED) {
1677                         switch(m->queue_mode) {
1678                         case DM_TYPE_BIO_BASED:
1679                                 DMEMIT("queue_mode bio ");
1680                                 break;
1681                         case DM_TYPE_MQ_REQUEST_BASED:
1682                                 DMEMIT("queue_mode mq ");
1683                                 break;
1684                         }
1685                 }
1686         }
1687
1688         if (!m->hw_handler_name || type == STATUSTYPE_INFO)
1689                 DMEMIT("0 ");
1690         else
1691                 DMEMIT("1 %s ", m->hw_handler_name);
1692
1693         DMEMIT("%u ", m->nr_priority_groups);
1694
1695         if (m->next_pg)
1696                 pg_num = m->next_pg->pg_num;
1697         else if (m->current_pg)
1698                 pg_num = m->current_pg->pg_num;
1699         else
1700                 pg_num = (m->nr_priority_groups ? 1 : 0);
1701
1702         DMEMIT("%u ", pg_num);
1703
1704         switch (type) {
1705         case STATUSTYPE_INFO:
1706                 list_for_each_entry(pg, &m->priority_groups, list) {
1707                         if (pg->bypassed)
1708                                 state = 'D';    /* Disabled */
1709                         else if (pg == m->current_pg)
1710                                 state = 'A';    /* Currently Active */
1711                         else
1712                                 state = 'E';    /* Enabled */
1713
1714                         DMEMIT("%c ", state);
1715
1716                         if (pg->ps.type->status)
1717                                 sz += pg->ps.type->status(&pg->ps, NULL, type,
1718                                                           result + sz,
1719                                                           maxlen - sz);
1720                         else
1721                                 DMEMIT("0 ");
1722
1723                         DMEMIT("%u %u ", pg->nr_pgpaths,
1724                                pg->ps.type->info_args);
1725
1726                         list_for_each_entry(p, &pg->pgpaths, list) {
1727                                 DMEMIT("%s %s %u ", p->path.dev->name,
1728                                        p->is_active ? "A" : "F",
1729                                        p->fail_count);
1730                                 if (pg->ps.type->status)
1731                                         sz += pg->ps.type->status(&pg->ps,
1732                                               &p->path, type, result + sz,
1733                                               maxlen - sz);
1734                         }
1735                 }
1736                 break;
1737
1738         case STATUSTYPE_TABLE:
1739                 list_for_each_entry(pg, &m->priority_groups, list) {
1740                         DMEMIT("%s ", pg->ps.type->name);
1741
1742                         if (pg->ps.type->status)
1743                                 sz += pg->ps.type->status(&pg->ps, NULL, type,
1744                                                           result + sz,
1745                                                           maxlen - sz);
1746                         else
1747                                 DMEMIT("0 ");
1748
1749                         DMEMIT("%u %u ", pg->nr_pgpaths,
1750                                pg->ps.type->table_args);
1751
1752                         list_for_each_entry(p, &pg->pgpaths, list) {
1753                                 DMEMIT("%s ", p->path.dev->name);
1754                                 if (pg->ps.type->status)
1755                                         sz += pg->ps.type->status(&pg->ps,
1756                                               &p->path, type, result + sz,
1757                                               maxlen - sz);
1758                         }
1759                 }
1760                 break;
1761         }
1762
1763         spin_unlock_irqrestore(&m->lock, flags);
1764 }
1765
1766 static int multipath_message(struct dm_target *ti, unsigned argc, char **argv)
1767 {
1768         int r = -EINVAL;
1769         struct dm_dev *dev;
1770         struct multipath *m = ti->private;
1771         action_fn action;
1772
1773         mutex_lock(&m->work_mutex);
1774
1775         if (dm_suspended(ti)) {
1776                 r = -EBUSY;
1777                 goto out;
1778         }
1779
1780         if (argc == 1) {
1781                 if (!strcasecmp(argv[0], "queue_if_no_path")) {
1782                         r = queue_if_no_path(m, true, false);
1783                         goto out;
1784                 } else if (!strcasecmp(argv[0], "fail_if_no_path")) {
1785                         r = queue_if_no_path(m, false, false);
1786                         goto out;
1787                 }
1788         }
1789
1790         if (argc != 2) {
1791                 DMWARN("Invalid multipath message arguments. Expected 2 arguments, got %d.", argc);
1792                 goto out;
1793         }
1794
1795         if (!strcasecmp(argv[0], "disable_group")) {
1796                 r = bypass_pg_num(m, argv[1], true);
1797                 goto out;
1798         } else if (!strcasecmp(argv[0], "enable_group")) {
1799                 r = bypass_pg_num(m, argv[1], false);
1800                 goto out;
1801         } else if (!strcasecmp(argv[0], "switch_group")) {
1802                 r = switch_pg_num(m, argv[1]);
1803                 goto out;
1804         } else if (!strcasecmp(argv[0], "reinstate_path"))
1805                 action = reinstate_path;
1806         else if (!strcasecmp(argv[0], "fail_path"))
1807                 action = fail_path;
1808         else {
1809                 DMWARN("Unrecognised multipath message received: %s", argv[0]);
1810                 goto out;
1811         }
1812
1813         r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev);
1814         if (r) {
1815                 DMWARN("message: error getting device %s",
1816                        argv[1]);
1817                 goto out;
1818         }
1819
1820         r = action_dev(m, dev, action);
1821
1822         dm_put_device(ti, dev);
1823
1824 out:
1825         mutex_unlock(&m->work_mutex);
1826         return r;
1827 }
1828
1829 static int multipath_prepare_ioctl(struct dm_target *ti,
1830                 struct block_device **bdev, fmode_t *mode)
1831 {
1832         struct multipath *m = ti->private;
1833         struct pgpath *current_pgpath;
1834         int r;
1835
1836         current_pgpath = lockless_dereference(m->current_pgpath);
1837         if (!current_pgpath)
1838                 current_pgpath = choose_pgpath(m, 0);
1839
1840         if (current_pgpath) {
1841                 if (!test_bit(MPATHF_QUEUE_IO, &m->flags)) {
1842                         *bdev = current_pgpath->path.dev->bdev;
1843                         *mode = current_pgpath->path.dev->mode;
1844                         r = 0;
1845                 } else {
1846                         /* pg_init has not started or completed */
1847                         r = -ENOTCONN;
1848                 }
1849         } else {
1850                 /* No path is available */
1851                 if (test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1852                         r = -ENOTCONN;
1853                 else
1854                         r = -EIO;
1855         }
1856
1857         if (r == -ENOTCONN) {
1858                 if (!lockless_dereference(m->current_pg)) {
1859                         /* Path status changed, redo selection */
1860                         (void) choose_pgpath(m, 0);
1861                 }
1862                 if (test_bit(MPATHF_PG_INIT_REQUIRED, &m->flags))
1863                         pg_init_all_paths(m);
1864                 dm_table_run_md_queue_async(m->ti->table);
1865                 process_queued_io_list(m);
1866         }
1867
1868         /*
1869          * Only pass ioctls through if the device sizes match exactly.
1870          */
1871         if (!r && ti->len != i_size_read((*bdev)->bd_inode) >> SECTOR_SHIFT)
1872                 return 1;
1873         return r;
1874 }
1875
1876 static int multipath_iterate_devices(struct dm_target *ti,
1877                                      iterate_devices_callout_fn fn, void *data)
1878 {
1879         struct multipath *m = ti->private;
1880         struct priority_group *pg;
1881         struct pgpath *p;
1882         int ret = 0;
1883
1884         list_for_each_entry(pg, &m->priority_groups, list) {
1885                 list_for_each_entry(p, &pg->pgpaths, list) {
1886                         ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
1887                         if (ret)
1888                                 goto out;
1889                 }
1890         }
1891
1892 out:
1893         return ret;
1894 }
1895
1896 static int pgpath_busy(struct pgpath *pgpath)
1897 {
1898         struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
1899
1900         return blk_lld_busy(q);
1901 }
1902
1903 /*
1904  * We return "busy", only when we can map I/Os but underlying devices
1905  * are busy (so even if we map I/Os now, the I/Os will wait on
1906  * the underlying queue).
1907  * In other words, if we want to kill I/Os or queue them inside us
1908  * due to map unavailability, we don't return "busy".  Otherwise,
1909  * dm core won't give us the I/Os and we can't do what we want.
1910  */
1911 static int multipath_busy(struct dm_target *ti)
1912 {
1913         bool busy = false, has_active = false;
1914         struct multipath *m = ti->private;
1915         struct priority_group *pg, *next_pg;
1916         struct pgpath *pgpath;
1917
1918         /* pg_init in progress */
1919         if (atomic_read(&m->pg_init_in_progress))
1920                 return true;
1921
1922         /* no paths available, for blk-mq: rely on IO mapping to delay requeue */
1923         if (!atomic_read(&m->nr_valid_paths) && test_bit(MPATHF_QUEUE_IF_NO_PATH, &m->flags))
1924                 return (m->queue_mode != DM_TYPE_MQ_REQUEST_BASED);
1925
1926         /* Guess which priority_group will be used at next mapping time */
1927         pg = lockless_dereference(m->current_pg);
1928         next_pg = lockless_dereference(m->next_pg);
1929         if (unlikely(!lockless_dereference(m->current_pgpath) && next_pg))
1930                 pg = next_pg;
1931
1932         if (!pg) {
1933                 /*
1934                  * We don't know which pg will be used at next mapping time.
1935                  * We don't call choose_pgpath() here to avoid to trigger
1936                  * pg_init just by busy checking.
1937                  * So we don't know whether underlying devices we will be using
1938                  * at next mapping time are busy or not. Just try mapping.
1939                  */
1940                 return busy;
1941         }
1942
1943         /*
1944          * If there is one non-busy active path at least, the path selector
1945          * will be able to select it. So we consider such a pg as not busy.
1946          */
1947         busy = true;
1948         list_for_each_entry(pgpath, &pg->pgpaths, list) {
1949                 if (pgpath->is_active) {
1950                         has_active = true;
1951                         if (!pgpath_busy(pgpath)) {
1952                                 busy = false;
1953                                 break;
1954                         }
1955                 }
1956         }
1957
1958         if (!has_active) {
1959                 /*
1960                  * No active path in this pg, so this pg won't be used and
1961                  * the current_pg will be changed at next mapping time.
1962                  * We need to try mapping to determine it.
1963                  */
1964                 busy = false;
1965         }
1966
1967         return busy;
1968 }
1969
1970 /*-----------------------------------------------------------------
1971  * Module setup
1972  *---------------------------------------------------------------*/
1973 static struct target_type multipath_target = {
1974         .name = "multipath",
1975         .version = {1, 12, 0},
1976         .features = DM_TARGET_SINGLETON | DM_TARGET_IMMUTABLE,
1977         .module = THIS_MODULE,
1978         .ctr = multipath_ctr,
1979         .dtr = multipath_dtr,
1980         .clone_and_map_rq = multipath_clone_and_map,
1981         .release_clone_rq = multipath_release_clone,
1982         .rq_end_io = multipath_end_io,
1983         .map = multipath_map_bio,
1984         .end_io = multipath_end_io_bio,
1985         .presuspend = multipath_presuspend,
1986         .postsuspend = multipath_postsuspend,
1987         .resume = multipath_resume,
1988         .status = multipath_status,
1989         .message = multipath_message,
1990         .prepare_ioctl = multipath_prepare_ioctl,
1991         .iterate_devices = multipath_iterate_devices,
1992         .busy = multipath_busy,
1993 };
1994
1995 static int __init dm_multipath_init(void)
1996 {
1997         int r;
1998
1999         r = dm_register_target(&multipath_target);
2000         if (r < 0) {
2001                 DMERR("request-based register failed %d", r);
2002                 r = -EINVAL;
2003                 goto bad_register_target;
2004         }
2005
2006         kmultipathd = alloc_workqueue("kmpathd", WQ_MEM_RECLAIM, 0);
2007         if (!kmultipathd) {
2008                 DMERR("failed to create workqueue kmpathd");
2009                 r = -ENOMEM;
2010                 goto bad_alloc_kmultipathd;
2011         }
2012
2013         /*
2014          * A separate workqueue is used to handle the device handlers
2015          * to avoid overloading existing workqueue. Overloading the
2016          * old workqueue would also create a bottleneck in the
2017          * path of the storage hardware device activation.
2018          */
2019         kmpath_handlerd = alloc_ordered_workqueue("kmpath_handlerd",
2020                                                   WQ_MEM_RECLAIM);
2021         if (!kmpath_handlerd) {
2022                 DMERR("failed to create workqueue kmpath_handlerd");
2023                 r = -ENOMEM;
2024                 goto bad_alloc_kmpath_handlerd;
2025         }
2026
2027         return 0;
2028
2029 bad_alloc_kmpath_handlerd:
2030         destroy_workqueue(kmultipathd);
2031 bad_alloc_kmultipathd:
2032         dm_unregister_target(&multipath_target);
2033 bad_register_target:
2034         return r;
2035 }
2036
2037 static void __exit dm_multipath_exit(void)
2038 {
2039         destroy_workqueue(kmpath_handlerd);
2040         destroy_workqueue(kmultipathd);
2041
2042         dm_unregister_target(&multipath_target);
2043 }
2044
2045 module_init(dm_multipath_init);
2046 module_exit(dm_multipath_exit);
2047
2048 MODULE_DESCRIPTION(DM_NAME " multipath target");
2049 MODULE_AUTHOR("Sistina Software <dm-devel@redhat.com>");
2050 MODULE_LICENSE("GPL");