drivers/block/null_blk.c

   1 #include <linux/module.h>
   2
   3 #include <linux/moduleparam.h>
   4 #include <linux/sched.h>
   5 #include <linux/fs.h>
   6 #include <linux/blkdev.h>
   7 #include <linux/init.h>
   8 #include <linux/slab.h>
   9 #include <linux/blk-mq.h>
  10 #include <linux/hrtimer.h>
  11 #include <linux/lightnvm.h>
  12
  13 struct nullb_cmd {
  14         struct list_head list;
  15         struct llist_node ll_list;
  16         struct call_single_data csd;
  17         struct request *rq;
  18         struct bio *bio;
  19         unsigned int tag;
  20         struct nullb_queue *nq;
  21         struct hrtimer timer;
  22 };
  23
  24 struct nullb_queue {
  25         unsigned long *tag_map;
  26         wait_queue_head_t wait;
  27         unsigned int queue_depth;
  28
  29         struct nullb_cmd *cmds;
  30 };
  31
  32 struct nullb {
  33         struct list_head list;
  34         unsigned int index;
  35         struct request_queue *q;
  36         struct gendisk *disk;
  37         struct nvm_dev *ndev;
  38         struct blk_mq_tag_set tag_set;
  39         struct hrtimer timer;
  40         unsigned int queue_depth;
  41         spinlock_t lock;
  42
  43         struct nullb_queue *queues;
  44         unsigned int nr_queues;
  45         char disk_name[DISK_NAME_LEN];
  46 };
  47
  48 static LIST_HEAD(nullb_list);
  49 static struct mutex lock;
  50 static int null_major;
  51 static int nullb_indexes;
  52 static struct kmem_cache *ppa_cache;
  53
  54 enum {
  55         NULL_IRQ_NONE           = 0,
  56         NULL_IRQ_SOFTIRQ        = 1,
  57         NULL_IRQ_TIMER          = 2,
  58 };
  59
  60 enum {
  61         NULL_Q_BIO              = 0,
  62         NULL_Q_RQ               = 1,
  63         NULL_Q_MQ               = 2,
  64 };
  65
  66 static int submit_queues;
  67 module_param(submit_queues, int, S_IRUGO);
  68 MODULE_PARM_DESC(submit_queues, "Number of submission queues");
  69
  70 static int home_node = NUMA_NO_NODE;
  71 module_param(home_node, int, S_IRUGO);
  72 MODULE_PARM_DESC(home_node, "Home node for the device");
  73
  74 static int queue_mode = NULL_Q_MQ;
  75
  76 static int null_param_store_val(const char *str, int *val, int min, int max)
  77 {
  78         int ret, new_val;
  79
  80         ret = kstrtoint(str, 10, &new_val);
  81         if (ret)
  82                 return -EINVAL;
  83
  84         if (new_val < min || new_val > max)
  85                 return -EINVAL;
  86
  87         *val = new_val;
  88         return 0;
  89 }
  90
  91 static int null_set_queue_mode(const char *str, const struct kernel_param *kp)
  92 {
  93         return null_param_store_val(str, &queue_mode, NULL_Q_BIO, NULL_Q_MQ);
  94 }
  95
  96 static const struct kernel_param_ops null_queue_mode_param_ops = {
  97         .set    = null_set_queue_mode,
  98         .get    = param_get_int,
  99 };
 100
 101 device_param_cb(queue_mode, &null_queue_mode_param_ops, &queue_mode, S_IRUGO);
 102 MODULE_PARM_DESC(queue_mode, "Block interface to use (0=bio,1=rq,2=multiqueue)");
 103
 104 static int gb = 250;
 105 module_param(gb, int, S_IRUGO);
 106 MODULE_PARM_DESC(gb, "Size in GB");
 107
 108 static int bs = 512;
 109 module_param(bs, int, S_IRUGO);
 110 MODULE_PARM_DESC(bs, "Block size (in bytes)");
 111
 112 static int nr_devices = 2;
 113 module_param(nr_devices, int, S_IRUGO);
 114 MODULE_PARM_DESC(nr_devices, "Number of devices to register");
 115
 116 static bool use_lightnvm;
 117 module_param(use_lightnvm, bool, S_IRUGO);
 118 MODULE_PARM_DESC(use_lightnvm, "Register as a LightNVM device");
 119
 120 static bool blocking;
 121 module_param(blocking, bool, S_IRUGO);
 122 MODULE_PARM_DESC(blocking, "Register as a blocking blk-mq driver device");
 123
 124 static int irqmode = NULL_IRQ_SOFTIRQ;
 125
 126 static int null_set_irqmode(const char *str, const struct kernel_param *kp)
 127 {
 128         return null_param_store_val(str, &irqmode, NULL_IRQ_NONE,
 129                                         NULL_IRQ_TIMER);
 130 }
 131
 132 static const struct kernel_param_ops null_irqmode_param_ops = {
 133         .set    = null_set_irqmode,
 134         .get    = param_get_int,
 135 };
 136
 137 device_param_cb(irqmode, &null_irqmode_param_ops, &irqmode, S_IRUGO);
 138 MODULE_PARM_DESC(irqmode, "IRQ completion handler. 0-none, 1-softirq, 2-timer");
 139
 140 static unsigned long completion_nsec = 10000;
 141 module_param(completion_nsec, ulong, S_IRUGO);
 142 MODULE_PARM_DESC(completion_nsec, "Time in ns to complete a request in hardware. Default: 10,000ns");
 143
 144 static int hw_queue_depth = 64;
 145 module_param(hw_queue_depth, int, S_IRUGO);
 146 MODULE_PARM_DESC(hw_queue_depth, "Queue depth for each hardware queue. Default: 64");
 147
 148 static bool use_per_node_hctx = false;
 149 module_param(use_per_node_hctx, bool, S_IRUGO);
 150 MODULE_PARM_DESC(use_per_node_hctx, "Use per-node allocation for hardware context queues. Default: false");
 151
 152 static void put_tag(struct nullb_queue *nq, unsigned int tag)
 153 {
 154         clear_bit_unlock(tag, nq->tag_map);
 155
 156         if (waitqueue_active(&nq->wait))
 157                 wake_up(&nq->wait);
 158 }
 159
 160 static unsigned int get_tag(struct nullb_queue *nq)
 161 {
 162         unsigned int tag;
 163
 164         do {
 165                 tag = find_first_zero_bit(nq->tag_map, nq->queue_depth);
 166                 if (tag >= nq->queue_depth)
 167                         return -1U;
 168         } while (test_and_set_bit_lock(tag, nq->tag_map));
 169
 170         return tag;
 171 }
 172
 173 static void free_cmd(struct nullb_cmd *cmd)
 174 {
 175         put_tag(cmd->nq, cmd->tag);
 176 }
 177
 178 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer);
 179
 180 static struct nullb_cmd *__alloc_cmd(struct nullb_queue *nq)
 181 {
 182         struct nullb_cmd *cmd;
 183         unsigned int tag;
 184
 185         tag = get_tag(nq);
 186         if (tag != -1U) {
 187                 cmd = &nq->cmds[tag];
 188                 cmd->tag = tag;
 189                 cmd->nq = nq;
 190                 if (irqmode == NULL_IRQ_TIMER) {
 191                         hrtimer_init(&cmd->timer, CLOCK_MONOTONIC,
 192                                      HRTIMER_MODE_REL);
 193                         cmd->timer.function = null_cmd_timer_expired;
 194                 }
 195                 return cmd;
 196         }
 197
 198         return NULL;
 199 }
 200
 201 static struct nullb_cmd *alloc_cmd(struct nullb_queue *nq, int can_wait)
 202 {
 203         struct nullb_cmd *cmd;
 204         DEFINE_WAIT(wait);
 205
 206         cmd = __alloc_cmd(nq);
 207         if (cmd || !can_wait)
 208                 return cmd;
 209
 210         do {
 211                 prepare_to_wait(&nq->wait, &wait, TASK_UNINTERRUPTIBLE);
 212                 cmd = __alloc_cmd(nq);
 213                 if (cmd)
 214                         break;
 215
 216                 io_schedule();
 217         } while (1);
 218
 219         finish_wait(&nq->wait, &wait);
 220         return cmd;
 221 }
 222
 223 static void end_cmd(struct nullb_cmd *cmd)
 224 {
 225         struct request_queue *q = NULL;
 226
 227         if (cmd->rq)
 228                 q = cmd->rq->q;
 229
 230         switch (queue_mode)  {
 231         case NULL_Q_MQ:
 232                 blk_mq_end_request(cmd->rq, 0);
 233                 return;
 234         case NULL_Q_RQ:
 235                 INIT_LIST_HEAD(&cmd->rq->queuelist);
 236                 blk_end_request_all(cmd->rq, 0);
 237                 break;
 238         case NULL_Q_BIO:
 239                 bio_endio(cmd->bio);
 240                 break;
 241         }
 242
 243         free_cmd(cmd);
 244
 245         /* Restart queue if needed, as we are freeing a tag */
 246         if (queue_mode == NULL_Q_RQ && blk_queue_stopped(q)) {
 247                 unsigned long flags;
 248
 249                 spin_lock_irqsave(q->queue_lock, flags);
 250                 blk_start_queue_async(q);
 251                 spin_unlock_irqrestore(q->queue_lock, flags);
 252         }
 253 }
 254
 255 static enum hrtimer_restart null_cmd_timer_expired(struct hrtimer *timer)
 256 {
 257         end_cmd(container_of(timer, struct nullb_cmd, timer));
 258
 259         return HRTIMER_NORESTART;
 260 }
 261
 262 static void null_cmd_end_timer(struct nullb_cmd *cmd)
 263 {
 264         ktime_t kt = completion_nsec;
 265
 266         hrtimer_start(&cmd->timer, kt, HRTIMER_MODE_REL);
 267 }
 268
 269 static void null_softirq_done_fn(struct request *rq)
 270 {
 271         if (queue_mode == NULL_Q_MQ)
 272                 end_cmd(blk_mq_rq_to_pdu(rq));
 273         else
 274                 end_cmd(rq->special);
 275 }
 276
 277 static inline void null_handle_cmd(struct nullb_cmd *cmd)
 278 {
 279         /* Complete IO by inline, softirq or timer */
 280         switch (irqmode) {
 281         case NULL_IRQ_SOFTIRQ:
 282                 switch (queue_mode)  {
 283                 case NULL_Q_MQ:
 284                         blk_mq_complete_request(cmd->rq, cmd->rq->errors);
 285                         break;
 286                 case NULL_Q_RQ:
 287                         blk_complete_request(cmd->rq);
 288                         break;
 289                 case NULL_Q_BIO:
 290                         /*
 291                          * XXX: no proper submitting cpu information available.
 292                          */
 293                         end_cmd(cmd);
 294                         break;
 295                 }
 296                 break;
 297         case NULL_IRQ_NONE:
 298                 end_cmd(cmd);
 299                 break;
 300         case NULL_IRQ_TIMER:
 301                 null_cmd_end_timer(cmd);
 302                 break;
 303         }
 304 }
 305
 306 static struct nullb_queue *nullb_to_queue(struct nullb *nullb)
 307 {
 308         int index = 0;
 309
 310         if (nullb->nr_queues != 1)
 311                 index = raw_smp_processor_id() / ((nr_cpu_ids + nullb->nr_queues - 1) / nullb->nr_queues);
 312
 313         return &nullb->queues[index];
 314 }
 315
 316 static blk_qc_t null_queue_bio(struct request_queue *q, struct bio *bio)
 317 {
 318         struct nullb *nullb = q->queuedata;
 319         struct nullb_queue *nq = nullb_to_queue(nullb);
 320         struct nullb_cmd *cmd;
 321
 322         cmd = alloc_cmd(nq, 1);
 323         cmd->bio = bio;
 324
 325         null_handle_cmd(cmd);
 326         return BLK_QC_T_NONE;
 327 }
 328
 329 static int null_rq_prep_fn(struct request_queue *q, struct request *req)
 330 {
 331         struct nullb *nullb = q->queuedata;
 332         struct nullb_queue *nq = nullb_to_queue(nullb);
 333         struct nullb_cmd *cmd;
 334
 335         cmd = alloc_cmd(nq, 0);
 336         if (cmd) {
 337                 cmd->rq = req;
 338                 req->special = cmd;
 339                 return BLKPREP_OK;
 340         }
 341         blk_stop_queue(q);
 342
 343         return BLKPREP_DEFER;
 344 }
 345
 346 static void null_request_fn(struct request_queue *q)
 347 {
 348         struct request *rq;
 349
 350         while ((rq = blk_fetch_request(q)) != NULL) {
 351                 struct nullb_cmd *cmd = rq->special;
 352
 353                 spin_unlock_irq(q->queue_lock);
 354                 null_handle_cmd(cmd);
 355                 spin_lock_irq(q->queue_lock);
 356         }
 357 }
 358
 359 static int null_queue_rq(struct blk_mq_hw_ctx *hctx,
 360                          const struct blk_mq_queue_data *bd)
 361 {
 362         struct nullb_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
 363
 364         might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
 365
 366         if (irqmode == NULL_IRQ_TIMER) {
 367                 hrtimer_init(&cmd->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
 368                 cmd->timer.function = null_cmd_timer_expired;
 369         }
 370         cmd->rq = bd->rq;
 371         cmd->nq = hctx->driver_data;
 372
 373         blk_mq_start_request(bd->rq);
 374
 375         null_handle_cmd(cmd);
 376         return BLK_MQ_RQ_QUEUE_OK;
 377 }
 378
 379 static void null_init_queue(struct nullb *nullb, struct nullb_queue *nq)
 380 {
 381         BUG_ON(!nullb);
 382         BUG_ON(!nq);
 383
 384         init_waitqueue_head(&nq->wait);
 385         nq->queue_depth = nullb->queue_depth;
 386 }
 387
 388 static int null_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 389                           unsigned int index)
 390 {
 391         struct nullb *nullb = data;
 392         struct nullb_queue *nq = &nullb->queues[index];
 393
 394         hctx->driver_data = nq;
 395         null_init_queue(nullb, nq);
 396         nullb->nr_queues++;
 397
 398         return 0;
 399 }
 400
 401 static struct blk_mq_ops null_mq_ops = {
 402         .queue_rq       = null_queue_rq,
 403         .init_hctx      = null_init_hctx,
 404         .complete       = null_softirq_done_fn,
 405 };
 406
 407 static void cleanup_queue(struct nullb_queue *nq)
 408 {
 409         kfree(nq->tag_map);
 410         kfree(nq->cmds);
 411 }
 412
 413 static void cleanup_queues(struct nullb *nullb)
 414 {
 415         int i;
 416
 417         for (i = 0; i < nullb->nr_queues; i++)
 418                 cleanup_queue(&nullb->queues[i]);
 419
 420         kfree(nullb->queues);
 421 }
 422
 423 #ifdef CONFIG_NVM
 424
 425 static void null_lnvm_end_io(struct request *rq, int error)
 426 {
 427         struct nvm_rq *rqd = rq->end_io_data;
 428
 429         rqd->error = error;
 430         nvm_end_io(rqd);
 431
 432         blk_put_request(rq);
 433 }
 434
 435 static int null_lnvm_submit_io(struct nvm_dev *dev, struct nvm_rq *rqd)
 436 {
 437         struct request_queue *q = dev->q;
 438         struct request *rq;
 439         struct bio *bio = rqd->bio;
 440
 441         rq = blk_mq_alloc_request(q,
 442                 op_is_write(bio_op(bio)) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN, 0);
 443         if (IS_ERR(rq))
 444                 return -ENOMEM;
 445
 446         rq->__sector = bio->bi_iter.bi_sector;
 447         rq->ioprio = bio_prio(bio);
 448
 449         if (bio_has_data(bio))
 450                 rq->nr_phys_segments = bio_phys_segments(q, bio);
 451
 452         rq->__data_len = bio->bi_iter.bi_size;
 453         rq->bio = rq->biotail = bio;
 454
 455         rq->end_io_data = rqd;
 456
 457         blk_execute_rq_nowait(q, NULL, rq, 0, null_lnvm_end_io);
 458
 459         return 0;
 460 }
 461
 462 static int null_lnvm_id(struct nvm_dev *dev, struct nvm_id *id)
 463 {
 464         sector_t size = gb * 1024 * 1024 * 1024ULL;
 465         sector_t blksize;
 466         struct nvm_id_group *grp;
 467
 468         id->ver_id = 0x1;
 469         id->vmnt = 0;
 470         id->cap = 0x2;
 471         id->dom = 0x1;
 472
 473         id->ppaf.blk_offset = 0;
 474         id->ppaf.blk_len = 16;
 475         id->ppaf.pg_offset = 16;
 476         id->ppaf.pg_len = 16;
 477         id->ppaf.sect_offset = 32;
 478         id->ppaf.sect_len = 8;
 479         id->ppaf.pln_offset = 40;
 480         id->ppaf.pln_len = 8;
 481         id->ppaf.lun_offset = 48;
 482         id->ppaf.lun_len = 8;
 483         id->ppaf.ch_offset = 56;
 484         id->ppaf.ch_len = 8;
 485
 486         sector_div(size, bs); /* convert size to pages */
 487         size >>= 8; /* concert size to pgs pr blk */
 488         grp = &id->grp;
 489         grp->mtype = 0;
 490         grp->fmtype = 0;
 491         grp->num_ch = 1;
 492         grp->num_pg = 256;
 493         blksize = size;
 494         size >>= 16;
 495         grp->num_lun = size + 1;
 496         sector_div(blksize, grp->num_lun);
 497         grp->num_blk = blksize;
 498         grp->num_pln = 1;
 499
 500         grp->fpg_sz = bs;
 501         grp->csecs = bs;
 502         grp->trdt = 25000;
 503         grp->trdm = 25000;
 504         grp->tprt = 500000;
 505         grp->tprm = 500000;
 506         grp->tbet = 1500000;
 507         grp->tbem = 1500000;
 508         grp->mpos = 0x010101; /* single plane rwe */
 509         grp->cpar = hw_queue_depth;
 510
 511         return 0;
 512 }
 513
 514 static void *null_lnvm_create_dma_pool(struct nvm_dev *dev, char *name)
 515 {
 516         mempool_t *virtmem_pool;
 517
 518         virtmem_pool = mempool_create_slab_pool(64, ppa_cache);
 519         if (!virtmem_pool) {
 520                 pr_err("null_blk: Unable to create virtual memory pool\n");
 521                 return NULL;
 522         }
 523
 524         return virtmem_pool;
 525 }
 526
 527 static void null_lnvm_destroy_dma_pool(void *pool)
 528 {
 529         mempool_destroy(pool);
 530 }
 531
 532 static void *null_lnvm_dev_dma_alloc(struct nvm_dev *dev, void *pool,
 533                                 gfp_t mem_flags, dma_addr_t *dma_handler)
 534 {
 535         return mempool_alloc(pool, mem_flags);
 536 }
 537
 538 static void null_lnvm_dev_dma_free(void *pool, void *entry,
 539                                                         dma_addr_t dma_handler)
 540 {
 541         mempool_free(entry, pool);
 542 }
 543
 544 static struct nvm_dev_ops null_lnvm_dev_ops = {
 545         .identity               = null_lnvm_id,
 546         .submit_io              = null_lnvm_submit_io,
 547
 548         .create_dma_pool        = null_lnvm_create_dma_pool,
 549         .destroy_dma_pool       = null_lnvm_destroy_dma_pool,
 550         .dev_dma_alloc          = null_lnvm_dev_dma_alloc,
 551         .dev_dma_free           = null_lnvm_dev_dma_free,
 552
 553         /* Simulate nvme protocol restriction */
 554         .max_phys_sect          = 64,
 555 };
 556
 557 static int null_nvm_register(struct nullb *nullb)
 558 {
 559         struct nvm_dev *dev;
 560         int rv;
 561
 562         dev = nvm_alloc_dev(0);
 563         if (!dev)
 564                 return -ENOMEM;
 565
 566         dev->q = nullb->q;
 567         memcpy(dev->name, nullb->disk_name, DISK_NAME_LEN);
 568         dev->ops = &null_lnvm_dev_ops;
 569
 570         rv = nvm_register(dev);
 571         if (rv) {
 572                 kfree(dev);
 573                 return rv;
 574         }
 575         nullb->ndev = dev;
 576         return 0;
 577 }
 578
 579 static void null_nvm_unregister(struct nullb *nullb)
 580 {
 581         nvm_unregister(nullb->ndev);
 582 }
 583 #else
 584 static int null_nvm_register(struct nullb *nullb)
 585 {
 586         pr_err("null_blk: CONFIG_NVM needs to be enabled for LightNVM\n");
 587         return -EINVAL;
 588 }
 589 static void null_nvm_unregister(struct nullb *nullb) {}
 590 #endif /* CONFIG_NVM */
 591
 592 static void null_del_dev(struct nullb *nullb)
 593 {
 594         list_del_init(&nullb->list);
 595
 596         if (use_lightnvm)
 597                 null_nvm_unregister(nullb);
 598         else
 599                 del_gendisk(nullb->disk);
 600         blk_cleanup_queue(nullb->q);
 601         if (queue_mode == NULL_Q_MQ)
 602                 blk_mq_free_tag_set(&nullb->tag_set);
 603         if (!use_lightnvm)
 604                 put_disk(nullb->disk);
 605         cleanup_queues(nullb);
 606         kfree(nullb);
 607 }
 608
 609 static int null_open(struct block_device *bdev, fmode_t mode)
 610 {
 611         return 0;
 612 }
 613
 614 static void null_release(struct gendisk *disk, fmode_t mode)
 615 {
 616 }
 617
 618 static const struct block_device_operations null_fops = {
 619         .owner =        THIS_MODULE,
 620         .open =         null_open,
 621         .release =      null_release,
 622 };
 623
 624 static int setup_commands(struct nullb_queue *nq)
 625 {
 626         struct nullb_cmd *cmd;
 627         int i, tag_size;
 628
 629         nq->cmds = kzalloc(nq->queue_depth * sizeof(*cmd), GFP_KERNEL);
 630         if (!nq->cmds)
 631                 return -ENOMEM;
 632
 633         tag_size = ALIGN(nq->queue_depth, BITS_PER_LONG) / BITS_PER_LONG;
 634         nq->tag_map = kzalloc(tag_size * sizeof(unsigned long), GFP_KERNEL);
 635         if (!nq->tag_map) {
 636                 kfree(nq->cmds);
 637                 return -ENOMEM;
 638         }
 639
 640         for (i = 0; i < nq->queue_depth; i++) {
 641                 cmd = &nq->cmds[i];
 642                 INIT_LIST_HEAD(&cmd->list);
 643                 cmd->ll_list.next = NULL;
 644                 cmd->tag = -1U;
 645         }
 646
 647         return 0;
 648 }
 649
 650 static int setup_queues(struct nullb *nullb)
 651 {
 652         nullb->queues = kzalloc(submit_queues * sizeof(struct nullb_queue),
 653                                                                 GFP_KERNEL);
 654         if (!nullb->queues)
 655                 return -ENOMEM;
 656
 657         nullb->nr_queues = 0;
 658         nullb->queue_depth = hw_queue_depth;
 659
 660         return 0;
 661 }
 662
 663 static int init_driver_queues(struct nullb *nullb)
 664 {
 665         struct nullb_queue *nq;
 666         int i, ret = 0;
 667
 668         for (i = 0; i < submit_queues; i++) {
 669                 nq = &nullb->queues[i];
 670
 671                 null_init_queue(nullb, nq);
 672
 673                 ret = setup_commands(nq);
 674                 if (ret)
 675                         return ret;
 676                 nullb->nr_queues++;
 677         }
 678         return 0;
 679 }
 680
 681 static int null_gendisk_register(struct nullb *nullb)
 682 {
 683         struct gendisk *disk;
 684         sector_t size;
 685
 686         disk = nullb->disk = alloc_disk_node(1, home_node);
 687         if (!disk)
 688                 return -ENOMEM;
 689         size = gb * 1024 * 1024 * 1024ULL;
 690         set_capacity(disk, size >> 9);
 691
 692         disk->flags |= GENHD_FL_EXT_DEVT | GENHD_FL_SUPPRESS_PARTITION_INFO;
 693         disk->major             = null_major;
 694         disk->first_minor       = nullb->index;
 695         disk->fops              = &null_fops;
 696         disk->private_data      = nullb;
 697         disk->queue             = nullb->q;
 698         strncpy(disk->disk_name, nullb->disk_name, DISK_NAME_LEN);
 699
 700         add_disk(disk);
 701         return 0;
 702 }
 703
 704 static int null_add_dev(void)
 705 {
 706         struct nullb *nullb;
 707         int rv;
 708
 709         nullb = kzalloc_node(sizeof(*nullb), GFP_KERNEL, home_node);
 710         if (!nullb) {
 711                 rv = -ENOMEM;
 712                 goto out;
 713         }
 714
 715         spin_lock_init(&nullb->lock);
 716
 717         if (queue_mode == NULL_Q_MQ && use_per_node_hctx)
 718                 submit_queues = nr_online_nodes;
 719
 720         rv = setup_queues(nullb);
 721         if (rv)
 722                 goto out_free_nullb;
 723
 724         if (queue_mode == NULL_Q_MQ) {
 725                 nullb->tag_set.ops = &null_mq_ops;
 726                 nullb->tag_set.nr_hw_queues = submit_queues;
 727                 nullb->tag_set.queue_depth = hw_queue_depth;
 728                 nullb->tag_set.numa_node = home_node;
 729                 nullb->tag_set.cmd_size = sizeof(struct nullb_cmd);
 730                 nullb->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
 731                 nullb->tag_set.driver_data = nullb;
 732
 733                 if (blocking)
 734                         nullb->tag_set.flags |= BLK_MQ_F_BLOCKING;
 735
 736                 rv = blk_mq_alloc_tag_set(&nullb->tag_set);
 737                 if (rv)
 738                         goto out_cleanup_queues;
 739
 740                 nullb->q = blk_mq_init_queue(&nullb->tag_set);
 741                 if (IS_ERR(nullb->q)) {
 742                         rv = -ENOMEM;
 743                         goto out_cleanup_tags;
 744                 }
 745         } else if (queue_mode == NULL_Q_BIO) {
 746                 nullb->q = blk_alloc_queue_node(GFP_KERNEL, home_node);
 747                 if (!nullb->q) {
 748                         rv = -ENOMEM;
 749                         goto out_cleanup_queues;
 750                 }
 751                 blk_queue_make_request(nullb->q, null_queue_bio);
 752                 rv = init_driver_queues(nullb);
 753                 if (rv)
 754                         goto out_cleanup_blk_queue;
 755         } else {
 756                 nullb->q = blk_init_queue_node(null_request_fn, &nullb->lock, home_node);
 757                 if (!nullb->q) {
 758                         rv = -ENOMEM;
 759                         goto out_cleanup_queues;
 760                 }
 761                 blk_queue_prep_rq(nullb->q, null_rq_prep_fn);
 762                 blk_queue_softirq_done(nullb->q, null_softirq_done_fn);
 763                 rv = init_driver_queues(nullb);
 764                 if (rv)
 765                         goto out_cleanup_blk_queue;
 766         }
 767
 768         nullb->q->queuedata = nullb;
 769         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, nullb->q);
 770         queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, nullb->q);
 771
 772         mutex_lock(&lock);
 773         nullb->index = nullb_indexes++;
 774         mutex_unlock(&lock);
 775
 776         blk_queue_logical_block_size(nullb->q, bs);
 777         blk_queue_physical_block_size(nullb->q, bs);
 778
 779         sprintf(nullb->disk_name, "nullb%d", nullb->index);
 780
 781         if (use_lightnvm)
 782                 rv = null_nvm_register(nullb);
 783         else
 784                 rv = null_gendisk_register(nullb);
 785
 786         if (rv)
 787                 goto out_cleanup_blk_queue;
 788
 789         mutex_lock(&lock);
 790         list_add_tail(&nullb->list, &nullb_list);
 791         mutex_unlock(&lock);
 792
 793         return 0;
 794 out_cleanup_blk_queue:
 795         blk_cleanup_queue(nullb->q);
 796 out_cleanup_tags:
 797         if (queue_mode == NULL_Q_MQ)
 798                 blk_mq_free_tag_set(&nullb->tag_set);
 799 out_cleanup_queues:
 800         cleanup_queues(nullb);
 801 out_free_nullb:
 802         kfree(nullb);
 803 out:
 804         return rv;
 805 }
 806
 807 static int __init null_init(void)
 808 {
 809         int ret = 0;
 810         unsigned int i;
 811         struct nullb *nullb;
 812
 813         if (bs > PAGE_SIZE) {
 814                 pr_warn("null_blk: invalid block size\n");
 815                 pr_warn("null_blk: defaults block size to %lu\n", PAGE_SIZE);
 816                 bs = PAGE_SIZE;
 817         }
 818
 819         if (use_lightnvm && bs != 4096) {
 820                 pr_warn("null_blk: LightNVM only supports 4k block size\n");
 821                 pr_warn("null_blk: defaults block size to 4k\n");
 822                 bs = 4096;
 823         }
 824
 825         if (use_lightnvm && queue_mode != NULL_Q_MQ) {
 826                 pr_warn("null_blk: LightNVM only supported for blk-mq\n");
 827                 pr_warn("null_blk: defaults queue mode to blk-mq\n");
 828                 queue_mode = NULL_Q_MQ;
 829         }
 830
 831         if (queue_mode == NULL_Q_MQ && use_per_node_hctx) {
 832                 if (submit_queues < nr_online_nodes) {
 833                         pr_warn("null_blk: submit_queues param is set to %u.",
 834                                                         nr_online_nodes);
 835                         submit_queues = nr_online_nodes;
 836                 }
 837         } else if (submit_queues > nr_cpu_ids)
 838                 submit_queues = nr_cpu_ids;
 839         else if (!submit_queues)
 840                 submit_queues = 1;
 841
 842         mutex_init(&lock);
 843
 844         null_major = register_blkdev(0, "nullb");
 845         if (null_major < 0)
 846                 return null_major;
 847
 848         if (use_lightnvm) {
 849                 ppa_cache = kmem_cache_create("ppa_cache", 64 * sizeof(u64),
 850                                                                 0, 0, NULL);
 851                 if (!ppa_cache) {
 852                         pr_err("null_blk: unable to create ppa cache\n");
 853                         ret = -ENOMEM;
 854                         goto err_ppa;
 855                 }
 856         }
 857
 858         for (i = 0; i < nr_devices; i++) {
 859                 ret = null_add_dev();
 860                 if (ret)
 861                         goto err_dev;
 862         }
 863
 864         pr_info("null: module loaded\n");
 865         return 0;
 866
 867 err_dev:
 868         while (!list_empty(&nullb_list)) {
 869                 nullb = list_entry(nullb_list.next, struct nullb, list);
 870                 null_del_dev(nullb);
 871         }
 872         kmem_cache_destroy(ppa_cache);
 873 err_ppa:
 874         unregister_blkdev(null_major, "nullb");
 875         return ret;
 876 }
 877
 878 static void __exit null_exit(void)
 879 {
 880         struct nullb *nullb;
 881
 882         unregister_blkdev(null_major, "nullb");
 883
 884         mutex_lock(&lock);
 885         while (!list_empty(&nullb_list)) {
 886                 nullb = list_entry(nullb_list.next, struct nullb, list);
 887                 null_del_dev(nullb);
 888         }
 889         mutex_unlock(&lock);
 890
 891         kmem_cache_destroy(ppa_cache);
 892 }
 893
 894 module_init(null_init);
 895 module_exit(null_exit);
 896
 897 MODULE_AUTHOR("Jens Axboe <jaxboe@fusionio.com>");
 898 MODULE_LICENSE("GPL");