drivers/s390/block/scm_blk.c

   1 /*
   2  * Block driver for s390 storage class memory.
   3  *
   4  * Copyright IBM Corp. 2012
   5  * Author(s): Sebastian Ott <sebott@linux.vnet.ibm.com>
   6  */
   7
   8 #define KMSG_COMPONENT "scm_block"
   9 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  10
  11 #include <linux/interrupt.h>
  12 #include <linux/spinlock.h>
  13 #include <linux/mempool.h>
  14 #include <linux/module.h>
  15 #include <linux/blkdev.h>
  16 #include <linux/blk-mq.h>
  17 #include <linux/genhd.h>
  18 #include <linux/slab.h>
  19 #include <linux/list.h>
  20 #include <asm/eadm.h>
  21 #include "scm_blk.h"
  22
  23 debug_info_t *scm_debug;
  24 static int scm_major;
  25 static mempool_t *aidaw_pool;
  26 static DEFINE_SPINLOCK(list_lock);
  27 static LIST_HEAD(inactive_requests);
  28 static unsigned int nr_requests = 64;
  29 static unsigned int nr_requests_per_io = 8;
  30 static atomic_t nr_devices = ATOMIC_INIT(0);
  31 module_param(nr_requests, uint, S_IRUGO);
  32 MODULE_PARM_DESC(nr_requests, "Number of parallel requests.");
  33
  34 module_param(nr_requests_per_io, uint, S_IRUGO);
  35 MODULE_PARM_DESC(nr_requests_per_io, "Number of requests per IO.");
  36
  37 MODULE_DESCRIPTION("Block driver for s390 storage class memory.");
  38 MODULE_LICENSE("GPL");
  39 MODULE_ALIAS("scm:scmdev*");
  40
  41 static void __scm_free_rq(struct scm_request *scmrq)
  42 {
  43         struct aob_rq_header *aobrq = to_aobrq(scmrq);
  44
  45         free_page((unsigned long) scmrq->aob);
  46         kfree(scmrq->request);
  47         kfree(aobrq);
  48 }
  49
  50 static void scm_free_rqs(void)
  51 {
  52         struct list_head *iter, *safe;
  53         struct scm_request *scmrq;
  54
  55         spin_lock_irq(&list_lock);
  56         list_for_each_safe(iter, safe, &inactive_requests) {
  57                 scmrq = list_entry(iter, struct scm_request, list);
  58                 list_del(&scmrq->list);
  59                 __scm_free_rq(scmrq);
  60         }
  61         spin_unlock_irq(&list_lock);
  62
  63         mempool_destroy(aidaw_pool);
  64 }
  65
  66 static int __scm_alloc_rq(void)
  67 {
  68         struct aob_rq_header *aobrq;
  69         struct scm_request *scmrq;
  70
  71         aobrq = kzalloc(sizeof(*aobrq) + sizeof(*scmrq), GFP_KERNEL);
  72         if (!aobrq)
  73                 return -ENOMEM;
  74
  75         scmrq = (void *) aobrq->data;
  76         scmrq->aob = (void *) get_zeroed_page(GFP_DMA);
  77         if (!scmrq->aob)
  78                 goto free;
  79
  80         scmrq->request = kcalloc(nr_requests_per_io, sizeof(scmrq->request[0]),
  81                                  GFP_KERNEL);
  82         if (!scmrq->request)
  83                 goto free;
  84
  85         INIT_LIST_HEAD(&scmrq->list);
  86         spin_lock_irq(&list_lock);
  87         list_add(&scmrq->list, &inactive_requests);
  88         spin_unlock_irq(&list_lock);
  89
  90         return 0;
  91 free:
  92         __scm_free_rq(scmrq);
  93         return -ENOMEM;
  94 }
  95
  96 static int scm_alloc_rqs(unsigned int nrqs)
  97 {
  98         int ret = 0;
  99
 100         aidaw_pool = mempool_create_page_pool(max(nrqs/8, 1U), 0);
 101         if (!aidaw_pool)
 102                 return -ENOMEM;
 103
 104         while (nrqs-- && !ret)
 105                 ret = __scm_alloc_rq();
 106
 107         return ret;
 108 }
 109
 110 static struct scm_request *scm_request_fetch(void)
 111 {
 112         struct scm_request *scmrq = NULL;
 113
 114         spin_lock_irq(&list_lock);
 115         if (list_empty(&inactive_requests))
 116                 goto out;
 117         scmrq = list_first_entry(&inactive_requests, struct scm_request, list);
 118         list_del(&scmrq->list);
 119 out:
 120         spin_unlock_irq(&list_lock);
 121         return scmrq;
 122 }
 123
 124 static void scm_request_done(struct scm_request *scmrq)
 125 {
 126         unsigned long flags;
 127         struct msb *msb;
 128         u64 aidaw;
 129         int i;
 130
 131         for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) {
 132                 msb = &scmrq->aob->msb[i];
 133                 aidaw = msb->data_addr;
 134
 135                 if ((msb->flags & MSB_FLAG_IDA) && aidaw &&
 136                     IS_ALIGNED(aidaw, PAGE_SIZE))
 137                         mempool_free(virt_to_page(aidaw), aidaw_pool);
 138         }
 139
 140         spin_lock_irqsave(&list_lock, flags);
 141         list_add(&scmrq->list, &inactive_requests);
 142         spin_unlock_irqrestore(&list_lock, flags);
 143 }
 144
 145 static bool scm_permit_request(struct scm_blk_dev *bdev, struct request *req)
 146 {
 147         return rq_data_dir(req) != WRITE || bdev->state != SCM_WR_PROHIBIT;
 148 }
 149
 150 static inline struct aidaw *scm_aidaw_alloc(void)
 151 {
 152         struct page *page = mempool_alloc(aidaw_pool, GFP_ATOMIC);
 153
 154         return page ? page_address(page) : NULL;
 155 }
 156
 157 static inline unsigned long scm_aidaw_bytes(struct aidaw *aidaw)
 158 {
 159         unsigned long _aidaw = (unsigned long) aidaw;
 160         unsigned long bytes = ALIGN(_aidaw, PAGE_SIZE) - _aidaw;
 161
 162         return (bytes / sizeof(*aidaw)) * PAGE_SIZE;
 163 }
 164
 165 struct aidaw *scm_aidaw_fetch(struct scm_request *scmrq, unsigned int bytes)
 166 {
 167         struct aidaw *aidaw;
 168
 169         if (scm_aidaw_bytes(scmrq->next_aidaw) >= bytes)
 170                 return scmrq->next_aidaw;
 171
 172         aidaw = scm_aidaw_alloc();
 173         if (aidaw)
 174                 memset(aidaw, 0, PAGE_SIZE);
 175         return aidaw;
 176 }
 177
 178 static int scm_request_prepare(struct scm_request *scmrq)
 179 {
 180         struct scm_blk_dev *bdev = scmrq->bdev;
 181         struct scm_device *scmdev = bdev->gendisk->private_data;
 182         int pos = scmrq->aob->request.msb_count;
 183         struct msb *msb = &scmrq->aob->msb[pos];
 184         struct request *req = scmrq->request[pos];
 185         struct req_iterator iter;
 186         struct aidaw *aidaw;
 187         struct bio_vec bv;
 188
 189         aidaw = scm_aidaw_fetch(scmrq, blk_rq_bytes(req));
 190         if (!aidaw)
 191                 return -ENOMEM;
 192
 193         msb->bs = MSB_BS_4K;
 194         scmrq->aob->request.msb_count++;
 195         msb->scm_addr = scmdev->address + ((u64) blk_rq_pos(req) << 9);
 196         msb->oc = (rq_data_dir(req) == READ) ? MSB_OC_READ : MSB_OC_WRITE;
 197         msb->flags |= MSB_FLAG_IDA;
 198         msb->data_addr = (u64) aidaw;
 199
 200         rq_for_each_segment(bv, req, iter) {
 201                 WARN_ON(bv.bv_offset);
 202                 msb->blk_count += bv.bv_len >> 12;
 203                 aidaw->data_addr = (u64) page_address(bv.bv_page);
 204                 aidaw++;
 205         }
 206
 207         scmrq->next_aidaw = aidaw;
 208         return 0;
 209 }
 210
 211 static inline void scm_request_set(struct scm_request *scmrq,
 212                                    struct request *req)
 213 {
 214         scmrq->request[scmrq->aob->request.msb_count] = req;
 215 }
 216
 217 static inline void scm_request_init(struct scm_blk_dev *bdev,
 218                                     struct scm_request *scmrq)
 219 {
 220         struct aob_rq_header *aobrq = to_aobrq(scmrq);
 221         struct aob *aob = scmrq->aob;
 222
 223         memset(scmrq->request, 0,
 224                nr_requests_per_io * sizeof(scmrq->request[0]));
 225         memset(aob, 0, sizeof(*aob));
 226         aobrq->scmdev = bdev->scmdev;
 227         aob->request.cmd_code = ARQB_CMD_MOVE;
 228         aob->request.data = (u64) aobrq;
 229         scmrq->bdev = bdev;
 230         scmrq->retries = 4;
 231         scmrq->error = 0;
 232         /* We don't use all msbs - place aidaws at the end of the aob page. */
 233         scmrq->next_aidaw = (void *) &aob->msb[nr_requests_per_io];
 234 }
 235
 236 static void scm_request_requeue(struct scm_request *scmrq)
 237 {
 238         struct scm_blk_dev *bdev = scmrq->bdev;
 239         int i;
 240
 241         for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++)
 242                 blk_mq_requeue_request(scmrq->request[i], false);
 243
 244         atomic_dec(&bdev->queued_reqs);
 245         scm_request_done(scmrq);
 246         blk_mq_kick_requeue_list(bdev->rq);
 247 }
 248
 249 static void scm_request_finish(struct scm_request *scmrq)
 250 {
 251         struct scm_blk_dev *bdev = scmrq->bdev;
 252         int i;
 253
 254         for (i = 0; i < nr_requests_per_io && scmrq->request[i]; i++) {
 255                 if (scmrq->error)
 256                         blk_mq_end_request(scmrq->request[i], scmrq->error);
 257                 else
 258                         blk_mq_complete_request(scmrq->request[i]);
 259         }
 260
 261         atomic_dec(&bdev->queued_reqs);
 262         scm_request_done(scmrq);
 263 }
 264
 265 static void scm_request_start(struct scm_request *scmrq)
 266 {
 267         struct scm_blk_dev *bdev = scmrq->bdev;
 268
 269         atomic_inc(&bdev->queued_reqs);
 270         if (eadm_start_aob(scmrq->aob)) {
 271                 SCM_LOG(5, "no subchannel");
 272                 scm_request_requeue(scmrq);
 273         }
 274 }
 275
 276 static int scm_blk_request(struct blk_mq_hw_ctx *hctx,
 277                            const struct blk_mq_queue_data *qd)
 278 {
 279         struct scm_device *scmdev = hctx->queue->queuedata;
 280         struct scm_blk_dev *bdev = dev_get_drvdata(&scmdev->dev);
 281         struct request *req = qd->rq;
 282         struct scm_request *scmrq;
 283
 284         spin_lock(&bdev->rq_lock);
 285         if (!scm_permit_request(bdev, req)) {
 286                 spin_unlock(&bdev->rq_lock);
 287                 return BLK_MQ_RQ_QUEUE_BUSY;
 288         }
 289
 290         scmrq = hctx->driver_data;
 291         if (!scmrq) {
 292                 scmrq = scm_request_fetch();
 293                 if (!scmrq) {
 294                         SCM_LOG(5, "no request");
 295                         spin_unlock(&bdev->rq_lock);
 296                         return BLK_MQ_RQ_QUEUE_BUSY;
 297                 }
 298                 scm_request_init(bdev, scmrq);
 299                 hctx->driver_data = scmrq;
 300         }
 301         scm_request_set(scmrq, req);
 302
 303         if (scm_request_prepare(scmrq)) {
 304                 SCM_LOG(5, "aidaw alloc failed");
 305                 scm_request_set(scmrq, NULL);
 306
 307                 if (scmrq->aob->request.msb_count)
 308                         scm_request_start(scmrq);
 309
 310                 hctx->driver_data = NULL;
 311                 spin_unlock(&bdev->rq_lock);
 312                 return BLK_MQ_RQ_QUEUE_BUSY;
 313         }
 314         blk_mq_start_request(req);
 315
 316         if (qd->last || scmrq->aob->request.msb_count == nr_requests_per_io) {
 317                 scm_request_start(scmrq);
 318                 hctx->driver_data = NULL;
 319         }
 320         spin_unlock(&bdev->rq_lock);
 321         return BLK_MQ_RQ_QUEUE_OK;
 322 }
 323
 324 static void __scmrq_log_error(struct scm_request *scmrq)
 325 {
 326         struct aob *aob = scmrq->aob;
 327
 328         if (scmrq->error == -ETIMEDOUT)
 329                 SCM_LOG(1, "Request timeout");
 330         else {
 331                 SCM_LOG(1, "Request error");
 332                 SCM_LOG_HEX(1, &aob->response, sizeof(aob->response));
 333         }
 334         if (scmrq->retries)
 335                 SCM_LOG(1, "Retry request");
 336         else
 337                 pr_err("An I/O operation to SCM failed with rc=%d\n",
 338                        scmrq->error);
 339 }
 340
 341 void scm_blk_irq(struct scm_device *scmdev, void *data, int error)
 342 {
 343         struct scm_request *scmrq = data;
 344         struct scm_blk_dev *bdev = scmrq->bdev;
 345
 346         scmrq->error = error;
 347         if (error)
 348                 __scmrq_log_error(scmrq);
 349
 350         spin_lock(&bdev->lock);
 351         list_add_tail(&scmrq->list, &bdev->finished_requests);
 352         spin_unlock(&bdev->lock);
 353         tasklet_hi_schedule(&bdev->tasklet);
 354 }
 355
 356 static void scm_blk_handle_error(struct scm_request *scmrq)
 357 {
 358         struct scm_blk_dev *bdev = scmrq->bdev;
 359         unsigned long flags;
 360
 361         if (scmrq->error != -EIO)
 362                 goto restart;
 363
 364         /* For -EIO the response block is valid. */
 365         switch (scmrq->aob->response.eqc) {
 366         case EQC_WR_PROHIBIT:
 367                 spin_lock_irqsave(&bdev->lock, flags);
 368                 if (bdev->state != SCM_WR_PROHIBIT)
 369                         pr_info("%lx: Write access to the SCM increment is suspended\n",
 370                                 (unsigned long) bdev->scmdev->address);
 371                 bdev->state = SCM_WR_PROHIBIT;
 372                 spin_unlock_irqrestore(&bdev->lock, flags);
 373                 goto requeue;
 374         default:
 375                 break;
 376         }
 377
 378 restart:
 379         if (!eadm_start_aob(scmrq->aob))
 380                 return;
 381
 382 requeue:
 383         scm_request_requeue(scmrq);
 384 }
 385
 386 static void scm_blk_tasklet(struct scm_blk_dev *bdev)
 387 {
 388         struct scm_request *scmrq;
 389         unsigned long flags;
 390
 391         spin_lock_irqsave(&bdev->lock, flags);
 392         while (!list_empty(&bdev->finished_requests)) {
 393                 scmrq = list_first_entry(&bdev->finished_requests,
 394                                          struct scm_request, list);
 395                 list_del(&scmrq->list);
 396                 spin_unlock_irqrestore(&bdev->lock, flags);
 397
 398                 if (scmrq->error && scmrq->retries-- > 0) {
 399                         scm_blk_handle_error(scmrq);
 400
 401                         /* Request restarted or requeued, handle next. */
 402                         spin_lock_irqsave(&bdev->lock, flags);
 403                         continue;
 404                 }
 405
 406                 scm_request_finish(scmrq);
 407                 spin_lock_irqsave(&bdev->lock, flags);
 408         }
 409         spin_unlock_irqrestore(&bdev->lock, flags);
 410 }
 411
 412 static const struct block_device_operations scm_blk_devops = {
 413         .owner = THIS_MODULE,
 414 };
 415
 416 static const struct blk_mq_ops scm_mq_ops = {
 417         .queue_rq = scm_blk_request,
 418 };
 419
 420 int scm_blk_dev_setup(struct scm_blk_dev *bdev, struct scm_device *scmdev)
 421 {
 422         unsigned int devindex, nr_max_blk;
 423         struct request_queue *rq;
 424         int len, ret;
 425
 426         devindex = atomic_inc_return(&nr_devices) - 1;
 427         /* scma..scmz + scmaa..scmzz */
 428         if (devindex > 701) {
 429                 ret = -ENODEV;
 430                 goto out;
 431         }
 432
 433         bdev->scmdev = scmdev;
 434         bdev->state = SCM_OPER;
 435         spin_lock_init(&bdev->rq_lock);
 436         spin_lock_init(&bdev->lock);
 437         INIT_LIST_HEAD(&bdev->finished_requests);
 438         atomic_set(&bdev->queued_reqs, 0);
 439         tasklet_init(&bdev->tasklet,
 440                      (void (*)(unsigned long)) scm_blk_tasklet,
 441                      (unsigned long) bdev);
 442
 443         bdev->tag_set.ops = &scm_mq_ops;
 444         bdev->tag_set.nr_hw_queues = 1;
 445         bdev->tag_set.queue_depth = nr_requests_per_io * nr_requests;
 446         bdev->tag_set.flags = BLK_MQ_F_SHOULD_MERGE;
 447
 448         ret = blk_mq_alloc_tag_set(&bdev->tag_set);
 449         if (ret)
 450                 goto out;
 451
 452         rq = blk_mq_init_queue(&bdev->tag_set);
 453         if (IS_ERR(rq)) {
 454                 ret = PTR_ERR(rq);
 455                 goto out_tag;
 456         }
 457         bdev->rq = rq;
 458         nr_max_blk = min(scmdev->nr_max_block,
 459                          (unsigned int) (PAGE_SIZE / sizeof(struct aidaw)));
 460
 461         blk_queue_logical_block_size(rq, 1 << 12);
 462         blk_queue_max_hw_sectors(rq, nr_max_blk << 3); /* 8 * 512 = blk_size */
 463         blk_queue_max_segments(rq, nr_max_blk);
 464         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, rq);
 465         queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, rq);
 466
 467         bdev->gendisk = alloc_disk(SCM_NR_PARTS);
 468         if (!bdev->gendisk) {
 469                 ret = -ENOMEM;
 470                 goto out_queue;
 471         }
 472         rq->queuedata = scmdev;
 473         bdev->gendisk->private_data = scmdev;
 474         bdev->gendisk->fops = &scm_blk_devops;
 475         bdev->gendisk->queue = rq;
 476         bdev->gendisk->major = scm_major;
 477         bdev->gendisk->first_minor = devindex * SCM_NR_PARTS;
 478
 479         len = snprintf(bdev->gendisk->disk_name, DISK_NAME_LEN, "scm");
 480         if (devindex > 25) {
 481                 len += snprintf(bdev->gendisk->disk_name + len,
 482                                 DISK_NAME_LEN - len, "%c",
 483                                 'a' + (devindex / 26) - 1);
 484                 devindex = devindex % 26;
 485         }
 486         snprintf(bdev->gendisk->disk_name + len, DISK_NAME_LEN - len, "%c",
 487                  'a' + devindex);
 488
 489         /* 512 byte sectors */
 490         set_capacity(bdev->gendisk, scmdev->size >> 9);
 491         device_add_disk(&scmdev->dev, bdev->gendisk);
 492         return 0;
 493
 494 out_queue:
 495         blk_cleanup_queue(rq);
 496 out_tag:
 497         blk_mq_free_tag_set(&bdev->tag_set);
 498 out:
 499         atomic_dec(&nr_devices);
 500         return ret;
 501 }
 502
 503 void scm_blk_dev_cleanup(struct scm_blk_dev *bdev)
 504 {
 505         tasklet_kill(&bdev->tasklet);
 506         del_gendisk(bdev->gendisk);
 507         blk_cleanup_queue(bdev->gendisk->queue);
 508         blk_mq_free_tag_set(&bdev->tag_set);
 509         put_disk(bdev->gendisk);
 510 }
 511
 512 void scm_blk_set_available(struct scm_blk_dev *bdev)
 513 {
 514         unsigned long flags;
 515
 516         spin_lock_irqsave(&bdev->lock, flags);
 517         if (bdev->state == SCM_WR_PROHIBIT)
 518                 pr_info("%lx: Write access to the SCM increment is restored\n",
 519                         (unsigned long) bdev->scmdev->address);
 520         bdev->state = SCM_OPER;
 521         spin_unlock_irqrestore(&bdev->lock, flags);
 522 }
 523
 524 static bool __init scm_blk_params_valid(void)
 525 {
 526         if (!nr_requests_per_io || nr_requests_per_io > 64)
 527                 return false;
 528
 529         return true;
 530 }
 531
 532 static int __init scm_blk_init(void)
 533 {
 534         int ret = -EINVAL;
 535
 536         if (!scm_blk_params_valid())
 537                 goto out;
 538
 539         ret = register_blkdev(0, "scm");
 540         if (ret < 0)
 541                 goto out;
 542
 543         scm_major = ret;
 544         ret = scm_alloc_rqs(nr_requests);
 545         if (ret)
 546                 goto out_free;
 547
 548         scm_debug = debug_register("scm_log", 16, 1, 16);
 549         if (!scm_debug) {
 550                 ret = -ENOMEM;
 551                 goto out_free;
 552         }
 553
 554         debug_register_view(scm_debug, &debug_hex_ascii_view);
 555         debug_set_level(scm_debug, 2);
 556
 557         ret = scm_drv_init();
 558         if (ret)
 559                 goto out_dbf;
 560
 561         return ret;
 562
 563 out_dbf:
 564         debug_unregister(scm_debug);
 565 out_free:
 566         scm_free_rqs();
 567         unregister_blkdev(scm_major, "scm");
 568 out:
 569         return ret;
 570 }
 571 module_init(scm_blk_init);
 572
 573 static void __exit scm_blk_cleanup(void)
 574 {
 575         scm_drv_cleanup();
 576         debug_unregister(scm_debug);
 577         scm_free_rqs();
 578         unregister_blkdev(scm_major, "scm");
 579 }
 580 module_exit(scm_blk_cleanup);