drivers/block/brd.c

   1 /*
   2  * Ram backed block device driver.
   3  *
   4  * Copyright (C) 2007 Nick Piggin
   5  * Copyright (C) 2007 Novell Inc.
   6  *
   7  * Parts derived from drivers/block/rd.c, and drivers/block/loop.c, copyright
   8  * of their respective owners.
   9  */
  10
  11 #include <linux/init.h>
  12 #include <linux/module.h>
  13 #include <linux/moduleparam.h>
  14 #include <linux/major.h>
  15 #include <linux/blkdev.h>
  16 #include <linux/bio.h>
  17 #include <linux/highmem.h>
  18 #include <linux/mutex.h>
  19 #include <linux/radix-tree.h>
  20 #include <linux/fs.h>
  21 #include <linux/slab.h>
  22 #ifdef CONFIG_BLK_DEV_RAM_DAX
  23 #include <linux/pfn_t.h>
  24 #include <linux/dax.h>
  25 #endif
  26
  27 #include <linux/uaccess.h>
  28
  29 #define SECTOR_SHIFT            9
  30 #define PAGE_SECTORS_SHIFT      (PAGE_SHIFT - SECTOR_SHIFT)
  31 #define PAGE_SECTORS            (1 << PAGE_SECTORS_SHIFT)
  32
  33 /*
  34  * Each block ramdisk device has a radix_tree brd_pages of pages that stores
  35  * the pages containing the block device's contents. A brd page's ->index is
  36  * its offset in PAGE_SIZE units. This is similar to, but in no way connected
  37  * with, the kernel's pagecache or buffer cache (which sit above our block
  38  * device).
  39  */
  40 struct brd_device {
  41         int             brd_number;
  42
  43         struct request_queue    *brd_queue;
  44         struct gendisk          *brd_disk;
  45 #ifdef CONFIG_BLK_DEV_RAM_DAX
  46         struct dax_device       *dax_dev;
  47 #endif
  48         struct list_head        brd_list;
  49
  50         /*
  51          * Backing store of pages and lock to protect it. This is the contents
  52          * of the block device.
  53          */
  54         spinlock_t              brd_lock;
  55         struct radix_tree_root  brd_pages;
  56 };
  57
  58 /*
  59  * Look up and return a brd's page for a given sector.
  60  */
  61 static DEFINE_MUTEX(brd_mutex);
  62 static struct page *brd_lookup_page(struct brd_device *brd, sector_t sector)
  63 {
  64         pgoff_t idx;
  65         struct page *page;
  66
  67         /*
  68          * The page lifetime is protected by the fact that we have opened the
  69          * device node -- brd pages will never be deleted under us, so we
  70          * don't need any further locking or refcounting.
  71          *
  72          * This is strictly true for the radix-tree nodes as well (ie. we
  73          * don't actually need the rcu_read_lock()), however that is not a
  74          * documented feature of the radix-tree API so it is better to be
  75          * safe here (we don't have total exclusion from radix tree updates
  76          * here, only deletes).
  77          */
  78         rcu_read_lock();
  79         idx = sector >> PAGE_SECTORS_SHIFT; /* sector to page index */
  80         page = radix_tree_lookup(&brd->brd_pages, idx);
  81         rcu_read_unlock();
  82
  83         BUG_ON(page && page->index != idx);
  84
  85         return page;
  86 }
  87
  88 /*
  89  * Look up and return a brd's page for a given sector.
  90  * If one does not exist, allocate an empty page, and insert that. Then
  91  * return it.
  92  */
  93 static struct page *brd_insert_page(struct brd_device *brd, sector_t sector)
  94 {
  95         pgoff_t idx;
  96         struct page *page;
  97         gfp_t gfp_flags;
  98
  99         page = brd_lookup_page(brd, sector);
 100         if (page)
 101                 return page;
 102
 103         /*
 104          * Must use NOIO because we don't want to recurse back into the
 105          * block or filesystem layers from page reclaim.
 106          *
 107          * Cannot support DAX and highmem, because our ->direct_access
 108          * routine for DAX must return memory that is always addressable.
 109          * If DAX was reworked to use pfns and kmap throughout, this
 110          * restriction might be able to be lifted.
 111          */
 112         gfp_flags = GFP_NOIO | __GFP_ZERO;
 113 #ifndef CONFIG_BLK_DEV_RAM_DAX
 114         gfp_flags |= __GFP_HIGHMEM;
 115 #endif
 116         page = alloc_page(gfp_flags);
 117         if (!page)
 118                 return NULL;
 119
 120         if (radix_tree_preload(GFP_NOIO)) {
 121                 __free_page(page);
 122                 return NULL;
 123         }
 124
 125         spin_lock(&brd->brd_lock);
 126         idx = sector >> PAGE_SECTORS_SHIFT;
 127         page->index = idx;
 128         if (radix_tree_insert(&brd->brd_pages, idx, page)) {
 129                 __free_page(page);
 130                 page = radix_tree_lookup(&brd->brd_pages, idx);
 131                 BUG_ON(!page);
 132                 BUG_ON(page->index != idx);
 133         }
 134         spin_unlock(&brd->brd_lock);
 135
 136         radix_tree_preload_end();
 137
 138         return page;
 139 }
 140
 141 static void brd_free_page(struct brd_device *brd, sector_t sector)
 142 {
 143         struct page *page;
 144         pgoff_t idx;
 145
 146         spin_lock(&brd->brd_lock);
 147         idx = sector >> PAGE_SECTORS_SHIFT;
 148         page = radix_tree_delete(&brd->brd_pages, idx);
 149         spin_unlock(&brd->brd_lock);
 150         if (page)
 151                 __free_page(page);
 152 }
 153
 154 static void brd_zero_page(struct brd_device *brd, sector_t sector)
 155 {
 156         struct page *page;
 157
 158         page = brd_lookup_page(brd, sector);
 159         if (page)
 160                 clear_highpage(page);
 161 }
 162
 163 /*
 164  * Free all backing store pages and radix tree. This must only be called when
 165  * there are no other users of the device.
 166  */
 167 #define FREE_BATCH 16
 168 static void brd_free_pages(struct brd_device *brd)
 169 {
 170         unsigned long pos = 0;
 171         struct page *pages[FREE_BATCH];
 172         int nr_pages;
 173
 174         do {
 175                 int i;
 176
 177                 nr_pages = radix_tree_gang_lookup(&brd->brd_pages,
 178                                 (void **)pages, pos, FREE_BATCH);
 179
 180                 for (i = 0; i < nr_pages; i++) {
 181                         void *ret;
 182
 183                         BUG_ON(pages[i]->index < pos);
 184                         pos = pages[i]->index;
 185                         ret = radix_tree_delete(&brd->brd_pages, pos);
 186                         BUG_ON(!ret || ret != pages[i]);
 187                         __free_page(pages[i]);
 188                 }
 189
 190                 pos++;
 191
 192                 /*
 193                  * This assumes radix_tree_gang_lookup always returns as
 194                  * many pages as possible. If the radix-tree code changes,
 195                  * so will this have to.
 196                  */
 197         } while (nr_pages == FREE_BATCH);
 198 }
 199
 200 /*
 201  * copy_to_brd_setup must be called before copy_to_brd. It may sleep.
 202  */
 203 static int copy_to_brd_setup(struct brd_device *brd, sector_t sector, size_t n)
 204 {
 205         unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT;
 206         size_t copy;
 207
 208         copy = min_t(size_t, n, PAGE_SIZE - offset);
 209         if (!brd_insert_page(brd, sector))
 210                 return -ENOSPC;
 211         if (copy < n) {
 212                 sector += copy >> SECTOR_SHIFT;
 213                 if (!brd_insert_page(brd, sector))
 214                         return -ENOSPC;
 215         }
 216         return 0;
 217 }
 218
 219 static void discard_from_brd(struct brd_device *brd,
 220                         sector_t sector, size_t n)
 221 {
 222         while (n >= PAGE_SIZE) {
 223                 /*
 224                  * Don't want to actually discard pages here because
 225                  * re-allocating the pages can result in writeback
 226                  * deadlocks under heavy load.
 227                  */
 228                 if (0)
 229                         brd_free_page(brd, sector);
 230                 else
 231                         brd_zero_page(brd, sector);
 232                 sector += PAGE_SIZE >> SECTOR_SHIFT;
 233                 n -= PAGE_SIZE;
 234         }
 235 }
 236
 237 /*
 238  * Copy n bytes from src to the brd starting at sector. Does not sleep.
 239  */
 240 static void copy_to_brd(struct brd_device *brd, const void *src,
 241                         sector_t sector, size_t n)
 242 {
 243         struct page *page;
 244         void *dst;
 245         unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT;
 246         size_t copy;
 247
 248         copy = min_t(size_t, n, PAGE_SIZE - offset);
 249         page = brd_lookup_page(brd, sector);
 250         BUG_ON(!page);
 251
 252         dst = kmap_atomic(page);
 253         memcpy(dst + offset, src, copy);
 254         kunmap_atomic(dst);
 255
 256         if (copy < n) {
 257                 src += copy;
 258                 sector += copy >> SECTOR_SHIFT;
 259                 copy = n - copy;
 260                 page = brd_lookup_page(brd, sector);
 261                 BUG_ON(!page);
 262
 263                 dst = kmap_atomic(page);
 264                 memcpy(dst, src, copy);
 265                 kunmap_atomic(dst);
 266         }
 267 }
 268
 269 /*
 270  * Copy n bytes to dst from the brd starting at sector. Does not sleep.
 271  */
 272 static void copy_from_brd(void *dst, struct brd_device *brd,
 273                         sector_t sector, size_t n)
 274 {
 275         struct page *page;
 276         void *src;
 277         unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT;
 278         size_t copy;
 279
 280         copy = min_t(size_t, n, PAGE_SIZE - offset);
 281         page = brd_lookup_page(brd, sector);
 282         if (page) {
 283                 src = kmap_atomic(page);
 284                 memcpy(dst, src + offset, copy);
 285                 kunmap_atomic(src);
 286         } else
 287                 memset(dst, 0, copy);
 288
 289         if (copy < n) {
 290                 dst += copy;
 291                 sector += copy >> SECTOR_SHIFT;
 292                 copy = n - copy;
 293                 page = brd_lookup_page(brd, sector);
 294                 if (page) {
 295                         src = kmap_atomic(page);
 296                         memcpy(dst, src, copy);
 297                         kunmap_atomic(src);
 298                 } else
 299                         memset(dst, 0, copy);
 300         }
 301 }
 302
 303 /*
 304  * Process a single bvec of a bio.
 305  */
 306 static int brd_do_bvec(struct brd_device *brd, struct page *page,
 307                         unsigned int len, unsigned int off, bool is_write,
 308                         sector_t sector)
 309 {
 310         void *mem;
 311         int err = 0;
 312
 313         if (is_write) {
 314                 err = copy_to_brd_setup(brd, sector, len);
 315                 if (err)
 316                         goto out;
 317         }
 318
 319         mem = kmap_atomic(page);
 320         if (!is_write) {
 321                 copy_from_brd(mem + off, brd, sector, len);
 322                 flush_dcache_page(page);
 323         } else {
 324                 flush_dcache_page(page);
 325                 copy_to_brd(brd, mem + off, sector, len);
 326         }
 327         kunmap_atomic(mem);
 328
 329 out:
 330         return err;
 331 }
 332
 333 static blk_qc_t brd_make_request(struct request_queue *q, struct bio *bio)
 334 {
 335         struct block_device *bdev = bio->bi_bdev;
 336         struct brd_device *brd = bdev->bd_disk->private_data;
 337         struct bio_vec bvec;
 338         sector_t sector;
 339         struct bvec_iter iter;
 340
 341         sector = bio->bi_iter.bi_sector;
 342         if (bio_end_sector(bio) > get_capacity(bdev->bd_disk))
 343                 goto io_error;
 344
 345         if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
 346                 if (sector & ((PAGE_SIZE >> SECTOR_SHIFT) - 1) ||
 347                     bio->bi_iter.bi_size & ~PAGE_MASK)
 348                         goto io_error;
 349                 discard_from_brd(brd, sector, bio->bi_iter.bi_size);
 350                 goto out;
 351         }
 352
 353         bio_for_each_segment(bvec, bio, iter) {
 354                 unsigned int len = bvec.bv_len;
 355                 int err;
 356
 357                 err = brd_do_bvec(brd, bvec.bv_page, len, bvec.bv_offset,
 358                                         op_is_write(bio_op(bio)), sector);
 359                 if (err)
 360                         goto io_error;
 361                 sector += len >> SECTOR_SHIFT;
 362         }
 363
 364 out:
 365         bio_endio(bio);
 366         return BLK_QC_T_NONE;
 367 io_error:
 368         bio_io_error(bio);
 369         return BLK_QC_T_NONE;
 370 }
 371
 372 static int brd_rw_page(struct block_device *bdev, sector_t sector,
 373                        struct page *page, bool is_write)
 374 {
 375         struct brd_device *brd = bdev->bd_disk->private_data;
 376         int err = brd_do_bvec(brd, page, PAGE_SIZE, 0, is_write, sector);
 377         page_endio(page, is_write, err);
 378         return err;
 379 }
 380
 381 #ifdef CONFIG_BLK_DEV_RAM_DAX
 382 static long __brd_direct_access(struct brd_device *brd, pgoff_t pgoff,
 383                 long nr_pages, void **kaddr, pfn_t *pfn)
 384 {
 385         struct page *page;
 386
 387         if (!brd)
 388                 return -ENODEV;
 389         page = brd_insert_page(brd, PFN_PHYS(pgoff) / 512);
 390         if (!page)
 391                 return -ENOSPC;
 392         *kaddr = page_address(page);
 393         *pfn = page_to_pfn_t(page);
 394
 395         return 1;
 396 }
 397
 398 static long brd_dax_direct_access(struct dax_device *dax_dev,
 399                 pgoff_t pgoff, long nr_pages, void **kaddr, pfn_t *pfn)
 400 {
 401         struct brd_device *brd = dax_get_private(dax_dev);
 402
 403         return __brd_direct_access(brd, pgoff, nr_pages, kaddr, pfn);
 404 }
 405
 406 static const struct dax_operations brd_dax_ops = {
 407         .direct_access = brd_dax_direct_access,
 408 };
 409 #endif
 410
 411 static const struct block_device_operations brd_fops = {
 412         .owner =                THIS_MODULE,
 413         .rw_page =              brd_rw_page,
 414 };
 415
 416 /*
 417  * And now the modules code and kernel interface.
 418  */
 419 static int rd_nr = CONFIG_BLK_DEV_RAM_COUNT;
 420 module_param(rd_nr, int, S_IRUGO);
 421 MODULE_PARM_DESC(rd_nr, "Maximum number of brd devices");
 422
 423 unsigned long rd_size = CONFIG_BLK_DEV_RAM_SIZE;
 424 module_param(rd_size, ulong, S_IRUGO);
 425 MODULE_PARM_DESC(rd_size, "Size of each RAM disk in kbytes.");
 426
 427 static int max_part = 1;
 428 module_param(max_part, int, S_IRUGO);
 429 MODULE_PARM_DESC(max_part, "Num Minors to reserve between devices");
 430
 431 MODULE_LICENSE("GPL");
 432 MODULE_ALIAS_BLOCKDEV_MAJOR(RAMDISK_MAJOR);
 433 MODULE_ALIAS("rd");
 434
 435 #ifndef MODULE
 436 /* Legacy boot options - nonmodular */
 437 static int __init ramdisk_size(char *str)
 438 {
 439         rd_size = simple_strtol(str, NULL, 0);
 440         return 1;
 441 }
 442 __setup("ramdisk_size=", ramdisk_size);
 443 #endif
 444
 445 /*
 446  * The device scheme is derived from loop.c. Keep them in synch where possible
 447  * (should share code eventually).
 448  */
 449 static LIST_HEAD(brd_devices);
 450 static DEFINE_MUTEX(brd_devices_mutex);
 451
 452 static struct brd_device *brd_alloc(int i)
 453 {
 454         struct brd_device *brd;
 455         struct gendisk *disk;
 456
 457         brd = kzalloc(sizeof(*brd), GFP_KERNEL);
 458         if (!brd)
 459                 goto out;
 460         brd->brd_number         = i;
 461         spin_lock_init(&brd->brd_lock);
 462         INIT_RADIX_TREE(&brd->brd_pages, GFP_ATOMIC);
 463
 464         brd->brd_queue = blk_alloc_queue(GFP_KERNEL);
 465         if (!brd->brd_queue)
 466                 goto out_free_dev;
 467
 468         blk_queue_make_request(brd->brd_queue, brd_make_request);
 469         blk_queue_max_hw_sectors(brd->brd_queue, 1024);
 470         blk_queue_bounce_limit(brd->brd_queue, BLK_BOUNCE_ANY);
 471
 472         /* This is so fdisk will align partitions on 4k, because of
 473          * direct_access API needing 4k alignment, returning a PFN
 474          * (This is only a problem on very small devices <= 4M,
 475          *  otherwise fdisk will align on 1M. Regardless this call
 476          *  is harmless)
 477          */
 478         blk_queue_physical_block_size(brd->brd_queue, PAGE_SIZE);
 479
 480         brd->brd_queue->limits.discard_granularity = PAGE_SIZE;
 481         blk_queue_max_discard_sectors(brd->brd_queue, UINT_MAX);
 482         brd->brd_queue->limits.discard_zeroes_data = 1;
 483         queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, brd->brd_queue);
 484         disk = brd->brd_disk = alloc_disk(max_part);
 485         if (!disk)
 486                 goto out_free_queue;
 487         disk->major             = RAMDISK_MAJOR;
 488         disk->first_minor       = i * max_part;
 489         disk->fops              = &brd_fops;
 490         disk->private_data      = brd;
 491         disk->queue             = brd->brd_queue;
 492         disk->flags             = GENHD_FL_EXT_DEVT;
 493         sprintf(disk->disk_name, "ram%d", i);
 494         set_capacity(disk, rd_size * 2);
 495
 496 #ifdef CONFIG_BLK_DEV_RAM_DAX
 497         queue_flag_set_unlocked(QUEUE_FLAG_DAX, brd->brd_queue);
 498         brd->dax_dev = alloc_dax(brd, disk->disk_name, &brd_dax_ops);
 499         if (!brd->dax_dev)
 500                 goto out_free_inode;
 501 #endif
 502
 503
 504         return brd;
 505
 506 #ifdef CONFIG_BLK_DEV_RAM_DAX
 507 out_free_inode:
 508         kill_dax(brd->dax_dev);
 509         put_dax(brd->dax_dev);
 510 #endif
 511 out_free_queue:
 512         blk_cleanup_queue(brd->brd_queue);
 513 out_free_dev:
 514         kfree(brd);
 515 out:
 516         return NULL;
 517 }
 518
 519 static void brd_free(struct brd_device *brd)
 520 {
 521         put_disk(brd->brd_disk);
 522         blk_cleanup_queue(brd->brd_queue);
 523         brd_free_pages(brd);
 524         kfree(brd);
 525 }
 526
 527 static struct brd_device *brd_init_one(int i, bool *new)
 528 {
 529         struct brd_device *brd;
 530
 531         *new = false;
 532         list_for_each_entry(brd, &brd_devices, brd_list) {
 533                 if (brd->brd_number == i)
 534                         goto out;
 535         }
 536
 537         brd = brd_alloc(i);
 538         if (brd) {
 539                 add_disk(brd->brd_disk);
 540                 list_add_tail(&brd->brd_list, &brd_devices);
 541         }
 542         *new = true;
 543 out:
 544         return brd;
 545 }
 546
 547 static void brd_del_one(struct brd_device *brd)
 548 {
 549         list_del(&brd->brd_list);
 550 #ifdef CONFIG_BLK_DEV_RAM_DAX
 551         kill_dax(brd->dax_dev);
 552         put_dax(brd->dax_dev);
 553 #endif
 554         del_gendisk(brd->brd_disk);
 555         brd_free(brd);
 556 }
 557
 558 static struct kobject *brd_probe(dev_t dev, int *part, void *data)
 559 {
 560         struct brd_device *brd;
 561         struct kobject *kobj;
 562         bool new;
 563
 564         mutex_lock(&brd_devices_mutex);
 565         brd = brd_init_one(MINOR(dev) / max_part, &new);
 566         kobj = brd ? get_disk(brd->brd_disk) : NULL;
 567         mutex_unlock(&brd_devices_mutex);
 568
 569         if (new)
 570                 *part = 0;
 571
 572         return kobj;
 573 }
 574
 575 static int __init brd_init(void)
 576 {
 577         struct brd_device *brd, *next;
 578         int i;
 579
 580         /*
 581          * brd module now has a feature to instantiate underlying device
 582          * structure on-demand, provided that there is an access dev node.
 583          *
 584          * (1) if rd_nr is specified, create that many upfront. else
 585          *     it defaults to CONFIG_BLK_DEV_RAM_COUNT
 586          * (2) User can further extend brd devices by create dev node themselves
 587          *     and have kernel automatically instantiate actual device
 588          *     on-demand. Example:
 589          *              mknod /path/devnod_name b 1 X   # 1 is the rd major
 590          *              fdisk -l /path/devnod_name
 591          *      If (X / max_part) was not already created it will be created
 592          *      dynamically.
 593          */
 594
 595         if (register_blkdev(RAMDISK_MAJOR, "ramdisk"))
 596                 return -EIO;
 597
 598         if (unlikely(!max_part))
 599                 max_part = 1;
 600
 601         for (i = 0; i < rd_nr; i++) {
 602                 brd = brd_alloc(i);
 603                 if (!brd)
 604                         goto out_free;
 605                 list_add_tail(&brd->brd_list, &brd_devices);
 606         }
 607
 608         /* point of no return */
 609
 610         list_for_each_entry(brd, &brd_devices, brd_list)
 611                 add_disk(brd->brd_disk);
 612
 613         blk_register_region(MKDEV(RAMDISK_MAJOR, 0), 1UL << MINORBITS,
 614                                   THIS_MODULE, brd_probe, NULL, NULL);
 615
 616         pr_info("brd: module loaded\n");
 617         return 0;
 618
 619 out_free:
 620         list_for_each_entry_safe(brd, next, &brd_devices, brd_list) {
 621                 list_del(&brd->brd_list);
 622                 brd_free(brd);
 623         }
 624         unregister_blkdev(RAMDISK_MAJOR, "ramdisk");
 625
 626         pr_info("brd: module NOT loaded !!!\n");
 627         return -ENOMEM;
 628 }
 629
 630 static void __exit brd_exit(void)
 631 {
 632         struct brd_device *brd, *next;
 633
 634         list_for_each_entry_safe(brd, next, &brd_devices, brd_list)
 635                 brd_del_one(brd);
 636
 637         blk_unregister_region(MKDEV(RAMDISK_MAJOR, 0), 1UL << MINORBITS);
 638         unregister_blkdev(RAMDISK_MAJOR, "ramdisk");
 639
 640         pr_info("brd: module unloaded\n");
 641 }
 642
 643 module_init(brd_init);
 644 module_exit(brd_exit);
 645