drivers/block/zram/zram_drv.c

   1 /*
   2  * Compressed RAM block device
   3  *
   4  * Copyright (C) 2008, 2009, 2010  Nitin Gupta
   5  *               2012, 2013 Minchan Kim
   6  *
   7  * This code is released using a dual license strategy: BSD/GPL
   8  * You can choose the licence that better fits your requirements.
   9  *
  10  * Released under the terms of 3-clause BSD License
  11  * Released under the terms of GNU General Public License Version 2.0
  12  *
  13  */
  14
  15 #define KMSG_COMPONENT "zram"
  16 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt
  17
  18 #ifdef CONFIG_ZRAM_DEBUG
  19 #define DEBUG
  20 #endif
  21
  22 #include <linux/module.h>
  23 #include <linux/kernel.h>
  24 #include <linux/bio.h>
  25 #include <linux/bitops.h>
  26 #include <linux/blkdev.h>
  27 #include <linux/buffer_head.h>
  28 #include <linux/device.h>
  29 #include <linux/genhd.h>
  30 #include <linux/highmem.h>
  31 #include <linux/slab.h>
  32 #include <linux/string.h>
  33 #include <linux/vmalloc.h>
  34 #include <linux/err.h>
  35
  36 #include "zram_drv.h"
  37
  38 /* Globals */
  39 static int zram_major;
  40 static struct zram *zram_devices;
  41 static const char *default_compressor = "lzo";
  42
  43 /* Module params (documentation at end) */
  44 static unsigned int num_devices = 1;
  45
  46 #define ZRAM_ATTR_RO(name)                                              \
  47 static ssize_t zram_attr_##name##_show(struct device *d,                \
  48                                 struct device_attribute *attr, char *b) \
  49 {                                                                       \
  50         struct zram *zram = dev_to_zram(d);                             \
  51         return sprintf(b, "%llu\n",                                     \
  52                 (u64)atomic64_read(&zram->stats.name));                 \
  53 }                                                                       \
  54 static struct device_attribute dev_attr_##name =                        \
  55         __ATTR(name, S_IRUGO, zram_attr_##name##_show, NULL);
  56
  57 static inline int init_done(struct zram *zram)
  58 {
  59         return zram->meta != NULL;
  60 }
  61
  62 static inline struct zram *dev_to_zram(struct device *dev)
  63 {
  64         return (struct zram *)dev_to_disk(dev)->private_data;
  65 }
  66
  67 static ssize_t disksize_show(struct device *dev,
  68                 struct device_attribute *attr, char *buf)
  69 {
  70         struct zram *zram = dev_to_zram(dev);
  71
  72         return sprintf(buf, "%llu\n", zram->disksize);
  73 }
  74
  75 static ssize_t initstate_show(struct device *dev,
  76                 struct device_attribute *attr, char *buf)
  77 {
  78         u32 val;
  79         struct zram *zram = dev_to_zram(dev);
  80
  81         down_read(&zram->init_lock);
  82         val = init_done(zram);
  83         up_read(&zram->init_lock);
  84
  85         return sprintf(buf, "%u\n", val);
  86 }
  87
  88 static ssize_t orig_data_size_show(struct device *dev,
  89                 struct device_attribute *attr, char *buf)
  90 {
  91         struct zram *zram = dev_to_zram(dev);
  92
  93         return sprintf(buf, "%llu\n",
  94                 (u64)(atomic64_read(&zram->stats.pages_stored)) << PAGE_SHIFT);
  95 }
  96
  97 static ssize_t mem_used_total_show(struct device *dev,
  98                 struct device_attribute *attr, char *buf)
  99 {
 100         u64 val = 0;
 101         struct zram *zram = dev_to_zram(dev);
 102         struct zram_meta *meta = zram->meta;
 103
 104         down_read(&zram->init_lock);
 105         if (init_done(zram))
 106                 val = zs_get_total_size_bytes(meta->mem_pool);
 107         up_read(&zram->init_lock);
 108
 109         return sprintf(buf, "%llu\n", val);
 110 }
 111
 112 static ssize_t max_comp_streams_show(struct device *dev,
 113                 struct device_attribute *attr, char *buf)
 114 {
 115         int val;
 116         struct zram *zram = dev_to_zram(dev);
 117
 118         down_read(&zram->init_lock);
 119         val = zram->max_comp_streams;
 120         up_read(&zram->init_lock);
 121
 122         return sprintf(buf, "%d\n", val);
 123 }
 124
 125 static ssize_t max_comp_streams_store(struct device *dev,
 126                 struct device_attribute *attr, const char *buf, size_t len)
 127 {
 128         int num;
 129         struct zram *zram = dev_to_zram(dev);
 130
 131         if (kstrtoint(buf, 0, &num))
 132                 return -EINVAL;
 133         if (num < 1)
 134                 return -EINVAL;
 135         down_write(&zram->init_lock);
 136         if (init_done(zram)) {
 137                 if (zcomp_set_max_streams(zram->comp, num))
 138                         pr_info("Cannot change max compression streams\n");
 139         }
 140         zram->max_comp_streams = num;
 141         up_write(&zram->init_lock);
 142         return len;
 143 }
 144
 145 static ssize_t comp_algorithm_show(struct device *dev,
 146                 struct device_attribute *attr, char *buf)
 147 {
 148         size_t sz;
 149         struct zram *zram = dev_to_zram(dev);
 150
 151         down_read(&zram->init_lock);
 152         sz = zcomp_available_show(zram->compressor, buf);
 153         up_read(&zram->init_lock);
 154
 155         return sz;
 156 }
 157
 158 static ssize_t comp_algorithm_store(struct device *dev,
 159                 struct device_attribute *attr, const char *buf, size_t len)
 160 {
 161         struct zram *zram = dev_to_zram(dev);
 162         down_write(&zram->init_lock);
 163         if (init_done(zram)) {
 164                 up_write(&zram->init_lock);
 165                 pr_info("Can't change algorithm for initialized device\n");
 166                 return -EBUSY;
 167         }
 168         strlcpy(zram->compressor, buf, sizeof(zram->compressor));
 169         up_write(&zram->init_lock);
 170         return len;
 171 }
 172
 173 /* flag operations needs meta->tb_lock */
 174 static int zram_test_flag(struct zram_meta *meta, u32 index,
 175                         enum zram_pageflags flag)
 176 {
 177         return meta->table[index].flags & BIT(flag);
 178 }
 179
 180 static void zram_set_flag(struct zram_meta *meta, u32 index,
 181                         enum zram_pageflags flag)
 182 {
 183         meta->table[index].flags |= BIT(flag);
 184 }
 185
 186 static void zram_clear_flag(struct zram_meta *meta, u32 index,
 187                         enum zram_pageflags flag)
 188 {
 189         meta->table[index].flags &= ~BIT(flag);
 190 }
 191
 192 static inline int is_partial_io(struct bio_vec *bvec)
 193 {
 194         return bvec->bv_len != PAGE_SIZE;
 195 }
 196
 197 /*
 198  * Check if request is within bounds and aligned on zram logical blocks.
 199  */
 200 static inline int valid_io_request(struct zram *zram, struct bio *bio)
 201 {
 202         u64 start, end, bound;
 203
 204         /* unaligned request */
 205         if (unlikely(bio->bi_iter.bi_sector &
 206                      (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)))
 207                 return 0;
 208         if (unlikely(bio->bi_iter.bi_size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))
 209                 return 0;
 210
 211         start = bio->bi_iter.bi_sector;
 212         end = start + (bio->bi_iter.bi_size >> SECTOR_SHIFT);
 213         bound = zram->disksize >> SECTOR_SHIFT;
 214         /* out of range range */
 215         if (unlikely(start >= bound || end > bound || start > end))
 216                 return 0;
 217
 218         /* I/O request is valid */
 219         return 1;
 220 }
 221
 222 static void zram_meta_free(struct zram_meta *meta)
 223 {
 224         zs_destroy_pool(meta->mem_pool);
 225         vfree(meta->table);
 226         kfree(meta);
 227 }
 228
 229 static struct zram_meta *zram_meta_alloc(u64 disksize)
 230 {
 231         size_t num_pages;
 232         struct zram_meta *meta = kmalloc(sizeof(*meta), GFP_KERNEL);
 233         if (!meta)
 234                 goto out;
 235
 236         num_pages = disksize >> PAGE_SHIFT;
 237         meta->table = vzalloc(num_pages * sizeof(*meta->table));
 238         if (!meta->table) {
 239                 pr_err("Error allocating zram address table\n");
 240                 goto free_meta;
 241         }
 242
 243         meta->mem_pool = zs_create_pool(GFP_NOIO | __GFP_HIGHMEM);
 244         if (!meta->mem_pool) {
 245                 pr_err("Error creating memory pool\n");
 246                 goto free_table;
 247         }
 248
 249         rwlock_init(&meta->tb_lock);
 250         return meta;
 251
 252 free_table:
 253         vfree(meta->table);
 254 free_meta:
 255         kfree(meta);
 256         meta = NULL;
 257 out:
 258         return meta;
 259 }
 260
 261 static void update_position(u32 *index, int *offset, struct bio_vec *bvec)
 262 {
 263         if (*offset + bvec->bv_len >= PAGE_SIZE)
 264                 (*index)++;
 265         *offset = (*offset + bvec->bv_len) % PAGE_SIZE;
 266 }
 267
 268 static int page_zero_filled(void *ptr)
 269 {
 270         unsigned int pos;
 271         unsigned long *page;
 272
 273         page = (unsigned long *)ptr;
 274
 275         for (pos = 0; pos != PAGE_SIZE / sizeof(*page); pos++) {
 276                 if (page[pos])
 277                         return 0;
 278         }
 279
 280         return 1;
 281 }
 282
 283 static void handle_zero_page(struct bio_vec *bvec)
 284 {
 285         struct page *page = bvec->bv_page;
 286         void *user_mem;
 287
 288         user_mem = kmap_atomic(page);
 289         if (is_partial_io(bvec))
 290                 memset(user_mem + bvec->bv_offset, 0, bvec->bv_len);
 291         else
 292                 clear_page(user_mem);
 293         kunmap_atomic(user_mem);
 294
 295         flush_dcache_page(page);
 296 }
 297
 298 /* NOTE: caller should hold meta->tb_lock with write-side */
 299 static void zram_free_page(struct zram *zram, size_t index)
 300 {
 301         struct zram_meta *meta = zram->meta;
 302         unsigned long handle = meta->table[index].handle;
 303
 304         if (unlikely(!handle)) {
 305                 /*
 306                  * No memory is allocated for zero filled pages.
 307                  * Simply clear zero page flag.
 308                  */
 309                 if (zram_test_flag(meta, index, ZRAM_ZERO)) {
 310                         zram_clear_flag(meta, index, ZRAM_ZERO);
 311                         atomic64_dec(&zram->stats.zero_pages);
 312                 }
 313                 return;
 314         }
 315
 316         zs_free(meta->mem_pool, handle);
 317
 318         atomic64_sub(meta->table[index].size, &zram->stats.compr_data_size);
 319         atomic64_dec(&zram->stats.pages_stored);
 320
 321         meta->table[index].handle = 0;
 322         meta->table[index].size = 0;
 323 }
 324
 325 static int zram_decompress_page(struct zram *zram, char *mem, u32 index)
 326 {
 327         int ret = 0;
 328         unsigned char *cmem;
 329         struct zram_meta *meta = zram->meta;
 330         unsigned long handle;
 331         u16 size;
 332
 333         read_lock(&meta->tb_lock);
 334         handle = meta->table[index].handle;
 335         size = meta->table[index].size;
 336
 337         if (!handle || zram_test_flag(meta, index, ZRAM_ZERO)) {
 338                 read_unlock(&meta->tb_lock);
 339                 clear_page(mem);
 340                 return 0;
 341         }
 342
 343         cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_RO);
 344         if (size == PAGE_SIZE)
 345                 copy_page(mem, cmem);
 346         else
 347                 ret = zcomp_decompress(zram->comp, cmem, size, mem);
 348         zs_unmap_object(meta->mem_pool, handle);
 349         read_unlock(&meta->tb_lock);
 350
 351         /* Should NEVER happen. Return bio error if it does. */
 352         if (unlikely(ret)) {
 353                 pr_err("Decompression failed! err=%d, page=%u\n", ret, index);
 354                 atomic64_inc(&zram->stats.failed_reads);
 355                 return ret;
 356         }
 357
 358         return 0;
 359 }
 360
 361 static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec,
 362                           u32 index, int offset, struct bio *bio)
 363 {
 364         int ret;
 365         struct page *page;
 366         unsigned char *user_mem, *uncmem = NULL;
 367         struct zram_meta *meta = zram->meta;
 368         page = bvec->bv_page;
 369
 370         read_lock(&meta->tb_lock);
 371         if (unlikely(!meta->table[index].handle) ||
 372                         zram_test_flag(meta, index, ZRAM_ZERO)) {
 373                 read_unlock(&meta->tb_lock);
 374                 handle_zero_page(bvec);
 375                 return 0;
 376         }
 377         read_unlock(&meta->tb_lock);
 378
 379         if (is_partial_io(bvec))
 380                 /* Use  a temporary buffer to decompress the page */
 381                 uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
 382
 383         user_mem = kmap_atomic(page);
 384         if (!is_partial_io(bvec))
 385                 uncmem = user_mem;
 386
 387         if (!uncmem) {
 388                 pr_info("Unable to allocate temp memory\n");
 389                 ret = -ENOMEM;
 390                 goto out_cleanup;
 391         }
 392
 393         ret = zram_decompress_page(zram, uncmem, index);
 394         /* Should NEVER happen. Return bio error if it does. */
 395         if (unlikely(ret))
 396                 goto out_cleanup;
 397
 398         if (is_partial_io(bvec))
 399                 memcpy(user_mem + bvec->bv_offset, uncmem + offset,
 400                                 bvec->bv_len);
 401
 402         flush_dcache_page(page);
 403         ret = 0;
 404 out_cleanup:
 405         kunmap_atomic(user_mem);
 406         if (is_partial_io(bvec))
 407                 kfree(uncmem);
 408         return ret;
 409 }
 410
 411 static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index,
 412                            int offset)
 413 {
 414         int ret = 0;
 415         size_t clen;
 416         unsigned long handle;
 417         struct page *page;
 418         unsigned char *user_mem, *cmem, *src, *uncmem = NULL;
 419         struct zram_meta *meta = zram->meta;
 420         struct zcomp_strm *zstrm;
 421         bool locked = false;
 422
 423         page = bvec->bv_page;
 424         if (is_partial_io(bvec)) {
 425                 /*
 426                  * This is a partial IO. We need to read the full page
 427                  * before to write the changes.
 428                  */
 429                 uncmem = kmalloc(PAGE_SIZE, GFP_NOIO);
 430                 if (!uncmem) {
 431                         ret = -ENOMEM;
 432                         goto out;
 433                 }
 434                 ret = zram_decompress_page(zram, uncmem, index);
 435                 if (ret)
 436                         goto out;
 437         }
 438
 439         zstrm = zcomp_strm_find(zram->comp);
 440         locked = true;
 441         user_mem = kmap_atomic(page);
 442
 443         if (is_partial_io(bvec)) {
 444                 memcpy(uncmem + offset, user_mem + bvec->bv_offset,
 445                        bvec->bv_len);
 446                 kunmap_atomic(user_mem);
 447                 user_mem = NULL;
 448         } else {
 449                 uncmem = user_mem;
 450         }
 451
 452         if (page_zero_filled(uncmem)) {
 453                 kunmap_atomic(user_mem);
 454                 /* Free memory associated with this sector now. */
 455                 write_lock(&zram->meta->tb_lock);
 456                 zram_free_page(zram, index);
 457                 zram_set_flag(meta, index, ZRAM_ZERO);
 458                 write_unlock(&zram->meta->tb_lock);
 459
 460                 atomic64_inc(&zram->stats.zero_pages);
 461                 ret = 0;
 462                 goto out;
 463         }
 464
 465         ret = zcomp_compress(zram->comp, zstrm, uncmem, &clen);
 466         if (!is_partial_io(bvec)) {
 467                 kunmap_atomic(user_mem);
 468                 user_mem = NULL;
 469                 uncmem = NULL;
 470         }
 471
 472         if (unlikely(ret)) {
 473                 pr_err("Compression failed! err=%d\n", ret);
 474                 goto out;
 475         }
 476         src = zstrm->buffer;
 477         if (unlikely(clen > max_zpage_size)) {
 478                 clen = PAGE_SIZE;
 479                 if (is_partial_io(bvec))
 480                         src = uncmem;
 481         }
 482
 483         handle = zs_malloc(meta->mem_pool, clen);
 484         if (!handle) {
 485                 pr_info("Error allocating memory for compressed page: %u, size=%zu\n",
 486                         index, clen);
 487                 ret = -ENOMEM;
 488                 goto out;
 489         }
 490         cmem = zs_map_object(meta->mem_pool, handle, ZS_MM_WO);
 491
 492         if ((clen == PAGE_SIZE) && !is_partial_io(bvec)) {
 493                 src = kmap_atomic(page);
 494                 copy_page(cmem, src);
 495                 kunmap_atomic(src);
 496         } else {
 497                 memcpy(cmem, src, clen);
 498         }
 499
 500         zcomp_strm_release(zram->comp, zstrm);
 501         locked = false;
 502         zs_unmap_object(meta->mem_pool, handle);
 503
 504         /*
 505          * Free memory associated with this sector
 506          * before overwriting unused sectors.
 507          */
 508         write_lock(&zram->meta->tb_lock);
 509         zram_free_page(zram, index);
 510
 511         meta->table[index].handle = handle;
 512         meta->table[index].size = clen;
 513         write_unlock(&zram->meta->tb_lock);
 514
 515         /* Update stats */
 516         atomic64_add(clen, &zram->stats.compr_data_size);
 517         atomic64_inc(&zram->stats.pages_stored);
 518 out:
 519         if (locked)
 520                 zcomp_strm_release(zram->comp, zstrm);
 521         if (is_partial_io(bvec))
 522                 kfree(uncmem);
 523         if (ret)
 524                 atomic64_inc(&zram->stats.failed_writes);
 525         return ret;
 526 }
 527
 528 static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index,
 529                         int offset, struct bio *bio)
 530 {
 531         int ret;
 532         int rw = bio_data_dir(bio);
 533
 534         if (rw == READ) {
 535                 atomic64_inc(&zram->stats.num_reads);
 536                 ret = zram_bvec_read(zram, bvec, index, offset, bio);
 537         } else {
 538                 atomic64_inc(&zram->stats.num_writes);
 539                 ret = zram_bvec_write(zram, bvec, index, offset);
 540         }
 541
 542         return ret;
 543 }
 544
 545 static void zram_reset_device(struct zram *zram, bool reset_capacity)
 546 {
 547         size_t index;
 548         struct zram_meta *meta;
 549
 550         down_write(&zram->init_lock);
 551         if (!init_done(zram)) {
 552                 up_write(&zram->init_lock);
 553                 return;
 554         }
 555
 556         meta = zram->meta;
 557         /* Free all pages that are still in this zram device */
 558         for (index = 0; index < zram->disksize >> PAGE_SHIFT; index++) {
 559                 unsigned long handle = meta->table[index].handle;
 560                 if (!handle)
 561                         continue;
 562
 563                 zs_free(meta->mem_pool, handle);
 564         }
 565
 566         zcomp_destroy(zram->comp);
 567         zram->max_comp_streams = 1;
 568
 569         zram_meta_free(zram->meta);
 570         zram->meta = NULL;
 571         /* Reset stats */
 572         memset(&zram->stats, 0, sizeof(zram->stats));
 573
 574         zram->disksize = 0;
 575         if (reset_capacity)
 576                 set_capacity(zram->disk, 0);
 577         up_write(&zram->init_lock);
 578 }
 579
 580 static ssize_t disksize_store(struct device *dev,
 581                 struct device_attribute *attr, const char *buf, size_t len)
 582 {
 583         u64 disksize;
 584         struct zcomp *comp;
 585         struct zram_meta *meta;
 586         struct zram *zram = dev_to_zram(dev);
 587         int err;
 588
 589         disksize = memparse(buf, NULL);
 590         if (!disksize)
 591                 return -EINVAL;
 592
 593         disksize = PAGE_ALIGN(disksize);
 594         meta = zram_meta_alloc(disksize);
 595         if (!meta)
 596                 return -ENOMEM;
 597
 598         comp = zcomp_create(zram->compressor, zram->max_comp_streams);
 599         if (IS_ERR(comp)) {
 600                 pr_info("Cannot initialise %s compressing backend\n",
 601                                 zram->compressor);
 602                 err = PTR_ERR(comp);
 603                 goto out_free_meta;
 604         }
 605
 606         down_write(&zram->init_lock);
 607         if (init_done(zram)) {
 608                 pr_info("Cannot change disksize for initialized device\n");
 609                 err = -EBUSY;
 610                 goto out_destroy_comp;
 611         }
 612
 613         zram->meta = meta;
 614         zram->comp = comp;
 615         zram->disksize = disksize;
 616         set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT);
 617         up_write(&zram->init_lock);
 618         return len;
 619
 620 out_destroy_comp:
 621         up_write(&zram->init_lock);
 622         zcomp_destroy(comp);
 623 out_free_meta:
 624         zram_meta_free(meta);
 625         return err;
 626 }
 627
 628 static ssize_t reset_store(struct device *dev,
 629                 struct device_attribute *attr, const char *buf, size_t len)
 630 {
 631         int ret;
 632         unsigned short do_reset;
 633         struct zram *zram;
 634         struct block_device *bdev;
 635
 636         zram = dev_to_zram(dev);
 637         bdev = bdget_disk(zram->disk, 0);
 638
 639         if (!bdev)
 640                 return -ENOMEM;
 641
 642         /* Do not reset an active device! */
 643         if (bdev->bd_holders) {
 644                 ret = -EBUSY;
 645                 goto out;
 646         }
 647
 648         ret = kstrtou16(buf, 10, &do_reset);
 649         if (ret)
 650                 goto out;
 651
 652         if (!do_reset) {
 653                 ret = -EINVAL;
 654                 goto out;
 655         }
 656
 657         /* Make sure all pending I/O is finished */
 658         fsync_bdev(bdev);
 659         bdput(bdev);
 660
 661         zram_reset_device(zram, true);
 662         return len;
 663
 664 out:
 665         bdput(bdev);
 666         return ret;
 667 }
 668
 669 static void __zram_make_request(struct zram *zram, struct bio *bio)
 670 {
 671         int offset;
 672         u32 index;
 673         struct bio_vec bvec;
 674         struct bvec_iter iter;
 675
 676         index = bio->bi_iter.bi_sector >> SECTORS_PER_PAGE_SHIFT;
 677         offset = (bio->bi_iter.bi_sector &
 678                   (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT;
 679
 680         bio_for_each_segment(bvec, bio, iter) {
 681                 int max_transfer_size = PAGE_SIZE - offset;
 682
 683                 if (bvec.bv_len > max_transfer_size) {
 684                         /*
 685                          * zram_bvec_rw() can only make operation on a single
 686                          * zram page. Split the bio vector.
 687                          */
 688                         struct bio_vec bv;
 689
 690                         bv.bv_page = bvec.bv_page;
 691                         bv.bv_len = max_transfer_size;
 692                         bv.bv_offset = bvec.bv_offset;
 693
 694                         if (zram_bvec_rw(zram, &bv, index, offset, bio) < 0)
 695                                 goto out;
 696
 697                         bv.bv_len = bvec.bv_len - max_transfer_size;
 698                         bv.bv_offset += max_transfer_size;
 699                         if (zram_bvec_rw(zram, &bv, index + 1, 0, bio) < 0)
 700                                 goto out;
 701                 } else
 702                         if (zram_bvec_rw(zram, &bvec, index, offset, bio) < 0)
 703                                 goto out;
 704
 705                 update_position(&index, &offset, &bvec);
 706         }
 707
 708         set_bit(BIO_UPTODATE, &bio->bi_flags);
 709         bio_endio(bio, 0);
 710         return;
 711
 712 out:
 713         bio_io_error(bio);
 714 }
 715
 716 /*
 717  * Handler function for all zram I/O requests.
 718  */
 719 static void zram_make_request(struct request_queue *queue, struct bio *bio)
 720 {
 721         struct zram *zram = queue->queuedata;
 722
 723         down_read(&zram->init_lock);
 724         if (unlikely(!init_done(zram)))
 725                 goto error;
 726
 727         if (!valid_io_request(zram, bio)) {
 728                 atomic64_inc(&zram->stats.invalid_io);
 729                 goto error;
 730         }
 731
 732         __zram_make_request(zram, bio);
 733         up_read(&zram->init_lock);
 734
 735         return;
 736
 737 error:
 738         up_read(&zram->init_lock);
 739         bio_io_error(bio);
 740 }
 741
 742 static void zram_slot_free_notify(struct block_device *bdev,
 743                                 unsigned long index)
 744 {
 745         struct zram *zram;
 746         struct zram_meta *meta;
 747
 748         zram = bdev->bd_disk->private_data;
 749         meta = zram->meta;
 750
 751         write_lock(&meta->tb_lock);
 752         zram_free_page(zram, index);
 753         write_unlock(&meta->tb_lock);
 754         atomic64_inc(&zram->stats.notify_free);
 755 }
 756
 757 static const struct block_device_operations zram_devops = {
 758         .swap_slot_free_notify = zram_slot_free_notify,
 759         .owner = THIS_MODULE
 760 };
 761
 762 static DEVICE_ATTR(disksize, S_IRUGO | S_IWUSR,
 763                 disksize_show, disksize_store);
 764 static DEVICE_ATTR(initstate, S_IRUGO, initstate_show, NULL);
 765 static DEVICE_ATTR(reset, S_IWUSR, NULL, reset_store);
 766 static DEVICE_ATTR(orig_data_size, S_IRUGO, orig_data_size_show, NULL);
 767 static DEVICE_ATTR(mem_used_total, S_IRUGO, mem_used_total_show, NULL);
 768 static DEVICE_ATTR(max_comp_streams, S_IRUGO | S_IWUSR,
 769                 max_comp_streams_show, max_comp_streams_store);
 770 static DEVICE_ATTR(comp_algorithm, S_IRUGO | S_IWUSR,
 771                 comp_algorithm_show, comp_algorithm_store);
 772
 773 ZRAM_ATTR_RO(num_reads);
 774 ZRAM_ATTR_RO(num_writes);
 775 ZRAM_ATTR_RO(failed_reads);
 776 ZRAM_ATTR_RO(failed_writes);
 777 ZRAM_ATTR_RO(invalid_io);
 778 ZRAM_ATTR_RO(notify_free);
 779 ZRAM_ATTR_RO(zero_pages);
 780 ZRAM_ATTR_RO(compr_data_size);
 781
 782 static struct attribute *zram_disk_attrs[] = {
 783         &dev_attr_disksize.attr,
 784         &dev_attr_initstate.attr,
 785         &dev_attr_reset.attr,
 786         &dev_attr_num_reads.attr,
 787         &dev_attr_num_writes.attr,
 788         &dev_attr_failed_reads.attr,
 789         &dev_attr_failed_writes.attr,
 790         &dev_attr_invalid_io.attr,
 791         &dev_attr_notify_free.attr,
 792         &dev_attr_zero_pages.attr,
 793         &dev_attr_orig_data_size.attr,
 794         &dev_attr_compr_data_size.attr,
 795         &dev_attr_mem_used_total.attr,
 796         &dev_attr_max_comp_streams.attr,
 797         &dev_attr_comp_algorithm.attr,
 798         NULL,
 799 };
 800
 801 static struct attribute_group zram_disk_attr_group = {
 802         .attrs = zram_disk_attrs,
 803 };
 804
 805 static int create_device(struct zram *zram, int device_id)
 806 {
 807         int ret = -ENOMEM;
 808
 809         init_rwsem(&zram->init_lock);
 810
 811         zram->queue = blk_alloc_queue(GFP_KERNEL);
 812         if (!zram->queue) {
 813                 pr_err("Error allocating disk queue for device %d\n",
 814                         device_id);
 815                 goto out;
 816         }
 817
 818         blk_queue_make_request(zram->queue, zram_make_request);
 819         zram->queue->queuedata = zram;
 820
 821          /* gendisk structure */
 822         zram->disk = alloc_disk(1);
 823         if (!zram->disk) {
 824                 pr_warn("Error allocating disk structure for device %d\n",
 825                         device_id);
 826                 goto out_free_queue;
 827         }
 828
 829         zram->disk->major = zram_major;
 830         zram->disk->first_minor = device_id;
 831         zram->disk->fops = &zram_devops;
 832         zram->disk->queue = zram->queue;
 833         zram->disk->private_data = zram;
 834         snprintf(zram->disk->disk_name, 16, "zram%d", device_id);
 835
 836         /* Actual capacity set using syfs (/sys/block/zram<id>/disksize */
 837         set_capacity(zram->disk, 0);
 838         /* zram devices sort of resembles non-rotational disks */
 839         queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue);
 840         /*
 841          * To ensure that we always get PAGE_SIZE aligned
 842          * and n*PAGE_SIZED sized I/O requests.
 843          */
 844         blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE);
 845         blk_queue_logical_block_size(zram->disk->queue,
 846                                         ZRAM_LOGICAL_BLOCK_SIZE);
 847         blk_queue_io_min(zram->disk->queue, PAGE_SIZE);
 848         blk_queue_io_opt(zram->disk->queue, PAGE_SIZE);
 849
 850         add_disk(zram->disk);
 851
 852         ret = sysfs_create_group(&disk_to_dev(zram->disk)->kobj,
 853                                 &zram_disk_attr_group);
 854         if (ret < 0) {
 855                 pr_warn("Error creating sysfs group");
 856                 goto out_free_disk;
 857         }
 858         strlcpy(zram->compressor, default_compressor, sizeof(zram->compressor));
 859         zram->meta = NULL;
 860         zram->max_comp_streams = 1;
 861         return 0;
 862
 863 out_free_disk:
 864         del_gendisk(zram->disk);
 865         put_disk(zram->disk);
 866 out_free_queue:
 867         blk_cleanup_queue(zram->queue);
 868 out:
 869         return ret;
 870 }
 871
 872 static void destroy_device(struct zram *zram)
 873 {
 874         sysfs_remove_group(&disk_to_dev(zram->disk)->kobj,
 875                         &zram_disk_attr_group);
 876
 877         del_gendisk(zram->disk);
 878         put_disk(zram->disk);
 879
 880         blk_cleanup_queue(zram->queue);
 881 }
 882
 883 static int __init zram_init(void)
 884 {
 885         int ret, dev_id;
 886
 887         if (num_devices > max_num_devices) {
 888                 pr_warn("Invalid value for num_devices: %u\n",
 889                                 num_devices);
 890                 ret = -EINVAL;
 891                 goto out;
 892         }
 893
 894         zram_major = register_blkdev(0, "zram");
 895         if (zram_major <= 0) {
 896                 pr_warn("Unable to get major number\n");
 897                 ret = -EBUSY;
 898                 goto out;
 899         }
 900
 901         /* Allocate the device array and initialize each one */
 902         zram_devices = kzalloc(num_devices * sizeof(struct zram), GFP_KERNEL);
 903         if (!zram_devices) {
 904                 ret = -ENOMEM;
 905                 goto unregister;
 906         }
 907
 908         for (dev_id = 0; dev_id < num_devices; dev_id++) {
 909                 ret = create_device(&zram_devices[dev_id], dev_id);
 910                 if (ret)
 911                         goto free_devices;
 912         }
 913
 914         pr_info("Created %u device(s) ...\n", num_devices);
 915
 916         return 0;
 917
 918 free_devices:
 919         while (dev_id)
 920                 destroy_device(&zram_devices[--dev_id]);
 921         kfree(zram_devices);
 922 unregister:
 923         unregister_blkdev(zram_major, "zram");
 924 out:
 925         return ret;
 926 }
 927
 928 static void __exit zram_exit(void)
 929 {
 930         int i;
 931         struct zram *zram;
 932
 933         for (i = 0; i < num_devices; i++) {
 934                 zram = &zram_devices[i];
 935
 936                 destroy_device(zram);
 937                 /*
 938                  * Shouldn't access zram->disk after destroy_device
 939                  * because destroy_device already released zram->disk.
 940                  */
 941                 zram_reset_device(zram, false);
 942         }
 943
 944         unregister_blkdev(zram_major, "zram");
 945
 946         kfree(zram_devices);
 947         pr_debug("Cleanup done!\n");
 948 }
 949
 950 module_init(zram_init);
 951 module_exit(zram_exit);
 952
 953 module_param(num_devices, uint, 0);
 954 MODULE_PARM_DESC(num_devices, "Number of zram devices");
 955
 956 MODULE_LICENSE("Dual BSD/GPL");
 957 MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
 958 MODULE_DESCRIPTION("Compressed RAM Block Device");