]> git.kernelconcepts.de Git - karo-tx-linux.git/blob - mm/page_io.c
Merge branch 'rc-fixes' of git://git.kernel.org/pub/scm/linux/kernel/git/mmarek/kbuild
[karo-tx-linux.git] / mm / page_io.c
1 /*
2  *  linux/mm/page_io.c
3  *
4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
5  *
6  *  Swap reorganised 29.12.95, 
7  *  Asynchronous swapping added 30.12.95. Stephen Tweedie
8  *  Removed race in async swapping. 14.4.1996. Bruno Haible
9  *  Add swap of shared pages through the page cache. 20.2.1998. Stephen Tweedie
10  *  Always use brw_page, life becomes simpler. 12 May 1998 Eric Biederman
11  */
12
13 #include <linux/mm.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/gfp.h>
16 #include <linux/pagemap.h>
17 #include <linux/swap.h>
18 #include <linux/bio.h>
19 #include <linux/swapops.h>
20 #include <linux/buffer_head.h>
21 #include <linux/writeback.h>
22 #include <linux/frontswap.h>
23 #include <linux/blkdev.h>
24 #include <linux/uio.h>
25 #include <asm/pgtable.h>
26
27 static struct bio *get_swap_bio(gfp_t gfp_flags,
28                                 struct page *page, bio_end_io_t end_io)
29 {
30         struct bio *bio;
31
32         bio = bio_alloc(gfp_flags, 1);
33         if (bio) {
34                 bio->bi_iter.bi_sector = map_swap_page(page, &bio->bi_bdev);
35                 bio->bi_iter.bi_sector <<= PAGE_SHIFT - 9;
36                 bio->bi_end_io = end_io;
37
38                 bio_add_page(bio, page, PAGE_SIZE, 0);
39                 BUG_ON(bio->bi_iter.bi_size != PAGE_SIZE);
40         }
41         return bio;
42 }
43
44 void end_swap_bio_write(struct bio *bio)
45 {
46         struct page *page = bio->bi_io_vec[0].bv_page;
47
48         if (bio->bi_error) {
49                 SetPageError(page);
50                 /*
51                  * We failed to write the page out to swap-space.
52                  * Re-dirty the page in order to avoid it being reclaimed.
53                  * Also print a dire warning that things will go BAD (tm)
54                  * very quickly.
55                  *
56                  * Also clear PG_reclaim to avoid rotate_reclaimable_page()
57                  */
58                 set_page_dirty(page);
59                 pr_alert("Write-error on swap-device (%u:%u:%llu)\n",
60                          imajor(bio->bi_bdev->bd_inode),
61                          iminor(bio->bi_bdev->bd_inode),
62                          (unsigned long long)bio->bi_iter.bi_sector);
63                 ClearPageReclaim(page);
64         }
65         end_page_writeback(page);
66         bio_put(bio);
67 }
68
69 static void swap_slot_free_notify(struct page *page)
70 {
71         struct swap_info_struct *sis;
72         struct gendisk *disk;
73
74         /*
75          * There is no guarantee that the page is in swap cache - the software
76          * suspend code (at least) uses end_swap_bio_read() against a non-
77          * swapcache page.  So we must check PG_swapcache before proceeding with
78          * this optimization.
79          */
80         if (unlikely(!PageSwapCache(page)))
81                 return;
82
83         sis = page_swap_info(page);
84         if (!(sis->flags & SWP_BLKDEV))
85                 return;
86
87         /*
88          * The swap subsystem performs lazy swap slot freeing,
89          * expecting that the page will be swapped out again.
90          * So we can avoid an unnecessary write if the page
91          * isn't redirtied.
92          * This is good for real swap storage because we can
93          * reduce unnecessary I/O and enhance wear-leveling
94          * if an SSD is used as the as swap device.
95          * But if in-memory swap device (eg zram) is used,
96          * this causes a duplicated copy between uncompressed
97          * data in VM-owned memory and compressed data in
98          * zram-owned memory.  So let's free zram-owned memory
99          * and make the VM-owned decompressed page *dirty*,
100          * so the page should be swapped out somewhere again if
101          * we again wish to reclaim it.
102          */
103         disk = sis->bdev->bd_disk;
104         if (disk->fops->swap_slot_free_notify) {
105                 swp_entry_t entry;
106                 unsigned long offset;
107
108                 entry.val = page_private(page);
109                 offset = swp_offset(entry);
110
111                 SetPageDirty(page);
112                 disk->fops->swap_slot_free_notify(sis->bdev,
113                                 offset);
114         }
115 }
116
117 static void end_swap_bio_read(struct bio *bio)
118 {
119         struct page *page = bio->bi_io_vec[0].bv_page;
120
121         if (bio->bi_error) {
122                 SetPageError(page);
123                 ClearPageUptodate(page);
124                 pr_alert("Read-error on swap-device (%u:%u:%llu)\n",
125                          imajor(bio->bi_bdev->bd_inode),
126                          iminor(bio->bi_bdev->bd_inode),
127                          (unsigned long long)bio->bi_iter.bi_sector);
128                 goto out;
129         }
130
131         SetPageUptodate(page);
132         swap_slot_free_notify(page);
133 out:
134         unlock_page(page);
135         bio_put(bio);
136 }
137
138 int generic_swapfile_activate(struct swap_info_struct *sis,
139                                 struct file *swap_file,
140                                 sector_t *span)
141 {
142         struct address_space *mapping = swap_file->f_mapping;
143         struct inode *inode = mapping->host;
144         unsigned blocks_per_page;
145         unsigned long page_no;
146         unsigned blkbits;
147         sector_t probe_block;
148         sector_t last_block;
149         sector_t lowest_block = -1;
150         sector_t highest_block = 0;
151         int nr_extents = 0;
152         int ret;
153
154         blkbits = inode->i_blkbits;
155         blocks_per_page = PAGE_SIZE >> blkbits;
156
157         /*
158          * Map all the blocks into the extent list.  This code doesn't try
159          * to be very smart.
160          */
161         probe_block = 0;
162         page_no = 0;
163         last_block = i_size_read(inode) >> blkbits;
164         while ((probe_block + blocks_per_page) <= last_block &&
165                         page_no < sis->max) {
166                 unsigned block_in_page;
167                 sector_t first_block;
168
169                 cond_resched();
170
171                 first_block = bmap(inode, probe_block);
172                 if (first_block == 0)
173                         goto bad_bmap;
174
175                 /*
176                  * It must be PAGE_SIZE aligned on-disk
177                  */
178                 if (first_block & (blocks_per_page - 1)) {
179                         probe_block++;
180                         goto reprobe;
181                 }
182
183                 for (block_in_page = 1; block_in_page < blocks_per_page;
184                                         block_in_page++) {
185                         sector_t block;
186
187                         block = bmap(inode, probe_block + block_in_page);
188                         if (block == 0)
189                                 goto bad_bmap;
190                         if (block != first_block + block_in_page) {
191                                 /* Discontiguity */
192                                 probe_block++;
193                                 goto reprobe;
194                         }
195                 }
196
197                 first_block >>= (PAGE_SHIFT - blkbits);
198                 if (page_no) {  /* exclude the header page */
199                         if (first_block < lowest_block)
200                                 lowest_block = first_block;
201                         if (first_block > highest_block)
202                                 highest_block = first_block;
203                 }
204
205                 /*
206                  * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
207                  */
208                 ret = add_swap_extent(sis, page_no, 1, first_block);
209                 if (ret < 0)
210                         goto out;
211                 nr_extents += ret;
212                 page_no++;
213                 probe_block += blocks_per_page;
214 reprobe:
215                 continue;
216         }
217         ret = nr_extents;
218         *span = 1 + highest_block - lowest_block;
219         if (page_no == 0)
220                 page_no = 1;    /* force Empty message */
221         sis->max = page_no;
222         sis->pages = page_no - 1;
223         sis->highest_bit = page_no - 1;
224 out:
225         return ret;
226 bad_bmap:
227         pr_err("swapon: swapfile has holes\n");
228         ret = -EINVAL;
229         goto out;
230 }
231
232 /*
233  * We may have stale swap cache pages in memory: notice
234  * them here and get rid of the unnecessary final write.
235  */
236 int swap_writepage(struct page *page, struct writeback_control *wbc)
237 {
238         int ret = 0;
239
240         if (try_to_free_swap(page)) {
241                 unlock_page(page);
242                 goto out;
243         }
244         if (frontswap_store(page) == 0) {
245                 set_page_writeback(page);
246                 unlock_page(page);
247                 end_page_writeback(page);
248                 goto out;
249         }
250         ret = __swap_writepage(page, wbc, end_swap_bio_write);
251 out:
252         return ret;
253 }
254
255 static sector_t swap_page_sector(struct page *page)
256 {
257         return (sector_t)__page_file_index(page) << (PAGE_SHIFT - 9);
258 }
259
260 int __swap_writepage(struct page *page, struct writeback_control *wbc,
261                 bio_end_io_t end_write_func)
262 {
263         struct bio *bio;
264         int ret;
265         struct swap_info_struct *sis = page_swap_info(page);
266
267         if (sis->flags & SWP_FILE) {
268                 struct kiocb kiocb;
269                 struct file *swap_file = sis->swap_file;
270                 struct address_space *mapping = swap_file->f_mapping;
271                 struct bio_vec bv = {
272                         .bv_page = page,
273                         .bv_len  = PAGE_SIZE,
274                         .bv_offset = 0
275                 };
276                 struct iov_iter from;
277
278                 iov_iter_bvec(&from, ITER_BVEC | WRITE, &bv, 1, PAGE_SIZE);
279                 init_sync_kiocb(&kiocb, swap_file);
280                 kiocb.ki_pos = page_file_offset(page);
281
282                 set_page_writeback(page);
283                 unlock_page(page);
284                 ret = mapping->a_ops->direct_IO(&kiocb, &from);
285                 if (ret == PAGE_SIZE) {
286                         count_vm_event(PSWPOUT);
287                         ret = 0;
288                 } else {
289                         /*
290                          * In the case of swap-over-nfs, this can be a
291                          * temporary failure if the system has limited
292                          * memory for allocating transmit buffers.
293                          * Mark the page dirty and avoid
294                          * rotate_reclaimable_page but rate-limit the
295                          * messages but do not flag PageError like
296                          * the normal direct-to-bio case as it could
297                          * be temporary.
298                          */
299                         set_page_dirty(page);
300                         ClearPageReclaim(page);
301                         pr_err_ratelimited("Write error on dio swapfile (%llu)\n",
302                                            page_file_offset(page));
303                 }
304                 end_page_writeback(page);
305                 return ret;
306         }
307
308         ret = bdev_write_page(sis->bdev, swap_page_sector(page), page, wbc);
309         if (!ret) {
310                 count_vm_event(PSWPOUT);
311                 return 0;
312         }
313
314         ret = 0;
315         bio = get_swap_bio(GFP_NOIO, page, end_write_func);
316         if (bio == NULL) {
317                 set_page_dirty(page);
318                 unlock_page(page);
319                 ret = -ENOMEM;
320                 goto out;
321         }
322         if (wbc->sync_mode == WB_SYNC_ALL)
323                 bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_SYNC);
324         else
325                 bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
326         count_vm_event(PSWPOUT);
327         set_page_writeback(page);
328         unlock_page(page);
329         submit_bio(bio);
330 out:
331         return ret;
332 }
333
334 int swap_readpage(struct page *page)
335 {
336         struct bio *bio;
337         int ret = 0;
338         struct swap_info_struct *sis = page_swap_info(page);
339
340         VM_BUG_ON_PAGE(!PageLocked(page), page);
341         VM_BUG_ON_PAGE(PageUptodate(page), page);
342         if (frontswap_load(page) == 0) {
343                 SetPageUptodate(page);
344                 unlock_page(page);
345                 goto out;
346         }
347
348         if (sis->flags & SWP_FILE) {
349                 struct file *swap_file = sis->swap_file;
350                 struct address_space *mapping = swap_file->f_mapping;
351
352                 ret = mapping->a_ops->readpage(swap_file, page);
353                 if (!ret)
354                         count_vm_event(PSWPIN);
355                 return ret;
356         }
357
358         ret = bdev_read_page(sis->bdev, swap_page_sector(page), page);
359         if (!ret) {
360                 if (trylock_page(page)) {
361                         swap_slot_free_notify(page);
362                         unlock_page(page);
363                 }
364
365                 count_vm_event(PSWPIN);
366                 return 0;
367         }
368
369         ret = 0;
370         bio = get_swap_bio(GFP_KERNEL, page, end_swap_bio_read);
371         if (bio == NULL) {
372                 unlock_page(page);
373                 ret = -ENOMEM;
374                 goto out;
375         }
376         bio_set_op_attrs(bio, REQ_OP_READ, 0);
377         count_vm_event(PSWPIN);
378         submit_bio(bio);
379 out:
380         return ret;
381 }
382
383 int swap_set_page_dirty(struct page *page)
384 {
385         struct swap_info_struct *sis = page_swap_info(page);
386
387         if (sis->flags & SWP_FILE) {
388                 struct address_space *mapping = sis->swap_file->f_mapping;
389                 return mapping->a_ops->set_page_dirty(page);
390         } else {
391                 return __set_page_dirty_no_writeback(page);
392         }
393 }