fs/xfs/xfs_buf.c

   1 /*
   2  * Copyright (c) 2000-2006 Silicon Graphics, Inc.
   3  * All Rights Reserved.
   4  *
   5  * This program is free software; you can redistribute it and/or
   6  * modify it under the terms of the GNU General Public License as
   7  * published by the Free Software Foundation.
   8  *
   9  * This program is distributed in the hope that it would be useful,
  10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  * GNU General Public License for more details.
  13  *
  14  * You should have received a copy of the GNU General Public License
  15  * along with this program; if not, write the Free Software Foundation,
  16  * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
  17  */
  18 #include "xfs.h"
  19 #include <linux/stddef.h>
  20 #include <linux/errno.h>
  21 #include <linux/gfp.h>
  22 #include <linux/pagemap.h>
  23 #include <linux/init.h>
  24 #include <linux/vmalloc.h>
  25 #include <linux/bio.h>
  26 #include <linux/sysctl.h>
  27 #include <linux/proc_fs.h>
  28 #include <linux/workqueue.h>
  29 #include <linux/percpu.h>
  30 #include <linux/blkdev.h>
  31 #include <linux/hash.h>
  32 #include <linux/kthread.h>
  33 #include <linux/migrate.h>
  34 #include <linux/backing-dev.h>
  35 #include <linux/freezer.h>
  36
  37 #include "xfs_sb.h"
  38 #include "xfs_inum.h"
  39 #include "xfs_log.h"
  40 #include "xfs_ag.h"
  41 #include "xfs_mount.h"
  42 #include "xfs_trace.h"
  43
  44 static kmem_zone_t *xfs_buf_zone;
  45 STATIC int xfsbufd(void *);
  46
  47 static struct workqueue_struct *xfslogd_workqueue;
  48 struct workqueue_struct *xfsdatad_workqueue;
  49 struct workqueue_struct *xfsconvertd_workqueue;
  50
  51 #ifdef XFS_BUF_LOCK_TRACKING
  52 # define XB_SET_OWNER(bp)       ((bp)->b_last_holder = current->pid)
  53 # define XB_CLEAR_OWNER(bp)     ((bp)->b_last_holder = -1)
  54 # define XB_GET_OWNER(bp)       ((bp)->b_last_holder)
  55 #else
  56 # define XB_SET_OWNER(bp)       do { } while (0)
  57 # define XB_CLEAR_OWNER(bp)     do { } while (0)
  58 # define XB_GET_OWNER(bp)       do { } while (0)
  59 #endif
  60
  61 #define xb_to_gfp(flags) \
  62         ((((flags) & XBF_READ_AHEAD) ? __GFP_NORETRY : \
  63           ((flags) & XBF_DONT_BLOCK) ? GFP_NOFS : GFP_KERNEL) | __GFP_NOWARN)
  64
  65 #define xb_to_km(flags) \
  66          (((flags) & XBF_DONT_BLOCK) ? KM_NOFS : KM_SLEEP)
  67
  68 #define xfs_buf_allocate(flags) \
  69         kmem_zone_alloc(xfs_buf_zone, xb_to_km(flags))
  70 #define xfs_buf_deallocate(bp) \
  71         kmem_zone_free(xfs_buf_zone, (bp));
  72
  73 static inline int
  74 xfs_buf_is_vmapped(
  75         struct xfs_buf  *bp)
  76 {
  77         /*
  78          * Return true if the buffer is vmapped.
  79          *
  80          * The XBF_MAPPED flag is set if the buffer should be mapped, but the
  81          * code is clever enough to know it doesn't have to map a single page,
  82          * so the check has to be both for XBF_MAPPED and bp->b_page_count > 1.
  83          */
  84         return (bp->b_flags & XBF_MAPPED) && bp->b_page_count > 1;
  85 }
  86
  87 static inline int
  88 xfs_buf_vmap_len(
  89         struct xfs_buf  *bp)
  90 {
  91         return (bp->b_page_count * PAGE_SIZE) - bp->b_offset;
  92 }
  93
  94 /*
  95  * xfs_buf_lru_add - add a buffer to the LRU.
  96  *
  97  * The LRU takes a new reference to the buffer so that it will only be freed
  98  * once the shrinker takes the buffer off the LRU.
  99  */
 100 STATIC void
 101 xfs_buf_lru_add(
 102         struct xfs_buf  *bp)
 103 {
 104         struct xfs_buftarg *btp = bp->b_target;
 105
 106         spin_lock(&btp->bt_lru_lock);
 107         if (list_empty(&bp->b_lru)) {
 108                 atomic_inc(&bp->b_hold);
 109                 list_add_tail(&bp->b_lru, &btp->bt_lru);
 110                 btp->bt_lru_nr++;
 111         }
 112         spin_unlock(&btp->bt_lru_lock);
 113 }
 114
 115 /*
 116  * xfs_buf_lru_del - remove a buffer from the LRU
 117  *
 118  * The unlocked check is safe here because it only occurs when there are not
 119  * b_lru_ref counts left on the inode under the pag->pag_buf_lock. it is there
 120  * to optimise the shrinker removing the buffer from the LRU and calling
 121  * xfs_buf_free(). i.e. it removes an unnecessary round trip on the
 122  * bt_lru_lock.
 123  */
 124 STATIC void
 125 xfs_buf_lru_del(
 126         struct xfs_buf  *bp)
 127 {
 128         struct xfs_buftarg *btp = bp->b_target;
 129
 130         if (list_empty(&bp->b_lru))
 131                 return;
 132
 133         spin_lock(&btp->bt_lru_lock);
 134         if (!list_empty(&bp->b_lru)) {
 135                 list_del_init(&bp->b_lru);
 136                 btp->bt_lru_nr--;
 137         }
 138         spin_unlock(&btp->bt_lru_lock);
 139 }
 140
 141 /*
 142  * When we mark a buffer stale, we remove the buffer from the LRU and clear the
 143  * b_lru_ref count so that the buffer is freed immediately when the buffer
 144  * reference count falls to zero. If the buffer is already on the LRU, we need
 145  * to remove the reference that LRU holds on the buffer.
 146  *
 147  * This prevents build-up of stale buffers on the LRU.
 148  */
 149 void
 150 xfs_buf_stale(
 151         struct xfs_buf  *bp)
 152 {
 153         bp->b_flags |= XBF_STALE;
 154         atomic_set(&(bp)->b_lru_ref, 0);
 155         if (!list_empty(&bp->b_lru)) {
 156                 struct xfs_buftarg *btp = bp->b_target;
 157
 158                 spin_lock(&btp->bt_lru_lock);
 159                 if (!list_empty(&bp->b_lru)) {
 160                         list_del_init(&bp->b_lru);
 161                         btp->bt_lru_nr--;
 162                         atomic_dec(&bp->b_hold);
 163                 }
 164                 spin_unlock(&btp->bt_lru_lock);
 165         }
 166         ASSERT(atomic_read(&bp->b_hold) >= 1);
 167 }
 168
 169 STATIC void
 170 _xfs_buf_initialize(
 171         xfs_buf_t               *bp,
 172         xfs_buftarg_t           *target,
 173         xfs_off_t               range_base,
 174         size_t                  range_length,
 175         xfs_buf_flags_t         flags)
 176 {
 177         /*
 178          * We don't want certain flags to appear in b_flags.
 179          */
 180         flags &= ~(XBF_LOCK|XBF_MAPPED|XBF_DONT_BLOCK|XBF_READ_AHEAD);
 181
 182         memset(bp, 0, sizeof(xfs_buf_t));
 183         atomic_set(&bp->b_hold, 1);
 184         atomic_set(&bp->b_lru_ref, 1);
 185         init_completion(&bp->b_iowait);
 186         INIT_LIST_HEAD(&bp->b_lru);
 187         INIT_LIST_HEAD(&bp->b_list);
 188         RB_CLEAR_NODE(&bp->b_rbnode);
 189         sema_init(&bp->b_sema, 0); /* held, no waiters */
 190         XB_SET_OWNER(bp);
 191         bp->b_target = target;
 192         bp->b_file_offset = range_base;
 193         /*
 194          * Set buffer_length and count_desired to the same value initially.
 195          * I/O routines should use count_desired, which will be the same in
 196          * most cases but may be reset (e.g. XFS recovery).
 197          */
 198         bp->b_buffer_length = bp->b_count_desired = range_length;
 199         bp->b_flags = flags;
 200         bp->b_bn = XFS_BUF_DADDR_NULL;
 201         atomic_set(&bp->b_pin_count, 0);
 202         init_waitqueue_head(&bp->b_waiters);
 203
 204         XFS_STATS_INC(xb_create);
 205
 206         trace_xfs_buf_init(bp, _RET_IP_);
 207 }
 208
 209 /*
 210  *      Allocate a page array capable of holding a specified number
 211  *      of pages, and point the page buf at it.
 212  */
 213 STATIC int
 214 _xfs_buf_get_pages(
 215         xfs_buf_t               *bp,
 216         int                     page_count,
 217         xfs_buf_flags_t         flags)
 218 {
 219         /* Make sure that we have a page list */
 220         if (bp->b_pages == NULL) {
 221                 bp->b_offset = xfs_buf_poff(bp->b_file_offset);
 222                 bp->b_page_count = page_count;
 223                 if (page_count <= XB_PAGES) {
 224                         bp->b_pages = bp->b_page_array;
 225                 } else {
 226                         bp->b_pages = kmem_alloc(sizeof(struct page *) *
 227                                         page_count, xb_to_km(flags));
 228                         if (bp->b_pages == NULL)
 229                                 return -ENOMEM;
 230                 }
 231                 memset(bp->b_pages, 0, sizeof(struct page *) * page_count);
 232         }
 233         return 0;
 234 }
 235
 236 /*
 237  *      Frees b_pages if it was allocated.
 238  */
 239 STATIC void
 240 _xfs_buf_free_pages(
 241         xfs_buf_t       *bp)
 242 {
 243         if (bp->b_pages != bp->b_page_array) {
 244                 kmem_free(bp->b_pages);
 245                 bp->b_pages = NULL;
 246         }
 247 }
 248
 249 /*
 250  *      Releases the specified buffer.
 251  *
 252  *      The modification state of any associated pages is left unchanged.
 253  *      The buffer most not be on any hash - use xfs_buf_rele instead for
 254  *      hashed and refcounted buffers
 255  */
 256 void
 257 xfs_buf_free(
 258         xfs_buf_t               *bp)
 259 {
 260         trace_xfs_buf_free(bp, _RET_IP_);
 261
 262         ASSERT(list_empty(&bp->b_lru));
 263
 264         if (bp->b_flags & _XBF_PAGES) {
 265                 uint            i;
 266
 267                 if (xfs_buf_is_vmapped(bp))
 268                         vm_unmap_ram(bp->b_addr - bp->b_offset,
 269                                         bp->b_page_count);
 270
 271                 for (i = 0; i < bp->b_page_count; i++) {
 272                         struct page     *page = bp->b_pages[i];
 273
 274                         __free_page(page);
 275                 }
 276         } else if (bp->b_flags & _XBF_KMEM)
 277                 kmem_free(bp->b_addr);
 278         _xfs_buf_free_pages(bp);
 279         xfs_buf_deallocate(bp);
 280 }
 281
 282 /*
 283  * Allocates all the pages for buffer in question and builds it's page list.
 284  */
 285 STATIC int
 286 xfs_buf_allocate_memory(
 287         xfs_buf_t               *bp,
 288         uint                    flags)
 289 {
 290         size_t                  size = bp->b_count_desired;
 291         size_t                  nbytes, offset;
 292         gfp_t                   gfp_mask = xb_to_gfp(flags);
 293         unsigned short          page_count, i;
 294         xfs_off_t               end;
 295         int                     error;
 296
 297         /*
 298          * for buffers that are contained within a single page, just allocate
 299          * the memory from the heap - there's no need for the complexity of
 300          * page arrays to keep allocation down to order 0.
 301          */
 302         if (bp->b_buffer_length < PAGE_SIZE) {
 303                 bp->b_addr = kmem_alloc(bp->b_buffer_length, xb_to_km(flags));
 304                 if (!bp->b_addr) {
 305                         /* low memory - use alloc_page loop instead */
 306                         goto use_alloc_page;
 307                 }
 308
 309                 if (((unsigned long)(bp->b_addr + bp->b_buffer_length - 1) &
 310                                                                 PAGE_MASK) !=
 311                     ((unsigned long)bp->b_addr & PAGE_MASK)) {
 312                         /* b_addr spans two pages - use alloc_page instead */
 313                         kmem_free(bp->b_addr);
 314                         bp->b_addr = NULL;
 315                         goto use_alloc_page;
 316                 }
 317                 bp->b_offset = offset_in_page(bp->b_addr);
 318                 bp->b_pages = bp->b_page_array;
 319                 bp->b_pages[0] = virt_to_page(bp->b_addr);
 320                 bp->b_page_count = 1;
 321                 bp->b_flags |= XBF_MAPPED | _XBF_KMEM;
 322                 return 0;
 323         }
 324
 325 use_alloc_page:
 326         end = bp->b_file_offset + bp->b_buffer_length;
 327         page_count = xfs_buf_btoc(end) - xfs_buf_btoct(bp->b_file_offset);
 328         error = _xfs_buf_get_pages(bp, page_count, flags);
 329         if (unlikely(error))
 330                 return error;
 331
 332         offset = bp->b_offset;
 333         bp->b_flags |= _XBF_PAGES;
 334
 335         for (i = 0; i < bp->b_page_count; i++) {
 336                 struct page     *page;
 337                 uint            retries = 0;
 338 retry:
 339                 page = alloc_page(gfp_mask);
 340                 if (unlikely(page == NULL)) {
 341                         if (flags & XBF_READ_AHEAD) {
 342                                 bp->b_page_count = i;
 343                                 error = ENOMEM;
 344                                 goto out_free_pages;
 345                         }
 346
 347                         /*
 348                          * This could deadlock.
 349                          *
 350                          * But until all the XFS lowlevel code is revamped to
 351                          * handle buffer allocation failures we can't do much.
 352                          */
 353                         if (!(++retries % 100))
 354                                 xfs_err(NULL,
 355                 "possible memory allocation deadlock in %s (mode:0x%x)",
 356                                         __func__, gfp_mask);
 357
 358                         XFS_STATS_INC(xb_page_retries);
 359                         congestion_wait(BLK_RW_ASYNC, HZ/50);
 360                         goto retry;
 361                 }
 362
 363                 XFS_STATS_INC(xb_page_found);
 364
 365                 nbytes = min_t(size_t, size, PAGE_SIZE - offset);
 366                 size -= nbytes;
 367                 bp->b_pages[i] = page;
 368                 offset = 0;
 369         }
 370         return 0;
 371
 372 out_free_pages:
 373         for (i = 0; i < bp->b_page_count; i++)
 374                 __free_page(bp->b_pages[i]);
 375         return error;
 376 }
 377
 378 /*
 379  *      Map buffer into kernel address-space if necessary.
 380  */
 381 STATIC int
 382 _xfs_buf_map_pages(
 383         xfs_buf_t               *bp,
 384         uint                    flags)
 385 {
 386         ASSERT(bp->b_flags & _XBF_PAGES);
 387         if (bp->b_page_count == 1) {
 388                 /* A single page buffer is always mappable */
 389                 bp->b_addr = page_address(bp->b_pages[0]) + bp->b_offset;
 390                 bp->b_flags |= XBF_MAPPED;
 391         } else if (flags & XBF_MAPPED) {
 392                 int retried = 0;
 393
 394                 do {
 395                         bp->b_addr = vm_map_ram(bp->b_pages, bp->b_page_count,
 396                                                 -1, PAGE_KERNEL);
 397                         if (bp->b_addr)
 398                                 break;
 399                         vm_unmap_aliases();
 400                 } while (retried++ <= 1);
 401
 402                 if (!bp->b_addr)
 403                         return -ENOMEM;
 404                 bp->b_addr += bp->b_offset;
 405                 bp->b_flags |= XBF_MAPPED;
 406         }
 407
 408         return 0;
 409 }
 410
 411 /*
 412  *      Finding and Reading Buffers
 413  */
 414
 415 /*
 416  *      Look up, and creates if absent, a lockable buffer for
 417  *      a given range of an inode.  The buffer is returned
 418  *      locked. No I/O is implied by this call.
 419  */
 420 xfs_buf_t *
 421 _xfs_buf_find(
 422         xfs_buftarg_t           *btp,   /* block device target          */
 423         xfs_off_t               ioff,   /* starting offset of range     */
 424         size_t                  isize,  /* length of range              */
 425         xfs_buf_flags_t         flags,
 426         xfs_buf_t               *new_bp)
 427 {
 428         xfs_off_t               range_base;
 429         size_t                  range_length;
 430         struct xfs_perag        *pag;
 431         struct rb_node          **rbp;
 432         struct rb_node          *parent;
 433         xfs_buf_t               *bp;
 434
 435         range_base = (ioff << BBSHIFT);
 436         range_length = (isize << BBSHIFT);
 437
 438         /* Check for IOs smaller than the sector size / not sector aligned */
 439         ASSERT(!(range_length < (1 << btp->bt_sshift)));
 440         ASSERT(!(range_base & (xfs_off_t)btp->bt_smask));
 441
 442         /* get tree root */
 443         pag = xfs_perag_get(btp->bt_mount,
 444                                 xfs_daddr_to_agno(btp->bt_mount, ioff));
 445
 446         /* walk tree */
 447         spin_lock(&pag->pag_buf_lock);
 448         rbp = &pag->pag_buf_tree.rb_node;
 449         parent = NULL;
 450         bp = NULL;
 451         while (*rbp) {
 452                 parent = *rbp;
 453                 bp = rb_entry(parent, struct xfs_buf, b_rbnode);
 454
 455                 if (range_base < bp->b_file_offset)
 456                         rbp = &(*rbp)->rb_left;
 457                 else if (range_base > bp->b_file_offset)
 458                         rbp = &(*rbp)->rb_right;
 459                 else {
 460                         /*
 461                          * found a block offset match. If the range doesn't
 462                          * match, the only way this is allowed is if the buffer
 463                          * in the cache is stale and the transaction that made
 464                          * it stale has not yet committed. i.e. we are
 465                          * reallocating a busy extent. Skip this buffer and
 466                          * continue searching to the right for an exact match.
 467                          */
 468                         if (bp->b_buffer_length != range_length) {
 469                                 ASSERT(bp->b_flags & XBF_STALE);
 470                                 rbp = &(*rbp)->rb_right;
 471                                 continue;
 472                         }
 473                         atomic_inc(&bp->b_hold);
 474                         goto found;
 475                 }
 476         }
 477
 478         /* No match found */
 479         if (new_bp) {
 480                 rb_link_node(&new_bp->b_rbnode, parent, rbp);
 481                 rb_insert_color(&new_bp->b_rbnode, &pag->pag_buf_tree);
 482                 /* the buffer keeps the perag reference until it is freed */
 483                 new_bp->b_pag = pag;
 484                 spin_unlock(&pag->pag_buf_lock);
 485         } else {
 486                 XFS_STATS_INC(xb_miss_locked);
 487                 spin_unlock(&pag->pag_buf_lock);
 488                 xfs_perag_put(pag);
 489         }
 490         return new_bp;
 491
 492 found:
 493         spin_unlock(&pag->pag_buf_lock);
 494         xfs_perag_put(pag);
 495
 496         if (!xfs_buf_trylock(bp)) {
 497                 if (flags & XBF_TRYLOCK) {
 498                         xfs_buf_rele(bp);
 499                         XFS_STATS_INC(xb_busy_locked);
 500                         return NULL;
 501                 }
 502                 xfs_buf_lock(bp);
 503                 XFS_STATS_INC(xb_get_locked_waited);
 504         }
 505
 506         /*
 507          * if the buffer is stale, clear all the external state associated with
 508          * it. We need to keep flags such as how we allocated the buffer memory
 509          * intact here.
 510          */
 511         if (bp->b_flags & XBF_STALE) {
 512                 ASSERT((bp->b_flags & _XBF_DELWRI_Q) == 0);
 513                 bp->b_flags &= XBF_MAPPED | _XBF_KMEM | _XBF_PAGES;
 514         }
 515
 516         trace_xfs_buf_find(bp, flags, _RET_IP_);
 517         XFS_STATS_INC(xb_get_locked);
 518         return bp;
 519 }
 520
 521 /*
 522  * Assembles a buffer covering the specified range. The code is optimised for
 523  * cache hits, as metadata intensive workloads will see 3 orders of magnitude
 524  * more hits than misses.
 525  */
 526 struct xfs_buf *
 527 xfs_buf_get(
 528         xfs_buftarg_t           *target,/* target for buffer            */
 529         xfs_off_t               ioff,   /* starting offset of range     */
 530         size_t                  isize,  /* length of range              */
 531         xfs_buf_flags_t         flags)
 532 {
 533         struct xfs_buf          *bp;
 534         struct xfs_buf          *new_bp;
 535         int                     error = 0;
 536
 537         bp = _xfs_buf_find(target, ioff, isize, flags, NULL);
 538         if (likely(bp))
 539                 goto found;
 540
 541         new_bp = xfs_buf_allocate(flags);
 542         if (unlikely(!new_bp))
 543                 return NULL;
 544
 545         _xfs_buf_initialize(new_bp, target,
 546                             ioff << BBSHIFT, isize << BBSHIFT, flags);
 547
 548         bp = _xfs_buf_find(target, ioff, isize, flags, new_bp);
 549         if (!bp) {
 550                 xfs_buf_deallocate(new_bp);
 551                 return NULL;
 552         }
 553
 554         if (bp == new_bp) {
 555                 error = xfs_buf_allocate_memory(bp, flags);
 556                 if (error)
 557                         goto no_buffer;
 558         } else
 559                 xfs_buf_deallocate(new_bp);
 560
 561         /*
 562          * Now we have a workable buffer, fill in the block number so
 563          * that we can do IO on it.
 564          */
 565         bp->b_bn = ioff;
 566         bp->b_count_desired = bp->b_buffer_length;
 567
 568 found:
 569         if (!(bp->b_flags & XBF_MAPPED)) {
 570                 error = _xfs_buf_map_pages(bp, flags);
 571                 if (unlikely(error)) {
 572                         xfs_warn(target->bt_mount,
 573                                 "%s: failed to map pages\n", __func__);
 574                         goto no_buffer;
 575                 }
 576         }
 577
 578         XFS_STATS_INC(xb_get);
 579         trace_xfs_buf_get(bp, flags, _RET_IP_);
 580         return bp;
 581
 582 no_buffer:
 583         if (flags & (XBF_LOCK | XBF_TRYLOCK))
 584                 xfs_buf_unlock(bp);
 585         xfs_buf_rele(bp);
 586         return NULL;
 587 }
 588
 589 STATIC int
 590 _xfs_buf_read(
 591         xfs_buf_t               *bp,
 592         xfs_buf_flags_t         flags)
 593 {
 594         int                     status;
 595
 596         ASSERT(!(flags & (XBF_DELWRI|XBF_WRITE)));
 597         ASSERT(bp->b_bn != XFS_BUF_DADDR_NULL);
 598
 599         bp->b_flags &= ~(XBF_WRITE | XBF_ASYNC | XBF_DELWRI | XBF_READ_AHEAD);
 600         bp->b_flags |= flags & (XBF_READ | XBF_ASYNC | XBF_READ_AHEAD);
 601
 602         status = xfs_buf_iorequest(bp);
 603         if (status || bp->b_error || (flags & XBF_ASYNC))
 604                 return status;
 605         return xfs_buf_iowait(bp);
 606 }
 607
 608 xfs_buf_t *
 609 xfs_buf_read(
 610         xfs_buftarg_t           *target,
 611         xfs_off_t               ioff,
 612         size_t                  isize,
 613         xfs_buf_flags_t         flags)
 614 {
 615         xfs_buf_t               *bp;
 616
 617         flags |= XBF_READ;
 618
 619         bp = xfs_buf_get(target, ioff, isize, flags);
 620         if (bp) {
 621                 trace_xfs_buf_read(bp, flags, _RET_IP_);
 622
 623                 if (!XFS_BUF_ISDONE(bp)) {
 624                         XFS_STATS_INC(xb_get_read);
 625                         _xfs_buf_read(bp, flags);
 626                 } else if (flags & XBF_ASYNC) {
 627                         /*
 628                          * Read ahead call which is already satisfied,
 629                          * drop the buffer
 630                          */
 631                         goto no_buffer;
 632                 } else {
 633                         /* We do not want read in the flags */
 634                         bp->b_flags &= ~XBF_READ;
 635                 }
 636         }
 637
 638         return bp;
 639
 640  no_buffer:
 641         if (flags & (XBF_LOCK | XBF_TRYLOCK))
 642                 xfs_buf_unlock(bp);
 643         xfs_buf_rele(bp);
 644         return NULL;
 645 }
 646
 647 /*
 648  *      If we are not low on memory then do the readahead in a deadlock
 649  *      safe manner.
 650  */
 651 void
 652 xfs_buf_readahead(
 653         xfs_buftarg_t           *target,
 654         xfs_off_t               ioff,
 655         size_t                  isize)
 656 {
 657         if (bdi_read_congested(target->bt_bdi))
 658                 return;
 659
 660         xfs_buf_read(target, ioff, isize,
 661                      XBF_TRYLOCK|XBF_ASYNC|XBF_READ_AHEAD|XBF_DONT_BLOCK);
 662 }
 663
 664 /*
 665  * Read an uncached buffer from disk. Allocates and returns a locked
 666  * buffer containing the disk contents or nothing.
 667  */
 668 struct xfs_buf *
 669 xfs_buf_read_uncached(
 670         struct xfs_mount        *mp,
 671         struct xfs_buftarg      *target,
 672         xfs_daddr_t             daddr,
 673         size_t                  length,
 674         int                     flags)
 675 {
 676         xfs_buf_t               *bp;
 677         int                     error;
 678
 679         bp = xfs_buf_get_uncached(target, length, flags);
 680         if (!bp)
 681                 return NULL;
 682
 683         /* set up the buffer for a read IO */
 684         XFS_BUF_SET_ADDR(bp, daddr);
 685         XFS_BUF_READ(bp);
 686
 687         xfsbdstrat(mp, bp);
 688         error = xfs_buf_iowait(bp);
 689         if (error || bp->b_error) {
 690                 xfs_buf_relse(bp);
 691                 return NULL;
 692         }
 693         return bp;
 694 }
 695
 696 xfs_buf_t *
 697 xfs_buf_get_empty(
 698         size_t                  len,
 699         xfs_buftarg_t           *target)
 700 {
 701         xfs_buf_t               *bp;
 702
 703         bp = xfs_buf_allocate(0);
 704         if (bp)
 705                 _xfs_buf_initialize(bp, target, 0, len, 0);
 706         return bp;
 707 }
 708
 709 /*
 710  * Return a buffer allocated as an empty buffer and associated to external
 711  * memory via xfs_buf_associate_memory() back to it's empty state.
 712  */
 713 void
 714 xfs_buf_set_empty(
 715         struct xfs_buf          *bp,
 716         size_t                  len)
 717 {
 718         if (bp->b_pages)
 719                 _xfs_buf_free_pages(bp);
 720
 721         bp->b_pages = NULL;
 722         bp->b_page_count = 0;
 723         bp->b_addr = NULL;
 724         bp->b_file_offset = 0;
 725         bp->b_buffer_length = bp->b_count_desired = len;
 726         bp->b_bn = XFS_BUF_DADDR_NULL;
 727         bp->b_flags &= ~XBF_MAPPED;
 728 }
 729
 730 static inline struct page *
 731 mem_to_page(
 732         void                    *addr)
 733 {
 734         if ((!is_vmalloc_addr(addr))) {
 735                 return virt_to_page(addr);
 736         } else {
 737                 return vmalloc_to_page(addr);
 738         }
 739 }
 740
 741 int
 742 xfs_buf_associate_memory(
 743         xfs_buf_t               *bp,
 744         void                    *mem,
 745         size_t                  len)
 746 {
 747         int                     rval;
 748         int                     i = 0;
 749         unsigned long           pageaddr;
 750         unsigned long           offset;
 751         size_t                  buflen;
 752         int                     page_count;
 753
 754         pageaddr = (unsigned long)mem & PAGE_MASK;
 755         offset = (unsigned long)mem - pageaddr;
 756         buflen = PAGE_ALIGN(len + offset);
 757         page_count = buflen >> PAGE_SHIFT;
 758
 759         /* Free any previous set of page pointers */
 760         if (bp->b_pages)
 761                 _xfs_buf_free_pages(bp);
 762
 763         bp->b_pages = NULL;
 764         bp->b_addr = mem;
 765
 766         rval = _xfs_buf_get_pages(bp, page_count, XBF_DONT_BLOCK);
 767         if (rval)
 768                 return rval;
 769
 770         bp->b_offset = offset;
 771
 772         for (i = 0; i < bp->b_page_count; i++) {
 773                 bp->b_pages[i] = mem_to_page((void *)pageaddr);
 774                 pageaddr += PAGE_SIZE;
 775         }
 776
 777         bp->b_count_desired = len;
 778         bp->b_buffer_length = buflen;
 779         bp->b_flags |= XBF_MAPPED;
 780
 781         return 0;
 782 }
 783
 784 xfs_buf_t *
 785 xfs_buf_get_uncached(
 786         struct xfs_buftarg      *target,
 787         size_t                  len,
 788         int                     flags)
 789 {
 790         unsigned long           page_count = PAGE_ALIGN(len) >> PAGE_SHIFT;
 791         int                     error, i;
 792         xfs_buf_t               *bp;
 793
 794         bp = xfs_buf_allocate(0);
 795         if (unlikely(bp == NULL))
 796                 goto fail;
 797         _xfs_buf_initialize(bp, target, 0, len, 0);
 798
 799         error = _xfs_buf_get_pages(bp, page_count, 0);
 800         if (error)
 801                 goto fail_free_buf;
 802
 803         for (i = 0; i < page_count; i++) {
 804                 bp->b_pages[i] = alloc_page(xb_to_gfp(flags));
 805                 if (!bp->b_pages[i])
 806                         goto fail_free_mem;
 807         }
 808         bp->b_flags |= _XBF_PAGES;
 809
 810         error = _xfs_buf_map_pages(bp, XBF_MAPPED);
 811         if (unlikely(error)) {
 812                 xfs_warn(target->bt_mount,
 813                         "%s: failed to map pages\n", __func__);
 814                 goto fail_free_mem;
 815         }
 816
 817         trace_xfs_buf_get_uncached(bp, _RET_IP_);
 818         return bp;
 819
 820  fail_free_mem:
 821         while (--i >= 0)
 822                 __free_page(bp->b_pages[i]);
 823         _xfs_buf_free_pages(bp);
 824  fail_free_buf:
 825         xfs_buf_deallocate(bp);
 826  fail:
 827         return NULL;
 828 }
 829
 830 /*
 831  *      Increment reference count on buffer, to hold the buffer concurrently
 832  *      with another thread which may release (free) the buffer asynchronously.
 833  *      Must hold the buffer already to call this function.
 834  */
 835 void
 836 xfs_buf_hold(
 837         xfs_buf_t               *bp)
 838 {
 839         trace_xfs_buf_hold(bp, _RET_IP_);
 840         atomic_inc(&bp->b_hold);
 841 }
 842
 843 /*
 844  *      Releases a hold on the specified buffer.  If the
 845  *      the hold count is 1, calls xfs_buf_free.
 846  */
 847 void
 848 xfs_buf_rele(
 849         xfs_buf_t               *bp)
 850 {
 851         struct xfs_perag        *pag = bp->b_pag;
 852
 853         trace_xfs_buf_rele(bp, _RET_IP_);
 854
 855         if (!pag) {
 856                 ASSERT(list_empty(&bp->b_lru));
 857                 ASSERT(RB_EMPTY_NODE(&bp->b_rbnode));
 858                 if (atomic_dec_and_test(&bp->b_hold))
 859                         xfs_buf_free(bp);
 860                 return;
 861         }
 862
 863         ASSERT(!RB_EMPTY_NODE(&bp->b_rbnode));
 864
 865         ASSERT(atomic_read(&bp->b_hold) > 0);
 866         if (atomic_dec_and_lock(&bp->b_hold, &pag->pag_buf_lock)) {
 867                 if (!(bp->b_flags & XBF_STALE) &&
 868                            atomic_read(&bp->b_lru_ref)) {
 869                         xfs_buf_lru_add(bp);
 870                         spin_unlock(&pag->pag_buf_lock);
 871                 } else {
 872                         xfs_buf_lru_del(bp);
 873                         ASSERT(!(bp->b_flags & (XBF_DELWRI|_XBF_DELWRI_Q)));
 874                         rb_erase(&bp->b_rbnode, &pag->pag_buf_tree);
 875                         spin_unlock(&pag->pag_buf_lock);
 876                         xfs_perag_put(pag);
 877                         xfs_buf_free(bp);
 878                 }
 879         }
 880 }
 881
 882
 883 /*
 884  *      Lock a buffer object, if it is not already locked.
 885  *
 886  *      If we come across a stale, pinned, locked buffer, we know that we are
 887  *      being asked to lock a buffer that has been reallocated. Because it is
 888  *      pinned, we know that the log has not been pushed to disk and hence it
 889  *      will still be locked.  Rather than continuing to have trylock attempts
 890  *      fail until someone else pushes the log, push it ourselves before
 891  *      returning.  This means that the xfsaild will not get stuck trying
 892  *      to push on stale inode buffers.
 893  */
 894 int
 895 xfs_buf_trylock(
 896         struct xfs_buf          *bp)
 897 {
 898         int                     locked;
 899
 900         locked = down_trylock(&bp->b_sema) == 0;
 901         if (locked)
 902                 XB_SET_OWNER(bp);
 903         else if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
 904                 xfs_log_force(bp->b_target->bt_mount, 0);
 905
 906         trace_xfs_buf_trylock(bp, _RET_IP_);
 907         return locked;
 908 }
 909
 910 /*
 911  *      Lock a buffer object.
 912  *
 913  *      If we come across a stale, pinned, locked buffer, we know that we
 914  *      are being asked to lock a buffer that has been reallocated. Because
 915  *      it is pinned, we know that the log has not been pushed to disk and
 916  *      hence it will still be locked. Rather than sleeping until someone
 917  *      else pushes the log, push it ourselves before trying to get the lock.
 918  */
 919 void
 920 xfs_buf_lock(
 921         struct xfs_buf          *bp)
 922 {
 923         trace_xfs_buf_lock(bp, _RET_IP_);
 924
 925         if (atomic_read(&bp->b_pin_count) && (bp->b_flags & XBF_STALE))
 926                 xfs_log_force(bp->b_target->bt_mount, 0);
 927         down(&bp->b_sema);
 928         XB_SET_OWNER(bp);
 929
 930         trace_xfs_buf_lock_done(bp, _RET_IP_);
 931 }
 932
 933 /*
 934  *      Releases the lock on the buffer object.
 935  *      If the buffer is marked delwri but is not queued, do so before we
 936  *      unlock the buffer as we need to set flags correctly.  We also need to
 937  *      take a reference for the delwri queue because the unlocker is going to
 938  *      drop their's and they don't know we just queued it.
 939  */
 940 void
 941 xfs_buf_unlock(
 942         struct xfs_buf          *bp)
 943 {
 944         XB_CLEAR_OWNER(bp);
 945         up(&bp->b_sema);
 946
 947         trace_xfs_buf_unlock(bp, _RET_IP_);
 948 }
 949
 950 STATIC void
 951 xfs_buf_wait_unpin(
 952         xfs_buf_t               *bp)
 953 {
 954         DECLARE_WAITQUEUE       (wait, current);
 955
 956         if (atomic_read(&bp->b_pin_count) == 0)
 957                 return;
 958
 959         add_wait_queue(&bp->b_waiters, &wait);
 960         for (;;) {
 961                 set_current_state(TASK_UNINTERRUPTIBLE);
 962                 if (atomic_read(&bp->b_pin_count) == 0)
 963                         break;
 964                 io_schedule();
 965         }
 966         remove_wait_queue(&bp->b_waiters, &wait);
 967         set_current_state(TASK_RUNNING);
 968 }
 969
 970 /*
 971  *      Buffer Utility Routines
 972  */
 973
 974 STATIC void
 975 xfs_buf_iodone_work(
 976         struct work_struct      *work)
 977 {
 978         xfs_buf_t               *bp =
 979                 container_of(work, xfs_buf_t, b_iodone_work);
 980
 981         if (bp->b_iodone)
 982                 (*(bp->b_iodone))(bp);
 983         else if (bp->b_flags & XBF_ASYNC)
 984                 xfs_buf_relse(bp);
 985 }
 986
 987 void
 988 xfs_buf_ioend(
 989         xfs_buf_t               *bp,
 990         int                     schedule)
 991 {
 992         trace_xfs_buf_iodone(bp, _RET_IP_);
 993
 994         bp->b_flags &= ~(XBF_READ | XBF_WRITE | XBF_READ_AHEAD);
 995         if (bp->b_error == 0)
 996                 bp->b_flags |= XBF_DONE;
 997
 998         if ((bp->b_iodone) || (bp->b_flags & XBF_ASYNC)) {
 999                 if (schedule) {
1000                         INIT_WORK(&bp->b_iodone_work, xfs_buf_iodone_work);
1001                         queue_work(xfslogd_workqueue, &bp->b_iodone_work);
1002                 } else {
1003                         xfs_buf_iodone_work(&bp->b_iodone_work);
1004                 }
1005         } else {
1006                 complete(&bp->b_iowait);
1007         }
1008 }
1009
1010 void
1011 xfs_buf_ioerror(
1012         xfs_buf_t               *bp,
1013         int                     error)
1014 {
1015         ASSERT(error >= 0 && error <= 0xffff);
1016         bp->b_error = (unsigned short)error;
1017         trace_xfs_buf_ioerror(bp, error, _RET_IP_);
1018 }
1019
1020 int
1021 xfs_bwrite(
1022         struct xfs_buf          *bp)
1023 {
1024         int                     error;
1025
1026         bp->b_flags |= XBF_WRITE;
1027         bp->b_flags &= ~(XBF_ASYNC | XBF_READ);
1028
1029         xfs_buf_delwri_dequeue(bp);
1030         xfs_bdstrat_cb(bp);
1031
1032         error = xfs_buf_iowait(bp);
1033         if (error) {
1034                 xfs_force_shutdown(bp->b_target->bt_mount,
1035                                    SHUTDOWN_META_IO_ERROR);
1036         }
1037         return error;
1038 }
1039
1040 /*
1041  * Called when we want to stop a buffer from getting written or read.
1042  * We attach the EIO error, muck with its flags, and call xfs_buf_ioend
1043  * so that the proper iodone callbacks get called.
1044  */
1045 STATIC int
1046 xfs_bioerror(
1047         xfs_buf_t *bp)
1048 {
1049 #ifdef XFSERRORDEBUG
1050         ASSERT(XFS_BUF_ISREAD(bp) || bp->b_iodone);
1051 #endif
1052
1053         /*
1054          * No need to wait until the buffer is unpinned, we aren't flushing it.
1055          */
1056         xfs_buf_ioerror(bp, EIO);
1057
1058         /*
1059          * We're calling xfs_buf_ioend, so delete XBF_DONE flag.
1060          */
1061         XFS_BUF_UNREAD(bp);
1062         xfs_buf_delwri_dequeue(bp);
1063         XFS_BUF_UNDONE(bp);
1064         XFS_BUF_STALE(bp);
1065
1066         xfs_buf_ioend(bp, 0);
1067
1068         return EIO;
1069 }
1070
1071 /*
1072  * Same as xfs_bioerror, except that we are releasing the buffer
1073  * here ourselves, and avoiding the xfs_buf_ioend call.
1074  * This is meant for userdata errors; metadata bufs come with
1075  * iodone functions attached, so that we can track down errors.
1076  */
1077 STATIC int
1078 xfs_bioerror_relse(
1079         struct xfs_buf  *bp)
1080 {
1081         int64_t         fl = bp->b_flags;
1082         /*
1083          * No need to wait until the buffer is unpinned.
1084          * We aren't flushing it.
1085          *
1086          * chunkhold expects B_DONE to be set, whether
1087          * we actually finish the I/O or not. We don't want to
1088          * change that interface.
1089          */
1090         XFS_BUF_UNREAD(bp);
1091         xfs_buf_delwri_dequeue(bp);
1092         XFS_BUF_DONE(bp);
1093         XFS_BUF_STALE(bp);
1094         bp->b_iodone = NULL;
1095         if (!(fl & XBF_ASYNC)) {
1096                 /*
1097                  * Mark b_error and B_ERROR _both_.
1098                  * Lot's of chunkcache code assumes that.
1099                  * There's no reason to mark error for
1100                  * ASYNC buffers.
1101                  */
1102                 xfs_buf_ioerror(bp, EIO);
1103                 XFS_BUF_FINISH_IOWAIT(bp);
1104         } else {
1105                 xfs_buf_relse(bp);
1106         }
1107
1108         return EIO;
1109 }
1110
1111
1112 /*
1113  * All xfs metadata buffers except log state machine buffers
1114  * get this attached as their b_bdstrat callback function.
1115  * This is so that we can catch a buffer
1116  * after prematurely unpinning it to forcibly shutdown the filesystem.
1117  */
1118 int
1119 xfs_bdstrat_cb(
1120         struct xfs_buf  *bp)
1121 {
1122         if (XFS_FORCED_SHUTDOWN(bp->b_target->bt_mount)) {
1123                 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1124                 /*
1125                  * Metadata write that didn't get logged but
1126                  * written delayed anyway. These aren't associated
1127                  * with a transaction, and can be ignored.
1128                  */
1129                 if (!bp->b_iodone && !XFS_BUF_ISREAD(bp))
1130                         return xfs_bioerror_relse(bp);
1131                 else
1132                         return xfs_bioerror(bp);
1133         }
1134
1135         xfs_buf_iorequest(bp);
1136         return 0;
1137 }
1138
1139 /*
1140  * Wrapper around bdstrat so that we can stop data from going to disk in case
1141  * we are shutting down the filesystem.  Typically user data goes thru this
1142  * path; one of the exceptions is the superblock.
1143  */
1144 void
1145 xfsbdstrat(
1146         struct xfs_mount        *mp,
1147         struct xfs_buf          *bp)
1148 {
1149         if (XFS_FORCED_SHUTDOWN(mp)) {
1150                 trace_xfs_bdstrat_shut(bp, _RET_IP_);
1151                 xfs_bioerror_relse(bp);
1152                 return;
1153         }
1154
1155         xfs_buf_iorequest(bp);
1156 }
1157
1158 STATIC void
1159 _xfs_buf_ioend(
1160         xfs_buf_t               *bp,
1161         int                     schedule)
1162 {
1163         if (atomic_dec_and_test(&bp->b_io_remaining) == 1)
1164                 xfs_buf_ioend(bp, schedule);
1165 }
1166
1167 STATIC void
1168 xfs_buf_bio_end_io(
1169         struct bio              *bio,
1170         int                     error)
1171 {
1172         xfs_buf_t               *bp = (xfs_buf_t *)bio->bi_private;
1173
1174         xfs_buf_ioerror(bp, -error);
1175
1176         if (!error && xfs_buf_is_vmapped(bp) && (bp->b_flags & XBF_READ))
1177                 invalidate_kernel_vmap_range(bp->b_addr, xfs_buf_vmap_len(bp));
1178
1179         _xfs_buf_ioend(bp, 1);
1180         bio_put(bio);
1181 }
1182
1183 STATIC void
1184 _xfs_buf_ioapply(
1185         xfs_buf_t               *bp)
1186 {
1187         int                     rw, map_i, total_nr_pages, nr_pages;
1188         struct bio              *bio;
1189         int                     offset = bp->b_offset;
1190         int                     size = bp->b_count_desired;
1191         sector_t                sector = bp->b_bn;
1192
1193         total_nr_pages = bp->b_page_count;
1194         map_i = 0;
1195
1196         if (bp->b_flags & XBF_WRITE) {
1197                 if (bp->b_flags & XBF_SYNCIO)
1198                         rw = WRITE_SYNC;
1199                 else
1200                         rw = WRITE;
1201                 if (bp->b_flags & XBF_FUA)
1202                         rw |= REQ_FUA;
1203                 if (bp->b_flags & XBF_FLUSH)
1204                         rw |= REQ_FLUSH;
1205         } else if (bp->b_flags & XBF_READ_AHEAD) {
1206                 rw = READA;
1207         } else {
1208                 rw = READ;
1209         }
1210
1211         /* we only use the buffer cache for meta-data */
1212         rw |= REQ_META;
1213
1214 next_chunk:
1215         atomic_inc(&bp->b_io_remaining);
1216         nr_pages = BIO_MAX_SECTORS >> (PAGE_SHIFT - BBSHIFT);
1217         if (nr_pages > total_nr_pages)
1218                 nr_pages = total_nr_pages;
1219
1220         bio = bio_alloc(GFP_NOIO, nr_pages);
1221         bio->bi_bdev = bp->b_target->bt_bdev;
1222         bio->bi_sector = sector;
1223         bio->bi_end_io = xfs_buf_bio_end_io;
1224         bio->bi_private = bp;
1225
1226
1227         for (; size && nr_pages; nr_pages--, map_i++) {
1228                 int     rbytes, nbytes = PAGE_SIZE - offset;
1229
1230                 if (nbytes > size)
1231                         nbytes = size;
1232
1233                 rbytes = bio_add_page(bio, bp->b_pages[map_i], nbytes, offset);
1234                 if (rbytes < nbytes)
1235                         break;
1236
1237                 offset = 0;
1238                 sector += nbytes >> BBSHIFT;
1239                 size -= nbytes;
1240                 total_nr_pages--;
1241         }
1242
1243         if (likely(bio->bi_size)) {
1244                 if (xfs_buf_is_vmapped(bp)) {
1245                         flush_kernel_vmap_range(bp->b_addr,
1246                                                 xfs_buf_vmap_len(bp));
1247                 }
1248                 submit_bio(rw, bio);
1249                 if (size)
1250                         goto next_chunk;
1251         } else {
1252                 xfs_buf_ioerror(bp, EIO);
1253                 bio_put(bio);
1254         }
1255 }
1256
1257 int
1258 xfs_buf_iorequest(
1259         xfs_buf_t               *bp)
1260 {
1261         trace_xfs_buf_iorequest(bp, _RET_IP_);
1262
1263         ASSERT(!(bp->b_flags & XBF_DELWRI));
1264
1265         if (bp->b_flags & XBF_WRITE)
1266                 xfs_buf_wait_unpin(bp);
1267         xfs_buf_hold(bp);
1268
1269         /* Set the count to 1 initially, this will stop an I/O
1270          * completion callout which happens before we have started
1271          * all the I/O from calling xfs_buf_ioend too early.
1272          */
1273         atomic_set(&bp->b_io_remaining, 1);
1274         _xfs_buf_ioapply(bp);
1275         _xfs_buf_ioend(bp, 0);
1276
1277         xfs_buf_rele(bp);
1278         return 0;
1279 }
1280
1281 /*
1282  *      Waits for I/O to complete on the buffer supplied.
1283  *      It returns immediately if no I/O is pending.
1284  *      It returns the I/O error code, if any, or 0 if there was no error.
1285  */
1286 int
1287 xfs_buf_iowait(
1288         xfs_buf_t               *bp)
1289 {
1290         trace_xfs_buf_iowait(bp, _RET_IP_);
1291
1292         wait_for_completion(&bp->b_iowait);
1293
1294         trace_xfs_buf_iowait_done(bp, _RET_IP_);
1295         return bp->b_error;
1296 }
1297
1298 xfs_caddr_t
1299 xfs_buf_offset(
1300         xfs_buf_t               *bp,
1301         size_t                  offset)
1302 {
1303         struct page             *page;
1304
1305         if (bp->b_flags & XBF_MAPPED)
1306                 return bp->b_addr + offset;
1307
1308         offset += bp->b_offset;
1309         page = bp->b_pages[offset >> PAGE_SHIFT];
1310         return (xfs_caddr_t)page_address(page) + (offset & (PAGE_SIZE-1));
1311 }
1312
1313 /*
1314  *      Move data into or out of a buffer.
1315  */
1316 void
1317 xfs_buf_iomove(
1318         xfs_buf_t               *bp,    /* buffer to process            */
1319         size_t                  boff,   /* starting buffer offset       */
1320         size_t                  bsize,  /* length to copy               */
1321         void                    *data,  /* data address                 */
1322         xfs_buf_rw_t            mode)   /* read/write/zero flag         */
1323 {
1324         size_t                  bend, cpoff, csize;
1325         struct page             *page;
1326
1327         bend = boff + bsize;
1328         while (boff < bend) {
1329                 page = bp->b_pages[xfs_buf_btoct(boff + bp->b_offset)];
1330                 cpoff = xfs_buf_poff(boff + bp->b_offset);
1331                 csize = min_t(size_t,
1332                               PAGE_SIZE-cpoff, bp->b_count_desired-boff);
1333
1334                 ASSERT(((csize + cpoff) <= PAGE_SIZE));
1335
1336                 switch (mode) {
1337                 case XBRW_ZERO:
1338                         memset(page_address(page) + cpoff, 0, csize);
1339                         break;
1340                 case XBRW_READ:
1341                         memcpy(data, page_address(page) + cpoff, csize);
1342                         break;
1343                 case XBRW_WRITE:
1344                         memcpy(page_address(page) + cpoff, data, csize);
1345                 }
1346
1347                 boff += csize;
1348                 data += csize;
1349         }
1350 }
1351
1352 /*
1353  *      Handling of buffer targets (buftargs).
1354  */
1355
1356 /*
1357  * Wait for any bufs with callbacks that have been submitted but have not yet
1358  * returned. These buffers will have an elevated hold count, so wait on those
1359  * while freeing all the buffers only held by the LRU.
1360  */
1361 void
1362 xfs_wait_buftarg(
1363         struct xfs_buftarg      *btp)
1364 {
1365         struct xfs_buf          *bp;
1366
1367 restart:
1368         spin_lock(&btp->bt_lru_lock);
1369         while (!list_empty(&btp->bt_lru)) {
1370                 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1371                 if (atomic_read(&bp->b_hold) > 1) {
1372                         spin_unlock(&btp->bt_lru_lock);
1373                         delay(100);
1374                         goto restart;
1375                 }
1376                 /*
1377                  * clear the LRU reference count so the bufer doesn't get
1378                  * ignored in xfs_buf_rele().
1379                  */
1380                 atomic_set(&bp->b_lru_ref, 0);
1381                 spin_unlock(&btp->bt_lru_lock);
1382                 xfs_buf_rele(bp);
1383                 spin_lock(&btp->bt_lru_lock);
1384         }
1385         spin_unlock(&btp->bt_lru_lock);
1386 }
1387
1388 int
1389 xfs_buftarg_shrink(
1390         struct shrinker         *shrink,
1391         struct shrink_control   *sc)
1392 {
1393         struct xfs_buftarg      *btp = container_of(shrink,
1394                                         struct xfs_buftarg, bt_shrinker);
1395         struct xfs_buf          *bp;
1396         int nr_to_scan = sc->nr_to_scan;
1397         LIST_HEAD(dispose);
1398
1399         if (!nr_to_scan)
1400                 return btp->bt_lru_nr;
1401
1402         spin_lock(&btp->bt_lru_lock);
1403         while (!list_empty(&btp->bt_lru)) {
1404                 if (nr_to_scan-- <= 0)
1405                         break;
1406
1407                 bp = list_first_entry(&btp->bt_lru, struct xfs_buf, b_lru);
1408
1409                 /*
1410                  * Decrement the b_lru_ref count unless the value is already
1411                  * zero. If the value is already zero, we need to reclaim the
1412                  * buffer, otherwise it gets another trip through the LRU.
1413                  */
1414                 if (!atomic_add_unless(&bp->b_lru_ref, -1, 0)) {
1415                         list_move_tail(&bp->b_lru, &btp->bt_lru);
1416                         continue;
1417                 }
1418
1419                 /*
1420                  * remove the buffer from the LRU now to avoid needing another
1421                  * lock round trip inside xfs_buf_rele().
1422                  */
1423                 list_move(&bp->b_lru, &dispose);
1424                 btp->bt_lru_nr--;
1425         }
1426         spin_unlock(&btp->bt_lru_lock);
1427
1428         while (!list_empty(&dispose)) {
1429                 bp = list_first_entry(&dispose, struct xfs_buf, b_lru);
1430                 list_del_init(&bp->b_lru);
1431                 xfs_buf_rele(bp);
1432         }
1433
1434         return btp->bt_lru_nr;
1435 }
1436
1437 void
1438 xfs_free_buftarg(
1439         struct xfs_mount        *mp,
1440         struct xfs_buftarg      *btp)
1441 {
1442         unregister_shrinker(&btp->bt_shrinker);
1443
1444         xfs_flush_buftarg(btp, 1);
1445         if (mp->m_flags & XFS_MOUNT_BARRIER)
1446                 xfs_blkdev_issue_flush(btp);
1447
1448         kthread_stop(btp->bt_task);
1449         kmem_free(btp);
1450 }
1451
1452 STATIC int
1453 xfs_setsize_buftarg_flags(
1454         xfs_buftarg_t           *btp,
1455         unsigned int            blocksize,
1456         unsigned int            sectorsize,
1457         int                     verbose)
1458 {
1459         btp->bt_bsize = blocksize;
1460         btp->bt_sshift = ffs(sectorsize) - 1;
1461         btp->bt_smask = sectorsize - 1;
1462
1463         if (set_blocksize(btp->bt_bdev, sectorsize)) {
1464                 xfs_warn(btp->bt_mount,
1465                         "Cannot set_blocksize to %u on device %s\n",
1466                         sectorsize, xfs_buf_target_name(btp));
1467                 return EINVAL;
1468         }
1469
1470         return 0;
1471 }
1472
1473 /*
1474  *      When allocating the initial buffer target we have not yet
1475  *      read in the superblock, so don't know what sized sectors
1476  *      are being used is at this early stage.  Play safe.
1477  */
1478 STATIC int
1479 xfs_setsize_buftarg_early(
1480         xfs_buftarg_t           *btp,
1481         struct block_device     *bdev)
1482 {
1483         return xfs_setsize_buftarg_flags(btp,
1484                         PAGE_SIZE, bdev_logical_block_size(bdev), 0);
1485 }
1486
1487 int
1488 xfs_setsize_buftarg(
1489         xfs_buftarg_t           *btp,
1490         unsigned int            blocksize,
1491         unsigned int            sectorsize)
1492 {
1493         return xfs_setsize_buftarg_flags(btp, blocksize, sectorsize, 1);
1494 }
1495
1496 STATIC int
1497 xfs_alloc_delwri_queue(
1498         xfs_buftarg_t           *btp,
1499         const char              *fsname)
1500 {
1501         INIT_LIST_HEAD(&btp->bt_delwri_queue);
1502         spin_lock_init(&btp->bt_delwri_lock);
1503         btp->bt_flags = 0;
1504         btp->bt_task = kthread_run(xfsbufd, btp, "xfsbufd/%s", fsname);
1505         if (IS_ERR(btp->bt_task))
1506                 return PTR_ERR(btp->bt_task);
1507         return 0;
1508 }
1509
1510 xfs_buftarg_t *
1511 xfs_alloc_buftarg(
1512         struct xfs_mount        *mp,
1513         struct block_device     *bdev,
1514         int                     external,
1515         const char              *fsname)
1516 {
1517         xfs_buftarg_t           *btp;
1518
1519         btp = kmem_zalloc(sizeof(*btp), KM_SLEEP);
1520
1521         btp->bt_mount = mp;
1522         btp->bt_dev =  bdev->bd_dev;
1523         btp->bt_bdev = bdev;
1524         btp->bt_bdi = blk_get_backing_dev_info(bdev);
1525         if (!btp->bt_bdi)
1526                 goto error;
1527
1528         INIT_LIST_HEAD(&btp->bt_lru);
1529         spin_lock_init(&btp->bt_lru_lock);
1530         if (xfs_setsize_buftarg_early(btp, bdev))
1531                 goto error;
1532         if (xfs_alloc_delwri_queue(btp, fsname))
1533                 goto error;
1534         btp->bt_shrinker.shrink = xfs_buftarg_shrink;
1535         btp->bt_shrinker.seeks = DEFAULT_SEEKS;
1536         register_shrinker(&btp->bt_shrinker);
1537         return btp;
1538
1539 error:
1540         kmem_free(btp);
1541         return NULL;
1542 }
1543
1544
1545 /*
1546  *      Delayed write buffer handling
1547  */
1548 void
1549 xfs_buf_delwri_queue(
1550         xfs_buf_t               *bp)
1551 {
1552         struct xfs_buftarg      *btp = bp->b_target;
1553
1554         trace_xfs_buf_delwri_queue(bp, _RET_IP_);
1555
1556         ASSERT(!(bp->b_flags & XBF_READ));
1557
1558         spin_lock(&btp->bt_delwri_lock);
1559         if (!list_empty(&bp->b_list)) {
1560                 /* if already in the queue, move it to the tail */
1561                 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1562                 list_move_tail(&bp->b_list, &btp->bt_delwri_queue);
1563         } else {
1564                 /* start xfsbufd as it is about to have something to do */
1565                 if (list_empty(&btp->bt_delwri_queue))
1566                         wake_up_process(bp->b_target->bt_task);
1567
1568                 atomic_inc(&bp->b_hold);
1569                 bp->b_flags |= XBF_DELWRI | _XBF_DELWRI_Q | XBF_ASYNC;
1570                 list_add_tail(&bp->b_list, &btp->bt_delwri_queue);
1571         }
1572         bp->b_queuetime = jiffies;
1573         spin_unlock(&btp->bt_delwri_lock);
1574 }
1575
1576 void
1577 xfs_buf_delwri_dequeue(
1578         xfs_buf_t               *bp)
1579 {
1580         int                     dequeued = 0;
1581
1582         spin_lock(&bp->b_target->bt_delwri_lock);
1583         if ((bp->b_flags & XBF_DELWRI) && !list_empty(&bp->b_list)) {
1584                 ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1585                 list_del_init(&bp->b_list);
1586                 dequeued = 1;
1587         }
1588         bp->b_flags &= ~(XBF_DELWRI|_XBF_DELWRI_Q);
1589         spin_unlock(&bp->b_target->bt_delwri_lock);
1590
1591         if (dequeued)
1592                 xfs_buf_rele(bp);
1593
1594         trace_xfs_buf_delwri_dequeue(bp, _RET_IP_);
1595 }
1596
1597 /*
1598  * If a delwri buffer needs to be pushed before it has aged out, then promote
1599  * it to the head of the delwri queue so that it will be flushed on the next
1600  * xfsbufd run. We do this by resetting the queuetime of the buffer to be older
1601  * than the age currently needed to flush the buffer. Hence the next time the
1602  * xfsbufd sees it is guaranteed to be considered old enough to flush.
1603  */
1604 void
1605 xfs_buf_delwri_promote(
1606         struct xfs_buf  *bp)
1607 {
1608         struct xfs_buftarg *btp = bp->b_target;
1609         long            age = xfs_buf_age_centisecs * msecs_to_jiffies(10) + 1;
1610
1611         ASSERT(bp->b_flags & XBF_DELWRI);
1612         ASSERT(bp->b_flags & _XBF_DELWRI_Q);
1613
1614         /*
1615          * Check the buffer age before locking the delayed write queue as we
1616          * don't need to promote buffers that are already past the flush age.
1617          */
1618         if (bp->b_queuetime < jiffies - age)
1619                 return;
1620         bp->b_queuetime = jiffies - age;
1621         spin_lock(&btp->bt_delwri_lock);
1622         list_move(&bp->b_list, &btp->bt_delwri_queue);
1623         spin_unlock(&btp->bt_delwri_lock);
1624 }
1625
1626 STATIC void
1627 xfs_buf_runall_queues(
1628         struct workqueue_struct *queue)
1629 {
1630         flush_workqueue(queue);
1631 }
1632
1633 /*
1634  * Move as many buffers as specified to the supplied list
1635  * idicating if we skipped any buffers to prevent deadlocks.
1636  */
1637 STATIC int
1638 xfs_buf_delwri_split(
1639         xfs_buftarg_t   *target,
1640         struct list_head *list,
1641         unsigned long   age)
1642 {
1643         xfs_buf_t       *bp, *n;
1644         int             skipped = 0;
1645         int             force;
1646
1647         force = test_and_clear_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1648         INIT_LIST_HEAD(list);
1649         spin_lock(&target->bt_delwri_lock);
1650         list_for_each_entry_safe(bp, n, &target->bt_delwri_queue, b_list) {
1651                 ASSERT(bp->b_flags & XBF_DELWRI);
1652
1653                 if (!xfs_buf_ispinned(bp) && xfs_buf_trylock(bp)) {
1654                         if (!force &&
1655                             time_before(jiffies, bp->b_queuetime + age)) {
1656                                 xfs_buf_unlock(bp);
1657                                 break;
1658                         }
1659
1660                         bp->b_flags &= ~(XBF_DELWRI | _XBF_DELWRI_Q);
1661                         bp->b_flags |= XBF_WRITE;
1662                         list_move_tail(&bp->b_list, list);
1663                         trace_xfs_buf_delwri_split(bp, _RET_IP_);
1664                 } else
1665                         skipped++;
1666         }
1667
1668         spin_unlock(&target->bt_delwri_lock);
1669         return skipped;
1670 }
1671
1672 /*
1673  * Compare function is more complex than it needs to be because
1674  * the return value is only 32 bits and we are doing comparisons
1675  * on 64 bit values
1676  */
1677 static int
1678 xfs_buf_cmp(
1679         void            *priv,
1680         struct list_head *a,
1681         struct list_head *b)
1682 {
1683         struct xfs_buf  *ap = container_of(a, struct xfs_buf, b_list);
1684         struct xfs_buf  *bp = container_of(b, struct xfs_buf, b_list);
1685         xfs_daddr_t             diff;
1686
1687         diff = ap->b_bn - bp->b_bn;
1688         if (diff < 0)
1689                 return -1;
1690         if (diff > 0)
1691                 return 1;
1692         return 0;
1693 }
1694
1695 STATIC int
1696 xfsbufd(
1697         void            *data)
1698 {
1699         xfs_buftarg_t   *target = (xfs_buftarg_t *)data;
1700
1701         current->flags |= PF_MEMALLOC;
1702
1703         set_freezable();
1704
1705         do {
1706                 long    age = xfs_buf_age_centisecs * msecs_to_jiffies(10);
1707                 long    tout = xfs_buf_timer_centisecs * msecs_to_jiffies(10);
1708                 struct list_head tmp;
1709                 struct blk_plug plug;
1710
1711                 if (unlikely(freezing(current))) {
1712                         set_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1713                         refrigerator();
1714                 } else {
1715                         clear_bit(XBT_FORCE_SLEEP, &target->bt_flags);
1716                 }
1717
1718                 /* sleep for a long time if there is nothing to do. */
1719                 if (list_empty(&target->bt_delwri_queue))
1720                         tout = MAX_SCHEDULE_TIMEOUT;
1721                 schedule_timeout_interruptible(tout);
1722
1723                 xfs_buf_delwri_split(target, &tmp, age);
1724                 list_sort(NULL, &tmp, xfs_buf_cmp);
1725
1726                 blk_start_plug(&plug);
1727                 while (!list_empty(&tmp)) {
1728                         struct xfs_buf *bp;
1729                         bp = list_first_entry(&tmp, struct xfs_buf, b_list);
1730                         list_del_init(&bp->b_list);
1731                         xfs_bdstrat_cb(bp);
1732                 }
1733                 blk_finish_plug(&plug);
1734         } while (!kthread_should_stop());
1735
1736         return 0;
1737 }
1738
1739 /*
1740  *      Go through all incore buffers, and release buffers if they belong to
1741  *      the given device. This is used in filesystem error handling to
1742  *      preserve the consistency of its metadata.
1743  */
1744 int
1745 xfs_flush_buftarg(
1746         xfs_buftarg_t   *target,
1747         int             wait)
1748 {
1749         xfs_buf_t       *bp;
1750         int             pincount = 0;
1751         LIST_HEAD(tmp_list);
1752         LIST_HEAD(wait_list);
1753         struct blk_plug plug;
1754
1755         xfs_buf_runall_queues(xfsconvertd_workqueue);
1756         xfs_buf_runall_queues(xfsdatad_workqueue);
1757         xfs_buf_runall_queues(xfslogd_workqueue);
1758
1759         set_bit(XBT_FORCE_FLUSH, &target->bt_flags);
1760         pincount = xfs_buf_delwri_split(target, &tmp_list, 0);
1761
1762         /*
1763          * Dropped the delayed write list lock, now walk the temporary list.
1764          * All I/O is issued async and then if we need to wait for completion
1765          * we do that after issuing all the IO.
1766          */
1767         list_sort(NULL, &tmp_list, xfs_buf_cmp);
1768
1769         blk_start_plug(&plug);
1770         while (!list_empty(&tmp_list)) {
1771                 bp = list_first_entry(&tmp_list, struct xfs_buf, b_list);
1772                 ASSERT(target == bp->b_target);
1773                 list_del_init(&bp->b_list);
1774                 if (wait) {
1775                         bp->b_flags &= ~XBF_ASYNC;
1776                         list_add(&bp->b_list, &wait_list);
1777                 }
1778                 xfs_bdstrat_cb(bp);
1779         }
1780         blk_finish_plug(&plug);
1781
1782         if (wait) {
1783                 /* Wait for IO to complete. */
1784                 while (!list_empty(&wait_list)) {
1785                         bp = list_first_entry(&wait_list, struct xfs_buf, b_list);
1786
1787                         list_del_init(&bp->b_list);
1788                         xfs_buf_iowait(bp);
1789                         xfs_buf_relse(bp);
1790                 }
1791         }
1792
1793         return pincount;
1794 }
1795
1796 int __init
1797 xfs_buf_init(void)
1798 {
1799         xfs_buf_zone = kmem_zone_init_flags(sizeof(xfs_buf_t), "xfs_buf",
1800                                                 KM_ZONE_HWALIGN, NULL);
1801         if (!xfs_buf_zone)
1802                 goto out;
1803
1804         xfslogd_workqueue = alloc_workqueue("xfslogd",
1805                                         WQ_MEM_RECLAIM | WQ_HIGHPRI, 1);
1806         if (!xfslogd_workqueue)
1807                 goto out_free_buf_zone;
1808
1809         xfsdatad_workqueue = alloc_workqueue("xfsdatad", WQ_MEM_RECLAIM, 1);
1810         if (!xfsdatad_workqueue)
1811                 goto out_destroy_xfslogd_workqueue;
1812
1813         xfsconvertd_workqueue = alloc_workqueue("xfsconvertd",
1814                                                 WQ_MEM_RECLAIM, 1);
1815         if (!xfsconvertd_workqueue)
1816                 goto out_destroy_xfsdatad_workqueue;
1817
1818         return 0;
1819
1820  out_destroy_xfsdatad_workqueue:
1821         destroy_workqueue(xfsdatad_workqueue);
1822  out_destroy_xfslogd_workqueue:
1823         destroy_workqueue(xfslogd_workqueue);
1824  out_free_buf_zone:
1825         kmem_zone_destroy(xfs_buf_zone);
1826  out:
1827         return -ENOMEM;
1828 }
1829
1830 void
1831 xfs_buf_terminate(void)
1832 {
1833         destroy_workqueue(xfsconvertd_workqueue);
1834         destroy_workqueue(xfsdatad_workqueue);
1835         destroy_workqueue(xfslogd_workqueue);
1836         kmem_zone_destroy(xfs_buf_zone);
1837 }
1838
1839 #ifdef CONFIG_KDB_MODULES
1840 struct list_head *
1841 xfs_get_buftarg_list(void)
1842 {
1843         return &xfs_buftarg_list;
1844 }
1845 #endif