6 * Partial copy of Linus' read cache modifications to fs/nfs/file.c
7 * modified for async RPC by okir@monad.swb.de
10 #include <linux/time.h>
11 #include <linux/kernel.h>
12 #include <linux/errno.h>
13 #include <linux/fcntl.h>
14 #include <linux/stat.h>
16 #include <linux/slab.h>
17 #include <linux/pagemap.h>
18 #include <linux/sunrpc/clnt.h>
19 #include <linux/nfs_fs.h>
20 #include <linux/nfs_page.h>
21 #include <linux/module.h>
23 #include <asm/system.h>
31 #define NFSDBG_FACILITY NFSDBG_PAGECACHE
33 static const struct nfs_pageio_ops nfs_pageio_read_ops;
34 static const struct rpc_call_ops nfs_read_partial_ops;
35 static const struct rpc_call_ops nfs_read_full_ops;
37 static struct kmem_cache *nfs_rdata_cachep;
38 static mempool_t *nfs_rdata_mempool;
40 #define MIN_POOL_READ (32)
42 struct nfs_read_data *nfs_readdata_alloc(unsigned int pagecount)
44 struct nfs_read_data *p = mempool_alloc(nfs_rdata_mempool, GFP_KERNEL);
47 memset(p, 0, sizeof(*p));
48 INIT_LIST_HEAD(&p->pages);
49 p->npages = pagecount;
50 if (pagecount <= ARRAY_SIZE(p->page_array))
51 p->pagevec = p->page_array;
53 p->pagevec = kcalloc(pagecount, sizeof(struct page *), GFP_KERNEL);
55 mempool_free(p, nfs_rdata_mempool);
63 void nfs_readdata_free(struct nfs_read_data *p)
65 if (p && (p->pagevec != &p->page_array[0]))
67 mempool_free(p, nfs_rdata_mempool);
70 void nfs_readdata_release(struct nfs_read_data *rdata)
72 put_lseg(rdata->lseg);
73 put_nfs_open_context(rdata->args.context);
74 nfs_readdata_free(rdata);
78 int nfs_return_empty_page(struct page *page)
80 zero_user(page, 0, PAGE_CACHE_SIZE);
81 SetPageUptodate(page);
86 static void nfs_readpage_truncate_uninitialised_page(struct nfs_read_data *data)
88 unsigned int remainder = data->args.count - data->res.count;
89 unsigned int base = data->args.pgbase + data->res.count;
93 if (data->res.eof == 0 || remainder == 0)
96 * Note: "remainder" can never be negative, since we check for
97 * this in the XDR code.
99 pages = &data->args.pages[base >> PAGE_CACHE_SHIFT];
100 base &= ~PAGE_CACHE_MASK;
101 pglen = PAGE_CACHE_SIZE - base;
103 if (remainder <= pglen) {
104 zero_user(*pages, base, remainder);
107 zero_user(*pages, base, pglen);
110 pglen = PAGE_CACHE_SIZE;
115 static void nfs_pageio_init_read_mds(struct nfs_pageio_descriptor *pgio,
118 nfs_pageio_init(pgio, inode, &nfs_pageio_read_ops,
119 NFS_SERVER(inode)->rsize, 0);
122 void nfs_pageio_reset_read_mds(struct nfs_pageio_descriptor *pgio)
124 pgio->pg_ops = &nfs_pageio_read_ops;
125 pgio->pg_bsize = NFS_SERVER(pgio->pg_inode)->rsize;
127 EXPORT_SYMBOL_GPL(nfs_pageio_reset_read_mds);
129 static void nfs_pageio_init_read(struct nfs_pageio_descriptor *pgio,
132 if (!pnfs_pageio_init_read(pgio, inode))
133 nfs_pageio_init_read_mds(pgio, inode);
136 int nfs_readpage_async(struct nfs_open_context *ctx, struct inode *inode,
139 struct nfs_page *new;
141 struct nfs_pageio_descriptor pgio;
143 len = nfs_page_length(page);
145 return nfs_return_empty_page(page);
146 new = nfs_create_request(ctx, inode, page, 0, len);
151 if (len < PAGE_CACHE_SIZE)
152 zero_user_segment(page, len, PAGE_CACHE_SIZE);
154 nfs_pageio_init_read(&pgio, inode);
155 nfs_pageio_add_request(&pgio, new);
156 nfs_pageio_complete(&pgio);
160 static void nfs_readpage_release(struct nfs_page *req)
162 struct inode *d_inode = req->wb_context->dentry->d_inode;
164 if (PageUptodate(req->wb_page))
165 nfs_readpage_to_fscache(d_inode, req->wb_page, 0);
167 unlock_page(req->wb_page);
169 dprintk("NFS: read done (%s/%Ld %d@%Ld)\n",
170 req->wb_context->dentry->d_inode->i_sb->s_id,
171 (long long)NFS_FILEID(req->wb_context->dentry->d_inode),
173 (long long)req_offset(req));
174 nfs_release_request(req);
177 int nfs_initiate_read(struct nfs_read_data *data, struct rpc_clnt *clnt,
178 const struct rpc_call_ops *call_ops)
180 struct inode *inode = data->inode;
181 int swap_flags = IS_SWAPFILE(inode) ? NFS_RPC_SWAPFLAGS : 0;
182 struct rpc_task *task;
183 struct rpc_message msg = {
184 .rpc_argp = &data->args,
185 .rpc_resp = &data->res,
186 .rpc_cred = data->cred,
188 struct rpc_task_setup task_setup_data = {
192 .callback_ops = call_ops,
193 .callback_data = data,
194 .workqueue = nfsiod_workqueue,
195 .flags = RPC_TASK_ASYNC | swap_flags,
198 /* Set up the initial task struct. */
199 NFS_PROTO(inode)->read_setup(data, &msg);
201 dprintk("NFS: %5u initiated read call (req %s/%lld, %u bytes @ "
205 (long long)NFS_FILEID(inode),
207 (unsigned long long)data->args.offset);
209 task = rpc_run_task(&task_setup_data);
211 return PTR_ERR(task);
215 EXPORT_SYMBOL_GPL(nfs_initiate_read);
218 * Set up the NFS read request struct
220 static void nfs_read_rpcsetup(struct nfs_page *req, struct nfs_read_data *data,
221 unsigned int count, unsigned int offset)
223 struct inode *inode = req->wb_context->dentry->d_inode;
227 data->cred = req->wb_context->cred;
229 data->args.fh = NFS_FH(inode);
230 data->args.offset = req_offset(req) + offset;
231 data->args.pgbase = req->wb_pgbase + offset;
232 data->args.pages = data->pagevec;
233 data->args.count = count;
234 data->args.context = get_nfs_open_context(req->wb_context);
235 data->args.lock_context = req->wb_lock_context;
237 data->res.fattr = &data->fattr;
238 data->res.count = count;
240 nfs_fattr_init(&data->fattr);
243 static int nfs_do_read(struct nfs_read_data *data,
244 const struct rpc_call_ops *call_ops)
246 struct inode *inode = data->args.context->dentry->d_inode;
248 return nfs_initiate_read(data, NFS_CLIENT(inode), call_ops);
252 nfs_do_multiple_reads(struct list_head *head,
253 const struct rpc_call_ops *call_ops)
255 struct nfs_read_data *data;
258 while (!list_empty(head)) {
261 data = list_entry(head->next, struct nfs_read_data, list);
262 list_del_init(&data->list);
264 ret2 = nfs_do_read(data, call_ops);
272 nfs_async_read_error(struct list_head *head)
274 struct nfs_page *req;
276 while (!list_empty(head)) {
277 req = nfs_list_entry(head->next);
278 nfs_list_remove_request(req);
279 SetPageError(req->wb_page);
280 nfs_readpage_release(req);
285 * Generate multiple requests to fill a single page.
287 * We optimize to reduce the number of read operations on the wire. If we
288 * detect that we're reading a page, or an area of a page, that is past the
289 * end of file, we do not generate NFS read operations but just clear the
290 * parts of the page that would have come back zero from the server anyway.
292 * We rely on the cached value of i_size to make this determination; another
293 * client can fill pages on the server past our cached end-of-file, but we
294 * won't see the new data until our attribute cache is updated. This is more
295 * or less conventional NFS client behavior.
297 static int nfs_pagein_multi(struct nfs_pageio_descriptor *desc, struct list_head *res)
299 struct nfs_page *req = nfs_list_entry(desc->pg_list.next);
300 struct page *page = req->wb_page;
301 struct nfs_read_data *data;
302 size_t rsize = desc->pg_bsize, nbytes;
307 nfs_list_remove_request(req);
310 nbytes = desc->pg_count;
312 size_t len = min(nbytes,rsize);
314 data = nfs_readdata_alloc(1);
317 data->pagevec[0] = page;
318 nfs_read_rpcsetup(req, data, len, offset);
319 list_add(&data->list, res);
323 } while(nbytes != 0);
324 atomic_set(&req->wb_complete, requests);
325 ClearPageError(page);
326 desc->pg_rpc_callops = &nfs_read_partial_ops;
329 while (!list_empty(res)) {
330 data = list_entry(res->next, struct nfs_read_data, list);
331 list_del(&data->list);
332 nfs_readdata_free(data);
335 nfs_readpage_release(req);
339 static int nfs_pagein_one(struct nfs_pageio_descriptor *desc, struct list_head *res)
341 struct nfs_page *req;
343 struct nfs_read_data *data;
344 struct list_head *head = &desc->pg_list;
347 data = nfs_readdata_alloc(nfs_page_array_len(desc->pg_base,
350 nfs_async_read_error(head);
355 pages = data->pagevec;
356 while (!list_empty(head)) {
357 req = nfs_list_entry(head->next);
358 nfs_list_remove_request(req);
359 nfs_list_add_request(req, &data->pages);
360 ClearPageError(req->wb_page);
361 *pages++ = req->wb_page;
363 req = nfs_list_entry(data->pages.next);
365 nfs_read_rpcsetup(req, data, desc->pg_count, 0);
366 list_add(&data->list, res);
367 desc->pg_rpc_callops = &nfs_read_full_ops;
372 int nfs_generic_pagein(struct nfs_pageio_descriptor *desc, struct list_head *head)
374 if (desc->pg_bsize < PAGE_CACHE_SIZE)
375 return nfs_pagein_multi(desc, head);
376 return nfs_pagein_one(desc, head);
379 static int nfs_generic_pg_readpages(struct nfs_pageio_descriptor *desc)
384 ret = nfs_generic_pagein(desc, &head);
386 ret = nfs_do_multiple_reads(&head, desc->pg_rpc_callops);
390 static const struct nfs_pageio_ops nfs_pageio_read_ops = {
391 .pg_test = nfs_generic_pg_test,
392 .pg_doio = nfs_generic_pg_readpages,
396 * This is the callback from RPC telling us whether a reply was
397 * received or some error occurred (timeout or socket shutdown).
399 int nfs_readpage_result(struct rpc_task *task, struct nfs_read_data *data)
403 dprintk("NFS: %s: %5u, (status %d)\n", __func__, task->tk_pid,
406 status = NFS_PROTO(data->inode)->read_done(task, data);
410 nfs_add_stats(data->inode, NFSIOS_SERVERREADBYTES, data->res.count);
412 if (task->tk_status == -ESTALE) {
413 set_bit(NFS_INO_STALE, &NFS_I(data->inode)->flags);
414 nfs_mark_for_revalidate(data->inode);
419 static void nfs_readpage_retry(struct rpc_task *task, struct nfs_read_data *data)
421 struct nfs_readargs *argp = &data->args;
422 struct nfs_readres *resp = &data->res;
424 if (resp->eof || resp->count == argp->count)
427 /* This is a short read! */
428 nfs_inc_stats(data->inode, NFSIOS_SHORTREAD);
429 /* Has the server at least made some progress? */
430 if (resp->count == 0)
433 /* Yes, so retry the read at the end of the data */
434 data->mds_offset += resp->count;
435 argp->offset += resp->count;
436 argp->pgbase += resp->count;
437 argp->count -= resp->count;
438 rpc_restart_call_prepare(task);
442 * Handle a read reply that fills part of a page.
444 static void nfs_readpage_result_partial(struct rpc_task *task, void *calldata)
446 struct nfs_read_data *data = calldata;
448 if (nfs_readpage_result(task, data) != 0)
450 if (task->tk_status < 0)
453 nfs_readpage_truncate_uninitialised_page(data);
454 nfs_readpage_retry(task, data);
457 static void nfs_readpage_release_partial(void *calldata)
459 struct nfs_read_data *data = calldata;
460 struct nfs_page *req = data->req;
461 struct page *page = req->wb_page;
462 int status = data->task.tk_status;
467 if (atomic_dec_and_test(&req->wb_complete)) {
468 if (!PageError(page))
469 SetPageUptodate(page);
470 nfs_readpage_release(req);
472 nfs_readdata_release(calldata);
475 #if defined(CONFIG_NFS_V4_1)
476 void nfs_read_prepare(struct rpc_task *task, void *calldata)
478 struct nfs_read_data *data = calldata;
480 if (nfs4_setup_sequence(NFS_SERVER(data->inode),
481 &data->args.seq_args, &data->res.seq_res,
484 rpc_call_start(task);
486 #endif /* CONFIG_NFS_V4_1 */
488 static const struct rpc_call_ops nfs_read_partial_ops = {
489 #if defined(CONFIG_NFS_V4_1)
490 .rpc_call_prepare = nfs_read_prepare,
491 #endif /* CONFIG_NFS_V4_1 */
492 .rpc_call_done = nfs_readpage_result_partial,
493 .rpc_release = nfs_readpage_release_partial,
496 static void nfs_readpage_set_pages_uptodate(struct nfs_read_data *data)
498 unsigned int count = data->res.count;
499 unsigned int base = data->args.pgbase;
503 count = data->args.count;
504 if (unlikely(count == 0))
506 pages = &data->args.pages[base >> PAGE_CACHE_SHIFT];
507 base &= ~PAGE_CACHE_MASK;
509 for (;count >= PAGE_CACHE_SIZE; count -= PAGE_CACHE_SIZE, pages++)
510 SetPageUptodate(*pages);
513 /* Was this a short read? */
514 if (data->res.eof || data->res.count == data->args.count)
515 SetPageUptodate(*pages);
519 * This is the callback from RPC telling us whether a reply was
520 * received or some error occurred (timeout or socket shutdown).
522 static void nfs_readpage_result_full(struct rpc_task *task, void *calldata)
524 struct nfs_read_data *data = calldata;
526 if (nfs_readpage_result(task, data) != 0)
528 if (task->tk_status < 0)
531 * Note: nfs_readpage_retry may change the values of
532 * data->args. In the multi-page case, we therefore need
533 * to ensure that we call nfs_readpage_set_pages_uptodate()
536 nfs_readpage_truncate_uninitialised_page(data);
537 nfs_readpage_set_pages_uptodate(data);
538 nfs_readpage_retry(task, data);
541 static void nfs_readpage_release_full(void *calldata)
543 struct nfs_read_data *data = calldata;
544 struct nfs_pageio_descriptor pgio;
546 if (data->pnfs_error) {
547 nfs_pageio_init_read_mds(&pgio, data->inode);
548 pgio.pg_recoalesce = 1;
550 while (!list_empty(&data->pages)) {
551 struct nfs_page *req = nfs_list_entry(data->pages.next);
553 nfs_list_remove_request(req);
554 if (!data->pnfs_error)
555 nfs_readpage_release(req);
557 nfs_pageio_add_request(&pgio, req);
559 if (data->pnfs_error)
560 nfs_pageio_complete(&pgio);
561 nfs_readdata_release(calldata);
564 static const struct rpc_call_ops nfs_read_full_ops = {
565 #if defined(CONFIG_NFS_V4_1)
566 .rpc_call_prepare = nfs_read_prepare,
567 #endif /* CONFIG_NFS_V4_1 */
568 .rpc_call_done = nfs_readpage_result_full,
569 .rpc_release = nfs_readpage_release_full,
573 * Read a page over NFS.
574 * We read the page synchronously in the following case:
575 * - The error flag is set for this page. This happens only when a
576 * previous async read operation failed.
578 int nfs_readpage(struct file *file, struct page *page)
580 struct nfs_open_context *ctx;
581 struct inode *inode = page->mapping->host;
584 dprintk("NFS: nfs_readpage (%p %ld@%lu)\n",
585 page, PAGE_CACHE_SIZE, page->index);
586 nfs_inc_stats(inode, NFSIOS_VFSREADPAGE);
587 nfs_add_stats(inode, NFSIOS_READPAGES, 1);
590 * Try to flush any pending writes to the file..
592 * NOTE! Because we own the page lock, there cannot
593 * be any new pending writes generated at this point
594 * for this page (other pages can be written to).
596 error = nfs_wb_page(inode, page);
599 if (PageUptodate(page))
603 if (NFS_STALE(inode))
608 ctx = nfs_find_open_context(inode, NULL, FMODE_READ);
612 ctx = get_nfs_open_context(nfs_file_open_context(file));
614 if (!IS_SYNC(inode)) {
615 error = nfs_readpage_from_fscache(ctx, inode, page);
620 error = nfs_readpage_async(ctx, inode, page);
623 put_nfs_open_context(ctx);
630 struct nfs_readdesc {
631 struct nfs_pageio_descriptor *pgio;
632 struct nfs_open_context *ctx;
636 readpage_async_filler(void *data, struct page *page)
638 struct nfs_readdesc *desc = (struct nfs_readdesc *)data;
639 struct inode *inode = page->mapping->host;
640 struct nfs_page *new;
644 len = nfs_page_length(page);
646 return nfs_return_empty_page(page);
648 new = nfs_create_request(desc->ctx, inode, page, 0, len);
652 if (len < PAGE_CACHE_SIZE)
653 zero_user_segment(page, len, PAGE_CACHE_SIZE);
654 if (!nfs_pageio_add_request(desc->pgio, new)) {
655 error = desc->pgio->pg_error;
660 error = PTR_ERR(new);
667 int nfs_readpages(struct file *filp, struct address_space *mapping,
668 struct list_head *pages, unsigned nr_pages)
670 struct nfs_pageio_descriptor pgio;
671 struct nfs_readdesc desc = {
674 struct inode *inode = mapping->host;
675 unsigned long npages;
678 dprintk("NFS: nfs_readpages (%s/%Ld %d)\n",
680 (long long)NFS_FILEID(inode),
682 nfs_inc_stats(inode, NFSIOS_VFSREADPAGES);
684 if (NFS_STALE(inode))
688 desc.ctx = nfs_find_open_context(inode, NULL, FMODE_READ);
689 if (desc.ctx == NULL)
692 desc.ctx = get_nfs_open_context(nfs_file_open_context(filp));
694 /* attempt to read as many of the pages as possible from the cache
695 * - this returns -ENOBUFS immediately if the cookie is negative
697 ret = nfs_readpages_from_fscache(desc.ctx, inode, mapping,
700 goto read_complete; /* all pages were read */
702 nfs_pageio_init_read(&pgio, inode);
704 ret = read_cache_pages(mapping, pages, readpage_async_filler, &desc);
706 nfs_pageio_complete(&pgio);
707 npages = (pgio.pg_bytes_written + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
708 nfs_add_stats(inode, NFSIOS_READPAGES, npages);
710 put_nfs_open_context(desc.ctx);
715 int __init nfs_init_readpagecache(void)
717 nfs_rdata_cachep = kmem_cache_create("nfs_read_data",
718 sizeof(struct nfs_read_data),
719 0, SLAB_HWCACHE_ALIGN,
721 if (nfs_rdata_cachep == NULL)
724 nfs_rdata_mempool = mempool_create_slab_pool(MIN_POOL_READ,
726 if (nfs_rdata_mempool == NULL)
732 void nfs_destroy_readpagecache(void)
734 mempool_destroy(nfs_rdata_mempool);
735 kmem_cache_destroy(nfs_rdata_cachep);