1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * linux/fs/nfs/write.c
   4 *
   5 * Write file data over NFS.
   6 *
   7 * Copyright (C) 1996, 1997, Olaf Kirch <okir@monad.swb.de>
   8 */
   9
  10#include <linux/types.h>
  11#include <linux/slab.h>
  12#include <linux/mm.h>
  13#include <linux/pagemap.h>
  14#include <linux/file.h>
  15#include <linux/writeback.h>
  16#include <linux/swap.h>
  17#include <linux/migrate.h>
  18
  19#include <linux/sunrpc/clnt.h>
  20#include <linux/nfs_fs.h>
  21#include <linux/nfs_mount.h>
  22#include <linux/nfs_page.h>
  23#include <linux/backing-dev.h>
  24#include <linux/export.h>
  25#include <linux/freezer.h>
  26#include <linux/wait.h>
  27#include <linux/iversion.h>
  28
  29#include <linux/uaccess.h>
  30#include <linux/sched/mm.h>
  31
  32#include "delegation.h"
  33#include "internal.h"
  34#include "iostat.h"
  35#include "nfs4_fs.h"
  36#include "fscache.h"
  37#include "pnfs.h"
  38
  39#include "nfstrace.h"
  40
  41#define NFSDBG_FACILITY		NFSDBG_PAGECACHE
  42
  43#define MIN_POOL_WRITE		(32)
  44#define MIN_POOL_COMMIT		(4)
  45
  46struct nfs_io_completion {
  47	void (*complete)(void *data);
  48	void *data;
  49	struct kref refcount;
  50};
  51
  52/*
  53 * Local function declarations
  54 */
  55static void nfs_redirty_request(struct nfs_page *req);
  56static const struct rpc_call_ops nfs_commit_ops;
  57static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops;
  58static const struct nfs_commit_completion_ops nfs_commit_completion_ops;
  59static const struct nfs_rw_ops nfs_rw_write_ops;
  60static void nfs_inode_remove_request(struct nfs_page *req);
  61static void nfs_clear_request_commit(struct nfs_page *req);
  62static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
  63				      struct inode *inode);
  64static struct nfs_page *
  65nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi,
  66						struct page *page);
  67
  68static struct kmem_cache *nfs_wdata_cachep;
  69static mempool_t *nfs_wdata_mempool;
  70static struct kmem_cache *nfs_cdata_cachep;
  71static mempool_t *nfs_commit_mempool;
  72
  73struct nfs_commit_data *nfs_commitdata_alloc(bool never_fail)
  74{
  75	struct nfs_commit_data *p;
  76
  77	if (never_fail)
  78		p = mempool_alloc(nfs_commit_mempool, GFP_NOIO);
  79	else {
  80		/* It is OK to do some reclaim, not no safe to wait
  81		 * for anything to be returned to the pool.
  82		 * mempool_alloc() cannot handle that particular combination,
  83		 * so we need two separate attempts.
  84		 */
  85		p = mempool_alloc(nfs_commit_mempool, GFP_NOWAIT);
  86		if (!p)
  87			p = kmem_cache_alloc(nfs_cdata_cachep, GFP_NOIO |
  88					     __GFP_NOWARN | __GFP_NORETRY);
  89		if (!p)
  90			return NULL;
  91	}
  92
  93	memset(p, 0, sizeof(*p));
  94	INIT_LIST_HEAD(&p->pages);
  95	return p;
  96}
  97EXPORT_SYMBOL_GPL(nfs_commitdata_alloc);
  98
  99void nfs_commit_free(struct nfs_commit_data *p)
 100{
 101	mempool_free(p, nfs_commit_mempool);
 102}
 103EXPORT_SYMBOL_GPL(nfs_commit_free);
 104
 105static struct nfs_pgio_header *nfs_writehdr_alloc(void)
 106{
 107	struct nfs_pgio_header *p = mempool_alloc(nfs_wdata_mempool, GFP_KERNEL);
 108
 109	memset(p, 0, sizeof(*p));
 110	p->rw_mode = FMODE_WRITE;
 111	return p;
 112}
 113
 114static void nfs_writehdr_free(struct nfs_pgio_header *hdr)
 115{
 116	mempool_free(hdr, nfs_wdata_mempool);
 117}
 118
 119static struct nfs_io_completion *nfs_io_completion_alloc(gfp_t gfp_flags)
 120{
 121	return kmalloc(sizeof(struct nfs_io_completion), gfp_flags);
 122}
 123
 124static void nfs_io_completion_init(struct nfs_io_completion *ioc,
 125		void (*complete)(void *), void *data)
 126{
 127	ioc->complete = complete;
 128	ioc->data = data;
 129	kref_init(&ioc->refcount);
 130}
 131
 132static void nfs_io_completion_release(struct kref *kref)
 133{
 134	struct nfs_io_completion *ioc = container_of(kref,
 135			struct nfs_io_completion, refcount);
 136	ioc->complete(ioc->data);
 137	kfree(ioc);
 138}
 139
 140static void nfs_io_completion_get(struct nfs_io_completion *ioc)
 141{
 142	if (ioc != NULL)
 143		kref_get(&ioc->refcount);
 144}
 145
 146static void nfs_io_completion_put(struct nfs_io_completion *ioc)
 147{
 148	if (ioc != NULL)
 149		kref_put(&ioc->refcount, nfs_io_completion_release);
 150}
 151
 152static void
 153nfs_page_set_inode_ref(struct nfs_page *req, struct inode *inode)
 154{
 155	if (!test_and_set_bit(PG_INODE_REF, &req->wb_flags)) {
 156		kref_get(&req->wb_kref);
 157		atomic_long_inc(&NFS_I(inode)->nrequests);
 158	}
 159}
 160
 161static int
 162nfs_cancel_remove_inode(struct nfs_page *req, struct inode *inode)
 163{
 164	int ret;
 165
 166	if (!test_bit(PG_REMOVE, &req->wb_flags))
 167		return 0;
 168	ret = nfs_page_group_lock(req);
 169	if (ret)
 170		return ret;
 171	if (test_and_clear_bit(PG_REMOVE, &req->wb_flags))
 172		nfs_page_set_inode_ref(req, inode);
 173	nfs_page_group_unlock(req);
 174	return 0;
 175}
 176
 177static struct nfs_page *
 178nfs_page_private_request(struct page *page)
 179{
 180	if (!PagePrivate(page))
 181		return NULL;
 182	return (struct nfs_page *)page_private(page);
 183}
 184
 185/*
 186 * nfs_page_find_head_request_locked - find head request associated with @page
 187 *
 188 * must be called while holding the inode lock.
 189 *
 190 * returns matching head request with reference held, or NULL if not found.
 191 */
 192static struct nfs_page *
 193nfs_page_find_private_request(struct page *page)
 194{
 195	struct address_space *mapping = page_file_mapping(page);
 196	struct nfs_page *req;
 197
 198	if (!PagePrivate(page))
 199		return NULL;
 200	spin_lock(&mapping->private_lock);
 201	req = nfs_page_private_request(page);
 202	if (req) {
 203		WARN_ON_ONCE(req->wb_head != req);
 204		kref_get(&req->wb_kref);
 205	}
 206	spin_unlock(&mapping->private_lock);
 207	return req;
 208}
 209
 210static struct nfs_page *
 211nfs_page_find_swap_request(struct page *page)
 212{
 213	struct inode *inode = page_file_mapping(page)->host;
 214	struct nfs_inode *nfsi = NFS_I(inode);
 215	struct nfs_page *req = NULL;
 216	if (!PageSwapCache(page))
 217		return NULL;
 218	mutex_lock(&nfsi->commit_mutex);
 219	if (PageSwapCache(page)) {
 220		req = nfs_page_search_commits_for_head_request_locked(nfsi,
 221			page);
 222		if (req) {
 223			WARN_ON_ONCE(req->wb_head != req);
 224			kref_get(&req->wb_kref);
 225		}
 226	}
 227	mutex_unlock(&nfsi->commit_mutex);
 228	return req;
 229}
 230
 231/*
 232 * nfs_page_find_head_request - find head request associated with @page
 233 *
 234 * returns matching head request with reference held, or NULL if not found.
 235 */
 236static struct nfs_page *nfs_page_find_head_request(struct page *page)
 237{
 238	struct nfs_page *req;
 239
 240	req = nfs_page_find_private_request(page);
 241	if (!req)
 242		req = nfs_page_find_swap_request(page);
 243	return req;
 244}
 245
 246static struct nfs_page *nfs_find_and_lock_page_request(struct page *page)
 247{
 248	struct inode *inode = page_file_mapping(page)->host;
 249	struct nfs_page *req, *head;
 250	int ret;
 251
 252	for (;;) {
 253		req = nfs_page_find_head_request(page);
 254		if (!req)
 255			return req;
 256		head = nfs_page_group_lock_head(req);
 257		if (head != req)
 258			nfs_release_request(req);
 259		if (IS_ERR(head))
 260			return head;
 261		ret = nfs_cancel_remove_inode(head, inode);
 262		if (ret < 0) {
 263			nfs_unlock_and_release_request(head);
 264			return ERR_PTR(ret);
 265		}
 266		/* Ensure that nobody removed the request before we locked it */
 267		if (head == nfs_page_private_request(page))
 268			break;
 269		if (PageSwapCache(page))
 270			break;
 271		nfs_unlock_and_release_request(head);
 272	}
 273	return head;
 274}
 275
 276/* Adjust the file length if we're writing beyond the end */
 277static void nfs_grow_file(struct page *page, unsigned int offset, unsigned int count)
 278{
 279	struct inode *inode = page_file_mapping(page)->host;
 280	loff_t end, i_size;
 281	pgoff_t end_index;
 282
 283	spin_lock(&inode->i_lock);
 284	i_size = i_size_read(inode);
 285	end_index = (i_size - 1) >> PAGE_SHIFT;
 286	if (i_size > 0 && page_index(page) < end_index)
 287		goto out;
 288	end = page_file_offset(page) + ((loff_t)offset+count);
 289	if (i_size >= end)
 290		goto out;
 291	i_size_write(inode, end);
 292	NFS_I(inode)->cache_validity &= ~NFS_INO_INVALID_SIZE;
 293	nfs_inc_stats(inode, NFSIOS_EXTENDWRITE);
 294out:
 295	spin_unlock(&inode->i_lock);
 296}
 297
 298/* A writeback failed: mark the page as bad, and invalidate the page cache */
 299static void nfs_set_pageerror(struct address_space *mapping)
 300{
 301	struct inode *inode = mapping->host;
 302
 303	nfs_zap_mapping(mapping->host, mapping);
 304	/* Force file size revalidation */
 305	spin_lock(&inode->i_lock);
 306	NFS_I(inode)->cache_validity |= NFS_INO_REVAL_FORCED |
 307					NFS_INO_REVAL_PAGECACHE |
 308					NFS_INO_INVALID_SIZE;
 309	spin_unlock(&inode->i_lock);
 310}
 311
 312static void nfs_mapping_set_error(struct page *page, int error)
 313{
 314	struct address_space *mapping = page_file_mapping(page);
 315
 316	SetPageError(page);
 317	mapping_set_error(mapping, error);
 318	nfs_set_pageerror(mapping);
 319}
 320
 321/*
 322 * nfs_page_group_search_locked
 323 * @head - head request of page group
 324 * @page_offset - offset into page
 325 *
 326 * Search page group with head @head to find a request that contains the
 327 * page offset @page_offset.
 328 *
 329 * Returns a pointer to the first matching nfs request, or NULL if no
 330 * match is found.
 331 *
 332 * Must be called with the page group lock held
 333 */
 334static struct nfs_page *
 335nfs_page_group_search_locked(struct nfs_page *head, unsigned int page_offset)
 336{
 337	struct nfs_page *req;
 338
 339	req = head;
 340	do {
 341		if (page_offset >= req->wb_pgbase &&
 342		    page_offset < (req->wb_pgbase + req->wb_bytes))
 343			return req;
 344
 345		req = req->wb_this_page;
 346	} while (req != head);
 347
 348	return NULL;
 349}
 350
 351/*
 352 * nfs_page_group_covers_page
 353 * @head - head request of page group
 354 *
 355 * Return true if the page group with head @head covers the whole page,
 356 * returns false otherwise
 357 */
 358static bool nfs_page_group_covers_page(struct nfs_page *req)
 359{
 360	struct nfs_page *tmp;
 361	unsigned int pos = 0;
 362	unsigned int len = nfs_page_length(req->wb_page);
 363
 364	nfs_page_group_lock(req);
 365
 366	for (;;) {
 367		tmp = nfs_page_group_search_locked(req->wb_head, pos);
 368		if (!tmp)
 369			break;
 370		pos = tmp->wb_pgbase + tmp->wb_bytes;
 371	}
 372
 373	nfs_page_group_unlock(req);
 374	return pos >= len;
 375}
 376
 377/* We can set the PG_uptodate flag if we see that a write request
 378 * covers the full page.
 379 */
 380static void nfs_mark_uptodate(struct nfs_page *req)
 381{
 382	if (PageUptodate(req->wb_page))
 383		return;
 384	if (!nfs_page_group_covers_page(req))
 385		return;
 386	SetPageUptodate(req->wb_page);
 387}
 388
 389static int wb_priority(struct writeback_control *wbc)
 390{
 391	int ret = 0;
 392
 393	if (wbc->sync_mode == WB_SYNC_ALL)
 394		ret = FLUSH_COND_STABLE;
 395	return ret;
 396}
 397
 398/*
 399 * NFS congestion control
 400 */
 401
 402int nfs_congestion_kb;
 403
 404#define NFS_CONGESTION_ON_THRESH 	(nfs_congestion_kb >> (PAGE_SHIFT-10))
 405#define NFS_CONGESTION_OFF_THRESH	\
 406	(NFS_CONGESTION_ON_THRESH - (NFS_CONGESTION_ON_THRESH >> 2))
 407
 408static void nfs_set_page_writeback(struct page *page)
 409{
 410	struct inode *inode = page_file_mapping(page)->host;
 411	struct nfs_server *nfss = NFS_SERVER(inode);
 412	int ret = test_set_page_writeback(page);
 413
 414	WARN_ON_ONCE(ret != 0);
 415
 416	if (atomic_long_inc_return(&nfss->writeback) >
 417			NFS_CONGESTION_ON_THRESH)
 418		set_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC);
 419}
 420
 421static void nfs_end_page_writeback(struct nfs_page *req)
 422{
 423	struct inode *inode = page_file_mapping(req->wb_page)->host;
 424	struct nfs_server *nfss = NFS_SERVER(inode);
 425	bool is_done;
 426
 427	is_done = nfs_page_group_sync_on_bit(req, PG_WB_END);
 428	nfs_unlock_request(req);
 429	if (!is_done)
 430		return;
 431
 432	end_page_writeback(req->wb_page);
 433	if (atomic_long_dec_return(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
 434		clear_bdi_congested(inode_to_bdi(inode), BLK_RW_ASYNC);
 435}
 436
 437/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 438 * nfs_destroy_unlinked_subrequests - destroy recently unlinked subrequests
 439 *
 440 * @destroy_list - request list (using wb_this_page) terminated by @old_head
 441 * @old_head - the old head of the list
 442 *
 443 * All subrequests must be locked and removed from all lists, so at this point
 444 * they are only "active" in this function, and possibly in nfs_wait_on_request
 445 * with a reference held by some other context.
 446 */
 447static void
 448nfs_destroy_unlinked_subrequests(struct nfs_page *destroy_list,
 449				 struct nfs_page *old_head,
 450				 struct inode *inode)
 451{
 452	while (destroy_list) {
 453		struct nfs_page *subreq = destroy_list;
 454
 455		destroy_list = (subreq->wb_this_page == old_head) ?
 456				   NULL : subreq->wb_this_page;
 457
 458		/* Note: lock subreq in order to change subreq->wb_head */
 459		nfs_page_set_headlock(subreq);
 460		WARN_ON_ONCE(old_head != subreq->wb_head);
 461
 462		/* make sure old group is not used */
 463		subreq->wb_this_page = subreq;
 464		subreq->wb_head = subreq;
 465
 466		clear_bit(PG_REMOVE, &subreq->wb_flags);
 467
 468		/* Note: races with nfs_page_group_destroy() */
 469		if (!kref_read(&subreq->wb_kref)) {
 470			/* Check if we raced with nfs_page_group_destroy() */
 471			if (test_and_clear_bit(PG_TEARDOWN, &subreq->wb_flags)) {
 472				nfs_page_clear_headlock(subreq);
 473				nfs_free_request(subreq);
 474			} else
 475				nfs_page_clear_headlock(subreq);
 476			continue;
 477		}
 478		nfs_page_clear_headlock(subreq);
 479
 480		nfs_release_request(old_head);
 481
 482		if (test_and_clear_bit(PG_INODE_REF, &subreq->wb_flags)) {
 483			nfs_release_request(subreq);
 484			atomic_long_dec(&NFS_I(inode)->nrequests);
 485		}
 486
 487		/* subreq is now totally disconnected from page group or any
 488		 * write / commit lists. last chance to wake any waiters */
 489		nfs_unlock_and_release_request(subreq);
 490	}
 491}
 492
 493/*
 494 * nfs_join_page_group - destroy subrequests of the head req
 495 * @head: the page used to lookup the "page group" of nfs_page structures
 496 * @inode: Inode to which the request belongs.
 
 
 497 *
 498 * This function joins all sub requests to the head request by first
 499 * locking all requests in the group, cancelling any pending operations
 500 * and finally updating the head request to cover the whole range covered by
 501 * the (former) group.  All subrequests are removed from any write or commit
 502 * lists, unlinked from the group and destroyed.
 
 
 
 
 
 503 */
 504void
 505nfs_join_page_group(struct nfs_page *head, struct inode *inode)
 506{
 507	struct nfs_page *subreq;
 
 508	struct nfs_page *destroy_list = NULL;
 509	unsigned int pgbase, off, bytes;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 510
 511	pgbase = head->wb_pgbase;
 512	bytes = head->wb_bytes;
 513	off = head->wb_offset;
 
 
 
 514	for (subreq = head->wb_this_page; subreq != head;
 515			subreq = subreq->wb_this_page) {
 516		/* Subrequests should always form a contiguous range */
 517		if (pgbase > subreq->wb_pgbase) {
 518			off -= pgbase - subreq->wb_pgbase;
 519			bytes += pgbase - subreq->wb_pgbase;
 520			pgbase = subreq->wb_pgbase;
 521		}
 522		bytes = max(subreq->wb_pgbase + subreq->wb_bytes
 523				- pgbase, bytes);
 524	}
 525
 526	/* Set the head request's range to cover the former page group */
 527	head->wb_pgbase = pgbase;
 528	head->wb_bytes = bytes;
 529	head->wb_offset = off;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 530
 531	/* Now that all requests are locked, make sure they aren't on any list.
 532	 * Commit list removal accounting is done after locks are dropped */
 533	subreq = head;
 534	do {
 535		nfs_clear_request_commit(subreq);
 536		subreq = subreq->wb_this_page;
 537	} while (subreq != head);
 538
 539	/* unlink subrequests from head, destroy them later */
 540	if (head->wb_this_page != head) {
 541		/* destroy list will be terminated by head */
 542		destroy_list = head->wb_this_page;
 543		head->wb_this_page = head;
 
 
 
 
 544	}
 545
 546	nfs_destroy_unlinked_subrequests(destroy_list, head, inode);
 547}
 
 
 
 
 548
 549/*
 550 * nfs_lock_and_join_requests - join all subreqs to the head req
 551 * @page: the page used to lookup the "page group" of nfs_page structures
 552 *
 553 * This function joins all sub requests to the head request by first
 554 * locking all requests in the group, cancelling any pending operations
 555 * and finally updating the head request to cover the whole range covered by
 556 * the (former) group.  All subrequests are removed from any write or commit
 557 * lists, unlinked from the group and destroyed.
 558 *
 559 * Returns a locked, referenced pointer to the head request - which after
 560 * this call is guaranteed to be the only request associated with the page.
 561 * Returns NULL if no requests are found for @page, or a ERR_PTR if an
 562 * error was encountered.
 563 */
 564static struct nfs_page *
 565nfs_lock_and_join_requests(struct page *page)
 566{
 567	struct inode *inode = page_file_mapping(page)->host;
 568	struct nfs_page *head;
 569	int ret;
 570
 571	/*
 572	 * A reference is taken only on the head request which acts as a
 573	 * reference to the whole page group - the group will not be destroyed
 574	 * until the head reference is released.
 575	 */
 576	head = nfs_find_and_lock_page_request(page);
 577	if (IS_ERR_OR_NULL(head))
 578		return head;
 579
 580	/* lock each request in the page group */
 581	ret = nfs_page_group_lock_subrequests(head);
 582	if (ret < 0) {
 583		nfs_unlock_and_release_request(head);
 584		return ERR_PTR(ret);
 585	}
 586
 587	nfs_join_page_group(head, inode);
 588
 589	return head;
 
 
 
 
 590}
 591
 592static void nfs_write_error(struct nfs_page *req, int error)
 593{
 594	trace_nfs_write_error(req, error);
 595	nfs_mapping_set_error(req->wb_page, error);
 596	nfs_inode_remove_request(req);
 597	nfs_end_page_writeback(req);
 598	nfs_release_request(req);
 599}
 600
 601/*
 602 * Find an associated nfs write request, and prepare to flush it out
 603 * May return an error if the user signalled nfs_wait_on_request().
 604 */
 605static int nfs_page_async_flush(struct nfs_pageio_descriptor *pgio,
 606				struct page *page)
 607{
 608	struct nfs_page *req;
 609	int ret = 0;
 610
 611	req = nfs_lock_and_join_requests(page);
 612	if (!req)
 613		goto out;
 614	ret = PTR_ERR(req);
 615	if (IS_ERR(req))
 616		goto out;
 617
 618	nfs_set_page_writeback(page);
 619	WARN_ON_ONCE(test_bit(PG_CLEAN, &req->wb_flags));
 620
 621	/* If there is a fatal error that covers this write, just exit */
 622	ret = pgio->pg_error;
 623	if (nfs_error_is_fatal_on_server(ret))
 624		goto out_launder;
 625
 626	ret = 0;
 627	if (!nfs_pageio_add_request(pgio, req)) {
 628		ret = pgio->pg_error;
 629		/*
 630		 * Remove the problematic req upon fatal errors on the server
 631		 */
 632		if (nfs_error_is_fatal(ret)) {
 633			if (nfs_error_is_fatal_on_server(ret))
 634				goto out_launder;
 635		} else
 636			ret = -EAGAIN;
 637		nfs_redirty_request(req);
 638		pgio->pg_error = 0;
 639	} else
 640		nfs_add_stats(page_file_mapping(page)->host,
 641				NFSIOS_WRITEPAGES, 1);
 642out:
 643	return ret;
 644out_launder:
 645	nfs_write_error(req, ret);
 646	return 0;
 647}
 648
 649static int nfs_do_writepage(struct page *page, struct writeback_control *wbc,
 650			    struct nfs_pageio_descriptor *pgio)
 651{
 652	int ret;
 653
 654	nfs_pageio_cond_complete(pgio, page_index(page));
 655	ret = nfs_page_async_flush(pgio, page);
 656	if (ret == -EAGAIN) {
 657		redirty_page_for_writepage(wbc, page);
 658		ret = AOP_WRITEPAGE_ACTIVATE;
 659	}
 660	return ret;
 661}
 662
 663/*
 664 * Write an mmapped page to the server.
 665 */
 666static int nfs_writepage_locked(struct page *page,
 667				struct writeback_control *wbc)
 668{
 669	struct nfs_pageio_descriptor pgio;
 670	struct inode *inode = page_file_mapping(page)->host;
 671	int err;
 672
 673	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGE);
 674	nfs_pageio_init_write(&pgio, inode, 0,
 675				false, &nfs_async_write_completion_ops);
 676	err = nfs_do_writepage(page, wbc, &pgio);
 677	pgio.pg_error = 0;
 678	nfs_pageio_complete(&pgio);
 679	if (err < 0)
 680		return err;
 681	if (nfs_error_is_fatal(pgio.pg_error))
 682		return pgio.pg_error;
 683	return 0;
 684}
 685
 686int nfs_writepage(struct page *page, struct writeback_control *wbc)
 687{
 688	int ret;
 689
 690	ret = nfs_writepage_locked(page, wbc);
 691	if (ret != AOP_WRITEPAGE_ACTIVATE)
 692		unlock_page(page);
 693	return ret;
 694}
 695
 696static int nfs_writepages_callback(struct page *page, struct writeback_control *wbc, void *data)
 697{
 698	int ret;
 699
 700	ret = nfs_do_writepage(page, wbc, data);
 701	if (ret != AOP_WRITEPAGE_ACTIVATE)
 702		unlock_page(page);
 703	return ret;
 704}
 705
 706static void nfs_io_completion_commit(void *inode)
 707{
 708	nfs_commit_inode(inode, 0);
 709}
 710
 711int nfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
 712{
 713	struct inode *inode = mapping->host;
 714	struct nfs_pageio_descriptor pgio;
 715	struct nfs_io_completion *ioc;
 716	int err;
 717
 718	nfs_inc_stats(inode, NFSIOS_VFSWRITEPAGES);
 719
 720	ioc = nfs_io_completion_alloc(GFP_KERNEL);
 721	if (ioc)
 722		nfs_io_completion_init(ioc, nfs_io_completion_commit, inode);
 723
 724	nfs_pageio_init_write(&pgio, inode, wb_priority(wbc), false,
 725				&nfs_async_write_completion_ops);
 726	pgio.pg_io_completion = ioc;
 727	err = write_cache_pages(mapping, wbc, nfs_writepages_callback, &pgio);
 728	pgio.pg_error = 0;
 729	nfs_pageio_complete(&pgio);
 730	nfs_io_completion_put(ioc);
 731
 732	if (err < 0)
 733		goto out_err;
 734	err = pgio.pg_error;
 735	if (nfs_error_is_fatal(err))
 736		goto out_err;
 737	return 0;
 738out_err:
 739	return err;
 740}
 741
 742/*
 743 * Insert a write request into an inode
 744 */
 745static void nfs_inode_add_request(struct inode *inode, struct nfs_page *req)
 746{
 747	struct address_space *mapping = page_file_mapping(req->wb_page);
 748	struct nfs_inode *nfsi = NFS_I(inode);
 749
 750	WARN_ON_ONCE(req->wb_this_page != req);
 751
 752	/* Lock the request! */
 753	nfs_lock_request(req);
 754
 755	/*
 756	 * Swap-space should not get truncated. Hence no need to plug the race
 757	 * with invalidate/truncate.
 758	 */
 759	spin_lock(&mapping->private_lock);
 760	if (!nfs_have_writebacks(inode) &&
 761	    NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
 762		inode_inc_iversion_raw(inode);
 763	if (likely(!PageSwapCache(req->wb_page))) {
 764		set_bit(PG_MAPPED, &req->wb_flags);
 765		SetPagePrivate(req->wb_page);
 766		set_page_private(req->wb_page, (unsigned long)req);
 767	}
 768	spin_unlock(&mapping->private_lock);
 769	atomic_long_inc(&nfsi->nrequests);
 770	/* this a head request for a page group - mark it as having an
 771	 * extra reference so sub groups can follow suit.
 772	 * This flag also informs pgio layer when to bump nrequests when
 773	 * adding subrequests. */
 774	WARN_ON(test_and_set_bit(PG_INODE_REF, &req->wb_flags));
 775	kref_get(&req->wb_kref);
 776}
 777
 778/*
 779 * Remove a write request from an inode
 780 */
 781static void nfs_inode_remove_request(struct nfs_page *req)
 782{
 783	struct address_space *mapping = page_file_mapping(req->wb_page);
 784	struct inode *inode = mapping->host;
 785	struct nfs_inode *nfsi = NFS_I(inode);
 786	struct nfs_page *head;
 787
 788	if (nfs_page_group_sync_on_bit(req, PG_REMOVE)) {
 789		head = req->wb_head;
 790
 791		spin_lock(&mapping->private_lock);
 792		if (likely(head->wb_page && !PageSwapCache(head->wb_page))) {
 793			set_page_private(head->wb_page, 0);
 794			ClearPagePrivate(head->wb_page);
 795			clear_bit(PG_MAPPED, &head->wb_flags);
 796		}
 797		spin_unlock(&mapping->private_lock);
 798	}
 799
 800	if (test_and_clear_bit(PG_INODE_REF, &req->wb_flags)) {
 801		nfs_release_request(req);
 802		atomic_long_dec(&nfsi->nrequests);
 803	}
 804}
 805
 806static void
 807nfs_mark_request_dirty(struct nfs_page *req)
 808{
 809	if (req->wb_page)
 810		__set_page_dirty_nobuffers(req->wb_page);
 811}
 812
 813/*
 814 * nfs_page_search_commits_for_head_request_locked
 815 *
 816 * Search through commit lists on @inode for the head request for @page.
 817 * Must be called while holding the inode (which is cinfo) lock.
 818 *
 819 * Returns the head request if found, or NULL if not found.
 820 */
 821static struct nfs_page *
 822nfs_page_search_commits_for_head_request_locked(struct nfs_inode *nfsi,
 823						struct page *page)
 824{
 825	struct nfs_page *freq, *t;
 826	struct nfs_commit_info cinfo;
 827	struct inode *inode = &nfsi->vfs_inode;
 828
 829	nfs_init_cinfo_from_inode(&cinfo, inode);
 830
 831	/* search through pnfs commit lists */
 832	freq = pnfs_search_commit_reqs(inode, &cinfo, page);
 833	if (freq)
 834		return freq->wb_head;
 835
 836	/* Linearly search the commit list for the correct request */
 837	list_for_each_entry_safe(freq, t, &cinfo.mds->list, wb_list) {
 838		if (freq->wb_page == page)
 839			return freq->wb_head;
 840	}
 841
 842	return NULL;
 843}
 844
 845/**
 846 * nfs_request_add_commit_list_locked - add request to a commit list
 847 * @req: pointer to a struct nfs_page
 848 * @dst: commit list head
 849 * @cinfo: holds list lock and accounting info
 850 *
 851 * This sets the PG_CLEAN bit, updates the cinfo count of
 852 * number of outstanding requests requiring a commit as well as
 853 * the MM page stats.
 854 *
 855 * The caller must hold NFS_I(cinfo->inode)->commit_mutex, and the
 856 * nfs_page lock.
 857 */
 858void
 859nfs_request_add_commit_list_locked(struct nfs_page *req, struct list_head *dst,
 860			    struct nfs_commit_info *cinfo)
 861{
 862	set_bit(PG_CLEAN, &req->wb_flags);
 863	nfs_list_add_request(req, dst);
 864	atomic_long_inc(&cinfo->mds->ncommit);
 865}
 866EXPORT_SYMBOL_GPL(nfs_request_add_commit_list_locked);
 867
 868/**
 869 * nfs_request_add_commit_list - add request to a commit list
 870 * @req: pointer to a struct nfs_page
 871 * @cinfo: holds list lock and accounting info
 872 *
 873 * This sets the PG_CLEAN bit, updates the cinfo count of
 874 * number of outstanding requests requiring a commit as well as
 875 * the MM page stats.
 876 *
 877 * The caller must _not_ hold the cinfo->lock, but must be
 878 * holding the nfs_page lock.
 879 */
 880void
 881nfs_request_add_commit_list(struct nfs_page *req, struct nfs_commit_info *cinfo)
 882{
 883	mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
 884	nfs_request_add_commit_list_locked(req, &cinfo->mds->list, cinfo);
 885	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
 886	if (req->wb_page)
 887		nfs_mark_page_unstable(req->wb_page, cinfo);
 888}
 889EXPORT_SYMBOL_GPL(nfs_request_add_commit_list);
 890
 891/**
 892 * nfs_request_remove_commit_list - Remove request from a commit list
 893 * @req: pointer to a nfs_page
 894 * @cinfo: holds list lock and accounting info
 895 *
 896 * This clears the PG_CLEAN bit, and updates the cinfo's count of
 897 * number of outstanding requests requiring a commit
 898 * It does not update the MM page stats.
 899 *
 900 * The caller _must_ hold the cinfo->lock and the nfs_page lock.
 901 */
 902void
 903nfs_request_remove_commit_list(struct nfs_page *req,
 904			       struct nfs_commit_info *cinfo)
 905{
 906	if (!test_and_clear_bit(PG_CLEAN, &(req)->wb_flags))
 907		return;
 908	nfs_list_remove_request(req);
 909	atomic_long_dec(&cinfo->mds->ncommit);
 910}
 911EXPORT_SYMBOL_GPL(nfs_request_remove_commit_list);
 912
 913static void nfs_init_cinfo_from_inode(struct nfs_commit_info *cinfo,
 914				      struct inode *inode)
 915{
 916	cinfo->inode = inode;
 917	cinfo->mds = &NFS_I(inode)->commit_info;
 918	cinfo->ds = pnfs_get_ds_info(inode);
 919	cinfo->dreq = NULL;
 920	cinfo->completion_ops = &nfs_commit_completion_ops;
 921}
 922
 923void nfs_init_cinfo(struct nfs_commit_info *cinfo,
 924		    struct inode *inode,
 925		    struct nfs_direct_req *dreq)
 926{
 927	if (dreq)
 928		nfs_init_cinfo_from_dreq(cinfo, dreq);
 929	else
 930		nfs_init_cinfo_from_inode(cinfo, inode);
 931}
 932EXPORT_SYMBOL_GPL(nfs_init_cinfo);
 933
 934/*
 935 * Add a request to the inode's commit list.
 936 */
 937void
 938nfs_mark_request_commit(struct nfs_page *req, struct pnfs_layout_segment *lseg,
 939			struct nfs_commit_info *cinfo, u32 ds_commit_idx)
 940{
 941	if (pnfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx))
 942		return;
 943	nfs_request_add_commit_list(req, cinfo);
 944}
 945
 946static void
 947nfs_clear_page_commit(struct page *page)
 948{
 949	dec_node_page_state(page, NR_WRITEBACK);
 950	dec_wb_stat(&inode_to_bdi(page_file_mapping(page)->host)->wb,
 951		    WB_WRITEBACK);
 952}
 953
 954/* Called holding the request lock on @req */
 955static void
 956nfs_clear_request_commit(struct nfs_page *req)
 957{
 958	if (test_bit(PG_CLEAN, &req->wb_flags)) {
 959		struct nfs_open_context *ctx = nfs_req_openctx(req);
 960		struct inode *inode = d_inode(ctx->dentry);
 961		struct nfs_commit_info cinfo;
 962
 963		nfs_init_cinfo_from_inode(&cinfo, inode);
 964		mutex_lock(&NFS_I(inode)->commit_mutex);
 965		if (!pnfs_clear_request_commit(req, &cinfo)) {
 966			nfs_request_remove_commit_list(req, &cinfo);
 967		}
 968		mutex_unlock(&NFS_I(inode)->commit_mutex);
 969		nfs_clear_page_commit(req->wb_page);
 970	}
 971}
 972
 973int nfs_write_need_commit(struct nfs_pgio_header *hdr)
 974{
 975	if (hdr->verf.committed == NFS_DATA_SYNC)
 976		return hdr->lseg == NULL;
 977	return hdr->verf.committed != NFS_FILE_SYNC;
 978}
 979
 980static void nfs_async_write_init(struct nfs_pgio_header *hdr)
 981{
 982	nfs_io_completion_get(hdr->io_completion);
 983}
 984
 985static void nfs_write_completion(struct nfs_pgio_header *hdr)
 986{
 987	struct nfs_commit_info cinfo;
 988	unsigned long bytes = 0;
 989
 990	if (test_bit(NFS_IOHDR_REDO, &hdr->flags))
 991		goto out;
 992	nfs_init_cinfo_from_inode(&cinfo, hdr->inode);
 993	while (!list_empty(&hdr->pages)) {
 994		struct nfs_page *req = nfs_list_entry(hdr->pages.next);
 995
 996		bytes += req->wb_bytes;
 997		nfs_list_remove_request(req);
 998		if (test_bit(NFS_IOHDR_ERROR, &hdr->flags) &&
 999		    (hdr->good_bytes < bytes)) {
1000			trace_nfs_comp_error(req, hdr->error);
1001			nfs_mapping_set_error(req->wb_page, hdr->error);
1002			goto remove_req;
1003		}
1004		if (nfs_write_need_commit(hdr)) {
1005			/* Reset wb_nio, since the write was successful. */
1006			req->wb_nio = 0;
1007			memcpy(&req->wb_verf, &hdr->verf.verifier, sizeof(req->wb_verf));
1008			nfs_mark_request_commit(req, hdr->lseg, &cinfo,
1009				hdr->pgio_mirror_idx);
1010			goto next;
1011		}
1012remove_req:
1013		nfs_inode_remove_request(req);
1014next:
1015		nfs_end_page_writeback(req);
1016		nfs_release_request(req);
1017	}
1018out:
1019	nfs_io_completion_put(hdr->io_completion);
1020	hdr->release(hdr);
1021}
1022
1023unsigned long
1024nfs_reqs_to_commit(struct nfs_commit_info *cinfo)
1025{
1026	return atomic_long_read(&cinfo->mds->ncommit);
1027}
1028
1029/* NFS_I(cinfo->inode)->commit_mutex held by caller */
1030int
1031nfs_scan_commit_list(struct list_head *src, struct list_head *dst,
1032		     struct nfs_commit_info *cinfo, int max)
1033{
1034	struct nfs_page *req, *tmp;
1035	int ret = 0;
1036
1037restart:
1038	list_for_each_entry_safe(req, tmp, src, wb_list) {
1039		kref_get(&req->wb_kref);
1040		if (!nfs_lock_request(req)) {
1041			int status;
1042
1043			/* Prevent deadlock with nfs_lock_and_join_requests */
1044			if (!list_empty(dst)) {
1045				nfs_release_request(req);
1046				continue;
1047			}
1048			/* Ensure we make progress to prevent livelock */
1049			mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
1050			status = nfs_wait_on_request(req);
1051			nfs_release_request(req);
1052			mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
1053			if (status < 0)
1054				break;
1055			goto restart;
1056		}
1057		nfs_request_remove_commit_list(req, cinfo);
1058		clear_bit(PG_COMMIT_TO_DS, &req->wb_flags);
1059		nfs_list_add_request(req, dst);
1060		ret++;
1061		if ((ret == max) && !cinfo->dreq)
1062			break;
1063		cond_resched();
1064	}
1065	return ret;
1066}
1067EXPORT_SYMBOL_GPL(nfs_scan_commit_list);
1068
1069/*
1070 * nfs_scan_commit - Scan an inode for commit requests
1071 * @inode: NFS inode to scan
1072 * @dst: mds destination list
1073 * @cinfo: mds and ds lists of reqs ready to commit
1074 *
1075 * Moves requests from the inode's 'commit' request list.
1076 * The requests are *not* checked to ensure that they form a contiguous set.
1077 */
1078int
1079nfs_scan_commit(struct inode *inode, struct list_head *dst,
1080		struct nfs_commit_info *cinfo)
1081{
1082	int ret = 0;
1083
1084	if (!atomic_long_read(&cinfo->mds->ncommit))
1085		return 0;
1086	mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
1087	if (atomic_long_read(&cinfo->mds->ncommit) > 0) {
1088		const int max = INT_MAX;
1089
1090		ret = nfs_scan_commit_list(&cinfo->mds->list, dst,
1091					   cinfo, max);
1092		ret += pnfs_scan_commit_lists(inode, cinfo, max - ret);
1093	}
1094	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
1095	return ret;
1096}
1097
1098/*
1099 * Search for an existing write request, and attempt to update
1100 * it to reflect a new dirty region on a given page.
1101 *
1102 * If the attempt fails, then the existing request is flushed out
1103 * to disk.
1104 */
1105static struct nfs_page *nfs_try_to_update_request(struct inode *inode,
1106		struct page *page,
1107		unsigned int offset,
1108		unsigned int bytes)
1109{
1110	struct nfs_page *req;
1111	unsigned int rqend;
1112	unsigned int end;
1113	int error;
1114
1115	end = offset + bytes;
1116
1117	req = nfs_lock_and_join_requests(page);
1118	if (IS_ERR_OR_NULL(req))
1119		return req;
1120
1121	rqend = req->wb_offset + req->wb_bytes;
1122	/*
1123	 * Tell the caller to flush out the request if
1124	 * the offsets are non-contiguous.
1125	 * Note: nfs_flush_incompatible() will already
1126	 * have flushed out requests having wrong owners.
1127	 */
1128	if (offset > rqend || end < req->wb_offset)
1129		goto out_flushme;
1130
1131	/* Okay, the request matches. Update the region */
1132	if (offset < req->wb_offset) {
1133		req->wb_offset = offset;
1134		req->wb_pgbase = offset;
1135	}
1136	if (end > rqend)
1137		req->wb_bytes = end - req->wb_offset;
1138	else
1139		req->wb_bytes = rqend - req->wb_offset;
1140	req->wb_nio = 0;
1141	return req;
1142out_flushme:
1143	/*
1144	 * Note: we mark the request dirty here because
1145	 * nfs_lock_and_join_requests() cannot preserve
1146	 * commit flags, so we have to replay the write.
1147	 */
1148	nfs_mark_request_dirty(req);
1149	nfs_unlock_and_release_request(req);
1150	error = nfs_wb_page(inode, page);
1151	return (error < 0) ? ERR_PTR(error) : NULL;
1152}
1153
1154/*
1155 * Try to update an existing write request, or create one if there is none.
1156 *
1157 * Note: Should always be called with the Page Lock held to prevent races
1158 * if we have to add a new request. Also assumes that the caller has
1159 * already called nfs_flush_incompatible() if necessary.
1160 */
1161static struct nfs_page * nfs_setup_write_request(struct nfs_open_context* ctx,
1162		struct page *page, unsigned int offset, unsigned int bytes)
1163{
1164	struct inode *inode = page_file_mapping(page)->host;
1165	struct nfs_page	*req;
1166
1167	req = nfs_try_to_update_request(inode, page, offset, bytes);
1168	if (req != NULL)
1169		goto out;
1170	req = nfs_create_request(ctx, page, offset, bytes);
1171	if (IS_ERR(req))
1172		goto out;
1173	nfs_inode_add_request(inode, req);
1174out:
1175	return req;
1176}
1177
1178static int nfs_writepage_setup(struct nfs_open_context *ctx, struct page *page,
1179		unsigned int offset, unsigned int count)
1180{
1181	struct nfs_page	*req;
1182
1183	req = nfs_setup_write_request(ctx, page, offset, count);
1184	if (IS_ERR(req))
1185		return PTR_ERR(req);
1186	/* Update file length */
1187	nfs_grow_file(page, offset, count);
1188	nfs_mark_uptodate(req);
1189	nfs_mark_request_dirty(req);
1190	nfs_unlock_and_release_request(req);
1191	return 0;
1192}
1193
1194int nfs_flush_incompatible(struct file *file, struct page *page)
1195{
1196	struct nfs_open_context *ctx = nfs_file_open_context(file);
1197	struct nfs_lock_context *l_ctx;
1198	struct file_lock_context *flctx = file_inode(file)->i_flctx;
1199	struct nfs_page	*req;
1200	int do_flush, status;
1201	/*
1202	 * Look for a request corresponding to this page. If there
1203	 * is one, and it belongs to another file, we flush it out
1204	 * before we try to copy anything into the page. Do this
1205	 * due to the lack of an ACCESS-type call in NFSv2.
1206	 * Also do the same if we find a request from an existing
1207	 * dropped page.
1208	 */
1209	do {
1210		req = nfs_page_find_head_request(page);
1211		if (req == NULL)
1212			return 0;
1213		l_ctx = req->wb_lock_context;
1214		do_flush = req->wb_page != page ||
1215			!nfs_match_open_context(nfs_req_openctx(req), ctx);
1216		if (l_ctx && flctx &&
1217		    !(list_empty_careful(&flctx->flc_posix) &&
1218		      list_empty_careful(&flctx->flc_flock))) {
1219			do_flush |= l_ctx->lockowner != current->files;
1220		}
1221		nfs_release_request(req);
1222		if (!do_flush)
1223			return 0;
1224		status = nfs_wb_page(page_file_mapping(page)->host, page);
1225	} while (status == 0);
1226	return status;
1227}
1228
1229/*
1230 * Avoid buffered writes when a open context credential's key would
1231 * expire soon.
1232 *
1233 * Returns -EACCES if the key will expire within RPC_KEY_EXPIRE_FAIL.
1234 *
1235 * Return 0 and set a credential flag which triggers the inode to flush
1236 * and performs  NFS_FILE_SYNC writes if the key will expired within
1237 * RPC_KEY_EXPIRE_TIMEO.
1238 */
1239int
1240nfs_key_timeout_notify(struct file *filp, struct inode *inode)
1241{
1242	struct nfs_open_context *ctx = nfs_file_open_context(filp);
1243
1244	if (nfs_ctx_key_to_expire(ctx, inode) &&
1245	    !ctx->ll_cred)
1246		/* Already expired! */
1247		return -EACCES;
1248	return 0;
1249}
1250
1251/*
1252 * Test if the open context credential key is marked to expire soon.
1253 */
1254bool nfs_ctx_key_to_expire(struct nfs_open_context *ctx, struct inode *inode)
1255{
1256	struct rpc_auth *auth = NFS_SERVER(inode)->client->cl_auth;
1257	struct rpc_cred *cred = ctx->ll_cred;
1258	struct auth_cred acred = {
1259		.cred = ctx->cred,
1260	};
1261
1262	if (cred && !cred->cr_ops->crmatch(&acred, cred, 0)) {
1263		put_rpccred(cred);
1264		ctx->ll_cred = NULL;
1265		cred = NULL;
1266	}
1267	if (!cred)
1268		cred = auth->au_ops->lookup_cred(auth, &acred, 0);
1269	if (!cred || IS_ERR(cred))
1270		return true;
1271	ctx->ll_cred = cred;
1272	return !!(cred->cr_ops->crkey_timeout &&
1273		  cred->cr_ops->crkey_timeout(cred));
1274}
1275
1276/*
1277 * If the page cache is marked as unsafe or invalid, then we can't rely on
1278 * the PageUptodate() flag. In this case, we will need to turn off
1279 * write optimisations that depend on the page contents being correct.
1280 */
1281static bool nfs_write_pageuptodate(struct page *page, struct inode *inode)
1282{
1283	struct nfs_inode *nfsi = NFS_I(inode);
1284
1285	if (nfs_have_delegated_attributes(inode))
1286		goto out;
1287	if (nfsi->cache_validity & NFS_INO_REVAL_PAGECACHE)
1288		return false;
1289	smp_rmb();
1290	if (test_bit(NFS_INO_INVALIDATING, &nfsi->flags))
1291		return false;
1292out:
1293	if (nfsi->cache_validity & NFS_INO_INVALID_DATA)
1294		return false;
1295	return PageUptodate(page) != 0;
1296}
1297
1298static bool
1299is_whole_file_wrlock(struct file_lock *fl)
1300{
1301	return fl->fl_start == 0 && fl->fl_end == OFFSET_MAX &&
1302			fl->fl_type == F_WRLCK;
1303}
1304
1305/* If we know the page is up to date, and we're not using byte range locks (or
1306 * if we have the whole file locked for writing), it may be more efficient to
1307 * extend the write to cover the entire page in order to avoid fragmentation
1308 * inefficiencies.
1309 *
1310 * If the file is opened for synchronous writes then we can just skip the rest
1311 * of the checks.
1312 */
1313static int nfs_can_extend_write(struct file *file, struct page *page, struct inode *inode)
1314{
1315	int ret;
1316	struct file_lock_context *flctx = inode->i_flctx;
1317	struct file_lock *fl;
1318
1319	if (file->f_flags & O_DSYNC)
1320		return 0;
1321	if (!nfs_write_pageuptodate(page, inode))
1322		return 0;
1323	if (NFS_PROTO(inode)->have_delegation(inode, FMODE_WRITE))
1324		return 1;
1325	if (!flctx || (list_empty_careful(&flctx->flc_flock) &&
1326		       list_empty_careful(&flctx->flc_posix)))
1327		return 1;
1328
1329	/* Check to see if there are whole file write locks */
1330	ret = 0;
1331	spin_lock(&flctx->flc_lock);
1332	if (!list_empty(&flctx->flc_posix)) {
1333		fl = list_first_entry(&flctx->flc_posix, struct file_lock,
1334					fl_list);
1335		if (is_whole_file_wrlock(fl))
1336			ret = 1;
1337	} else if (!list_empty(&flctx->flc_flock)) {
1338		fl = list_first_entry(&flctx->flc_flock, struct file_lock,
1339					fl_list);
1340		if (fl->fl_type == F_WRLCK)
1341			ret = 1;
1342	}
1343	spin_unlock(&flctx->flc_lock);
1344	return ret;
1345}
1346
1347/*
1348 * Update and possibly write a cached page of an NFS file.
1349 *
1350 * XXX: Keep an eye on generic_file_read to make sure it doesn't do bad
1351 * things with a page scheduled for an RPC call (e.g. invalidate it).
1352 */
1353int nfs_updatepage(struct file *file, struct page *page,
1354		unsigned int offset, unsigned int count)
1355{
1356	struct nfs_open_context *ctx = nfs_file_open_context(file);
1357	struct address_space *mapping = page_file_mapping(page);
1358	struct inode	*inode = mapping->host;
1359	int		status = 0;
1360
1361	nfs_inc_stats(inode, NFSIOS_VFSUPDATEPAGE);
1362
1363	dprintk("NFS:       nfs_updatepage(%pD2 %d@%lld)\n",
1364		file, count, (long long)(page_file_offset(page) + offset));
1365
1366	if (!count)
1367		goto out;
1368
1369	if (nfs_can_extend_write(file, page, inode)) {
1370		count = max(count + offset, nfs_page_length(page));
1371		offset = 0;
1372	}
1373
1374	status = nfs_writepage_setup(ctx, page, offset, count);
1375	if (status < 0)
1376		nfs_set_pageerror(mapping);
1377	else
1378		__set_page_dirty_nobuffers(page);
1379out:
1380	dprintk("NFS:       nfs_updatepage returns %d (isize %lld)\n",
1381			status, (long long)i_size_read(inode));
1382	return status;
1383}
1384
1385static int flush_task_priority(int how)
1386{
1387	switch (how & (FLUSH_HIGHPRI|FLUSH_LOWPRI)) {
1388		case FLUSH_HIGHPRI:
1389			return RPC_PRIORITY_HIGH;
1390		case FLUSH_LOWPRI:
1391			return RPC_PRIORITY_LOW;
1392	}
1393	return RPC_PRIORITY_NORMAL;
1394}
1395
1396static void nfs_initiate_write(struct nfs_pgio_header *hdr,
1397			       struct rpc_message *msg,
1398			       const struct nfs_rpc_ops *rpc_ops,
1399			       struct rpc_task_setup *task_setup_data, int how)
1400{
1401	int priority = flush_task_priority(how);
1402
1403	task_setup_data->priority = priority;
1404	rpc_ops->write_setup(hdr, msg, &task_setup_data->rpc_client);
1405	trace_nfs_initiate_write(hdr);
 
1406}
1407
1408/* If a nfs_flush_* function fails, it should remove reqs from @head and
1409 * call this on each, which will prepare them to be retried on next
1410 * writeback using standard nfs.
1411 */
1412static void nfs_redirty_request(struct nfs_page *req)
1413{
1414	/* Bump the transmission count */
1415	req->wb_nio++;
1416	nfs_mark_request_dirty(req);
1417	set_bit(NFS_CONTEXT_RESEND_WRITES, &nfs_req_openctx(req)->flags);
1418	nfs_end_page_writeback(req);
1419	nfs_release_request(req);
1420}
1421
1422static void nfs_async_write_error(struct list_head *head, int error)
1423{
1424	struct nfs_page	*req;
1425
1426	while (!list_empty(head)) {
1427		req = nfs_list_entry(head->next);
1428		nfs_list_remove_request(req);
1429		if (nfs_error_is_fatal(error))
1430			nfs_write_error(req, error);
1431		else
1432			nfs_redirty_request(req);
1433	}
1434}
1435
1436static void nfs_async_write_reschedule_io(struct nfs_pgio_header *hdr)
1437{
1438	nfs_async_write_error(&hdr->pages, 0);
1439	filemap_fdatawrite_range(hdr->inode->i_mapping, hdr->args.offset,
1440			hdr->args.offset + hdr->args.count - 1);
1441}
1442
1443static const struct nfs_pgio_completion_ops nfs_async_write_completion_ops = {
1444	.init_hdr = nfs_async_write_init,
1445	.error_cleanup = nfs_async_write_error,
1446	.completion = nfs_write_completion,
1447	.reschedule_io = nfs_async_write_reschedule_io,
1448};
1449
1450void nfs_pageio_init_write(struct nfs_pageio_descriptor *pgio,
1451			       struct inode *inode, int ioflags, bool force_mds,
1452			       const struct nfs_pgio_completion_ops *compl_ops)
1453{
1454	struct nfs_server *server = NFS_SERVER(inode);
1455	const struct nfs_pageio_ops *pg_ops = &nfs_pgio_rw_ops;
1456
1457#ifdef CONFIG_NFS_V4_1
1458	if (server->pnfs_curr_ld && !force_mds)
1459		pg_ops = server->pnfs_curr_ld->pg_write_ops;
1460#endif
1461	nfs_pageio_init(pgio, inode, pg_ops, compl_ops, &nfs_rw_write_ops,
1462			server->wsize, ioflags);
1463}
1464EXPORT_SYMBOL_GPL(nfs_pageio_init_write);
1465
1466void nfs_pageio_reset_write_mds(struct nfs_pageio_descriptor *pgio)
1467{
1468	struct nfs_pgio_mirror *mirror;
1469
1470	if (pgio->pg_ops && pgio->pg_ops->pg_cleanup)
1471		pgio->pg_ops->pg_cleanup(pgio);
1472
1473	pgio->pg_ops = &nfs_pgio_rw_ops;
1474
1475	nfs_pageio_stop_mirroring(pgio);
1476
1477	mirror = &pgio->pg_mirrors[0];
1478	mirror->pg_bsize = NFS_SERVER(pgio->pg_inode)->wsize;
1479}
1480EXPORT_SYMBOL_GPL(nfs_pageio_reset_write_mds);
1481
1482
1483void nfs_commit_prepare(struct rpc_task *task, void *calldata)
1484{
1485	struct nfs_commit_data *data = calldata;
1486
1487	NFS_PROTO(data->inode)->commit_rpc_prepare(task, data);
1488}
1489
1490/*
1491 * Special version of should_remove_suid() that ignores capabilities.
1492 */
1493static int nfs_should_remove_suid(const struct inode *inode)
1494{
1495	umode_t mode = inode->i_mode;
1496	int kill = 0;
1497
1498	/* suid always must be killed */
1499	if (unlikely(mode & S_ISUID))
1500		kill = ATTR_KILL_SUID;
1501
1502	/*
1503	 * sgid without any exec bits is just a mandatory locking mark; leave
1504	 * it alone.  If some exec bits are set, it's a real sgid; kill it.
1505	 */
1506	if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
1507		kill |= ATTR_KILL_SGID;
1508
1509	if (unlikely(kill && S_ISREG(mode)))
1510		return kill;
1511
1512	return 0;
1513}
1514
1515static void nfs_writeback_check_extend(struct nfs_pgio_header *hdr,
1516		struct nfs_fattr *fattr)
1517{
1518	struct nfs_pgio_args *argp = &hdr->args;
1519	struct nfs_pgio_res *resp = &hdr->res;
1520	u64 size = argp->offset + resp->count;
1521
1522	if (!(fattr->valid & NFS_ATTR_FATTR_SIZE))
1523		fattr->size = size;
1524	if (nfs_size_to_loff_t(fattr->size) < i_size_read(hdr->inode)) {
1525		fattr->valid &= ~NFS_ATTR_FATTR_SIZE;
1526		return;
1527	}
1528	if (size != fattr->size)
1529		return;
1530	/* Set attribute barrier */
1531	nfs_fattr_set_barrier(fattr);
1532	/* ...and update size */
1533	fattr->valid |= NFS_ATTR_FATTR_SIZE;
1534}
1535
1536void nfs_writeback_update_inode(struct nfs_pgio_header *hdr)
1537{
1538	struct nfs_fattr *fattr = &hdr->fattr;
1539	struct inode *inode = hdr->inode;
1540
1541	spin_lock(&inode->i_lock);
1542	nfs_writeback_check_extend(hdr, fattr);
1543	nfs_post_op_update_inode_force_wcc_locked(inode, fattr);
1544	spin_unlock(&inode->i_lock);
1545}
1546EXPORT_SYMBOL_GPL(nfs_writeback_update_inode);
1547
1548/*
1549 * This function is called when the WRITE call is complete.
1550 */
1551static int nfs_writeback_done(struct rpc_task *task,
1552			      struct nfs_pgio_header *hdr,
1553			      struct inode *inode)
1554{
1555	int status;
1556
1557	/*
1558	 * ->write_done will attempt to use post-op attributes to detect
1559	 * conflicting writes by other clients.  A strict interpretation
1560	 * of close-to-open would allow us to continue caching even if
1561	 * another writer had changed the file, but some applications
1562	 * depend on tighter cache coherency when writing.
1563	 */
1564	status = NFS_PROTO(inode)->write_done(task, hdr);
1565	if (status != 0)
1566		return status;
1567
1568	nfs_add_stats(inode, NFSIOS_SERVERWRITTENBYTES, hdr->res.count);
1569	trace_nfs_writeback_done(task, hdr);
 
1570
1571	if (hdr->res.verf->committed < hdr->args.stable &&
1572	    task->tk_status >= 0) {
1573		/* We tried a write call, but the server did not
1574		 * commit data to stable storage even though we
1575		 * requested it.
1576		 * Note: There is a known bug in Tru64 < 5.0 in which
1577		 *	 the server reports NFS_DATA_SYNC, but performs
1578		 *	 NFS_FILE_SYNC. We therefore implement this checking
1579		 *	 as a dprintk() in order to avoid filling syslog.
1580		 */
1581		static unsigned long    complain;
1582
1583		/* Note this will print the MDS for a DS write */
1584		if (time_before(complain, jiffies)) {
1585			dprintk("NFS:       faulty NFS server %s:"
1586				" (committed = %d) != (stable = %d)\n",
1587				NFS_SERVER(inode)->nfs_client->cl_hostname,
1588				hdr->res.verf->committed, hdr->args.stable);
1589			complain = jiffies + 300 * HZ;
1590		}
1591	}
1592
1593	/* Deal with the suid/sgid bit corner case */
1594	if (nfs_should_remove_suid(inode)) {
1595		spin_lock(&inode->i_lock);
1596		NFS_I(inode)->cache_validity |= NFS_INO_INVALID_OTHER;
1597		spin_unlock(&inode->i_lock);
1598	}
1599	return 0;
1600}
1601
1602/*
1603 * This function is called when the WRITE call is complete.
1604 */
1605static void nfs_writeback_result(struct rpc_task *task,
1606				 struct nfs_pgio_header *hdr)
1607{
1608	struct nfs_pgio_args	*argp = &hdr->args;
1609	struct nfs_pgio_res	*resp = &hdr->res;
1610
1611	if (resp->count < argp->count) {
1612		static unsigned long    complain;
1613
1614		/* This a short write! */
1615		nfs_inc_stats(hdr->inode, NFSIOS_SHORTWRITE);
1616
1617		/* Has the server at least made some progress? */
1618		if (resp->count == 0) {
1619			if (time_before(complain, jiffies)) {
1620				printk(KERN_WARNING
1621				       "NFS: Server wrote zero bytes, expected %u.\n",
1622				       argp->count);
1623				complain = jiffies + 300 * HZ;
1624			}
1625			nfs_set_pgio_error(hdr, -EIO, argp->offset);
1626			task->tk_status = -EIO;
1627			return;
1628		}
1629
1630		/* For non rpc-based layout drivers, retry-through-MDS */
1631		if (!task->tk_ops) {
1632			hdr->pnfs_error = -EAGAIN;
1633			return;
1634		}
1635
1636		/* Was this an NFSv2 write or an NFSv3 stable write? */
1637		if (resp->verf->committed != NFS_UNSTABLE) {
1638			/* Resend from where the server left off */
1639			hdr->mds_offset += resp->count;
1640			argp->offset += resp->count;
1641			argp->pgbase += resp->count;
1642			argp->count -= resp->count;
1643		} else {
1644			/* Resend as a stable write in order to avoid
1645			 * headaches in the case of a server crash.
1646			 */
1647			argp->stable = NFS_FILE_SYNC;
1648		}
1649		resp->count = 0;
1650		resp->verf->committed = 0;
1651		rpc_restart_call_prepare(task);
1652	}
1653}
1654
1655static int wait_on_commit(struct nfs_mds_commit_info *cinfo)
1656{
1657	return wait_var_event_killable(&cinfo->rpcs_out,
1658				       !atomic_read(&cinfo->rpcs_out));
1659}
1660
1661static void nfs_commit_begin(struct nfs_mds_commit_info *cinfo)
1662{
1663	atomic_inc(&cinfo->rpcs_out);
1664}
1665
1666static void nfs_commit_end(struct nfs_mds_commit_info *cinfo)
1667{
1668	if (atomic_dec_and_test(&cinfo->rpcs_out))
1669		wake_up_var(&cinfo->rpcs_out);
1670}
1671
1672void nfs_commitdata_release(struct nfs_commit_data *data)
1673{
1674	put_nfs_open_context(data->context);
1675	nfs_commit_free(data);
1676}
1677EXPORT_SYMBOL_GPL(nfs_commitdata_release);
1678
1679int nfs_initiate_commit(struct rpc_clnt *clnt, struct nfs_commit_data *data,
1680			const struct nfs_rpc_ops *nfs_ops,
1681			const struct rpc_call_ops *call_ops,
1682			int how, int flags)
1683{
1684	struct rpc_task *task;
1685	int priority = flush_task_priority(how);
1686	struct rpc_message msg = {
1687		.rpc_argp = &data->args,
1688		.rpc_resp = &data->res,
1689		.rpc_cred = data->cred,
1690	};
1691	struct rpc_task_setup task_setup_data = {
1692		.task = &data->task,
1693		.rpc_client = clnt,
1694		.rpc_message = &msg,
1695		.callback_ops = call_ops,
1696		.callback_data = data,
1697		.workqueue = nfsiod_workqueue,
1698		.flags = RPC_TASK_ASYNC | flags,
1699		.priority = priority,
1700	};
1701	/* Set up the initial task struct.  */
1702	nfs_ops->commit_setup(data, &msg, &task_setup_data.rpc_client);
1703	trace_nfs_initiate_commit(data);
1704
1705	dprintk("NFS: initiated commit call\n");
1706
1707	task = rpc_run_task(&task_setup_data);
1708	if (IS_ERR(task))
1709		return PTR_ERR(task);
1710	if (how & FLUSH_SYNC)
1711		rpc_wait_for_completion_task(task);
1712	rpc_put_task(task);
1713	return 0;
1714}
1715EXPORT_SYMBOL_GPL(nfs_initiate_commit);
1716
1717static loff_t nfs_get_lwb(struct list_head *head)
1718{
1719	loff_t lwb = 0;
1720	struct nfs_page *req;
1721
1722	list_for_each_entry(req, head, wb_list)
1723		if (lwb < (req_offset(req) + req->wb_bytes))
1724			lwb = req_offset(req) + req->wb_bytes;
1725
1726	return lwb;
1727}
1728
1729/*
1730 * Set up the argument/result storage required for the RPC call.
1731 */
1732void nfs_init_commit(struct nfs_commit_data *data,
1733		     struct list_head *head,
1734		     struct pnfs_layout_segment *lseg,
1735		     struct nfs_commit_info *cinfo)
1736{
1737	struct nfs_page *first;
1738	struct nfs_open_context *ctx;
1739	struct inode *inode;
1740
1741	/* Set up the RPC argument and reply structs
1742	 * NB: take care not to mess about with data->commit et al. */
1743
1744	if (head)
1745		list_splice_init(head, &data->pages);
1746
1747	first = nfs_list_entry(data->pages.next);
1748	ctx = nfs_req_openctx(first);
1749	inode = d_inode(ctx->dentry);
1750
1751	data->inode	  = inode;
1752	data->cred	  = ctx->cred;
1753	data->lseg	  = lseg; /* reference transferred */
1754	/* only set lwb for pnfs commit */
1755	if (lseg)
1756		data->lwb = nfs_get_lwb(&data->pages);
1757	data->mds_ops     = &nfs_commit_ops;
1758	data->completion_ops = cinfo->completion_ops;
1759	data->dreq	  = cinfo->dreq;
1760
1761	data->args.fh     = NFS_FH(data->inode);
1762	/* Note: we always request a commit of the entire inode */
1763	data->args.offset = 0;
1764	data->args.count  = 0;
1765	data->context     = get_nfs_open_context(ctx);
1766	data->res.fattr   = &data->fattr;
1767	data->res.verf    = &data->verf;
1768	nfs_fattr_init(&data->fattr);
1769}
1770EXPORT_SYMBOL_GPL(nfs_init_commit);
1771
1772void nfs_retry_commit(struct list_head *page_list,
1773		      struct pnfs_layout_segment *lseg,
1774		      struct nfs_commit_info *cinfo,
1775		      u32 ds_commit_idx)
1776{
1777	struct nfs_page *req;
1778
1779	while (!list_empty(page_list)) {
1780		req = nfs_list_entry(page_list->next);
1781		nfs_list_remove_request(req);
1782		nfs_mark_request_commit(req, lseg, cinfo, ds_commit_idx);
1783		if (!cinfo->dreq)
1784			nfs_clear_page_commit(req->wb_page);
1785		nfs_unlock_and_release_request(req);
1786	}
1787}
1788EXPORT_SYMBOL_GPL(nfs_retry_commit);
1789
1790static void
1791nfs_commit_resched_write(struct nfs_commit_info *cinfo,
1792		struct nfs_page *req)
1793{
1794	__set_page_dirty_nobuffers(req->wb_page);
1795}
1796
1797/*
1798 * Commit dirty pages
1799 */
1800static int
1801nfs_commit_list(struct inode *inode, struct list_head *head, int how,
1802		struct nfs_commit_info *cinfo)
1803{
1804	struct nfs_commit_data	*data;
1805
1806	/* another commit raced with us */
1807	if (list_empty(head))
1808		return 0;
1809
1810	data = nfs_commitdata_alloc(true);
1811
1812	/* Set up the argument struct */
1813	nfs_init_commit(data, head, NULL, cinfo);
1814	atomic_inc(&cinfo->mds->rpcs_out);
1815	return nfs_initiate_commit(NFS_CLIENT(inode), data, NFS_PROTO(inode),
1816				   data->mds_ops, how, RPC_TASK_CRED_NOREF);
1817}
1818
1819/*
1820 * COMMIT call returned
1821 */
1822static void nfs_commit_done(struct rpc_task *task, void *calldata)
1823{
1824	struct nfs_commit_data	*data = calldata;
1825
1826        dprintk("NFS: %5u nfs_commit_done (status %d)\n",
1827                                task->tk_pid, task->tk_status);
1828
1829	/* Call the NFS version-specific code */
1830	NFS_PROTO(data->inode)->commit_done(task, data);
1831	trace_nfs_commit_done(task, data);
1832}
1833
1834static void nfs_commit_release_pages(struct nfs_commit_data *data)
1835{
1836	const struct nfs_writeverf *verf = data->res.verf;
1837	struct nfs_page	*req;
1838	int status = data->task.tk_status;
1839	struct nfs_commit_info cinfo;
1840	struct nfs_server *nfss;
1841
1842	while (!list_empty(&data->pages)) {
1843		req = nfs_list_entry(data->pages.next);
1844		nfs_list_remove_request(req);
1845		if (req->wb_page)
1846			nfs_clear_page_commit(req->wb_page);
1847
1848		dprintk("NFS:       commit (%s/%llu %d@%lld)",
1849			nfs_req_openctx(req)->dentry->d_sb->s_id,
1850			(unsigned long long)NFS_FILEID(d_inode(nfs_req_openctx(req)->dentry)),
1851			req->wb_bytes,
1852			(long long)req_offset(req));
1853		if (status < 0) {
1854			if (req->wb_page) {
1855				trace_nfs_commit_error(req, status);
1856				nfs_mapping_set_error(req->wb_page, status);
1857				nfs_inode_remove_request(req);
1858			}
1859			dprintk_cont(", error = %d\n", status);
1860			goto next;
1861		}
1862
1863		/* Okay, COMMIT succeeded, apparently. Check the verifier
1864		 * returned by the server against all stored verfs. */
1865		if (nfs_write_match_verf(verf, req)) {
1866			/* We have a match */
1867			if (req->wb_page)
1868				nfs_inode_remove_request(req);
1869			dprintk_cont(" OK\n");
1870			goto next;
1871		}
1872		/* We have a mismatch. Write the page again */
1873		dprintk_cont(" mismatch\n");
1874		nfs_mark_request_dirty(req);
1875		set_bit(NFS_CONTEXT_RESEND_WRITES, &nfs_req_openctx(req)->flags);
1876	next:
1877		nfs_unlock_and_release_request(req);
1878		/* Latency breaker */
1879		cond_resched();
1880	}
1881	nfss = NFS_SERVER(data->inode);
1882	if (atomic_long_read(&nfss->writeback) < NFS_CONGESTION_OFF_THRESH)
1883		clear_bdi_congested(inode_to_bdi(data->inode), BLK_RW_ASYNC);
1884
1885	nfs_init_cinfo(&cinfo, data->inode, data->dreq);
1886	nfs_commit_end(cinfo.mds);
1887}
1888
1889static void nfs_commit_release(void *calldata)
1890{
1891	struct nfs_commit_data *data = calldata;
1892
1893	data->completion_ops->completion(data);
1894	nfs_commitdata_release(calldata);
1895}
1896
1897static const struct rpc_call_ops nfs_commit_ops = {
1898	.rpc_call_prepare = nfs_commit_prepare,
1899	.rpc_call_done = nfs_commit_done,
1900	.rpc_release = nfs_commit_release,
1901};
1902
1903static const struct nfs_commit_completion_ops nfs_commit_completion_ops = {
1904	.completion = nfs_commit_release_pages,
1905	.resched_write = nfs_commit_resched_write,
1906};
1907
1908int nfs_generic_commit_list(struct inode *inode, struct list_head *head,
1909			    int how, struct nfs_commit_info *cinfo)
1910{
1911	int status;
1912
1913	status = pnfs_commit_list(inode, head, how, cinfo);
1914	if (status == PNFS_NOT_ATTEMPTED)
1915		status = nfs_commit_list(inode, head, how, cinfo);
1916	return status;
1917}
1918
1919static int __nfs_commit_inode(struct inode *inode, int how,
1920		struct writeback_control *wbc)
1921{
1922	LIST_HEAD(head);
1923	struct nfs_commit_info cinfo;
1924	int may_wait = how & FLUSH_SYNC;
1925	int ret, nscan;
1926
1927	nfs_init_cinfo_from_inode(&cinfo, inode);
1928	nfs_commit_begin(cinfo.mds);
1929	for (;;) {
1930		ret = nscan = nfs_scan_commit(inode, &head, &cinfo);
1931		if (ret <= 0)
1932			break;
1933		ret = nfs_generic_commit_list(inode, &head, how, &cinfo);
1934		if (ret < 0)
1935			break;
1936		ret = 0;
1937		if (wbc && wbc->sync_mode == WB_SYNC_NONE) {
1938			if (nscan < wbc->nr_to_write)
1939				wbc->nr_to_write -= nscan;
1940			else
1941				wbc->nr_to_write = 0;
1942		}
1943		if (nscan < INT_MAX)
1944			break;
1945		cond_resched();
1946	}
1947	nfs_commit_end(cinfo.mds);
1948	if (ret || !may_wait)
1949		return ret;
1950	return wait_on_commit(cinfo.mds);
1951}
1952
1953int nfs_commit_inode(struct inode *inode, int how)
1954{
1955	return __nfs_commit_inode(inode, how, NULL);
1956}
1957EXPORT_SYMBOL_GPL(nfs_commit_inode);
1958
1959int nfs_write_inode(struct inode *inode, struct writeback_control *wbc)
1960{
1961	struct nfs_inode *nfsi = NFS_I(inode);
1962	int flags = FLUSH_SYNC;
1963	int ret = 0;
1964
1965	if (wbc->sync_mode == WB_SYNC_NONE) {
1966		/* no commits means nothing needs to be done */
1967		if (!atomic_long_read(&nfsi->commit_info.ncommit))
1968			goto check_requests_outstanding;
1969
1970		/* Don't commit yet if this is a non-blocking flush and there
1971		 * are a lot of outstanding writes for this mapping.
1972		 */
1973		if (mapping_tagged(inode->i_mapping, PAGECACHE_TAG_WRITEBACK))
1974			goto out_mark_dirty;
1975
1976		/* don't wait for the COMMIT response */
1977		flags = 0;
1978	}
1979
1980	ret = __nfs_commit_inode(inode, flags, wbc);
1981	if (!ret) {
1982		if (flags & FLUSH_SYNC)
1983			return 0;
1984	} else if (atomic_long_read(&nfsi->commit_info.ncommit))
1985		goto out_mark_dirty;
1986
1987check_requests_outstanding:
1988	if (!atomic_read(&nfsi->commit_info.rpcs_out))
1989		return ret;
1990out_mark_dirty:
1991	__mark_inode_dirty(inode, I_DIRTY_DATASYNC);
1992	return ret;
1993}
1994EXPORT_SYMBOL_GPL(nfs_write_inode);
1995
1996/*
1997 * Wrapper for filemap_write_and_wait_range()
1998 *
1999 * Needed for pNFS in order to ensure data becomes visible to the
2000 * client.
2001 */
2002int nfs_filemap_write_and_wait_range(struct address_space *mapping,
2003		loff_t lstart, loff_t lend)
2004{
2005	int ret;
2006
2007	ret = filemap_write_and_wait_range(mapping, lstart, lend);
2008	if (ret == 0)
2009		ret = pnfs_sync_inode(mapping->host, true);
2010	return ret;
2011}
2012EXPORT_SYMBOL_GPL(nfs_filemap_write_and_wait_range);
2013
2014/*
2015 * flush the inode to disk.
2016 */
2017int nfs_wb_all(struct inode *inode)
2018{
2019	int ret;
2020
2021	trace_nfs_writeback_inode_enter(inode);
2022
2023	ret = filemap_write_and_wait(inode->i_mapping);
2024	if (ret)
2025		goto out;
2026	ret = nfs_commit_inode(inode, FLUSH_SYNC);
2027	if (ret < 0)
2028		goto out;
2029	pnfs_sync_inode(inode, true);
2030	ret = 0;
2031
2032out:
2033	trace_nfs_writeback_inode_exit(inode, ret);
2034	return ret;
2035}
2036EXPORT_SYMBOL_GPL(nfs_wb_all);
2037
2038int nfs_wb_page_cancel(struct inode *inode, struct page *page)
2039{
2040	struct nfs_page *req;
2041	int ret = 0;
2042
2043	wait_on_page_writeback(page);
2044
2045	/* blocking call to cancel all requests and join to a single (head)
2046	 * request */
2047	req = nfs_lock_and_join_requests(page);
2048
2049	if (IS_ERR(req)) {
2050		ret = PTR_ERR(req);
2051	} else if (req) {
2052		/* all requests from this page have been cancelled by
2053		 * nfs_lock_and_join_requests, so just remove the head
2054		 * request from the inode / page_private pointer and
2055		 * release it */
2056		nfs_inode_remove_request(req);
2057		nfs_unlock_and_release_request(req);
2058	}
2059
2060	return ret;
2061}
2062
2063/*
2064 * Write back all requests on one page - we do this before reading it.
2065 */
2066int nfs_wb_page(struct inode *inode, struct page *page)
2067{
2068	loff_t range_start = page_file_offset(page);
2069	loff_t range_end = range_start + (loff_t)(PAGE_SIZE - 1);
2070	struct writeback_control wbc = {
2071		.sync_mode = WB_SYNC_ALL,
2072		.nr_to_write = 0,
2073		.range_start = range_start,
2074		.range_end = range_end,
2075	};
2076	int ret;
2077
2078	trace_nfs_writeback_page_enter(inode);
2079
2080	for (;;) {
2081		wait_on_page_writeback(page);
2082		if (clear_page_dirty_for_io(page)) {
2083			ret = nfs_writepage_locked(page, &wbc);
2084			if (ret < 0)
2085				goto out_error;
2086			continue;
2087		}
2088		ret = 0;
2089		if (!PagePrivate(page))
2090			break;
2091		ret = nfs_commit_inode(inode, FLUSH_SYNC);
2092		if (ret < 0)
2093			goto out_error;
2094	}
2095out_error:
2096	trace_nfs_writeback_page_exit(inode, ret);
2097	return ret;
2098}
2099
2100#ifdef CONFIG_MIGRATION
2101int nfs_migrate_page(struct address_space *mapping, struct page *newpage,
2102		struct page *page, enum migrate_mode mode)
2103{
2104	/*
2105	 * If PagePrivate is set, then the page is currently associated with
2106	 * an in-progress read or write request. Don't try to migrate it.
2107	 *
2108	 * FIXME: we could do this in principle, but we'll need a way to ensure
2109	 *        that we can safely release the inode reference while holding
2110	 *        the page lock.
2111	 */
2112	if (PagePrivate(page))
2113		return -EBUSY;
2114
2115	if (!nfs_fscache_release_page(page, GFP_KERNEL))
2116		return -EBUSY;
2117
2118	return migrate_page(mapping, newpage, page, mode);
2119}
2120#endif
2121
2122int __init nfs_init_writepagecache(void)
2123{
2124	nfs_wdata_cachep = kmem_cache_create("nfs_write_data",
2125					     sizeof(struct nfs_pgio_header),
2126					     0, SLAB_HWCACHE_ALIGN,
2127					     NULL);
2128	if (nfs_wdata_cachep == NULL)
2129		return -ENOMEM;
2130
2131	nfs_wdata_mempool = mempool_create_slab_pool(MIN_POOL_WRITE,
2132						     nfs_wdata_cachep);
2133	if (nfs_wdata_mempool == NULL)
2134		goto out_destroy_write_cache;
2135
2136	nfs_cdata_cachep = kmem_cache_create("nfs_commit_data",
2137					     sizeof(struct nfs_commit_data),
2138					     0, SLAB_HWCACHE_ALIGN,
2139					     NULL);
2140	if (nfs_cdata_cachep == NULL)
2141		goto out_destroy_write_mempool;
2142
2143	nfs_commit_mempool = mempool_create_slab_pool(MIN_POOL_COMMIT,
2144						      nfs_cdata_cachep);
2145	if (nfs_commit_mempool == NULL)
2146		goto out_destroy_commit_cache;
2147
2148	/*
2149	 * NFS congestion size, scale with available memory.
2150	 *
2151	 *  64MB:    8192k
2152	 * 128MB:   11585k
2153	 * 256MB:   16384k
2154	 * 512MB:   23170k
2155	 *   1GB:   32768k
2156	 *   2GB:   46340k
2157	 *   4GB:   65536k
2158	 *   8GB:   92681k
2159	 *  16GB:  131072k
2160	 *
2161	 * This allows larger machines to have larger/more transfers.
2162	 * Limit the default to 256M
2163	 */
2164	nfs_congestion_kb = (16*int_sqrt(totalram_pages())) << (PAGE_SHIFT-10);
2165	if (nfs_congestion_kb > 256*1024)
2166		nfs_congestion_kb = 256*1024;
2167
2168	return 0;
2169
2170out_destroy_commit_cache:
2171	kmem_cache_destroy(nfs_cdata_cachep);
2172out_destroy_write_mempool:
2173	mempool_destroy(nfs_wdata_mempool);
2174out_destroy_write_cache:
2175	kmem_cache_destroy(nfs_wdata_cachep);
2176	return -ENOMEM;
2177}
2178
2179void nfs_destroy_writepagecache(void)
2180{
2181	mempool_destroy(nfs_commit_mempool);
2182	kmem_cache_destroy(nfs_cdata_cachep);
2183	mempool_destroy(nfs_wdata_mempool);
2184	kmem_cache_destroy(nfs_wdata_cachep);
2185}
2186
2187static const struct nfs_rw_ops nfs_rw_write_ops = {
2188	.rw_alloc_header	= nfs_writehdr_alloc,
2189	.rw_free_header		= nfs_writehdr_free,
2190	.rw_done		= nfs_writeback_done,
2191	.rw_result		= nfs_writeback_result,
2192	.rw_initiate		= nfs_initiate_write,
2193};