1 // SPDX-License-Identifier: GPL-2.0-only
44 	i->req = req;  in nfs_page_iter_page_init()
45 	i->count = 0;  in nfs_page_iter_page_init()
50 	const struct nfs_page *req = i->req;  in nfs_page_iter_page_advance()
51 	size_t tmp = i->count + sz;  in nfs_page_iter_page_advance()
53 	i->count = (tmp < req->wb_bytes) ? tmp : req->wb_bytes;  in nfs_page_iter_page_advance()
58 	const struct nfs_page *req = i->req;  in nfs_page_iter_page_get()
61 	if (i->count != req->wb_bytes) {  in nfs_page_iter_page_get()
62 		size_t base = i->count + req->wb_pgbase;  in nfs_page_iter_page_get()
63 		size_t len = PAGE_SIZE - offset_in_page(base);  in nfs_page_iter_page_get()
75 	if (desc->pg_ops->pg_get_mirror)  in nfs_pgio_get_mirror()
76 		return desc->pg_ops->pg_get_mirror(desc, idx);  in nfs_pgio_get_mirror()
77 	return &desc->pg_mirrors[0];  in nfs_pgio_get_mirror()
83 	return nfs_pgio_get_mirror(desc, desc->pg_mirror_idx);  in nfs_pgio_current_mirror()
90 	if (desc->pg_ops->pg_set_mirror)  in nfs_pgio_set_current_mirror()
91 		return desc->pg_ops->pg_set_mirror(desc, idx);  in nfs_pgio_set_current_mirror()
92 	return desc->pg_mirror_idx;  in nfs_pgio_set_current_mirror()
99 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_pgheader_init()  local
102 	hdr->req = nfs_list_entry(mirror->pg_list.next);  in nfs_pgheader_init()
103 	hdr->inode = desc->pg_inode;  in nfs_pgheader_init()
104 	hdr->cred = nfs_req_openctx(hdr->req)->cred;  in nfs_pgheader_init()
105 	hdr->io_start = req_offset(hdr->req);  in nfs_pgheader_init()
106 	hdr->good_bytes = mirror->pg_count;  in nfs_pgheader_init()
107 	hdr->io_completion = desc->pg_io_completion;  in nfs_pgheader_init()
108 	hdr->dreq = desc->pg_dreq;  in nfs_pgheader_init()
110 	hdr->release = release;  in nfs_pgheader_init()
111 	hdr->completion_ops = desc->pg_completion_ops;  in nfs_pgheader_init()
112 	if (hdr->completion_ops->init_hdr)  in nfs_pgheader_init()
113 		hdr->completion_ops->init_hdr(hdr);  in nfs_pgheader_init()
115 	hdr->pgio_mirror_idx = desc->pg_mirror_idx;  in nfs_pgheader_init()
121 	unsigned int new = pos - hdr->io_start;  in nfs_set_pgio_error()
124 	if (hdr->good_bytes > new) {  in nfs_set_pgio_error()
125 		hdr->good_bytes = new;  in nfs_set_pgio_error()
126 		clear_bit(NFS_IOHDR_EOF, &hdr->flags);  in nfs_set_pgio_error()
127 		if (!test_and_set_bit(NFS_IOHDR_ERROR, &hdr->flags))  in nfs_set_pgio_error()
128 			hdr->error = error;  in nfs_set_pgio_error()
137 		INIT_LIST_HEAD(&p->wb_list);  in nfs_page_alloc()
148  * nfs_iocounter_wait - wait for i/o to complete
151  * returns -ERESTARTSYS if interrupted by a fatal signal.
157 	return wait_var_event_killable(&l_ctx->io_count,  in nfs_iocounter_wait()
158 				       !atomic_read(&l_ctx->io_count));  in nfs_iocounter_wait()
162  * nfs_async_iocounter_wait - wait on a rpc_waitqueue for I/O
173 	struct inode *inode = d_inode(l_ctx->open_context->dentry);  in nfs_async_iocounter_wait()
176 	if (atomic_read(&l_ctx->io_count) > 0) {  in nfs_async_iocounter_wait()
177 		rpc_sleep_on(&NFS_SERVER(inode)->uoc_rpcwaitq, task, NULL);  in nfs_async_iocounter_wait()
181 	if (atomic_read(&l_ctx->io_count) == 0) {  in nfs_async_iocounter_wait()
182 		rpc_wake_up_queued_task(&NFS_SERVER(inode)->uoc_rpcwaitq, task);  in nfs_async_iocounter_wait()
191  * nfs_page_set_headlock - set the request PG_HEADLOCK
194  * this lock must be held when modifying req->wb_head
201 	if (!test_and_set_bit(PG_HEADLOCK, &req->wb_flags))  in nfs_page_set_headlock()
204 	set_bit(PG_CONTENDED1, &req->wb_flags);  in nfs_page_set_headlock()
206 	return wait_on_bit_lock(&req->wb_flags, PG_HEADLOCK,  in nfs_page_set_headlock()
211  * nfs_page_clear_headlock - clear the request PG_HEADLOCK
217 	clear_bit_unlock(PG_HEADLOCK, &req->wb_flags);  in nfs_page_clear_headlock()
219 	if (!test_bit(PG_CONTENDED1, &req->wb_flags))  in nfs_page_clear_headlock()
221 	wake_up_bit(&req->wb_flags, PG_HEADLOCK);  in nfs_page_clear_headlock()
225  * nfs_page_group_lock - lock the head of the page group
239 	if (ret || req->wb_head == req)  in nfs_page_group_lock()
241 	return nfs_page_set_headlock(req->wb_head);  in nfs_page_group_lock()
245  * nfs_page_group_unlock - unlock the head of the page group
251 	if (req != req->wb_head)  in nfs_page_group_unlock()
252 		nfs_page_clear_headlock(req->wb_head);  in nfs_page_group_unlock()
264 	struct nfs_page *head = req->wb_head;  in nfs_page_group_sync_on_bit_locked()
267 	WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_flags));  in nfs_page_group_sync_on_bit_locked()
268 	WARN_ON_ONCE(test_and_set_bit(bit, &req->wb_flags));  in nfs_page_group_sync_on_bit_locked()
270 	tmp = req->wb_this_page;  in nfs_page_group_sync_on_bit_locked()
272 		if (!test_bit(bit, &tmp->wb_flags))  in nfs_page_group_sync_on_bit_locked()
274 		tmp = tmp->wb_this_page;  in nfs_page_group_sync_on_bit_locked()
280 		clear_bit(bit, &tmp->wb_flags);  in nfs_page_group_sync_on_bit_locked()
281 		tmp = tmp->wb_this_page;  in nfs_page_group_sync_on_bit_locked()
288  * nfs_page_group_sync_on_bit - set bit on current request, but only
290  * @req - request in page group
291  * @bit - PG_* bit that is used to sync page group
305  * nfs_page_group_init - Initialize the page group linkage for @req
306  * @req - a new nfs request
307  * @prev - the previous request in page group, or NULL if @req is the first
318 		req->wb_head = req;  in nfs_page_group_init()
319 		req->wb_this_page = req;  in nfs_page_group_init()
322 		WARN_ON_ONCE(prev->wb_this_page != prev->wb_head);  in nfs_page_group_init()
323 		WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &prev->wb_head->wb_flags));  in nfs_page_group_init()
324 		req->wb_head = prev->wb_head;  in nfs_page_group_init()
325 		req->wb_this_page = prev->wb_this_page;  in nfs_page_group_init()
326 		prev->wb_this_page = req;  in nfs_page_group_init()
330 		kref_get(&req->wb_head->wb_kref);  in nfs_page_group_init()
335 		if (test_bit(PG_INODE_REF, &prev->wb_head->wb_flags)) {  in nfs_page_group_init()
337 			set_bit(PG_INODE_REF, &req->wb_flags);  in nfs_page_group_init()
338 			kref_get(&req->wb_kref);  in nfs_page_group_init()
339 			atomic_long_inc(&NFS_I(inode)->nrequests);  in nfs_page_group_init()
345  * nfs_page_group_destroy - sync the destruction of page groups
346  * @req - request that no longer needs the page group
355 	struct nfs_page *head = req->wb_head;  in nfs_page_group_destroy()
363 		next = tmp->wb_this_page;  in nfs_page_group_destroy()
365 		tmp->wb_this_page = tmp;  in nfs_page_group_destroy()
366 		tmp->wb_head = tmp;  in nfs_page_group_destroy()
381 	struct nfs_open_context *ctx = l_ctx->open_context;  in nfs_page_create()
383 	if (test_bit(NFS_CONTEXT_BAD, &ctx->flags))  in nfs_page_create()
384 		return ERR_PTR(-EBADF);  in nfs_page_create()
388 		return ERR_PTR(-ENOMEM);  in nfs_page_create()
390 	req->wb_lock_context = l_ctx;  in nfs_page_create()
391 	refcount_inc(&l_ctx->count);  in nfs_page_create()
392 	atomic_inc(&l_ctx->io_count);  in nfs_page_create()
395 	 * long write-back delay. This will be adjusted in  in nfs_page_create()
397 	req->wb_pgbase = pgbase;  in nfs_page_create()
398 	req->wb_index = index;  in nfs_page_create()
399 	req->wb_offset = offset;  in nfs_page_create()
400 	req->wb_bytes = count;  in nfs_page_create()
401 	kref_init(&req->wb_kref);  in nfs_page_create()
402 	req->wb_nio = 0;  in nfs_page_create()
409 		req->wb_folio = folio;  in nfs_page_assign_folio()
411 		set_bit(PG_FOLIO, &req->wb_flags);  in nfs_page_assign_folio()
418 		req->wb_page = page;  in nfs_page_assign_page()
424  * nfs_page_create_from_page - Create an NFS read/write request.
456  * nfs_page_create_from_folio - Create an NFS read/write request.
476 	ret = nfs_page_create(l_ctx, offset, folio->index, offset, count);  in nfs_page_create_from_folio()
496 	ret = nfs_page_create(req->wb_lock_context, pgbase, req->wb_index,  in nfs_create_subreq()
504 		for (last = req->wb_head;  in nfs_create_subreq()
505 		     last->wb_this_page != req->wb_head;  in nfs_create_subreq()
506 		     last = last->wb_this_page)  in nfs_create_subreq()
511 		ret->wb_nio = req->wb_nio;  in nfs_create_subreq()
517  * nfs_unlock_request - Unlock request and wake up sleepers.
522 	clear_bit_unlock(PG_BUSY, &req->wb_flags);  in nfs_unlock_request()
524 	if (!test_bit(PG_CONTENDED2, &req->wb_flags))  in nfs_unlock_request()
526 	wake_up_bit(&req->wb_flags, PG_BUSY);  in nfs_unlock_request()
530  * nfs_unlock_and_release_request - Unlock request and release the nfs_page
540  * nfs_clear_request - Free up all resources allocated to the request
549 	struct page *page = req->wb_page;  in nfs_clear_request()
550 	struct nfs_lock_context *l_ctx = req->wb_lock_context;  in nfs_clear_request()
555 		req->wb_folio = NULL;  in nfs_clear_request()
556 		clear_bit(PG_FOLIO, &req->wb_flags);  in nfs_clear_request()
559 		req->wb_page = NULL;  in nfs_clear_request()
562 		if (atomic_dec_and_test(&l_ctx->io_count)) {  in nfs_clear_request()
563 			wake_up_var(&l_ctx->io_count);  in nfs_clear_request()
564 			ctx = l_ctx->open_context;  in nfs_clear_request()
565 			if (test_bit(NFS_CONTEXT_UNLOCK, &ctx->flags))  in nfs_clear_request()
566 				rpc_wake_up(&NFS_SERVER(d_inode(ctx->dentry))->uoc_rpcwaitq);  in nfs_clear_request()
569 		req->wb_lock_context = NULL;  in nfs_clear_request()
574  * nfs_free_request - Release the count on an NFS read/write request
581 	WARN_ON_ONCE(req->wb_this_page != req);  in nfs_free_request()
584 	WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags));  in nfs_free_request()
585 	WARN_ON_ONCE(test_bit(PG_UNLOCKPAGE, &req->wb_flags));  in nfs_free_request()
586 	WARN_ON_ONCE(test_bit(PG_UPTODATE, &req->wb_flags));  in nfs_free_request()
587 	WARN_ON_ONCE(test_bit(PG_WB_END, &req->wb_flags));  in nfs_free_request()
588 	WARN_ON_ONCE(test_bit(PG_REMOVE, &req->wb_flags));  in nfs_free_request()
597 	kref_put(&req->wb_kref, nfs_page_group_destroy);  in nfs_release_request()
602  * nfs_generic_pg_test - determine if requests can be coalesced
613 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_generic_pg_test()  local
616 	if (mirror->pg_count > mirror->pg_bsize) {  in nfs_generic_pg_test()
626 	if (((mirror->pg_count + req->wb_bytes) >> PAGE_SHIFT) *  in nfs_generic_pg_test()
630 	return min(mirror->pg_bsize - mirror->pg_count, (size_t)req->wb_bytes);  in nfs_generic_pg_test()
636 	struct nfs_pgio_header *hdr = ops->rw_alloc_header();  in nfs_pgio_header_alloc()
639 		INIT_LIST_HEAD(&hdr->pages);  in nfs_pgio_header_alloc()
640 		hdr->rw_ops = ops;  in nfs_pgio_header_alloc()
647  * nfs_pgio_data_destroy - make @hdr suitable for reuse
656 	if (hdr->args.context)  in nfs_pgio_data_destroy()
657 		put_nfs_open_context(hdr->args.context);  in nfs_pgio_data_destroy()
658 	if (hdr->page_array.pagevec != hdr->page_array.page_array)  in nfs_pgio_data_destroy()
659 		kfree(hdr->page_array.pagevec);  in nfs_pgio_data_destroy()
663  * nfs_pgio_header_free - Free a read or write header
669 	hdr->rw_ops->rw_free_header(hdr);  in nfs_pgio_header_free()
674  * nfs_pgio_rpcsetup - Set up arguments for a pageio call
678  * @how: How to commit data (writes only)
685 	struct nfs_page *req = hdr->req;  in nfs_pgio_rpcsetup()
688 	 * NB: take care not to mess about with hdr->commit et al. */  in nfs_pgio_rpcsetup()
690 	hdr->args.fh     = NFS_FH(hdr->inode);  in nfs_pgio_rpcsetup()
691 	hdr->args.offset = req_offset(req);  in nfs_pgio_rpcsetup()
693 	hdr->mds_offset = hdr->args.offset;  in nfs_pgio_rpcsetup()
694 	hdr->args.pgbase = pgbase;  in nfs_pgio_rpcsetup()
695 	hdr->args.pages  = hdr->page_array.pagevec;  in nfs_pgio_rpcsetup()
696 	hdr->args.count  = count;  in nfs_pgio_rpcsetup()
697 	hdr->args.context = get_nfs_open_context(nfs_req_openctx(req));  in nfs_pgio_rpcsetup()
698 	hdr->args.lock_context = req->wb_lock_context;  in nfs_pgio_rpcsetup()
699 	hdr->args.stable  = NFS_UNSTABLE;  in nfs_pgio_rpcsetup()
708 		hdr->args.stable = NFS_FILE_SYNC;  in nfs_pgio_rpcsetup()
711 	hdr->res.fattr   = &hdr->fattr;  in nfs_pgio_rpcsetup()
712 	hdr->res.count   = 0;  in nfs_pgio_rpcsetup()
713 	hdr->res.eof     = 0;  in nfs_pgio_rpcsetup()
714 	hdr->res.verf    = &hdr->verf;  in nfs_pgio_rpcsetup()
715 	nfs_fattr_init(&hdr->fattr);  in nfs_pgio_rpcsetup()
719  * nfs_pgio_prepare - Prepare pageio hdr to go over the wire
727 	err = NFS_PROTO(hdr->inode)->pgio_rpc_prepare(task, hdr);  in nfs_pgio_prepare()
739 		.rpc_argp = &hdr->args,  in nfs_initiate_pgio()
740 		.rpc_resp = &hdr->res,  in nfs_initiate_pgio()
745 		.task = &hdr->task,  in nfs_initiate_pgio()
753 	if (nfs_server_capable(hdr->inode, NFS_CAP_MOVEABLE))  in nfs_initiate_pgio()
756 	hdr->rw_ops->rw_initiate(hdr, &msg, rpc_ops, &task_setup_data, how);  in nfs_initiate_pgio()
760 		hdr->inode->i_sb->s_id,  in nfs_initiate_pgio()
761 		(unsigned long long)NFS_FILEID(hdr->inode),  in nfs_initiate_pgio()
762 		hdr->args.count,  in nfs_initiate_pgio()
763 		(unsigned long long)hdr->args.offset);  in nfs_initiate_pgio()
766 		return nfs_local_doio(NFS_SERVER(hdr->inode)->nfs_client,  in nfs_initiate_pgio()
778  * nfs_pgio_error - Clean up from a pageio error
783 	set_bit(NFS_IOHDR_REDO, &hdr->flags);  in nfs_pgio_error()
784 	hdr->completion_ops->completion(hdr);  in nfs_pgio_error()
788  * nfs_pgio_release - Release pageio data
794 	hdr->completion_ops->completion(hdr);  in nfs_pgio_release()
797 static void nfs_pageio_mirror_init(struct nfs_pgio_mirror *mirror,  in nfs_pageio_mirror_init()  argument
800 	INIT_LIST_HEAD(&mirror->pg_list);  in nfs_pageio_mirror_init()
801 	mirror->pg_bytes_written = 0;  in nfs_pageio_mirror_init()
802 	mirror->pg_count = 0;  in nfs_pageio_mirror_init()
803 	mirror->pg_bsize = bsize;  in nfs_pageio_mirror_init()
804 	mirror->pg_base = 0;  in nfs_pageio_mirror_init()
805 	mirror->pg_recoalesce = 0;  in nfs_pageio_mirror_init()
809  * nfs_pageio_init - initialise a page io descriptor
826 	desc->pg_moreio = 0;  in nfs_pageio_init()
827 	desc->pg_inode = inode;  in nfs_pageio_init()
828 	desc->pg_ops = pg_ops;  in nfs_pageio_init()
829 	desc->pg_completion_ops = compl_ops;  in nfs_pageio_init()
830 	desc->pg_rw_ops = rw_ops;  in nfs_pageio_init()
831 	desc->pg_ioflags = io_flags;  in nfs_pageio_init()
832 	desc->pg_error = 0;  in nfs_pageio_init()
833 	desc->pg_lseg = NULL;  in nfs_pageio_init()
834 	desc->pg_io_completion = NULL;  in nfs_pageio_init()
835 	desc->pg_dreq = NULL;  in nfs_pageio_init()
837 	desc->pg_bsize = bsize;  in nfs_pageio_init()
839 	desc->pg_mirror_count = 1;  in nfs_pageio_init()
840 	desc->pg_mirror_idx = 0;  in nfs_pageio_init()
842 	desc->pg_mirrors_dynamic = NULL;  in nfs_pageio_init()
843 	desc->pg_mirrors = desc->pg_mirrors_static;  in nfs_pageio_init()
844 	nfs_pageio_mirror_init(&desc->pg_mirrors[0], bsize);  in nfs_pageio_init()
845 	desc->pg_maxretrans = 0;  in nfs_pageio_init()
849  * nfs_pgio_result - Basic pageio error handling
856 	struct inode *inode = hdr->inode;  in nfs_pgio_result()
858 	if (hdr->rw_ops->rw_done(task, hdr, inode) != 0)  in nfs_pgio_result()
860 	if (task->tk_status < 0)  in nfs_pgio_result()
861 		nfs_set_pgio_error(hdr, task->tk_status, hdr->args.offset);  in nfs_pgio_result()
863 		hdr->rw_ops->rw_result(task, hdr);  in nfs_pgio_result()
877 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_generic_pgio()  local
882 	struct list_head *head = &mirror->pg_list;  in nfs_generic_pgio()
884 	struct nfs_page_array *pg_array = &hdr->page_array;  in nfs_generic_pgio()
886 	unsigned int pg_base = offset_in_page(mirror->pg_base);  in nfs_generic_pgio()
889 	pagecount = nfs_page_array_len(pg_base, mirror->pg_count);  in nfs_generic_pgio()
890 	pg_array->npages = pagecount;  in nfs_generic_pgio()
892 	if (pagecount <= ARRAY_SIZE(pg_array->page_array))  in nfs_generic_pgio()
893 		pg_array->pagevec = pg_array->page_array;  in nfs_generic_pgio()
895 		pg_array->pagevec = kcalloc(pagecount, sizeof(struct page *), gfp_flags);  in nfs_generic_pgio()
896 		if (!pg_array->pagevec) {  in nfs_generic_pgio()
897 			pg_array->npages = 0;  in nfs_generic_pgio()
899 			desc->pg_error = -ENOMEM;  in nfs_generic_pgio()
900 			return desc->pg_error;  in nfs_generic_pgio()
904 	nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq);  in nfs_generic_pgio()
905 	pages = hdr->page_array.pagevec;  in nfs_generic_pgio()
912 		req = nfs_list_entry(head->next);  in nfs_generic_pgio()
913 		nfs_list_move_request(req, &hdr->pages);  in nfs_generic_pgio()
915 		if (req->wb_pgbase == 0)  in nfs_generic_pgio()
931 		desc->pg_error = -EINVAL;  in nfs_generic_pgio()
932 		return desc->pg_error;  in nfs_generic_pgio()
935 	if ((desc->pg_ioflags & FLUSH_COND_STABLE) &&  in nfs_generic_pgio()
936 	    (desc->pg_moreio || nfs_reqs_to_commit(&cinfo)))  in nfs_generic_pgio()
937 		desc->pg_ioflags &= ~FLUSH_COND_STABLE;  in nfs_generic_pgio()
940 	nfs_pgio_rpcsetup(hdr, pg_base, mirror->pg_count, desc->pg_ioflags,  in nfs_generic_pgio()
942 	desc->pg_rpc_callops = &nfs_pgio_common_ops;  in nfs_generic_pgio()
953 	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);  in nfs_generic_pg_pgios()
955 		desc->pg_error = -ENOMEM;  in nfs_generic_pg_pgios()
956 		return desc->pg_error;  in nfs_generic_pg_pgios()
961 		struct nfs_client *clp = NFS_SERVER(hdr->inode)->nfs_client;  in nfs_generic_pg_pgios()
964 			nfs_local_open_fh(clp, hdr->cred, hdr->args.fh,  in nfs_generic_pg_pgios()
965 					  &hdr->args.context->nfl,  in nfs_generic_pg_pgios()
966 					  hdr->args.context->mode);  in nfs_generic_pg_pgios()
968 		if (NFS_SERVER(hdr->inode)->nfs_client->cl_minorversion)  in nfs_generic_pg_pgios()
970 		ret = nfs_initiate_pgio(NFS_CLIENT(hdr->inode),  in nfs_generic_pg_pgios()
972 					hdr->cred,  in nfs_generic_pg_pgios()
973 					NFS_PROTO(hdr->inode),  in nfs_generic_pg_pgios()
974 					desc->pg_rpc_callops,  in nfs_generic_pg_pgios()
975 					desc->pg_ioflags,  in nfs_generic_pg_pgios()
989 	kfree(desc->pg_mirrors_dynamic);  in nfs_pageio_alloc_mirrors()
990 	desc->pg_mirrors_dynamic = NULL;  in nfs_pageio_alloc_mirrors()
992 		return desc->pg_mirrors_static;  in nfs_pageio_alloc_mirrors()
996 			nfs_pageio_mirror_init(&ret[i], desc->pg_bsize);  in nfs_pageio_alloc_mirrors()
997 		desc->pg_mirrors_dynamic = ret;  in nfs_pageio_alloc_mirrors()
1003  * nfs_pageio_setup_mirroring - determine if mirroring is to be used
1011 	if (pgio->pg_ops->pg_get_mirror_count)  in nfs_pageio_setup_mirroring()
1012 		mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req);  in nfs_pageio_setup_mirroring()
1013 	if (mirror_count == pgio->pg_mirror_count || pgio->pg_error < 0)  in nfs_pageio_setup_mirroring()
1017 		pgio->pg_error = -EINVAL;  in nfs_pageio_setup_mirroring()
1021 	pgio->pg_mirrors = nfs_pageio_alloc_mirrors(pgio, mirror_count);  in nfs_pageio_setup_mirroring()
1022 	if (pgio->pg_mirrors == NULL) {  in nfs_pageio_setup_mirroring()
1023 		pgio->pg_error = -ENOMEM;  in nfs_pageio_setup_mirroring()
1024 		pgio->pg_mirrors = pgio->pg_mirrors_static;  in nfs_pageio_setup_mirroring()
1027 	pgio->pg_mirror_count = mirror_count;  in nfs_pageio_setup_mirroring()
1032 	pgio->pg_mirror_count = 1;  in nfs_pageio_cleanup_mirroring()
1033 	pgio->pg_mirror_idx = 0;  in nfs_pageio_cleanup_mirroring()
1034 	pgio->pg_mirrors = pgio->pg_mirrors_static;  in nfs_pageio_cleanup_mirroring()
1035 	kfree(pgio->pg_mirrors_dynamic);  in nfs_pageio_cleanup_mirroring()
1036 	pgio->pg_mirrors_dynamic = NULL;  in nfs_pageio_cleanup_mirroring()
1042 	return l1->lockowner == l2->lockowner;  in nfs_match_lock_context()
1048 	size_t prev_end = prev->wb_pgbase + prev->wb_bytes;  in nfs_page_is_contiguous()
1050 	if (req_offset(req) != req_offset(prev) + prev->wb_bytes)  in nfs_page_is_contiguous()
1052 	if (req->wb_pgbase == 0)  in nfs_page_is_contiguous()
1054 	if (req->wb_pgbase == prev_end) {  in nfs_page_is_contiguous()
1058 		return req->wb_page == prev->wb_page;  in nfs_page_is_contiguous()
1064  * nfs_coalesce_size - test two requests for compatibility
1070  * page data area they describe is contiguous, and that their RPC
1084 		flctx = locks_inode_context(d_inode(nfs_req_openctx(req)->dentry));  in nfs_coalesce_size()
1086 		    !(list_empty_careful(&flctx->flc_posix) &&  in nfs_coalesce_size()
1087 		      list_empty_careful(&flctx->flc_flock)) &&  in nfs_coalesce_size()
1088 		    !nfs_match_lock_context(req->wb_lock_context,  in nfs_coalesce_size()
1089 					    prev->wb_lock_context))  in nfs_coalesce_size()
1094 	return pgio->pg_ops->pg_test(pgio, prev, req);  in nfs_coalesce_size()
1098  * nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
1109 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_pageio_do_add_request()  local
1113 	if (list_empty(&mirror->pg_list)) {  in nfs_pageio_do_add_request()
1114 		if (desc->pg_ops->pg_init)  in nfs_pageio_do_add_request()
1115 			desc->pg_ops->pg_init(desc, req);  in nfs_pageio_do_add_request()
1116 		if (desc->pg_error < 0)  in nfs_pageio_do_add_request()
1118 		mirror->pg_base = req->wb_pgbase;  in nfs_pageio_do_add_request()
1119 		mirror->pg_count = 0;  in nfs_pageio_do_add_request()
1120 		mirror->pg_recoalesce = 0;  in nfs_pageio_do_add_request()
1122 		prev = nfs_list_entry(mirror->pg_list.prev);  in nfs_pageio_do_add_request()
1124 	if (desc->pg_maxretrans && req->wb_nio > desc->pg_maxretrans) {  in nfs_pageio_do_add_request()
1125 		if (NFS_SERVER(desc->pg_inode)->flags & NFS_MOUNT_SOFTERR)  in nfs_pageio_do_add_request()
1126 			desc->pg_error = -ETIMEDOUT;  in nfs_pageio_do_add_request()
1128 			desc->pg_error = -EIO;  in nfs_pageio_do_add_request()
1133 	if (size < req->wb_bytes)  in nfs_pageio_do_add_request()
1135 	nfs_list_move_request(req, &mirror->pg_list);  in nfs_pageio_do_add_request()
1136 	mirror->pg_count += req->wb_bytes;  in nfs_pageio_do_add_request()
1137 	return req->wb_bytes;  in nfs_pageio_do_add_request()
1145 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_pageio_doio()  local
1147 	if (!list_empty(&mirror->pg_list)) {  in nfs_pageio_doio()
1148 		int error = desc->pg_ops->pg_doio(desc);  in nfs_pageio_doio()
1150 			desc->pg_error = error;  in nfs_pageio_doio()
1151 		if (list_empty(&mirror->pg_list))  in nfs_pageio_doio()
1152 			mirror->pg_bytes_written += mirror->pg_count;  in nfs_pageio_doio()
1163 	desc->pg_completion_ops->error_cleanup(&head, desc->pg_error);  in nfs_pageio_cleanup_request()
1167  * __nfs_pageio_add_request - Attempt to coalesce a request into a page list.
1181 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in __nfs_pageio_add_request()  local
1188 	subreq_size = subreq->wb_bytes;  in __nfs_pageio_add_request()
1195 			req->wb_pgbase += size;  in __nfs_pageio_add_request()
1196 			req->wb_bytes -= size;  in __nfs_pageio_add_request()
1197 			req->wb_offset += size;  in __nfs_pageio_add_request()
1198 			subreq_size = req->wb_bytes;  in __nfs_pageio_add_request()
1206 			subreq_size = req->wb_bytes;  in __nfs_pageio_add_request()
1212 			desc->pg_moreio = 1;  in __nfs_pageio_add_request()
1214 			if (desc->pg_error < 0 || mirror->pg_recoalesce)  in __nfs_pageio_add_request()
1220 		subreq = nfs_create_subreq(req, req->wb_pgbase,  in __nfs_pageio_add_request()
1221 				req->wb_offset, size);  in __nfs_pageio_add_request()
1230 	desc->pg_error = PTR_ERR(subreq);  in __nfs_pageio_add_request()
1237 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_do_recoalesce()  local
1241 		list_splice_init(&mirror->pg_list, &head);  in nfs_do_recoalesce()
1242 		mirror->pg_recoalesce = 0;  in nfs_do_recoalesce()
1250 			if (desc->pg_error < 0) {  in nfs_do_recoalesce()
1251 				list_splice_tail(&head, &mirror->pg_list);  in nfs_do_recoalesce()
1252 				mirror->pg_recoalesce = 1;  in nfs_do_recoalesce()
1257 	} while (mirror->pg_recoalesce);  in nfs_do_recoalesce()
1270 		if (desc->pg_error < 0)  in nfs_pageio_add_request_mirror()
1281 	struct nfs_pgio_mirror *mirror;  in nfs_pageio_error_cleanup()  local
1283 	if (!desc->pg_error)  in nfs_pageio_error_cleanup()
1286 	for (midx = 0; midx < desc->pg_mirror_count; midx++) {  in nfs_pageio_error_cleanup()
1287 		mirror = nfs_pgio_get_mirror(desc, midx);  in nfs_pageio_error_cleanup()
1288 		desc->pg_completion_ops->error_cleanup(&mirror->pg_list,  in nfs_pageio_error_cleanup()
1289 				desc->pg_error);  in nfs_pageio_error_cleanup()
1300 	pgbase = req->wb_pgbase;  in nfs_pageio_add_request()
1301 	offset = req->wb_offset;  in nfs_pageio_add_request()
1302 	bytes = req->wb_bytes;  in nfs_pageio_add_request()
1305 	if (desc->pg_error < 0)  in nfs_pageio_add_request()
1308 	/* Create the mirror instances first, and fire them off */  in nfs_pageio_add_request()
1309 	for (midx = 1; midx < desc->pg_mirror_count; midx++) {  in nfs_pageio_add_request()
1317 			desc->pg_error = PTR_ERR(dupreq);  in nfs_pageio_add_request()
1340  * nfs_pageio_complete_mirror - Complete I/O on the current mirror of an
1343  * @mirror_idx: pointer to mirror index
1348 	struct nfs_pgio_mirror *mirror;  in nfs_pageio_complete_mirror()  local
1352 	mirror = nfs_pgio_current_mirror(desc);  in nfs_pageio_complete_mirror()
1356 		if (desc->pg_error < 0 || !mirror->pg_recoalesce)  in nfs_pageio_complete_mirror()
1365  * nfs_pageio_resend - Transfer requests to new descriptor and resend
1366  * @hdr - the pgio header to move request from
1367  * @desc - the pageio descriptor to add requests to
1379 	desc->pg_io_completion = hdr->io_completion;  in nfs_pageio_resend()
1380 	desc->pg_dreq = hdr->dreq;  in nfs_pageio_resend()
1382 	list_splice_init(&hdr->pages, &pages);  in nfs_pageio_resend()
1391 		int err = desc->pg_error < 0 ? desc->pg_error : -EIO;  in nfs_pageio_resend()
1392 		hdr->completion_ops->error_cleanup(&pages, err);  in nfs_pageio_resend()
1393 		nfs_set_pgio_error(hdr, err, hdr->io_start);  in nfs_pageio_resend()
1401  * nfs_pageio_complete - Complete I/O then cleanup an nfs_pageio_descriptor
1408 	for (midx = 0; midx < desc->pg_mirror_count; midx++)  in nfs_pageio_complete()
1411 	if (desc->pg_error < 0)  in nfs_pageio_complete()
1413 	if (desc->pg_ops->pg_cleanup)  in nfs_pageio_complete()
1414 		desc->pg_ops->pg_cleanup(desc);  in nfs_pageio_complete()
1419  * nfs_pageio_cond_complete - Conditional I/O completion
1424  * on non-contiguous ranges of pages as that might deadlock. This
1431 	struct nfs_pgio_mirror *mirror;  in nfs_pageio_cond_complete()  local
1436 	for (midx = 0; midx < desc->pg_mirror_count; midx++) {  in nfs_pageio_cond_complete()
1437 		mirror = nfs_pgio_get_mirror(desc, midx);  in nfs_pageio_cond_complete()
1438 		if (!list_empty(&mirror->pg_list)) {  in nfs_pageio_cond_complete()
1439 			prev = nfs_list_entry(mirror->pg_list.prev);  in nfs_pageio_cond_complete()
1444 			} else if (index == prev->wb_index + 1)  in nfs_pageio_cond_complete()
1450 			desc->pg_moreio = 1;  in nfs_pageio_cond_complete()
1458  * nfs_pageio_stop_mirroring - stop using mirroring (set mirror count to 1)
1472 		return -ENOMEM;  in nfs_init_nfspagecache()