73 nfs_pgio_get_mirror(struct nfs_pageio_descriptor *desc, u32 idx)  in nfs_pgio_get_mirror()  argument
75 	if (desc->pg_ops->pg_get_mirror)  in nfs_pgio_get_mirror()
76 		return desc->pg_ops->pg_get_mirror(desc, idx);  in nfs_pgio_get_mirror()
77 	return &desc->pg_mirrors[0];  in nfs_pgio_get_mirror()
81 nfs_pgio_current_mirror(struct nfs_pageio_descriptor *desc)  in nfs_pgio_current_mirror()  argument
83 	return nfs_pgio_get_mirror(desc, desc->pg_mirror_idx);  in nfs_pgio_current_mirror()
88 nfs_pgio_set_current_mirror(struct nfs_pageio_descriptor *desc, u32 idx)  in nfs_pgio_set_current_mirror()  argument
90 	if (desc->pg_ops->pg_set_mirror)  in nfs_pgio_set_current_mirror()
91 		return desc->pg_ops->pg_set_mirror(desc, idx);  in nfs_pgio_set_current_mirror()
92 	return desc->pg_mirror_idx;  in nfs_pgio_set_current_mirror()
95 void nfs_pgheader_init(struct nfs_pageio_descriptor *desc,  in nfs_pgheader_init()  argument
99 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_pgheader_init()
103 	hdr->inode = desc->pg_inode;  in nfs_pgheader_init()
107 	hdr->io_completion = desc->pg_io_completion;  in nfs_pgheader_init()
108 	hdr->dreq = desc->pg_dreq;  in nfs_pgheader_init()
109 	nfs_netfs_set_pgio_header(hdr, desc);  in nfs_pgheader_init()
111 	hdr->completion_ops = desc->pg_completion_ops;  in nfs_pgheader_init()
115 	hdr->pgio_mirror_idx = desc->pg_mirror_idx;  in nfs_pgheader_init()
725 size_t nfs_generic_pg_test(struct nfs_pageio_descriptor *desc,  in nfs_generic_pg_test()  argument
728 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_generic_pg_test()
928 void nfs_pageio_init(struct nfs_pageio_descriptor *desc,  in nfs_pageio_init()  argument
936 	desc->pg_moreio = 0;  in nfs_pageio_init()
937 	desc->pg_inode = inode;  in nfs_pageio_init()
938 	desc->pg_ops = pg_ops;  in nfs_pageio_init()
939 	desc->pg_completion_ops = compl_ops;  in nfs_pageio_init()
940 	desc->pg_rw_ops = rw_ops;  in nfs_pageio_init()
941 	desc->pg_ioflags = io_flags;  in nfs_pageio_init()
942 	desc->pg_error = 0;  in nfs_pageio_init()
943 	desc->pg_lseg = NULL;  in nfs_pageio_init()
944 	desc->pg_io_completion = NULL;  in nfs_pageio_init()
945 	desc->pg_dreq = NULL;  in nfs_pageio_init()
946 	nfs_netfs_reset_pageio_descriptor(desc);  in nfs_pageio_init()
947 	desc->pg_bsize = bsize;  in nfs_pageio_init()
949 	desc->pg_mirror_count = 1;  in nfs_pageio_init()
950 	desc->pg_mirror_idx = 0;  in nfs_pageio_init()
952 	desc->pg_mirrors_dynamic = NULL;  in nfs_pageio_init()
953 	desc->pg_mirrors = desc->pg_mirrors_static;  in nfs_pageio_init()
954 	nfs_pageio_mirror_init(&desc->pg_mirrors[0], bsize);  in nfs_pageio_init()
955 	desc->pg_maxretrans = 0;  in nfs_pageio_init()
984 int nfs_generic_pgio(struct nfs_pageio_descriptor *desc,  in nfs_generic_pgio()  argument
987 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_generic_pgio()
1009 			desc->pg_error = -ENOMEM;  in nfs_generic_pgio()
1010 			return desc->pg_error;  in nfs_generic_pgio()
1014 	nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq);  in nfs_generic_pgio()
1041 		desc->pg_error = -EINVAL;  in nfs_generic_pgio()
1042 		return desc->pg_error;  in nfs_generic_pgio()
1045 	if ((desc->pg_ioflags & FLUSH_COND_STABLE) &&  in nfs_generic_pgio()
1046 	    (desc->pg_moreio || nfs_reqs_to_commit(&cinfo)))  in nfs_generic_pgio()
1047 		desc->pg_ioflags &= ~FLUSH_COND_STABLE;  in nfs_generic_pgio()
1050 	nfs_pgio_rpcsetup(hdr, pg_base, mirror->pg_count, desc->pg_ioflags,  in nfs_generic_pgio()
1052 	desc->pg_rpc_callops = &nfs_pgio_common_ops;  in nfs_generic_pgio()
1057 static int nfs_generic_pg_pgios(struct nfs_pageio_descriptor *desc)  in nfs_generic_pg_pgios()  argument
1063 	hdr = nfs_pgio_header_alloc(desc->pg_rw_ops);  in nfs_generic_pg_pgios()
1065 		desc->pg_error = -ENOMEM;  in nfs_generic_pg_pgios()
1066 		return desc->pg_error;  in nfs_generic_pg_pgios()
1068 	nfs_pgheader_init(desc, hdr, nfs_pgio_header_free);  in nfs_generic_pg_pgios()
1069 	ret = nfs_generic_pgio(desc, hdr);  in nfs_generic_pg_pgios()
1077 					desc->pg_rpc_callops,  in nfs_generic_pg_pgios()
1078 					desc->pg_ioflags,  in nfs_generic_pg_pgios()
1085 nfs_pageio_alloc_mirrors(struct nfs_pageio_descriptor *desc,  in nfs_pageio_alloc_mirrors()  argument
1091 	kfree(desc->pg_mirrors_dynamic);  in nfs_pageio_alloc_mirrors()
1092 	desc->pg_mirrors_dynamic = NULL;  in nfs_pageio_alloc_mirrors()
1094 		return desc->pg_mirrors_static;  in nfs_pageio_alloc_mirrors()
1098 			nfs_pageio_mirror_init(&ret[i], desc->pg_bsize);  in nfs_pageio_alloc_mirrors()
1099 		desc->pg_mirrors_dynamic = ret;  in nfs_pageio_alloc_mirrors()
1208 nfs_pageio_do_add_request(struct nfs_pageio_descriptor *desc,  in nfs_pageio_do_add_request()  argument
1211 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_pageio_do_add_request()
1216 		if (desc->pg_ops->pg_init)  in nfs_pageio_do_add_request()
1217 			desc->pg_ops->pg_init(desc, req);  in nfs_pageio_do_add_request()
1218 		if (desc->pg_error < 0)  in nfs_pageio_do_add_request()
1226 	if (desc->pg_maxretrans && req->wb_nio > desc->pg_maxretrans) {  in nfs_pageio_do_add_request()
1227 		if (NFS_SERVER(desc->pg_inode)->flags & NFS_MOUNT_SOFTERR)  in nfs_pageio_do_add_request()
1228 			desc->pg_error = -ETIMEDOUT;  in nfs_pageio_do_add_request()
1230 			desc->pg_error = -EIO;  in nfs_pageio_do_add_request()
1234 	size = nfs_coalesce_size(prev, req, desc);  in nfs_pageio_do_add_request()
1245 static void nfs_pageio_doio(struct nfs_pageio_descriptor *desc)  in nfs_pageio_doio()  argument
1247 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_pageio_doio()
1250 		int error = desc->pg_ops->pg_doio(desc);  in nfs_pageio_doio()
1252 			desc->pg_error = error;  in nfs_pageio_doio()
1259 nfs_pageio_cleanup_request(struct nfs_pageio_descriptor *desc,  in nfs_pageio_cleanup_request()  argument
1265 	desc->pg_completion_ops->error_cleanup(&head, desc->pg_error);  in nfs_pageio_cleanup_request()
1280 static int __nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,  in __nfs_pageio_add_request()  argument
1283 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in __nfs_pageio_add_request()
1292 		size = nfs_pageio_do_add_request(desc, subreq);  in __nfs_pageio_add_request()
1306 			nfs_pageio_cleanup_request(desc, subreq);  in __nfs_pageio_add_request()
1314 			desc->pg_moreio = 1;  in __nfs_pageio_add_request()
1315 			nfs_pageio_doio(desc);  in __nfs_pageio_add_request()
1316 			if (desc->pg_error < 0 || mirror->pg_recoalesce)  in __nfs_pageio_add_request()
1332 	desc->pg_error = PTR_ERR(subreq);  in __nfs_pageio_add_request()
1337 static int nfs_do_recoalesce(struct nfs_pageio_descriptor *desc)  in nfs_do_recoalesce()  argument
1339 	struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc);  in nfs_do_recoalesce()
1350 			if (__nfs_pageio_add_request(desc, req))  in nfs_do_recoalesce()
1352 			if (desc->pg_error < 0) {  in nfs_do_recoalesce()
1363 static int nfs_pageio_add_request_mirror(struct nfs_pageio_descriptor *desc,  in nfs_pageio_add_request_mirror()  argument
1369 		ret = __nfs_pageio_add_request(desc, req);  in nfs_pageio_add_request_mirror()
1372 		if (desc->pg_error < 0)  in nfs_pageio_add_request_mirror()
1374 		ret = nfs_do_recoalesce(desc);  in nfs_pageio_add_request_mirror()
1380 static void nfs_pageio_error_cleanup(struct nfs_pageio_descriptor *desc)  in nfs_pageio_error_cleanup()  argument
1385 	if (!desc->pg_error)  in nfs_pageio_error_cleanup()
1388 	for (midx = 0; midx < desc->pg_mirror_count; midx++) {  in nfs_pageio_error_cleanup()
1389 		mirror = nfs_pgio_get_mirror(desc, midx);  in nfs_pageio_error_cleanup()
1390 		desc->pg_completion_ops->error_cleanup(&mirror->pg_list,  in nfs_pageio_error_cleanup()
1391 				desc->pg_error);  in nfs_pageio_error_cleanup()
1395 int nfs_pageio_add_request(struct nfs_pageio_descriptor *desc,  in nfs_pageio_add_request()  argument
1406 	nfs_pageio_setup_mirroring(desc, req);  in nfs_pageio_add_request()
1407 	if (desc->pg_error < 0)  in nfs_pageio_add_request()
1411 	for (midx = 1; midx < desc->pg_mirror_count; midx++) {  in nfs_pageio_add_request()
1419 			desc->pg_error = PTR_ERR(dupreq);  in nfs_pageio_add_request()
1423 		nfs_pgio_set_current_mirror(desc, midx);  in nfs_pageio_add_request()
1424 		if (!nfs_pageio_add_request_mirror(desc, dupreq))  in nfs_pageio_add_request()
1428 	nfs_pgio_set_current_mirror(desc, 0);  in nfs_pageio_add_request()
1429 	if (!nfs_pageio_add_request_mirror(desc, req))  in nfs_pageio_add_request()
1435 	nfs_pageio_cleanup_request(desc, dupreq);  in nfs_pageio_add_request()
1437 	nfs_pageio_error_cleanup(desc);  in nfs_pageio_add_request()
1447 static void nfs_pageio_complete_mirror(struct nfs_pageio_descriptor *desc,  in nfs_pageio_complete_mirror()  argument
1453 	restore_idx = nfs_pgio_set_current_mirror(desc, mirror_idx);  in nfs_pageio_complete_mirror()
1454 	mirror = nfs_pgio_current_mirror(desc);  in nfs_pageio_complete_mirror()
1457 		nfs_pageio_doio(desc);  in nfs_pageio_complete_mirror()
1458 		if (desc->pg_error < 0 || !mirror->pg_recoalesce)  in nfs_pageio_complete_mirror()
1460 		if (!nfs_do_recoalesce(desc))  in nfs_pageio_complete_mirror()
1463 	nfs_pgio_set_current_mirror(desc, restore_idx);  in nfs_pageio_complete_mirror()
1476 int nfs_pageio_resend(struct nfs_pageio_descriptor *desc,  in nfs_pageio_resend()  argument
1481 	desc->pg_io_completion = hdr->io_completion;  in nfs_pageio_resend()
1482 	desc->pg_dreq = hdr->dreq;  in nfs_pageio_resend()
1483 	nfs_netfs_set_pageio_descriptor(desc, hdr);  in nfs_pageio_resend()
1488 		if (!nfs_pageio_add_request(desc, req))  in nfs_pageio_resend()
1491 	nfs_pageio_complete(desc);  in nfs_pageio_resend()
1493 		int err = desc->pg_error < 0 ? desc->pg_error : -EIO;  in nfs_pageio_resend()
1506 void nfs_pageio_complete(struct nfs_pageio_descriptor *desc)  in nfs_pageio_complete()  argument
1510 	for (midx = 0; midx < desc->pg_mirror_count; midx++)  in nfs_pageio_complete()
1511 		nfs_pageio_complete_mirror(desc, midx);  in nfs_pageio_complete()
1513 	if (desc->pg_error < 0)  in nfs_pageio_complete()
1514 		nfs_pageio_error_cleanup(desc);  in nfs_pageio_complete()
1515 	if (desc->pg_ops->pg_cleanup)  in nfs_pageio_complete()
1516 		desc->pg_ops->pg_cleanup(desc);  in nfs_pageio_complete()
1517 	nfs_pageio_cleanup_mirroring(desc);  in nfs_pageio_complete()
1531 void nfs_pageio_cond_complete(struct nfs_pageio_descriptor *desc, pgoff_t index)  in nfs_pageio_cond_complete()  argument
1538 	for (midx = 0; midx < desc->pg_mirror_count; midx++) {  in nfs_pageio_cond_complete()
1539 		mirror = nfs_pgio_get_mirror(desc, midx);  in nfs_pageio_cond_complete()
1552 			desc->pg_moreio = 1;  in nfs_pageio_cond_complete()
1553 			nfs_pageio_complete(desc);  in nfs_pageio_cond_complete()