58 		 "Enable inter server to server copy offload. Default: false");
408 	 * field isn't set for some reason, throw warning and just copy  in set_change_info()
957 	 * If we do a zero copy read, then a client will see read data  in nfsd4_read()
961 	 * To ensure proper ordering, we therefore turn off zero copy if  in nfsd4_read()
1272 static void nfs4_put_copy(struct nfsd4_copy *copy)  in nfs4_put_copy()  argument
1274 	if (!refcount_dec_and_test(&copy->refcount))  in nfs4_put_copy()
1276 	atomic_dec(&copy->cp_nn->pending_async_copies);  in nfs4_put_copy()
1277 	kfree(copy->cp_src);  in nfs4_put_copy()
1278 	kfree(copy);  in nfs4_put_copy()
1281 static void nfsd4_stop_copy(struct nfsd4_copy *copy)  in nfsd4_stop_copy()  argument
1283 	if (!test_and_set_bit(NFSD4_COPY_F_STOPPED, &copy->cp_flags))  in nfsd4_stop_copy()
1284 		kthread_stop(copy->copy_task);  in nfsd4_stop_copy()
1285 	nfs4_put_copy(copy);  in nfsd4_stop_copy()
1290 	struct nfsd4_copy *copy = NULL;  in nfsd4_unhash_copy()  local
1294 		copy = list_first_entry(&clp->async_copies, struct nfsd4_copy,  in nfsd4_unhash_copy()
1296 		refcount_inc(&copy->refcount);  in nfsd4_unhash_copy()
1297 		copy->cp_clp = NULL;  in nfsd4_unhash_copy()
1298 		if (!list_empty(&copy->copies))  in nfsd4_unhash_copy()
1299 			list_del_init(&copy->copies);  in nfsd4_unhash_copy()
1302 	return copy;  in nfsd4_unhash_copy()
1307 	struct nfsd4_copy *copy;  in nfsd4_shutdown_copy()  local
1309 	while ((copy = nfsd4_unhash_copy(clp)) != NULL)  in nfsd4_shutdown_copy()
1310 		nfsd4_stop_copy(copy);  in nfsd4_shutdown_copy()
1400  * Support one copy source server for now.
1494  * Verify COPY destination stateid.
1498  * Called with COPY cstate:
1505 		      struct nfsd4_copy *copy)  in nfsd4_setup_inter_ssc()  argument
1508 	stateid_t *s_stid = &copy->cp_src_stateid;  in nfsd4_setup_inter_ssc()
1513 					    &copy->cp_dst_stateid,  in nfsd4_setup_inter_ssc()
1514 					    WR_STATE, &copy->nf_dst, NULL);  in nfsd4_setup_inter_ssc()
1518 	status = nfsd4_interssc_connect(copy->cp_src, rqstp, &copy->ss_nsui);  in nfsd4_setup_inter_ssc()
1524 	copy->c_fh.size = s_fh->fh_handle.fh_size;  in nfsd4_setup_inter_ssc()
1525 	memcpy(copy->c_fh.data, &s_fh->fh_handle.fh_raw, copy->c_fh.size);  in nfsd4_setup_inter_ssc()
1526 	copy->stateid.seqid = cpu_to_be32(s_stid->si_generation);  in nfsd4_setup_inter_ssc()
1527 	memcpy(copy->stateid.other, (void *)&s_stid->si_opaque,  in nfsd4_setup_inter_ssc()
1563 		      struct nfsd4_copy *copy)  in nfsd4_setup_inter_ssc()  argument
1585 		      struct nfsd4_copy *copy)  in nfsd4_setup_intra_ssc()  argument
1587 	return nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,  in nfsd4_setup_intra_ssc()
1588 				 &copy->nf_src, &copy->cp_dst_stateid,  in nfsd4_setup_intra_ssc()
1589 				 &copy->nf_dst);  in nfsd4_setup_intra_ssc()
1615 static void nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)  in nfsd4_init_copy_res()  argument
1617 	copy->cp_res.wr_stable_how =  in nfsd4_init_copy_res()
1618 		test_bit(NFSD4_COPY_F_COMMITTED, &copy->cp_flags) ?  in nfsd4_init_copy_res()
1620 	nfsd4_copy_set_sync(copy, sync);  in nfsd4_init_copy_res()
1623 static ssize_t _nfsd_copy_file_range(struct nfsd4_copy *copy,  in _nfsd_copy_file_range()  argument
1629 	u64 bytes_total = copy->cp_count;  in _nfsd_copy_file_range()
1630 	u64 src_pos = copy->cp_src_pos;  in _nfsd_copy_file_range()
1631 	u64 dst_pos = copy->cp_dst_pos;  in _nfsd_copy_file_range()
1646 		copy->cp_res.wr_bytes_written += bytes_copied;  in _nfsd_copy_file_range()
1649 	} while (bytes_total > 0 && nfsd4_copy_is_async(copy));  in _nfsd_copy_file_range()
1650 	/* for a non-zero asynchronous copy do a commit of data */  in _nfsd_copy_file_range()
1651 	if (nfsd4_copy_is_async(copy) && copy->cp_res.wr_bytes_written > 0) {  in _nfsd_copy_file_range()
1653 		end = copy->cp_dst_pos + copy->cp_res.wr_bytes_written - 1;  in _nfsd_copy_file_range()
1654 		status = vfs_fsync_range(dst, copy->cp_dst_pos, end, 0);  in _nfsd_copy_file_range()
1658 			set_bit(NFSD4_COPY_F_COMMITTED, &copy->cp_flags);  in _nfsd_copy_file_range()
1663 static __be32 nfsd4_do_copy(struct nfsd4_copy *copy,  in nfsd4_do_copy()  argument
1670 	bytes = _nfsd_copy_file_range(copy, dst, src);  in nfsd4_do_copy()
1672 	/* for async copy, we ignore the error, client can always retry  in nfsd4_do_copy()
1675 	if (bytes < 0 && !copy->cp_res.wr_bytes_written)  in nfsd4_do_copy()
1678 		nfsd4_init_copy_res(copy, sync);  in nfsd4_do_copy()
1705 static void release_copy_files(struct nfsd4_copy *copy)  in release_copy_files()  argument
1707 	if (copy->nf_src)  in release_copy_files()
1708 		nfsd_file_put(copy->nf_src);  in release_copy_files()
1709 	if (copy->nf_dst)  in release_copy_files()
1710 		nfsd_file_put(copy->nf_dst);  in release_copy_files()
1713 static void cleanup_async_copy(struct nfsd4_copy *copy)  in cleanup_async_copy()  argument
1715 	nfs4_free_copy_state(copy);  in cleanup_async_copy()
1716 	release_copy_files(copy);  in cleanup_async_copy()
1717 	if (copy->cp_clp) {  in cleanup_async_copy()
1718 		spin_lock(&copy->cp_clp->async_lock);  in cleanup_async_copy()
1719 		if (!list_empty(&copy->copies))  in cleanup_async_copy()
1720 			list_del_init(&copy->copies);  in cleanup_async_copy()
1721 		spin_unlock(&copy->cp_clp->async_lock);  in cleanup_async_copy()
1723 	nfs4_put_copy(copy);  in cleanup_async_copy()
1726 static void nfsd4_send_cb_offload(struct nfsd4_copy *copy, __be32 nfserr)  in nfsd4_send_cb_offload()  argument
1734 	memcpy(&cbo->co_res, &copy->cp_res, sizeof(copy->cp_res));  in nfsd4_send_cb_offload()
1735 	memcpy(&cbo->co_fh, &copy->fh, sizeof(copy->fh));  in nfsd4_send_cb_offload()
1738 	nfsd4_init_cb(&cbo->co_cb, copy->cp_clp, &nfsd4_cb_offload_ops,  in nfsd4_send_cb_offload()
1740 	trace_nfsd_cb_offload(copy->cp_clp, &cbo->co_res.cb_stateid,  in nfsd4_send_cb_offload()
1741 			      &cbo->co_fh, copy->cp_count, nfserr);  in nfsd4_send_cb_offload()
1746  * nfsd4_do_async_copy - kthread function for background server-side COPY
1747  * @data: arguments for COPY operation
1750  *   %0: Copy operation is done.
1754 	struct nfsd4_copy *copy = (struct nfsd4_copy *)data;  in nfsd4_do_async_copy()  local
1757 	if (nfsd4_ssc_is_inter(copy)) {  in nfsd4_do_async_copy()
1760 		filp = nfs42_ssc_open(copy->ss_nsui->nsui_vfsmount,  in nfsd4_do_async_copy()
1761 				      &copy->c_fh, &copy->stateid);  in nfsd4_do_async_copy()
1773 		nfserr = nfsd4_do_copy(copy, filp, copy->nf_dst->nf_file,  in nfsd4_do_async_copy()
1775 		nfsd4_cleanup_inter_ssc(copy->ss_nsui, filp, copy->nf_dst);  in nfsd4_do_async_copy()
1777 		nfserr = nfsd4_do_copy(copy, copy->nf_src->nf_file,  in nfsd4_do_async_copy()
1778 				       copy->nf_dst->nf_file, false);  in nfsd4_do_async_copy()
1782 	nfsd4_send_cb_offload(copy, nfserr);  in nfsd4_do_async_copy()
1783 	cleanup_async_copy(copy);  in nfsd4_do_async_copy()
1793 	struct nfsd4_copy *copy = &u->copy;  in nfsd4_copy()  local
1797 	result = &copy->cp_res;  in nfsd4_copy()
1800 	copy->cp_clp = cstate->clp;  in nfsd4_copy()
1801 	if (nfsd4_ssc_is_inter(copy)) {  in nfsd4_copy()
1802 		if (!inter_copy_offload_enable || nfsd4_copy_is_sync(copy)) {  in nfsd4_copy()
1806 		status = nfsd4_setup_inter_ssc(rqstp, cstate, copy);  in nfsd4_copy()
1810 		status = nfsd4_setup_intra_ssc(rqstp, cstate, copy);  in nfsd4_copy()
1815 	memcpy(&copy->fh, &cstate->current_fh.fh_handle,  in nfsd4_copy()
1817 	if (nfsd4_copy_is_async(copy)) {  in nfsd4_copy()
1824 		/* Arbitrary cap on number of pending async copy operations */  in nfsd4_copy()
1831 		if (!nfs4_init_copy_state(nn, copy))  in nfsd4_copy()
1833 		memcpy(&result->cb_stateid, &copy->cp_stateid.cs_stid,  in nfsd4_copy()
1835 		dup_copy_fields(copy, async_copy);  in nfsd4_copy()
1837 				async_copy, "%s", "copy thread");  in nfsd4_copy()
1847 		status = nfsd4_do_copy(copy, copy->nf_src->nf_file,  in nfsd4_copy()
1848 				       copy->nf_dst->nf_file, true);  in nfsd4_copy()
1851 	release_copy_files(copy);  in nfsd4_copy()
1854 	if (nfsd4_ssc_is_inter(copy)) {  in nfsd4_copy()
1856 		 * Source's vfsmount of inter-copy will be unmounted  in nfsd4_copy()
1857 		 * by the laundromat. Use copy instead of async_copy  in nfsd4_copy()
1860 		refcount_dec(&copy->ss_nsui->nsui_refcnt);  in nfsd4_copy()
1871 	struct nfsd4_copy *copy;  in find_async_copy_locked()  local
1875 	list_for_each_entry(copy, &clp->async_copies, copies) {  in find_async_copy_locked()
1876 		if (memcmp(&copy->cp_stateid.cs_stid, stateid, NFS4_STATEID_SIZE))  in find_async_copy_locked()
1878 		return copy;  in find_async_copy_locked()
1886 	struct nfsd4_copy *copy;  in find_async_copy()  local
1889 	copy = find_async_copy_locked(clp, stateid);  in find_async_copy()
1890 	if (copy)  in find_async_copy()
1891 		refcount_inc(&copy->refcount);  in find_async_copy()
1893 	return copy;  in find_async_copy()
1902 	struct nfsd4_copy *copy;  in nfsd4_offload_cancel()  local
1905 	copy = find_async_copy(clp, &os->stateid);  in nfsd4_offload_cancel()
1906 	if (!copy) {  in nfsd4_offload_cancel()
1911 		nfsd4_stop_copy(copy);  in nfsd4_offload_cancel()
1988 	struct nfsd4_copy *copy;  in nfsd4_offload_status()  local
1992 	copy = find_async_copy_locked(clp, &os->stateid);  in nfsd4_offload_status()
1993 	if (copy)  in nfsd4_offload_status()
1994 		os->count = copy->cp_res.wr_bytes_written;  in nfsd4_offload_status()
2447 	 * Get the entire list, then copy out only the user attributes  in nfsd4_listxattrs()
2580 	struct nfsd4_copy *copy;  in check_if_stalefh_allowed()  local
2584 	/* traverse all operation and if it's a COPY compound, mark the  in check_if_stalefh_allowed()
2596 			copy = (struct nfsd4_copy *)&op->u;  in check_if_stalefh_allowed()
2602 			if (nfsd4_ssc_is_inter(copy))  in check_if_stalefh_allowed()