169 		u32 meta_seed, void **metap, struct io_uring_cmd *ioucmd,  in nvme_map_user_request()  argument
184 	if (ioucmd && (ioucmd->flags & IORING_URING_CMD_FIXED)) {  in nvme_map_user_request()
191 				rq_data_dir(req), &iter, ioucmd);  in nvme_map_user_request()
472 		struct io_uring_cmd *ioucmd)  in nvme_uring_cmd_pdu()  argument
474 	return (struct nvme_uring_cmd_pdu *)&ioucmd->pdu;  in nvme_uring_cmd_pdu()
477 static void nvme_uring_task_meta_cb(struct io_uring_cmd *ioucmd,  in nvme_uring_task_meta_cb()  argument
480 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);  in nvme_uring_task_meta_cb()
499 	io_uring_cmd_done(ioucmd, status, result, issue_flags);  in nvme_uring_task_meta_cb()
502 static void nvme_uring_task_cb(struct io_uring_cmd *ioucmd,  in nvme_uring_task_cb()  argument
505 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);  in nvme_uring_task_cb()
510 	io_uring_cmd_done(ioucmd, pdu->nvme_status, pdu->u.result, issue_flags);  in nvme_uring_task_cb()
516 	struct io_uring_cmd *ioucmd = req->end_io_data;  in nvme_uring_cmd_end_io()  local
517 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);  in nvme_uring_cmd_end_io()
534 		WRITE_ONCE(ioucmd->cookie, NULL);  in nvme_uring_cmd_end_io()
535 		nvme_uring_task_cb(ioucmd, IO_URING_F_UNLOCKED);  in nvme_uring_cmd_end_io()
537 		io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_cb);  in nvme_uring_cmd_end_io()
546 	struct io_uring_cmd *ioucmd = req->end_io_data;  in nvme_uring_cmd_end_io_meta()  local
547 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);  in nvme_uring_cmd_end_io_meta()
557 		WRITE_ONCE(ioucmd->cookie, NULL);  in nvme_uring_cmd_end_io_meta()
558 		nvme_uring_task_meta_cb(ioucmd, IO_URING_F_UNLOCKED);  in nvme_uring_cmd_end_io_meta()
560 		io_uring_cmd_do_in_task_lazy(ioucmd, nvme_uring_task_meta_cb);  in nvme_uring_cmd_end_io_meta()
567 		struct io_uring_cmd *ioucmd, unsigned int issue_flags, bool vec)  in nvme_uring_cmd_io()  argument
569 	struct nvme_uring_cmd_pdu *pdu = nvme_uring_cmd_pdu(ioucmd);  in nvme_uring_cmd_io()
570 	const struct nvme_uring_cmd *cmd = io_uring_sqe_cmd(ioucmd->sqe);  in nvme_uring_cmd_io()
601 	if (!nvme_cmd_allowed(ns, &c, 0, ioucmd->file->f_mode & FMODE_WRITE))  in nvme_uring_cmd_io()
625 			d.metadata_len, 0, &meta, ioucmd, vec);  in nvme_uring_cmd_io()
631 		ioucmd->flags |= IORING_URING_CMD_POLLED;  in nvme_uring_cmd_io()
632 		WRITE_ONCE(ioucmd->cookie, req);  in nvme_uring_cmd_io()
638 	req->end_io_data = ioucmd;  in nvme_uring_cmd_io()
758 static int nvme_ns_uring_cmd(struct nvme_ns *ns, struct io_uring_cmd *ioucmd,  in nvme_ns_uring_cmd()  argument
764 	BUILD_BUG_ON(sizeof(struct nvme_uring_cmd_pdu) > sizeof(ioucmd->pdu));  in nvme_ns_uring_cmd()
770 	switch (ioucmd->cmd_op) {  in nvme_ns_uring_cmd()
772 		ret = nvme_uring_cmd_io(ctrl, ns, ioucmd, issue_flags, false);  in nvme_ns_uring_cmd()
775 		ret = nvme_uring_cmd_io(ctrl, ns, ioucmd, issue_flags, true);  in nvme_ns_uring_cmd()
784 int nvme_ns_chr_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)  in nvme_ns_chr_uring_cmd()  argument
786 	struct nvme_ns *ns = container_of(file_inode(ioucmd->file)->i_cdev,  in nvme_ns_chr_uring_cmd()
789 	return nvme_ns_uring_cmd(ns, ioucmd, issue_flags);  in nvme_ns_chr_uring_cmd()
792 int nvme_ns_chr_uring_cmd_iopoll(struct io_uring_cmd *ioucmd,  in nvme_ns_chr_uring_cmd_iopoll()  argument
799 	if (!(ioucmd->flags & IORING_URING_CMD_POLLED))  in nvme_ns_chr_uring_cmd_iopoll()
802 	req = READ_ONCE(ioucmd->cookie);  in nvme_ns_chr_uring_cmd_iopoll()
883 int nvme_ns_head_chr_uring_cmd(struct io_uring_cmd *ioucmd,  in nvme_ns_head_chr_uring_cmd()  argument
886 	struct cdev *cdev = file_inode(ioucmd->file)->i_cdev;  in nvme_ns_head_chr_uring_cmd()
893 		ret = nvme_ns_uring_cmd(ns, ioucmd, issue_flags);  in nvme_ns_head_chr_uring_cmd()
899 int nvme_dev_uring_cmd(struct io_uring_cmd *ioucmd, unsigned int issue_flags)  in nvme_dev_uring_cmd()  argument
901 	struct nvme_ctrl *ctrl = ioucmd->file->private_data;  in nvme_dev_uring_cmd()
912 	switch (ioucmd->cmd_op) {  in nvme_dev_uring_cmd()
914 		ret = nvme_uring_cmd_io(ctrl, NULL, ioucmd, issue_flags, false);  in nvme_dev_uring_cmd()
917 		ret = nvme_uring_cmd_io(ctrl, NULL, ioucmd, issue_flags, true);  in nvme_dev_uring_cmd()