153 		struct nvme_command *cmd, blk_opf_t rq_flags,  in nvme_alloc_user_request()  argument
158 	req = blk_mq_alloc_request(q, nvme_req_op(cmd) | rq_flags, blk_flags);  in nvme_alloc_user_request()
161 	nvme_init_request(req, cmd);  in nvme_alloc_user_request()
222 		struct nvme_command *cmd, u64 ubuffer, unsigned bufflen,  in nvme_submit_user_cmd()  argument
234 	req = nvme_alloc_user_request(q, cmd, 0, 0);  in nvme_submit_user_cmd()
249 	effects = nvme_passthru_start(ctrl, ns, cmd->common.opcode);  in nvme_submit_user_cmd()
261 		nvme_passthru_end(ctrl, ns, effects, cmd, ret);  in nvme_submit_user_cmd()
346 	struct nvme_passthru_cmd cmd;  in nvme_user_cmd()  local
352 	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))  in nvme_user_cmd()
354 	if (cmd.flags)  in nvme_user_cmd()
356 	if (!nvme_validate_passthru_nsid(ctrl, ns, cmd.nsid))  in nvme_user_cmd()
360 	c.common.opcode = cmd.opcode;  in nvme_user_cmd()
361 	c.common.flags = cmd.flags;  in nvme_user_cmd()
362 	c.common.nsid = cpu_to_le32(cmd.nsid);  in nvme_user_cmd()
363 	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);  in nvme_user_cmd()
364 	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);  in nvme_user_cmd()
365 	c.common.cdw10 = cpu_to_le32(cmd.cdw10);  in nvme_user_cmd()
366 	c.common.cdw11 = cpu_to_le32(cmd.cdw11);  in nvme_user_cmd()
367 	c.common.cdw12 = cpu_to_le32(cmd.cdw12);  in nvme_user_cmd()
368 	c.common.cdw13 = cpu_to_le32(cmd.cdw13);  in nvme_user_cmd()
369 	c.common.cdw14 = cpu_to_le32(cmd.cdw14);  in nvme_user_cmd()
370 	c.common.cdw15 = cpu_to_le32(cmd.cdw15);  in nvme_user_cmd()
375 	if (cmd.timeout_ms)  in nvme_user_cmd()
376 		timeout = msecs_to_jiffies(cmd.timeout_ms);  in nvme_user_cmd()
379 			cmd.addr, cmd.data_len, nvme_to_user_ptr(cmd.metadata),  in nvme_user_cmd()
380 			cmd.metadata_len, 0, &result, timeout, 0);  in nvme_user_cmd()
394 	struct nvme_passthru_cmd64 cmd;  in nvme_user_cmd64()  local
399 	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))  in nvme_user_cmd64()
401 	if (cmd.flags)  in nvme_user_cmd64()
403 	if (!nvme_validate_passthru_nsid(ctrl, ns, cmd.nsid))  in nvme_user_cmd64()
407 	c.common.opcode = cmd.opcode;  in nvme_user_cmd64()
408 	c.common.flags = cmd.flags;  in nvme_user_cmd64()
409 	c.common.nsid = cpu_to_le32(cmd.nsid);  in nvme_user_cmd64()
410 	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);  in nvme_user_cmd64()
411 	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);  in nvme_user_cmd64()
412 	c.common.cdw10 = cpu_to_le32(cmd.cdw10);  in nvme_user_cmd64()
413 	c.common.cdw11 = cpu_to_le32(cmd.cdw11);  in nvme_user_cmd64()
414 	c.common.cdw12 = cpu_to_le32(cmd.cdw12);  in nvme_user_cmd64()
415 	c.common.cdw13 = cpu_to_le32(cmd.cdw13);  in nvme_user_cmd64()
416 	c.common.cdw14 = cpu_to_le32(cmd.cdw14);  in nvme_user_cmd64()
417 	c.common.cdw15 = cpu_to_le32(cmd.cdw15);  in nvme_user_cmd64()
422 	if (cmd.timeout_ms)  in nvme_user_cmd64()
423 		timeout = msecs_to_jiffies(cmd.timeout_ms);  in nvme_user_cmd64()
426 			cmd.addr, cmd.data_len, nvme_to_user_ptr(cmd.metadata),  in nvme_user_cmd64()
427 			cmd.metadata_len, 0, &cmd.result, timeout, flags);  in nvme_user_cmd64()
430 		if (put_user(cmd.result, &ucmd->result))  in nvme_user_cmd64()
564 	const struct nvme_uring_cmd *cmd = io_uring_sqe_cmd(ioucmd->sqe);  in nvme_uring_cmd_io()  local
574 	c.common.opcode = READ_ONCE(cmd->opcode);  in nvme_uring_cmd_io()
575 	c.common.flags = READ_ONCE(cmd->flags);  in nvme_uring_cmd_io()
580 	c.common.nsid = cpu_to_le32(cmd->nsid);  in nvme_uring_cmd_io()
584 	c.common.cdw2[0] = cpu_to_le32(READ_ONCE(cmd->cdw2));  in nvme_uring_cmd_io()
585 	c.common.cdw2[1] = cpu_to_le32(READ_ONCE(cmd->cdw3));  in nvme_uring_cmd_io()
588 	c.common.cdw10 = cpu_to_le32(READ_ONCE(cmd->cdw10));  in nvme_uring_cmd_io()
589 	c.common.cdw11 = cpu_to_le32(READ_ONCE(cmd->cdw11));  in nvme_uring_cmd_io()
590 	c.common.cdw12 = cpu_to_le32(READ_ONCE(cmd->cdw12));  in nvme_uring_cmd_io()
591 	c.common.cdw13 = cpu_to_le32(READ_ONCE(cmd->cdw13));  in nvme_uring_cmd_io()
592 	c.common.cdw14 = cpu_to_le32(READ_ONCE(cmd->cdw14));  in nvme_uring_cmd_io()
593 	c.common.cdw15 = cpu_to_le32(READ_ONCE(cmd->cdw15));  in nvme_uring_cmd_io()
598 	d.metadata = READ_ONCE(cmd->metadata);  in nvme_uring_cmd_io()
599 	d.addr = READ_ONCE(cmd->addr);  in nvme_uring_cmd_io()
600 	d.data_len = READ_ONCE(cmd->data_len);  in nvme_uring_cmd_io()
601 	d.metadata_len = READ_ONCE(cmd->metadata_len);  in nvme_uring_cmd_io()
602 	d.timeout_ms = READ_ONCE(cmd->timeout_ms);  in nvme_uring_cmd_io()
644 static bool is_ctrl_ioctl(unsigned int cmd)  in is_ctrl_ioctl()  argument
646 	if (cmd == NVME_IOCTL_ADMIN_CMD || cmd == NVME_IOCTL_ADMIN64_CMD)  in is_ctrl_ioctl()
648 	if (is_sed_ioctl(cmd))  in is_ctrl_ioctl()
653 static int nvme_ctrl_ioctl(struct nvme_ctrl *ctrl, unsigned int cmd,  in nvme_ctrl_ioctl()  argument
656 	switch (cmd) {  in nvme_ctrl_ioctl()
662 		return sed_ioctl(ctrl->opal_dev, cmd, argp);  in nvme_ctrl_ioctl()
684 static int nvme_ns_ioctl(struct nvme_ns *ns, unsigned int cmd,  in nvme_ns_ioctl()  argument
687 	switch (cmd) {  in nvme_ns_ioctl()
715 		unsigned int cmd, unsigned long arg)  in nvme_ioctl()  argument
725 	if (is_ctrl_ioctl(cmd))  in nvme_ioctl()
726 		return nvme_ctrl_ioctl(ns->ctrl, cmd, argp, open_for_write);  in nvme_ioctl()
727 	return nvme_ns_ioctl(ns, cmd, argp, flags, open_for_write);  in nvme_ioctl()
730 long nvme_ns_chr_ioctl(struct file *file, unsigned int cmd, unsigned long arg)  in nvme_ns_chr_ioctl()  argument
737 	if (is_ctrl_ioctl(cmd))  in nvme_ns_chr_ioctl()
738 		return nvme_ctrl_ioctl(ns->ctrl, cmd, argp, open_for_write);  in nvme_ns_chr_ioctl()
739 	return nvme_ns_ioctl(ns, cmd, argp, 0, open_for_write);  in nvme_ns_chr_ioctl()
802 static int nvme_ns_head_ctrl_ioctl(struct nvme_ns *ns, unsigned int cmd,  in nvme_ns_head_ctrl_ioctl()  argument
812 	ret = nvme_ctrl_ioctl(ns->ctrl, cmd, argp, open_for_write);  in nvme_ns_head_ctrl_ioctl()
819 		unsigned int cmd, unsigned long arg)  in nvme_ns_head_ioctl()  argument
841 	if (is_ctrl_ioctl(cmd))  in nvme_ns_head_ioctl()
842 		return nvme_ns_head_ctrl_ioctl(ns, cmd, argp, head, srcu_idx,  in nvme_ns_head_ioctl()
845 	ret = nvme_ns_ioctl(ns, cmd, argp, flags, open_for_write);  in nvme_ns_head_ioctl()
851 long nvme_ns_head_chr_ioctl(struct file *file, unsigned int cmd,  in nvme_ns_head_chr_ioctl()  argument
867 	if (is_ctrl_ioctl(cmd))  in nvme_ns_head_chr_ioctl()
868 		return nvme_ns_head_ctrl_ioctl(ns, cmd, argp, head, srcu_idx,  in nvme_ns_head_chr_ioctl()
871 	ret = nvme_ns_ioctl(ns, cmd, argp, 0, open_for_write);  in nvme_ns_head_chr_ioctl()
957 long nvme_dev_ioctl(struct file *file, unsigned int cmd,  in nvme_dev_ioctl()  argument
964 	switch (cmd) {  in nvme_dev_ioctl()