1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * NVMe over Fabrics DH-HMAC-CHAP authentication command handling.
4  * Copyright (c) 2020 Hannes Reinecke, SUSE Software Solutions.
5  * All rights reserved.
6  */
7 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
8 #include <linux/blkdev.h>
9 #include <linux/random.h>
10 #include <linux/nvme-auth.h>
11 #include <crypto/hash.h>
12 #include <crypto/kpp.h>
13 #include "nvmet.h"
14 
15 static void nvmet_auth_expired_work(struct work_struct *work)
16 {
17 	struct nvmet_sq *sq = container_of(to_delayed_work(work),
18 			struct nvmet_sq, auth_expired_work);
19 
20 	pr_debug("%s: ctrl %d qid %d transaction %u expired, resetting\n",
21 		 __func__, sq->ctrl->cntlid, sq->qid, sq->dhchap_tid);
22 	sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
23 	sq->dhchap_tid = -1;
24 }
25 
26 void nvmet_auth_sq_init(struct nvmet_sq *sq)
27 {
28 	/* Initialize in-band authentication */
29 	INIT_DELAYED_WORK(&sq->auth_expired_work, nvmet_auth_expired_work);
30 	sq->authenticated = false;
31 	sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
32 }
33 
34 static u16 nvmet_auth_negotiate(struct nvmet_req *req, void *d)
35 {
36 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
37 	struct nvmf_auth_dhchap_negotiate_data *data = d;
38 	int i, hash_id = 0, fallback_hash_id = 0, dhgid, fallback_dhgid;
39 
40 	pr_debug("%s: ctrl %d qid %d: data sc_d %d napd %d authid %d halen %d dhlen %d\n",
41 		 __func__, ctrl->cntlid, req->sq->qid,
42 		 data->sc_c, data->napd, data->auth_protocol[0].dhchap.authid,
43 		 data->auth_protocol[0].dhchap.halen,
44 		 data->auth_protocol[0].dhchap.dhlen);
45 	req->sq->dhchap_tid = le16_to_cpu(data->t_id);
46 	if (data->sc_c)
47 		return NVME_AUTH_DHCHAP_FAILURE_CONCAT_MISMATCH;
48 
49 	if (data->napd != 1)
50 		return NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
51 
52 	if (data->auth_protocol[0].dhchap.authid !=
53 	    NVME_AUTH_DHCHAP_AUTH_ID)
54 		return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
55 
56 	for (i = 0; i < data->auth_protocol[0].dhchap.halen; i++) {
57 		u8 host_hmac_id = data->auth_protocol[0].dhchap.idlist[i];
58 
59 		if (!fallback_hash_id &&
60 		    crypto_has_shash(nvme_auth_hmac_name(host_hmac_id), 0, 0))
61 			fallback_hash_id = host_hmac_id;
62 		if (ctrl->shash_id != host_hmac_id)
63 			continue;
64 		hash_id = ctrl->shash_id;
65 		break;
66 	}
67 	if (hash_id == 0) {
68 		if (fallback_hash_id == 0) {
69 			pr_debug("%s: ctrl %d qid %d: no usable hash found\n",
70 				 __func__, ctrl->cntlid, req->sq->qid);
71 			return NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
72 		}
73 		pr_debug("%s: ctrl %d qid %d: no usable hash found, falling back to %s\n",
74 			 __func__, ctrl->cntlid, req->sq->qid,
75 			 nvme_auth_hmac_name(fallback_hash_id));
76 		ctrl->shash_id = fallback_hash_id;
77 	}
78 
79 	dhgid = -1;
80 	fallback_dhgid = -1;
81 	for (i = 0; i < data->auth_protocol[0].dhchap.dhlen; i++) {
82 		int tmp_dhgid = data->auth_protocol[0].dhchap.idlist[i + 30];
83 
84 		if (tmp_dhgid != ctrl->dh_gid) {
85 			dhgid = tmp_dhgid;
86 			break;
87 		}
88 		if (fallback_dhgid < 0) {
89 			const char *kpp = nvme_auth_dhgroup_kpp(tmp_dhgid);
90 
91 			if (crypto_has_kpp(kpp, 0, 0))
92 				fallback_dhgid = tmp_dhgid;
93 		}
94 	}
95 	if (dhgid < 0) {
96 		if (fallback_dhgid < 0) {
97 			pr_debug("%s: ctrl %d qid %d: no usable DH group found\n",
98 				 __func__, ctrl->cntlid, req->sq->qid);
99 			return NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
100 		}
101 		pr_debug("%s: ctrl %d qid %d: configured DH group %s not found\n",
102 			 __func__, ctrl->cntlid, req->sq->qid,
103 			 nvme_auth_dhgroup_name(fallback_dhgid));
104 		ctrl->dh_gid = fallback_dhgid;
105 	}
106 	pr_debug("%s: ctrl %d qid %d: selected DH group %s (%d)\n",
107 		 __func__, ctrl->cntlid, req->sq->qid,
108 		 nvme_auth_dhgroup_name(ctrl->dh_gid), ctrl->dh_gid);
109 	return 0;
110 }
111 
112 static u16 nvmet_auth_reply(struct nvmet_req *req, void *d)
113 {
114 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
115 	struct nvmf_auth_dhchap_reply_data *data = d;
116 	u16 dhvlen = le16_to_cpu(data->dhvlen);
117 	u8 *response;
118 
119 	pr_debug("%s: ctrl %d qid %d: data hl %d cvalid %d dhvlen %u\n",
120 		 __func__, ctrl->cntlid, req->sq->qid,
121 		 data->hl, data->cvalid, dhvlen);
122 
123 	if (dhvlen) {
124 		if (!ctrl->dh_tfm)
125 			return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
126 		if (nvmet_auth_ctrl_sesskey(req, data->rval + 2 * data->hl,
127 					    dhvlen) < 0)
128 			return NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE;
129 	}
130 
131 	response = kmalloc(data->hl, GFP_KERNEL);
132 	if (!response)
133 		return NVME_AUTH_DHCHAP_FAILURE_FAILED;
134 
135 	if (!ctrl->host_key) {
136 		pr_warn("ctrl %d qid %d no host key\n",
137 			ctrl->cntlid, req->sq->qid);
138 		kfree(response);
139 		return NVME_AUTH_DHCHAP_FAILURE_FAILED;
140 	}
141 	if (nvmet_auth_host_hash(req, response, data->hl) < 0) {
142 		pr_debug("ctrl %d qid %d host hash failed\n",
143 			 ctrl->cntlid, req->sq->qid);
144 		kfree(response);
145 		return NVME_AUTH_DHCHAP_FAILURE_FAILED;
146 	}
147 
148 	if (memcmp(data->rval, response, data->hl)) {
149 		pr_info("ctrl %d qid %d host response mismatch\n",
150 			ctrl->cntlid, req->sq->qid);
151 		kfree(response);
152 		return NVME_AUTH_DHCHAP_FAILURE_FAILED;
153 	}
154 	kfree(response);
155 	pr_debug("%s: ctrl %d qid %d host authenticated\n",
156 		 __func__, ctrl->cntlid, req->sq->qid);
157 	if (data->cvalid) {
158 		req->sq->dhchap_c2 = kmemdup(data->rval + data->hl, data->hl,
159 					     GFP_KERNEL);
160 		if (!req->sq->dhchap_c2)
161 			return NVME_AUTH_DHCHAP_FAILURE_FAILED;
162 
163 		pr_debug("%s: ctrl %d qid %d challenge %*ph\n",
164 			 __func__, ctrl->cntlid, req->sq->qid, data->hl,
165 			 req->sq->dhchap_c2);
166 		req->sq->dhchap_s2 = le32_to_cpu(data->seqnum);
167 	} else {
168 		req->sq->authenticated = true;
169 		req->sq->dhchap_c2 = NULL;
170 	}
171 
172 	return 0;
173 }
174 
175 static u16 nvmet_auth_failure2(void *d)
176 {
177 	struct nvmf_auth_dhchap_failure_data *data = d;
178 
179 	return data->rescode_exp;
180 }
181 
182 void nvmet_execute_auth_send(struct nvmet_req *req)
183 {
184 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
185 	struct nvmf_auth_dhchap_success2_data *data;
186 	void *d;
187 	u32 tl;
188 	u16 status = 0;
189 
190 	if (req->cmd->auth_send.secp != NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER) {
191 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
192 		req->error_loc =
193 			offsetof(struct nvmf_auth_send_command, secp);
194 		goto done;
195 	}
196 	if (req->cmd->auth_send.spsp0 != 0x01) {
197 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
198 		req->error_loc =
199 			offsetof(struct nvmf_auth_send_command, spsp0);
200 		goto done;
201 	}
202 	if (req->cmd->auth_send.spsp1 != 0x01) {
203 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
204 		req->error_loc =
205 			offsetof(struct nvmf_auth_send_command, spsp1);
206 		goto done;
207 	}
208 	tl = le32_to_cpu(req->cmd->auth_send.tl);
209 	if (!tl) {
210 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
211 		req->error_loc =
212 			offsetof(struct nvmf_auth_send_command, tl);
213 		goto done;
214 	}
215 	if (!nvmet_check_transfer_len(req, tl)) {
216 		pr_debug("%s: transfer length mismatch (%u)\n", __func__, tl);
217 		return;
218 	}
219 
220 	d = kmalloc(tl, GFP_KERNEL);
221 	if (!d) {
222 		status = NVME_SC_INTERNAL;
223 		goto done;
224 	}
225 
226 	status = nvmet_copy_from_sgl(req, 0, d, tl);
227 	if (status)
228 		goto done_kfree;
229 
230 	data = d;
231 	pr_debug("%s: ctrl %d qid %d type %d id %d step %x\n", __func__,
232 		 ctrl->cntlid, req->sq->qid, data->auth_type, data->auth_id,
233 		 req->sq->dhchap_step);
234 	if (data->auth_type != NVME_AUTH_COMMON_MESSAGES &&
235 	    data->auth_type != NVME_AUTH_DHCHAP_MESSAGES)
236 		goto done_failure1;
237 	if (data->auth_type == NVME_AUTH_COMMON_MESSAGES) {
238 		if (data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE) {
239 			/* Restart negotiation */
240 			pr_debug("%s: ctrl %d qid %d reset negotiation\n", __func__,
241 				 ctrl->cntlid, req->sq->qid);
242 			if (!req->sq->qid) {
243 				if (nvmet_setup_auth(ctrl) < 0) {
244 					status = NVME_SC_INTERNAL;
245 					pr_err("ctrl %d qid 0 failed to setup"
246 					       "re-authentication",
247 					       ctrl->cntlid);
248 					goto done_failure1;
249 				}
250 			}
251 			req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE;
252 		} else if (data->auth_id != req->sq->dhchap_step)
253 			goto done_failure1;
254 		/* Validate negotiation parameters */
255 		status = nvmet_auth_negotiate(req, d);
256 		if (status == 0)
257 			req->sq->dhchap_step =
258 				NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE;
259 		else {
260 			req->sq->dhchap_step =
261 				NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
262 			req->sq->dhchap_status = status;
263 			status = 0;
264 		}
265 		goto done_kfree;
266 	}
267 	if (data->auth_id != req->sq->dhchap_step) {
268 		pr_debug("%s: ctrl %d qid %d step mismatch (%d != %d)\n",
269 			 __func__, ctrl->cntlid, req->sq->qid,
270 			 data->auth_id, req->sq->dhchap_step);
271 		goto done_failure1;
272 	}
273 	if (le16_to_cpu(data->t_id) != req->sq->dhchap_tid) {
274 		pr_debug("%s: ctrl %d qid %d invalid transaction %d (expected %d)\n",
275 			 __func__, ctrl->cntlid, req->sq->qid,
276 			 le16_to_cpu(data->t_id),
277 			 req->sq->dhchap_tid);
278 		req->sq->dhchap_step =
279 			NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
280 		req->sq->dhchap_status =
281 			NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD;
282 		goto done_kfree;
283 	}
284 
285 	switch (data->auth_id) {
286 	case NVME_AUTH_DHCHAP_MESSAGE_REPLY:
287 		status = nvmet_auth_reply(req, d);
288 		if (status == 0)
289 			req->sq->dhchap_step =
290 				NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1;
291 		else {
292 			req->sq->dhchap_step =
293 				NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
294 			req->sq->dhchap_status = status;
295 			status = 0;
296 		}
297 		goto done_kfree;
298 	case NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2:
299 		req->sq->authenticated = true;
300 		pr_debug("%s: ctrl %d qid %d ctrl authenticated\n",
301 			 __func__, ctrl->cntlid, req->sq->qid);
302 		goto done_kfree;
303 	case NVME_AUTH_DHCHAP_MESSAGE_FAILURE2:
304 		status = nvmet_auth_failure2(d);
305 		if (status) {
306 			pr_warn("ctrl %d qid %d: authentication failed (%d)\n",
307 				ctrl->cntlid, req->sq->qid, status);
308 			req->sq->dhchap_status = status;
309 			req->sq->authenticated = false;
310 			status = 0;
311 		}
312 		goto done_kfree;
313 	default:
314 		req->sq->dhchap_status =
315 			NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
316 		req->sq->dhchap_step =
317 			NVME_AUTH_DHCHAP_MESSAGE_FAILURE2;
318 		req->sq->authenticated = false;
319 		goto done_kfree;
320 	}
321 done_failure1:
322 	req->sq->dhchap_status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE;
323 	req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2;
324 
325 done_kfree:
326 	kfree(d);
327 done:
328 	pr_debug("%s: ctrl %d qid %d dhchap status %x step %x\n", __func__,
329 		 ctrl->cntlid, req->sq->qid,
330 		 req->sq->dhchap_status, req->sq->dhchap_step);
331 	if (status)
332 		pr_debug("%s: ctrl %d qid %d nvme status %x error loc %d\n",
333 			 __func__, ctrl->cntlid, req->sq->qid,
334 			 status, req->error_loc);
335 	req->cqe->result.u64 = 0;
336 	if (req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2 &&
337 	    req->sq->dhchap_step != NVME_AUTH_DHCHAP_MESSAGE_FAILURE2) {
338 		unsigned long auth_expire_secs = ctrl->kato ? ctrl->kato : 120;
339 
340 		mod_delayed_work(system_wq, &req->sq->auth_expired_work,
341 				 auth_expire_secs * HZ);
342 		goto complete;
343 	}
344 	/* Final states, clear up variables */
345 	nvmet_auth_sq_free(req->sq);
346 	if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE2)
347 		nvmet_ctrl_fatal_error(ctrl);
348 
349 complete:
350 	nvmet_req_complete(req, status);
351 }
352 
353 static int nvmet_auth_challenge(struct nvmet_req *req, void *d, int al)
354 {
355 	struct nvmf_auth_dhchap_challenge_data *data = d;
356 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
357 	int ret = 0;
358 	int hash_len = nvme_auth_hmac_hash_len(ctrl->shash_id);
359 	int data_size = sizeof(*d) + hash_len;
360 
361 	if (ctrl->dh_tfm)
362 		data_size += ctrl->dh_keysize;
363 	if (al < data_size) {
364 		pr_debug("%s: buffer too small (al %d need %d)\n", __func__,
365 			 al, data_size);
366 		return -EINVAL;
367 	}
368 	memset(data, 0, data_size);
369 	req->sq->dhchap_s1 = nvme_auth_get_seqnum();
370 	data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
371 	data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE;
372 	data->t_id = cpu_to_le16(req->sq->dhchap_tid);
373 	data->hashid = ctrl->shash_id;
374 	data->hl = hash_len;
375 	data->seqnum = cpu_to_le32(req->sq->dhchap_s1);
376 	req->sq->dhchap_c1 = kmalloc(data->hl, GFP_KERNEL);
377 	if (!req->sq->dhchap_c1)
378 		return -ENOMEM;
379 	get_random_bytes(req->sq->dhchap_c1, data->hl);
380 	memcpy(data->cval, req->sq->dhchap_c1, data->hl);
381 	if (ctrl->dh_tfm) {
382 		data->dhgid = ctrl->dh_gid;
383 		data->dhvlen = cpu_to_le16(ctrl->dh_keysize);
384 		ret = nvmet_auth_ctrl_exponential(req, data->cval + data->hl,
385 						  ctrl->dh_keysize);
386 	}
387 	pr_debug("%s: ctrl %d qid %d seq %d transaction %d hl %d dhvlen %zu\n",
388 		 __func__, ctrl->cntlid, req->sq->qid, req->sq->dhchap_s1,
389 		 req->sq->dhchap_tid, data->hl, ctrl->dh_keysize);
390 	return ret;
391 }
392 
393 static int nvmet_auth_success1(struct nvmet_req *req, void *d, int al)
394 {
395 	struct nvmf_auth_dhchap_success1_data *data = d;
396 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
397 	int hash_len = nvme_auth_hmac_hash_len(ctrl->shash_id);
398 
399 	WARN_ON(al < sizeof(*data));
400 	memset(data, 0, sizeof(*data));
401 	data->auth_type = NVME_AUTH_DHCHAP_MESSAGES;
402 	data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1;
403 	data->t_id = cpu_to_le16(req->sq->dhchap_tid);
404 	data->hl = hash_len;
405 	if (req->sq->dhchap_c2) {
406 		if (!ctrl->ctrl_key) {
407 			pr_warn("ctrl %d qid %d no ctrl key\n",
408 				ctrl->cntlid, req->sq->qid);
409 			return NVME_AUTH_DHCHAP_FAILURE_FAILED;
410 		}
411 		if (nvmet_auth_ctrl_hash(req, data->rval, data->hl))
412 			return NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE;
413 		data->rvalid = 1;
414 		pr_debug("ctrl %d qid %d response %*ph\n",
415 			 ctrl->cntlid, req->sq->qid, data->hl, data->rval);
416 	}
417 	return 0;
418 }
419 
420 static void nvmet_auth_failure1(struct nvmet_req *req, void *d, int al)
421 {
422 	struct nvmf_auth_dhchap_failure_data *data = d;
423 
424 	WARN_ON(al < sizeof(*data));
425 	data->auth_type = NVME_AUTH_COMMON_MESSAGES;
426 	data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
427 	data->t_id = cpu_to_le16(req->sq->dhchap_tid);
428 	data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED;
429 	data->rescode_exp = req->sq->dhchap_status;
430 }
431 
432 void nvmet_execute_auth_receive(struct nvmet_req *req)
433 {
434 	struct nvmet_ctrl *ctrl = req->sq->ctrl;
435 	void *d;
436 	u32 al;
437 	u16 status = 0;
438 
439 	if (req->cmd->auth_receive.secp != NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER) {
440 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
441 		req->error_loc =
442 			offsetof(struct nvmf_auth_receive_command, secp);
443 		goto done;
444 	}
445 	if (req->cmd->auth_receive.spsp0 != 0x01) {
446 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
447 		req->error_loc =
448 			offsetof(struct nvmf_auth_receive_command, spsp0);
449 		goto done;
450 	}
451 	if (req->cmd->auth_receive.spsp1 != 0x01) {
452 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
453 		req->error_loc =
454 			offsetof(struct nvmf_auth_receive_command, spsp1);
455 		goto done;
456 	}
457 	al = le32_to_cpu(req->cmd->auth_receive.al);
458 	if (!al) {
459 		status = NVME_SC_INVALID_FIELD | NVME_SC_DNR;
460 		req->error_loc =
461 			offsetof(struct nvmf_auth_receive_command, al);
462 		goto done;
463 	}
464 	if (!nvmet_check_transfer_len(req, al)) {
465 		pr_debug("%s: transfer length mismatch (%u)\n", __func__, al);
466 		return;
467 	}
468 
469 	d = kmalloc(al, GFP_KERNEL);
470 	if (!d) {
471 		status = NVME_SC_INTERNAL;
472 		goto done;
473 	}
474 	pr_debug("%s: ctrl %d qid %d step %x\n", __func__,
475 		 ctrl->cntlid, req->sq->qid, req->sq->dhchap_step);
476 	switch (req->sq->dhchap_step) {
477 	case NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE:
478 		if (nvmet_auth_challenge(req, d, al) < 0) {
479 			pr_warn("ctrl %d qid %d: challenge error (%d)\n",
480 				ctrl->cntlid, req->sq->qid, status);
481 			status = NVME_SC_INTERNAL;
482 			break;
483 		}
484 		req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_REPLY;
485 		break;
486 	case NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1:
487 		status = nvmet_auth_success1(req, d, al);
488 		if (status) {
489 			req->sq->dhchap_status = status;
490 			req->sq->authenticated = false;
491 			nvmet_auth_failure1(req, d, al);
492 			pr_warn("ctrl %d qid %d: success1 status (%x)\n",
493 				ctrl->cntlid, req->sq->qid,
494 				req->sq->dhchap_status);
495 			break;
496 		}
497 		req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2;
498 		break;
499 	case NVME_AUTH_DHCHAP_MESSAGE_FAILURE1:
500 		req->sq->authenticated = false;
501 		nvmet_auth_failure1(req, d, al);
502 		pr_warn("ctrl %d qid %d failure1 (%x)\n",
503 			ctrl->cntlid, req->sq->qid, req->sq->dhchap_status);
504 		break;
505 	default:
506 		pr_warn("ctrl %d qid %d unhandled step (%d)\n",
507 			ctrl->cntlid, req->sq->qid, req->sq->dhchap_step);
508 		req->sq->dhchap_step = NVME_AUTH_DHCHAP_MESSAGE_FAILURE1;
509 		req->sq->dhchap_status = NVME_AUTH_DHCHAP_FAILURE_FAILED;
510 		nvmet_auth_failure1(req, d, al);
511 		status = 0;
512 		break;
513 	}
514 
515 	status = nvmet_copy_to_sgl(req, 0, d, al);
516 	kfree(d);
517 done:
518 	req->cqe->result.u64 = 0;
519 
520 	if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2)
521 		nvmet_auth_sq_free(req->sq);
522 	else if (req->sq->dhchap_step == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) {
523 		nvmet_auth_sq_free(req->sq);
524 		nvmet_ctrl_fatal_error(ctrl);
525 	}
526 	nvmet_req_complete(req, status);
527 }
528