1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2020 Hannes Reinecke, SUSE Linux 4 */ 5 6 #include <linux/crc32.h> 7 #include <linux/base64.h> 8 #include <linux/prandom.h> 9 #include <asm/unaligned.h> 10 #include <crypto/hash.h> 11 #include <crypto/dh.h> 12 #include "nvme.h" 13 #include "fabrics.h" 14 #include <linux/nvme-auth.h> 15 16 struct nvme_dhchap_queue_context { 17 struct list_head entry; 18 struct work_struct auth_work; 19 struct nvme_ctrl *ctrl; 20 struct crypto_shash *shash_tfm; 21 struct crypto_kpp *dh_tfm; 22 void *buf; 23 size_t buf_size; 24 int qid; 25 int error; 26 u32 s1; 27 u32 s2; 28 u16 transaction; 29 u8 status; 30 u8 hash_id; 31 size_t hash_len; 32 u8 dhgroup_id; 33 u8 c1[64]; 34 u8 c2[64]; 35 u8 response[64]; 36 u8 *host_response; 37 u8 *ctrl_key; 38 int ctrl_key_len; 39 u8 *host_key; 40 int host_key_len; 41 u8 *sess_key; 42 int sess_key_len; 43 }; 44 45 #define nvme_auth_flags_from_qid(qid) \ 46 (qid == 0) ? 0 : BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED 47 #define nvme_auth_queue_from_qid(ctrl, qid) \ 48 (qid == 0) ? (ctrl)->fabrics_q : (ctrl)->connect_q 49 50 static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid, 51 void *data, size_t data_len, bool auth_send) 52 { 53 struct nvme_command cmd = {}; 54 blk_mq_req_flags_t flags = nvme_auth_flags_from_qid(qid); 55 struct request_queue *q = nvme_auth_queue_from_qid(ctrl, qid); 56 int ret; 57 58 cmd.auth_common.opcode = nvme_fabrics_command; 59 cmd.auth_common.secp = NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER; 60 cmd.auth_common.spsp0 = 0x01; 61 cmd.auth_common.spsp1 = 0x01; 62 if (auth_send) { 63 cmd.auth_send.fctype = nvme_fabrics_type_auth_send; 64 cmd.auth_send.tl = cpu_to_le32(data_len); 65 } else { 66 cmd.auth_receive.fctype = nvme_fabrics_type_auth_receive; 67 cmd.auth_receive.al = cpu_to_le32(data_len); 68 } 69 70 ret = __nvme_submit_sync_cmd(q, &cmd, NULL, data, data_len, 71 qid == 0 ? NVME_QID_ANY : qid, 72 0, flags); 73 if (ret > 0) 74 dev_warn(ctrl->device, 75 "qid %d auth_send failed with status %d\n", qid, ret); 76 else if (ret < 0) 77 dev_err(ctrl->device, 78 "qid %d auth_send failed with error %d\n", qid, ret); 79 return ret; 80 } 81 82 static int nvme_auth_receive_validate(struct nvme_ctrl *ctrl, int qid, 83 struct nvmf_auth_dhchap_failure_data *data, 84 u16 transaction, u8 expected_msg) 85 { 86 dev_dbg(ctrl->device, "%s: qid %d auth_type %d auth_id %x\n", 87 __func__, qid, data->auth_type, data->auth_id); 88 89 if (data->auth_type == NVME_AUTH_COMMON_MESSAGES && 90 data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) { 91 return data->rescode_exp; 92 } 93 if (data->auth_type != NVME_AUTH_DHCHAP_MESSAGES || 94 data->auth_id != expected_msg) { 95 dev_warn(ctrl->device, 96 "qid %d invalid message %02x/%02x\n", 97 qid, data->auth_type, data->auth_id); 98 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE; 99 } 100 if (le16_to_cpu(data->t_id) != transaction) { 101 dev_warn(ctrl->device, 102 "qid %d invalid transaction ID %d\n", 103 qid, le16_to_cpu(data->t_id)); 104 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE; 105 } 106 return 0; 107 } 108 109 static int nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl *ctrl, 110 struct nvme_dhchap_queue_context *chap) 111 { 112 struct nvmf_auth_dhchap_negotiate_data *data = chap->buf; 113 size_t size = sizeof(*data) + sizeof(union nvmf_auth_protocol); 114 115 if (chap->buf_size < size) { 116 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 117 return -EINVAL; 118 } 119 memset((u8 *)chap->buf, 0, size); 120 data->auth_type = NVME_AUTH_COMMON_MESSAGES; 121 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE; 122 data->t_id = cpu_to_le16(chap->transaction); 123 data->sc_c = 0; /* No secure channel concatenation */ 124 data->napd = 1; 125 data->auth_protocol[0].dhchap.authid = NVME_AUTH_DHCHAP_AUTH_ID; 126 data->auth_protocol[0].dhchap.halen = 3; 127 data->auth_protocol[0].dhchap.dhlen = 6; 128 data->auth_protocol[0].dhchap.idlist[0] = NVME_AUTH_HASH_SHA256; 129 data->auth_protocol[0].dhchap.idlist[1] = NVME_AUTH_HASH_SHA384; 130 data->auth_protocol[0].dhchap.idlist[2] = NVME_AUTH_HASH_SHA512; 131 data->auth_protocol[0].dhchap.idlist[30] = NVME_AUTH_DHGROUP_NULL; 132 data->auth_protocol[0].dhchap.idlist[31] = NVME_AUTH_DHGROUP_2048; 133 data->auth_protocol[0].dhchap.idlist[32] = NVME_AUTH_DHGROUP_3072; 134 data->auth_protocol[0].dhchap.idlist[33] = NVME_AUTH_DHGROUP_4096; 135 data->auth_protocol[0].dhchap.idlist[34] = NVME_AUTH_DHGROUP_6144; 136 data->auth_protocol[0].dhchap.idlist[35] = NVME_AUTH_DHGROUP_8192; 137 138 return size; 139 } 140 141 static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl, 142 struct nvme_dhchap_queue_context *chap) 143 { 144 struct nvmf_auth_dhchap_challenge_data *data = chap->buf; 145 u16 dhvlen = le16_to_cpu(data->dhvlen); 146 size_t size = sizeof(*data) + data->hl + dhvlen; 147 const char *gid_name = nvme_auth_dhgroup_name(data->dhgid); 148 const char *hmac_name, *kpp_name; 149 150 if (chap->buf_size < size) { 151 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 152 return NVME_SC_INVALID_FIELD; 153 } 154 155 hmac_name = nvme_auth_hmac_name(data->hashid); 156 if (!hmac_name) { 157 dev_warn(ctrl->device, 158 "qid %d: invalid HASH ID %d\n", 159 chap->qid, data->hashid); 160 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 161 return NVME_SC_INVALID_FIELD; 162 } 163 164 if (chap->hash_id == data->hashid && chap->shash_tfm && 165 !strcmp(crypto_shash_alg_name(chap->shash_tfm), hmac_name) && 166 crypto_shash_digestsize(chap->shash_tfm) == data->hl) { 167 dev_dbg(ctrl->device, 168 "qid %d: reuse existing hash %s\n", 169 chap->qid, hmac_name); 170 goto select_kpp; 171 } 172 173 /* Reset if hash cannot be reused */ 174 if (chap->shash_tfm) { 175 crypto_free_shash(chap->shash_tfm); 176 chap->hash_id = 0; 177 chap->hash_len = 0; 178 } 179 chap->shash_tfm = crypto_alloc_shash(hmac_name, 0, 180 CRYPTO_ALG_ALLOCATES_MEMORY); 181 if (IS_ERR(chap->shash_tfm)) { 182 dev_warn(ctrl->device, 183 "qid %d: failed to allocate hash %s, error %ld\n", 184 chap->qid, hmac_name, PTR_ERR(chap->shash_tfm)); 185 chap->shash_tfm = NULL; 186 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 187 return NVME_SC_AUTH_REQUIRED; 188 } 189 190 if (crypto_shash_digestsize(chap->shash_tfm) != data->hl) { 191 dev_warn(ctrl->device, 192 "qid %d: invalid hash length %d\n", 193 chap->qid, data->hl); 194 crypto_free_shash(chap->shash_tfm); 195 chap->shash_tfm = NULL; 196 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 197 return NVME_SC_AUTH_REQUIRED; 198 } 199 200 /* Reset host response if the hash had been changed */ 201 if (chap->hash_id != data->hashid) { 202 kfree(chap->host_response); 203 chap->host_response = NULL; 204 } 205 206 chap->hash_id = data->hashid; 207 chap->hash_len = data->hl; 208 dev_dbg(ctrl->device, "qid %d: selected hash %s\n", 209 chap->qid, hmac_name); 210 211 select_kpp: 212 kpp_name = nvme_auth_dhgroup_kpp(data->dhgid); 213 if (!kpp_name) { 214 dev_warn(ctrl->device, 215 "qid %d: invalid DH group id %d\n", 216 chap->qid, data->dhgid); 217 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 218 /* Leave previous dh_tfm intact */ 219 return NVME_SC_AUTH_REQUIRED; 220 } 221 222 /* Clear host and controller key to avoid accidental reuse */ 223 kfree_sensitive(chap->host_key); 224 chap->host_key = NULL; 225 chap->host_key_len = 0; 226 kfree_sensitive(chap->ctrl_key); 227 chap->ctrl_key = NULL; 228 chap->ctrl_key_len = 0; 229 230 if (chap->dhgroup_id == data->dhgid && 231 (data->dhgid == NVME_AUTH_DHGROUP_NULL || chap->dh_tfm)) { 232 dev_dbg(ctrl->device, 233 "qid %d: reuse existing DH group %s\n", 234 chap->qid, gid_name); 235 goto skip_kpp; 236 } 237 238 /* Reset dh_tfm if it can't be reused */ 239 if (chap->dh_tfm) { 240 crypto_free_kpp(chap->dh_tfm); 241 chap->dh_tfm = NULL; 242 } 243 244 if (data->dhgid != NVME_AUTH_DHGROUP_NULL) { 245 if (dhvlen == 0) { 246 dev_warn(ctrl->device, 247 "qid %d: empty DH value\n", 248 chap->qid); 249 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 250 return NVME_SC_INVALID_FIELD; 251 } 252 253 chap->dh_tfm = crypto_alloc_kpp(kpp_name, 0, 0); 254 if (IS_ERR(chap->dh_tfm)) { 255 int ret = PTR_ERR(chap->dh_tfm); 256 257 dev_warn(ctrl->device, 258 "qid %d: error %d initializing DH group %s\n", 259 chap->qid, ret, gid_name); 260 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 261 chap->dh_tfm = NULL; 262 return NVME_SC_AUTH_REQUIRED; 263 } 264 dev_dbg(ctrl->device, "qid %d: selected DH group %s\n", 265 chap->qid, gid_name); 266 } else if (dhvlen != 0) { 267 dev_warn(ctrl->device, 268 "qid %d: invalid DH value for NULL DH\n", 269 chap->qid); 270 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 271 return NVME_SC_INVALID_FIELD; 272 } 273 chap->dhgroup_id = data->dhgid; 274 275 skip_kpp: 276 chap->s1 = le32_to_cpu(data->seqnum); 277 memcpy(chap->c1, data->cval, chap->hash_len); 278 if (dhvlen) { 279 chap->ctrl_key = kmalloc(dhvlen, GFP_KERNEL); 280 if (!chap->ctrl_key) { 281 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 282 return NVME_SC_AUTH_REQUIRED; 283 } 284 chap->ctrl_key_len = dhvlen; 285 memcpy(chap->ctrl_key, data->cval + chap->hash_len, 286 dhvlen); 287 dev_dbg(ctrl->device, "ctrl public key %*ph\n", 288 (int)chap->ctrl_key_len, chap->ctrl_key); 289 } 290 291 return 0; 292 } 293 294 static int nvme_auth_set_dhchap_reply_data(struct nvme_ctrl *ctrl, 295 struct nvme_dhchap_queue_context *chap) 296 { 297 struct nvmf_auth_dhchap_reply_data *data = chap->buf; 298 size_t size = sizeof(*data); 299 300 size += 2 * chap->hash_len; 301 302 if (chap->host_key_len) 303 size += chap->host_key_len; 304 305 if (chap->buf_size < size) { 306 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 307 return -EINVAL; 308 } 309 310 memset(chap->buf, 0, size); 311 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES; 312 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_REPLY; 313 data->t_id = cpu_to_le16(chap->transaction); 314 data->hl = chap->hash_len; 315 data->dhvlen = cpu_to_le16(chap->host_key_len); 316 memcpy(data->rval, chap->response, chap->hash_len); 317 if (ctrl->ctrl_key) { 318 get_random_bytes(chap->c2, chap->hash_len); 319 data->cvalid = 1; 320 chap->s2 = nvme_auth_get_seqnum(); 321 memcpy(data->rval + chap->hash_len, chap->c2, 322 chap->hash_len); 323 dev_dbg(ctrl->device, "%s: qid %d ctrl challenge %*ph\n", 324 __func__, chap->qid, (int)chap->hash_len, chap->c2); 325 } else { 326 memset(chap->c2, 0, chap->hash_len); 327 chap->s2 = 0; 328 } 329 data->seqnum = cpu_to_le32(chap->s2); 330 if (chap->host_key_len) { 331 dev_dbg(ctrl->device, "%s: qid %d host public key %*ph\n", 332 __func__, chap->qid, 333 chap->host_key_len, chap->host_key); 334 memcpy(data->rval + 2 * chap->hash_len, chap->host_key, 335 chap->host_key_len); 336 } 337 338 return size; 339 } 340 341 static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl, 342 struct nvme_dhchap_queue_context *chap) 343 { 344 struct nvmf_auth_dhchap_success1_data *data = chap->buf; 345 size_t size = sizeof(*data); 346 347 if (ctrl->ctrl_key) 348 size += chap->hash_len; 349 350 if (chap->buf_size < size) { 351 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 352 return NVME_SC_INVALID_FIELD; 353 } 354 355 if (data->hl != chap->hash_len) { 356 dev_warn(ctrl->device, 357 "qid %d: invalid hash length %u\n", 358 chap->qid, data->hl); 359 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 360 return NVME_SC_INVALID_FIELD; 361 } 362 363 /* Just print out information for the admin queue */ 364 if (chap->qid == 0) 365 dev_info(ctrl->device, 366 "qid 0: authenticated with hash %s dhgroup %s\n", 367 nvme_auth_hmac_name(chap->hash_id), 368 nvme_auth_dhgroup_name(chap->dhgroup_id)); 369 370 if (!data->rvalid) 371 return 0; 372 373 /* Validate controller response */ 374 if (memcmp(chap->response, data->rval, data->hl)) { 375 dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n", 376 __func__, chap->qid, (int)chap->hash_len, data->rval); 377 dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n", 378 __func__, chap->qid, (int)chap->hash_len, 379 chap->response); 380 dev_warn(ctrl->device, 381 "qid %d: controller authentication failed\n", 382 chap->qid); 383 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 384 return NVME_SC_AUTH_REQUIRED; 385 } 386 387 /* Just print out information for the admin queue */ 388 if (chap->qid == 0) 389 dev_info(ctrl->device, 390 "qid 0: controller authenticated\n"); 391 return 0; 392 } 393 394 static int nvme_auth_set_dhchap_success2_data(struct nvme_ctrl *ctrl, 395 struct nvme_dhchap_queue_context *chap) 396 { 397 struct nvmf_auth_dhchap_success2_data *data = chap->buf; 398 size_t size = sizeof(*data); 399 400 memset(chap->buf, 0, size); 401 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES; 402 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2; 403 data->t_id = cpu_to_le16(chap->transaction); 404 405 return size; 406 } 407 408 static int nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl *ctrl, 409 struct nvme_dhchap_queue_context *chap) 410 { 411 struct nvmf_auth_dhchap_failure_data *data = chap->buf; 412 size_t size = sizeof(*data); 413 414 memset(chap->buf, 0, size); 415 data->auth_type = NVME_AUTH_COMMON_MESSAGES; 416 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2; 417 data->t_id = cpu_to_le16(chap->transaction); 418 data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED; 419 data->rescode_exp = chap->status; 420 421 return size; 422 } 423 424 static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl, 425 struct nvme_dhchap_queue_context *chap) 426 { 427 SHASH_DESC_ON_STACK(shash, chap->shash_tfm); 428 u8 buf[4], *challenge = chap->c1; 429 int ret; 430 431 dev_dbg(ctrl->device, "%s: qid %d host response seq %u transaction %d\n", 432 __func__, chap->qid, chap->s1, chap->transaction); 433 434 if (!chap->host_response) { 435 chap->host_response = nvme_auth_transform_key(ctrl->host_key, 436 ctrl->opts->host->nqn); 437 if (IS_ERR(chap->host_response)) { 438 ret = PTR_ERR(chap->host_response); 439 chap->host_response = NULL; 440 return ret; 441 } 442 } else { 443 dev_dbg(ctrl->device, "%s: qid %d re-using host response\n", 444 __func__, chap->qid); 445 } 446 447 ret = crypto_shash_setkey(chap->shash_tfm, 448 chap->host_response, ctrl->host_key->len); 449 if (ret) { 450 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n", 451 chap->qid, ret); 452 goto out; 453 } 454 455 if (chap->dh_tfm) { 456 challenge = kmalloc(chap->hash_len, GFP_KERNEL); 457 if (!challenge) { 458 ret = -ENOMEM; 459 goto out; 460 } 461 ret = nvme_auth_augmented_challenge(chap->hash_id, 462 chap->sess_key, 463 chap->sess_key_len, 464 chap->c1, challenge, 465 chap->hash_len); 466 if (ret) 467 goto out; 468 } 469 470 shash->tfm = chap->shash_tfm; 471 ret = crypto_shash_init(shash); 472 if (ret) 473 goto out; 474 ret = crypto_shash_update(shash, challenge, chap->hash_len); 475 if (ret) 476 goto out; 477 put_unaligned_le32(chap->s1, buf); 478 ret = crypto_shash_update(shash, buf, 4); 479 if (ret) 480 goto out; 481 put_unaligned_le16(chap->transaction, buf); 482 ret = crypto_shash_update(shash, buf, 2); 483 if (ret) 484 goto out; 485 memset(buf, 0, sizeof(buf)); 486 ret = crypto_shash_update(shash, buf, 1); 487 if (ret) 488 goto out; 489 ret = crypto_shash_update(shash, "HostHost", 8); 490 if (ret) 491 goto out; 492 ret = crypto_shash_update(shash, ctrl->opts->host->nqn, 493 strlen(ctrl->opts->host->nqn)); 494 if (ret) 495 goto out; 496 ret = crypto_shash_update(shash, buf, 1); 497 if (ret) 498 goto out; 499 ret = crypto_shash_update(shash, ctrl->opts->subsysnqn, 500 strlen(ctrl->opts->subsysnqn)); 501 if (ret) 502 goto out; 503 ret = crypto_shash_final(shash, chap->response); 504 out: 505 if (challenge != chap->c1) 506 kfree(challenge); 507 return ret; 508 } 509 510 static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl, 511 struct nvme_dhchap_queue_context *chap) 512 { 513 SHASH_DESC_ON_STACK(shash, chap->shash_tfm); 514 u8 *ctrl_response; 515 u8 buf[4], *challenge = chap->c2; 516 int ret; 517 518 ctrl_response = nvme_auth_transform_key(ctrl->ctrl_key, 519 ctrl->opts->subsysnqn); 520 if (IS_ERR(ctrl_response)) { 521 ret = PTR_ERR(ctrl_response); 522 return ret; 523 } 524 ret = crypto_shash_setkey(chap->shash_tfm, 525 ctrl_response, ctrl->ctrl_key->len); 526 if (ret) { 527 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n", 528 chap->qid, ret); 529 goto out; 530 } 531 532 if (chap->dh_tfm) { 533 challenge = kmalloc(chap->hash_len, GFP_KERNEL); 534 if (!challenge) { 535 ret = -ENOMEM; 536 goto out; 537 } 538 ret = nvme_auth_augmented_challenge(chap->hash_id, 539 chap->sess_key, 540 chap->sess_key_len, 541 chap->c2, challenge, 542 chap->hash_len); 543 if (ret) 544 goto out; 545 } 546 dev_dbg(ctrl->device, "%s: qid %d ctrl response seq %u transaction %d\n", 547 __func__, chap->qid, chap->s2, chap->transaction); 548 dev_dbg(ctrl->device, "%s: qid %d challenge %*ph\n", 549 __func__, chap->qid, (int)chap->hash_len, challenge); 550 dev_dbg(ctrl->device, "%s: qid %d subsysnqn %s\n", 551 __func__, chap->qid, ctrl->opts->subsysnqn); 552 dev_dbg(ctrl->device, "%s: qid %d hostnqn %s\n", 553 __func__, chap->qid, ctrl->opts->host->nqn); 554 shash->tfm = chap->shash_tfm; 555 ret = crypto_shash_init(shash); 556 if (ret) 557 goto out; 558 ret = crypto_shash_update(shash, challenge, chap->hash_len); 559 if (ret) 560 goto out; 561 put_unaligned_le32(chap->s2, buf); 562 ret = crypto_shash_update(shash, buf, 4); 563 if (ret) 564 goto out; 565 put_unaligned_le16(chap->transaction, buf); 566 ret = crypto_shash_update(shash, buf, 2); 567 if (ret) 568 goto out; 569 memset(buf, 0, 4); 570 ret = crypto_shash_update(shash, buf, 1); 571 if (ret) 572 goto out; 573 ret = crypto_shash_update(shash, "Controller", 10); 574 if (ret) 575 goto out; 576 ret = crypto_shash_update(shash, ctrl->opts->subsysnqn, 577 strlen(ctrl->opts->subsysnqn)); 578 if (ret) 579 goto out; 580 ret = crypto_shash_update(shash, buf, 1); 581 if (ret) 582 goto out; 583 ret = crypto_shash_update(shash, ctrl->opts->host->nqn, 584 strlen(ctrl->opts->host->nqn)); 585 if (ret) 586 goto out; 587 ret = crypto_shash_final(shash, chap->response); 588 out: 589 if (challenge != chap->c2) 590 kfree(challenge); 591 kfree(ctrl_response); 592 return ret; 593 } 594 595 static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl, 596 struct nvme_dhchap_queue_context *chap) 597 { 598 int ret; 599 600 if (chap->host_key && chap->host_key_len) { 601 dev_dbg(ctrl->device, 602 "qid %d: reusing host key\n", chap->qid); 603 goto gen_sesskey; 604 } 605 ret = nvme_auth_gen_privkey(chap->dh_tfm, chap->dhgroup_id); 606 if (ret < 0) { 607 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 608 return ret; 609 } 610 611 chap->host_key_len = crypto_kpp_maxsize(chap->dh_tfm); 612 613 chap->host_key = kzalloc(chap->host_key_len, GFP_KERNEL); 614 if (!chap->host_key) { 615 chap->host_key_len = 0; 616 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 617 return -ENOMEM; 618 } 619 ret = nvme_auth_gen_pubkey(chap->dh_tfm, 620 chap->host_key, chap->host_key_len); 621 if (ret) { 622 dev_dbg(ctrl->device, 623 "failed to generate public key, error %d\n", ret); 624 kfree(chap->host_key); 625 chap->host_key = NULL; 626 chap->host_key_len = 0; 627 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 628 return ret; 629 } 630 631 gen_sesskey: 632 chap->sess_key_len = chap->host_key_len; 633 chap->sess_key = kmalloc(chap->sess_key_len, GFP_KERNEL); 634 if (!chap->sess_key) { 635 chap->sess_key_len = 0; 636 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 637 return -ENOMEM; 638 } 639 640 ret = nvme_auth_gen_shared_secret(chap->dh_tfm, 641 chap->ctrl_key, chap->ctrl_key_len, 642 chap->sess_key, chap->sess_key_len); 643 if (ret) { 644 dev_dbg(ctrl->device, 645 "failed to generate shared secret, error %d\n", ret); 646 kfree_sensitive(chap->sess_key); 647 chap->sess_key = NULL; 648 chap->sess_key_len = 0; 649 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 650 return ret; 651 } 652 dev_dbg(ctrl->device, "shared secret %*ph\n", 653 (int)chap->sess_key_len, chap->sess_key); 654 return 0; 655 } 656 657 static void __nvme_auth_reset(struct nvme_dhchap_queue_context *chap) 658 { 659 kfree_sensitive(chap->host_response); 660 chap->host_response = NULL; 661 kfree_sensitive(chap->host_key); 662 chap->host_key = NULL; 663 chap->host_key_len = 0; 664 kfree_sensitive(chap->ctrl_key); 665 chap->ctrl_key = NULL; 666 chap->ctrl_key_len = 0; 667 kfree_sensitive(chap->sess_key); 668 chap->sess_key = NULL; 669 chap->sess_key_len = 0; 670 chap->status = 0; 671 chap->error = 0; 672 chap->s1 = 0; 673 chap->s2 = 0; 674 chap->transaction = 0; 675 memset(chap->c1, 0, sizeof(chap->c1)); 676 memset(chap->c2, 0, sizeof(chap->c2)); 677 } 678 679 static void __nvme_auth_free(struct nvme_dhchap_queue_context *chap) 680 { 681 __nvme_auth_reset(chap); 682 if (chap->shash_tfm) 683 crypto_free_shash(chap->shash_tfm); 684 if (chap->dh_tfm) 685 crypto_free_kpp(chap->dh_tfm); 686 kfree_sensitive(chap->ctrl_key); 687 kfree_sensitive(chap->host_key); 688 kfree_sensitive(chap->sess_key); 689 kfree_sensitive(chap->host_response); 690 kfree(chap->buf); 691 kfree(chap); 692 } 693 694 static void __nvme_auth_work(struct work_struct *work) 695 { 696 struct nvme_dhchap_queue_context *chap = 697 container_of(work, struct nvme_dhchap_queue_context, auth_work); 698 struct nvme_ctrl *ctrl = chap->ctrl; 699 size_t tl; 700 int ret = 0; 701 702 chap->transaction = ctrl->transaction++; 703 704 /* DH-HMAC-CHAP Step 1: send negotiate */ 705 dev_dbg(ctrl->device, "%s: qid %d send negotiate\n", 706 __func__, chap->qid); 707 ret = nvme_auth_set_dhchap_negotiate_data(ctrl, chap); 708 if (ret < 0) { 709 chap->error = ret; 710 return; 711 } 712 tl = ret; 713 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 714 if (ret) { 715 chap->error = ret; 716 return; 717 } 718 719 /* DH-HMAC-CHAP Step 2: receive challenge */ 720 dev_dbg(ctrl->device, "%s: qid %d receive challenge\n", 721 __func__, chap->qid); 722 723 memset(chap->buf, 0, chap->buf_size); 724 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, chap->buf_size, false); 725 if (ret) { 726 dev_warn(ctrl->device, 727 "qid %d failed to receive challenge, %s %d\n", 728 chap->qid, ret < 0 ? "error" : "nvme status", ret); 729 chap->error = ret; 730 return; 731 } 732 ret = nvme_auth_receive_validate(ctrl, chap->qid, chap->buf, chap->transaction, 733 NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE); 734 if (ret) { 735 chap->status = ret; 736 chap->error = NVME_SC_AUTH_REQUIRED; 737 return; 738 } 739 740 ret = nvme_auth_process_dhchap_challenge(ctrl, chap); 741 if (ret) { 742 /* Invalid challenge parameters */ 743 chap->error = ret; 744 goto fail2; 745 } 746 747 if (chap->ctrl_key_len) { 748 dev_dbg(ctrl->device, 749 "%s: qid %d DH exponential\n", 750 __func__, chap->qid); 751 ret = nvme_auth_dhchap_exponential(ctrl, chap); 752 if (ret) { 753 chap->error = ret; 754 goto fail2; 755 } 756 } 757 758 dev_dbg(ctrl->device, "%s: qid %d host response\n", 759 __func__, chap->qid); 760 ret = nvme_auth_dhchap_setup_host_response(ctrl, chap); 761 if (ret) { 762 chap->error = ret; 763 goto fail2; 764 } 765 766 /* DH-HMAC-CHAP Step 3: send reply */ 767 dev_dbg(ctrl->device, "%s: qid %d send reply\n", 768 __func__, chap->qid); 769 ret = nvme_auth_set_dhchap_reply_data(ctrl, chap); 770 if (ret < 0) { 771 chap->error = ret; 772 goto fail2; 773 } 774 775 tl = ret; 776 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 777 if (ret) { 778 chap->error = ret; 779 goto fail2; 780 } 781 782 /* DH-HMAC-CHAP Step 4: receive success1 */ 783 dev_dbg(ctrl->device, "%s: qid %d receive success1\n", 784 __func__, chap->qid); 785 786 memset(chap->buf, 0, chap->buf_size); 787 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, chap->buf_size, false); 788 if (ret) { 789 dev_warn(ctrl->device, 790 "qid %d failed to receive success1, %s %d\n", 791 chap->qid, ret < 0 ? "error" : "nvme status", ret); 792 chap->error = ret; 793 return; 794 } 795 ret = nvme_auth_receive_validate(ctrl, chap->qid, 796 chap->buf, chap->transaction, 797 NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1); 798 if (ret) { 799 chap->status = ret; 800 chap->error = NVME_SC_AUTH_REQUIRED; 801 return; 802 } 803 804 if (ctrl->ctrl_key) { 805 dev_dbg(ctrl->device, 806 "%s: qid %d controller response\n", 807 __func__, chap->qid); 808 ret = nvme_auth_dhchap_setup_ctrl_response(ctrl, chap); 809 if (ret) { 810 chap->error = ret; 811 goto fail2; 812 } 813 } 814 815 ret = nvme_auth_process_dhchap_success1(ctrl, chap); 816 if (ret) { 817 /* Controller authentication failed */ 818 chap->error = NVME_SC_AUTH_REQUIRED; 819 goto fail2; 820 } 821 822 if (ctrl->ctrl_key) { 823 /* DH-HMAC-CHAP Step 5: send success2 */ 824 dev_dbg(ctrl->device, "%s: qid %d send success2\n", 825 __func__, chap->qid); 826 tl = nvme_auth_set_dhchap_success2_data(ctrl, chap); 827 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 828 if (ret) 829 chap->error = ret; 830 } 831 if (!ret) { 832 chap->error = 0; 833 return; 834 } 835 836 fail2: 837 dev_dbg(ctrl->device, "%s: qid %d send failure2, status %x\n", 838 __func__, chap->qid, chap->status); 839 tl = nvme_auth_set_dhchap_failure2_data(ctrl, chap); 840 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 841 /* 842 * only update error if send failure2 failed and no other 843 * error had been set during authentication. 844 */ 845 if (ret && !chap->error) 846 chap->error = ret; 847 } 848 849 int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid) 850 { 851 struct nvme_dhchap_queue_context *chap; 852 853 if (!ctrl->host_key) { 854 dev_warn(ctrl->device, "qid %d: no key\n", qid); 855 return -ENOKEY; 856 } 857 858 if (ctrl->opts->dhchap_ctrl_secret && !ctrl->ctrl_key) { 859 dev_warn(ctrl->device, "qid %d: invalid ctrl key\n", qid); 860 return -ENOKEY; 861 } 862 863 mutex_lock(&ctrl->dhchap_auth_mutex); 864 /* Check if the context is already queued */ 865 list_for_each_entry(chap, &ctrl->dhchap_auth_list, entry) { 866 WARN_ON(!chap->buf); 867 if (chap->qid == qid) { 868 dev_dbg(ctrl->device, "qid %d: re-using context\n", qid); 869 mutex_unlock(&ctrl->dhchap_auth_mutex); 870 flush_work(&chap->auth_work); 871 __nvme_auth_reset(chap); 872 queue_work(nvme_wq, &chap->auth_work); 873 return 0; 874 } 875 } 876 chap = kzalloc(sizeof(*chap), GFP_KERNEL); 877 if (!chap) { 878 mutex_unlock(&ctrl->dhchap_auth_mutex); 879 return -ENOMEM; 880 } 881 chap->qid = (qid == NVME_QID_ANY) ? 0 : qid; 882 chap->ctrl = ctrl; 883 884 /* 885 * Allocate a large enough buffer for the entire negotiation: 886 * 4k should be enough to ffdhe8192. 887 */ 888 chap->buf_size = 4096; 889 chap->buf = kzalloc(chap->buf_size, GFP_KERNEL); 890 if (!chap->buf) { 891 mutex_unlock(&ctrl->dhchap_auth_mutex); 892 kfree(chap); 893 return -ENOMEM; 894 } 895 896 INIT_WORK(&chap->auth_work, __nvme_auth_work); 897 list_add(&chap->entry, &ctrl->dhchap_auth_list); 898 mutex_unlock(&ctrl->dhchap_auth_mutex); 899 queue_work(nvme_wq, &chap->auth_work); 900 return 0; 901 } 902 EXPORT_SYMBOL_GPL(nvme_auth_negotiate); 903 904 int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid) 905 { 906 struct nvme_dhchap_queue_context *chap; 907 int ret; 908 909 mutex_lock(&ctrl->dhchap_auth_mutex); 910 list_for_each_entry(chap, &ctrl->dhchap_auth_list, entry) { 911 if (chap->qid != qid) 912 continue; 913 mutex_unlock(&ctrl->dhchap_auth_mutex); 914 flush_work(&chap->auth_work); 915 ret = chap->error; 916 return ret; 917 } 918 mutex_unlock(&ctrl->dhchap_auth_mutex); 919 return -ENXIO; 920 } 921 EXPORT_SYMBOL_GPL(nvme_auth_wait); 922 923 void nvme_auth_reset(struct nvme_ctrl *ctrl) 924 { 925 struct nvme_dhchap_queue_context *chap; 926 927 mutex_lock(&ctrl->dhchap_auth_mutex); 928 list_for_each_entry(chap, &ctrl->dhchap_auth_list, entry) { 929 mutex_unlock(&ctrl->dhchap_auth_mutex); 930 flush_work(&chap->auth_work); 931 __nvme_auth_reset(chap); 932 } 933 mutex_unlock(&ctrl->dhchap_auth_mutex); 934 } 935 EXPORT_SYMBOL_GPL(nvme_auth_reset); 936 937 static void nvme_dhchap_auth_work(struct work_struct *work) 938 { 939 struct nvme_ctrl *ctrl = 940 container_of(work, struct nvme_ctrl, dhchap_auth_work); 941 int ret, q; 942 943 /* Authenticate admin queue first */ 944 ret = nvme_auth_negotiate(ctrl, 0); 945 if (ret) { 946 dev_warn(ctrl->device, 947 "qid 0: error %d setting up authentication\n", ret); 948 return; 949 } 950 ret = nvme_auth_wait(ctrl, 0); 951 if (ret) { 952 dev_warn(ctrl->device, 953 "qid 0: authentication failed\n"); 954 return; 955 } 956 957 for (q = 1; q < ctrl->queue_count; q++) { 958 ret = nvme_auth_negotiate(ctrl, q); 959 if (ret) { 960 dev_warn(ctrl->device, 961 "qid %d: error %d setting up authentication\n", 962 q, ret); 963 break; 964 } 965 } 966 967 /* 968 * Failure is a soft-state; credentials remain valid until 969 * the controller terminates the connection. 970 */ 971 } 972 973 void nvme_auth_init_ctrl(struct nvme_ctrl *ctrl) 974 { 975 INIT_LIST_HEAD(&ctrl->dhchap_auth_list); 976 INIT_WORK(&ctrl->dhchap_auth_work, nvme_dhchap_auth_work); 977 mutex_init(&ctrl->dhchap_auth_mutex); 978 if (!ctrl->opts) 979 return; 980 nvme_auth_generate_key(ctrl->opts->dhchap_secret, &ctrl->host_key); 981 nvme_auth_generate_key(ctrl->opts->dhchap_ctrl_secret, &ctrl->ctrl_key); 982 } 983 EXPORT_SYMBOL_GPL(nvme_auth_init_ctrl); 984 985 void nvme_auth_stop(struct nvme_ctrl *ctrl) 986 { 987 struct nvme_dhchap_queue_context *chap = NULL, *tmp; 988 989 cancel_work_sync(&ctrl->dhchap_auth_work); 990 mutex_lock(&ctrl->dhchap_auth_mutex); 991 list_for_each_entry_safe(chap, tmp, &ctrl->dhchap_auth_list, entry) 992 cancel_work_sync(&chap->auth_work); 993 mutex_unlock(&ctrl->dhchap_auth_mutex); 994 } 995 EXPORT_SYMBOL_GPL(nvme_auth_stop); 996 997 void nvme_auth_free(struct nvme_ctrl *ctrl) 998 { 999 struct nvme_dhchap_queue_context *chap = NULL, *tmp; 1000 1001 mutex_lock(&ctrl->dhchap_auth_mutex); 1002 list_for_each_entry_safe(chap, tmp, &ctrl->dhchap_auth_list, entry) { 1003 list_del_init(&chap->entry); 1004 flush_work(&chap->auth_work); 1005 __nvme_auth_free(chap); 1006 } 1007 mutex_unlock(&ctrl->dhchap_auth_mutex); 1008 if (ctrl->host_key) { 1009 nvme_auth_free_key(ctrl->host_key); 1010 ctrl->host_key = NULL; 1011 } 1012 if (ctrl->ctrl_key) { 1013 nvme_auth_free_key(ctrl->ctrl_key); 1014 ctrl->ctrl_key = NULL; 1015 } 1016 } 1017 EXPORT_SYMBOL_GPL(nvme_auth_free); 1018