1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (c) 2020 Hannes Reinecke, SUSE Linux 4 */ 5 6 #include <linux/crc32.h> 7 #include <linux/base64.h> 8 #include <linux/prandom.h> 9 #include <asm/unaligned.h> 10 #include <crypto/hash.h> 11 #include <crypto/dh.h> 12 #include "nvme.h" 13 #include "fabrics.h" 14 #include <linux/nvme-auth.h> 15 16 #define CHAP_BUF_SIZE 4096 17 static struct kmem_cache *nvme_chap_buf_cache; 18 static mempool_t *nvme_chap_buf_pool; 19 20 struct nvme_dhchap_queue_context { 21 struct list_head entry; 22 struct work_struct auth_work; 23 struct nvme_ctrl *ctrl; 24 struct crypto_shash *shash_tfm; 25 struct crypto_kpp *dh_tfm; 26 void *buf; 27 int qid; 28 int error; 29 u32 s1; 30 u32 s2; 31 u16 transaction; 32 u8 status; 33 u8 dhgroup_id; 34 u8 hash_id; 35 size_t hash_len; 36 u8 c1[64]; 37 u8 c2[64]; 38 u8 response[64]; 39 u8 *host_response; 40 u8 *ctrl_key; 41 u8 *host_key; 42 u8 *sess_key; 43 int ctrl_key_len; 44 int host_key_len; 45 int sess_key_len; 46 }; 47 48 static struct workqueue_struct *nvme_auth_wq; 49 50 #define nvme_auth_flags_from_qid(qid) \ 51 (qid == 0) ? 0 : BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED 52 #define nvme_auth_queue_from_qid(ctrl, qid) \ 53 (qid == 0) ? (ctrl)->fabrics_q : (ctrl)->connect_q 54 55 static inline int ctrl_max_dhchaps(struct nvme_ctrl *ctrl) 56 { 57 return ctrl->opts->nr_io_queues + ctrl->opts->nr_write_queues + 58 ctrl->opts->nr_poll_queues + 1; 59 } 60 61 static int nvme_auth_submit(struct nvme_ctrl *ctrl, int qid, 62 void *data, size_t data_len, bool auth_send) 63 { 64 struct nvme_command cmd = {}; 65 blk_mq_req_flags_t flags = nvme_auth_flags_from_qid(qid); 66 struct request_queue *q = nvme_auth_queue_from_qid(ctrl, qid); 67 int ret; 68 69 cmd.auth_common.opcode = nvme_fabrics_command; 70 cmd.auth_common.secp = NVME_AUTH_DHCHAP_PROTOCOL_IDENTIFIER; 71 cmd.auth_common.spsp0 = 0x01; 72 cmd.auth_common.spsp1 = 0x01; 73 if (auth_send) { 74 cmd.auth_send.fctype = nvme_fabrics_type_auth_send; 75 cmd.auth_send.tl = cpu_to_le32(data_len); 76 } else { 77 cmd.auth_receive.fctype = nvme_fabrics_type_auth_receive; 78 cmd.auth_receive.al = cpu_to_le32(data_len); 79 } 80 81 ret = __nvme_submit_sync_cmd(q, &cmd, NULL, data, data_len, 82 qid == 0 ? NVME_QID_ANY : qid, 83 0, flags); 84 if (ret > 0) 85 dev_warn(ctrl->device, 86 "qid %d auth_send failed with status %d\n", qid, ret); 87 else if (ret < 0) 88 dev_err(ctrl->device, 89 "qid %d auth_send failed with error %d\n", qid, ret); 90 return ret; 91 } 92 93 static int nvme_auth_receive_validate(struct nvme_ctrl *ctrl, int qid, 94 struct nvmf_auth_dhchap_failure_data *data, 95 u16 transaction, u8 expected_msg) 96 { 97 dev_dbg(ctrl->device, "%s: qid %d auth_type %d auth_id %x\n", 98 __func__, qid, data->auth_type, data->auth_id); 99 100 if (data->auth_type == NVME_AUTH_COMMON_MESSAGES && 101 data->auth_id == NVME_AUTH_DHCHAP_MESSAGE_FAILURE1) { 102 return data->rescode_exp; 103 } 104 if (data->auth_type != NVME_AUTH_DHCHAP_MESSAGES || 105 data->auth_id != expected_msg) { 106 dev_warn(ctrl->device, 107 "qid %d invalid message %02x/%02x\n", 108 qid, data->auth_type, data->auth_id); 109 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE; 110 } 111 if (le16_to_cpu(data->t_id) != transaction) { 112 dev_warn(ctrl->device, 113 "qid %d invalid transaction ID %d\n", 114 qid, le16_to_cpu(data->t_id)); 115 return NVME_AUTH_DHCHAP_FAILURE_INCORRECT_MESSAGE; 116 } 117 return 0; 118 } 119 120 static int nvme_auth_set_dhchap_negotiate_data(struct nvme_ctrl *ctrl, 121 struct nvme_dhchap_queue_context *chap) 122 { 123 struct nvmf_auth_dhchap_negotiate_data *data = chap->buf; 124 size_t size = sizeof(*data) + sizeof(union nvmf_auth_protocol); 125 126 if (size > CHAP_BUF_SIZE) { 127 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 128 return -EINVAL; 129 } 130 memset((u8 *)chap->buf, 0, size); 131 data->auth_type = NVME_AUTH_COMMON_MESSAGES; 132 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_NEGOTIATE; 133 data->t_id = cpu_to_le16(chap->transaction); 134 data->sc_c = 0; /* No secure channel concatenation */ 135 data->napd = 1; 136 data->auth_protocol[0].dhchap.authid = NVME_AUTH_DHCHAP_AUTH_ID; 137 data->auth_protocol[0].dhchap.halen = 3; 138 data->auth_protocol[0].dhchap.dhlen = 6; 139 data->auth_protocol[0].dhchap.idlist[0] = NVME_AUTH_HASH_SHA256; 140 data->auth_protocol[0].dhchap.idlist[1] = NVME_AUTH_HASH_SHA384; 141 data->auth_protocol[0].dhchap.idlist[2] = NVME_AUTH_HASH_SHA512; 142 data->auth_protocol[0].dhchap.idlist[30] = NVME_AUTH_DHGROUP_NULL; 143 data->auth_protocol[0].dhchap.idlist[31] = NVME_AUTH_DHGROUP_2048; 144 data->auth_protocol[0].dhchap.idlist[32] = NVME_AUTH_DHGROUP_3072; 145 data->auth_protocol[0].dhchap.idlist[33] = NVME_AUTH_DHGROUP_4096; 146 data->auth_protocol[0].dhchap.idlist[34] = NVME_AUTH_DHGROUP_6144; 147 data->auth_protocol[0].dhchap.idlist[35] = NVME_AUTH_DHGROUP_8192; 148 149 return size; 150 } 151 152 static int nvme_auth_process_dhchap_challenge(struct nvme_ctrl *ctrl, 153 struct nvme_dhchap_queue_context *chap) 154 { 155 struct nvmf_auth_dhchap_challenge_data *data = chap->buf; 156 u16 dhvlen = le16_to_cpu(data->dhvlen); 157 size_t size = sizeof(*data) + data->hl + dhvlen; 158 const char *gid_name = nvme_auth_dhgroup_name(data->dhgid); 159 const char *hmac_name, *kpp_name; 160 161 if (size > CHAP_BUF_SIZE) { 162 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 163 return -EINVAL; 164 } 165 166 hmac_name = nvme_auth_hmac_name(data->hashid); 167 if (!hmac_name) { 168 dev_warn(ctrl->device, 169 "qid %d: invalid HASH ID %d\n", 170 chap->qid, data->hashid); 171 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 172 return -EPROTO; 173 } 174 175 if (chap->hash_id == data->hashid && chap->shash_tfm && 176 !strcmp(crypto_shash_alg_name(chap->shash_tfm), hmac_name) && 177 crypto_shash_digestsize(chap->shash_tfm) == data->hl) { 178 dev_dbg(ctrl->device, 179 "qid %d: reuse existing hash %s\n", 180 chap->qid, hmac_name); 181 goto select_kpp; 182 } 183 184 /* Reset if hash cannot be reused */ 185 if (chap->shash_tfm) { 186 crypto_free_shash(chap->shash_tfm); 187 chap->hash_id = 0; 188 chap->hash_len = 0; 189 } 190 chap->shash_tfm = crypto_alloc_shash(hmac_name, 0, 191 CRYPTO_ALG_ALLOCATES_MEMORY); 192 if (IS_ERR(chap->shash_tfm)) { 193 dev_warn(ctrl->device, 194 "qid %d: failed to allocate hash %s, error %ld\n", 195 chap->qid, hmac_name, PTR_ERR(chap->shash_tfm)); 196 chap->shash_tfm = NULL; 197 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 198 return -ENOMEM; 199 } 200 201 if (crypto_shash_digestsize(chap->shash_tfm) != data->hl) { 202 dev_warn(ctrl->device, 203 "qid %d: invalid hash length %d\n", 204 chap->qid, data->hl); 205 crypto_free_shash(chap->shash_tfm); 206 chap->shash_tfm = NULL; 207 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 208 return -EPROTO; 209 } 210 211 chap->hash_id = data->hashid; 212 chap->hash_len = data->hl; 213 dev_dbg(ctrl->device, "qid %d: selected hash %s\n", 214 chap->qid, hmac_name); 215 216 select_kpp: 217 kpp_name = nvme_auth_dhgroup_kpp(data->dhgid); 218 if (!kpp_name) { 219 dev_warn(ctrl->device, 220 "qid %d: invalid DH group id %d\n", 221 chap->qid, data->dhgid); 222 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 223 /* Leave previous dh_tfm intact */ 224 return -EPROTO; 225 } 226 227 if (chap->dhgroup_id == data->dhgid && 228 (data->dhgid == NVME_AUTH_DHGROUP_NULL || chap->dh_tfm)) { 229 dev_dbg(ctrl->device, 230 "qid %d: reuse existing DH group %s\n", 231 chap->qid, gid_name); 232 goto skip_kpp; 233 } 234 235 /* Reset dh_tfm if it can't be reused */ 236 if (chap->dh_tfm) { 237 crypto_free_kpp(chap->dh_tfm); 238 chap->dh_tfm = NULL; 239 } 240 241 if (data->dhgid != NVME_AUTH_DHGROUP_NULL) { 242 if (dhvlen == 0) { 243 dev_warn(ctrl->device, 244 "qid %d: empty DH value\n", 245 chap->qid); 246 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 247 return -EPROTO; 248 } 249 250 chap->dh_tfm = crypto_alloc_kpp(kpp_name, 0, 0); 251 if (IS_ERR(chap->dh_tfm)) { 252 int ret = PTR_ERR(chap->dh_tfm); 253 254 dev_warn(ctrl->device, 255 "qid %d: error %d initializing DH group %s\n", 256 chap->qid, ret, gid_name); 257 chap->status = NVME_AUTH_DHCHAP_FAILURE_DHGROUP_UNUSABLE; 258 chap->dh_tfm = NULL; 259 return ret; 260 } 261 dev_dbg(ctrl->device, "qid %d: selected DH group %s\n", 262 chap->qid, gid_name); 263 } else if (dhvlen != 0) { 264 dev_warn(ctrl->device, 265 "qid %d: invalid DH value for NULL DH\n", 266 chap->qid); 267 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 268 return -EPROTO; 269 } 270 chap->dhgroup_id = data->dhgid; 271 272 skip_kpp: 273 chap->s1 = le32_to_cpu(data->seqnum); 274 memcpy(chap->c1, data->cval, chap->hash_len); 275 if (dhvlen) { 276 chap->ctrl_key = kmalloc(dhvlen, GFP_KERNEL); 277 if (!chap->ctrl_key) { 278 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 279 return -ENOMEM; 280 } 281 chap->ctrl_key_len = dhvlen; 282 memcpy(chap->ctrl_key, data->cval + chap->hash_len, 283 dhvlen); 284 dev_dbg(ctrl->device, "ctrl public key %*ph\n", 285 (int)chap->ctrl_key_len, chap->ctrl_key); 286 } 287 288 return 0; 289 } 290 291 static int nvme_auth_set_dhchap_reply_data(struct nvme_ctrl *ctrl, 292 struct nvme_dhchap_queue_context *chap) 293 { 294 struct nvmf_auth_dhchap_reply_data *data = chap->buf; 295 size_t size = sizeof(*data); 296 297 size += 2 * chap->hash_len; 298 299 if (chap->host_key_len) 300 size += chap->host_key_len; 301 302 if (size > CHAP_BUF_SIZE) { 303 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 304 return -EINVAL; 305 } 306 307 memset(chap->buf, 0, size); 308 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES; 309 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_REPLY; 310 data->t_id = cpu_to_le16(chap->transaction); 311 data->hl = chap->hash_len; 312 data->dhvlen = cpu_to_le16(chap->host_key_len); 313 memcpy(data->rval, chap->response, chap->hash_len); 314 if (ctrl->ctrl_key) { 315 get_random_bytes(chap->c2, chap->hash_len); 316 data->cvalid = 1; 317 chap->s2 = nvme_auth_get_seqnum(); 318 memcpy(data->rval + chap->hash_len, chap->c2, 319 chap->hash_len); 320 dev_dbg(ctrl->device, "%s: qid %d ctrl challenge %*ph\n", 321 __func__, chap->qid, (int)chap->hash_len, chap->c2); 322 } else { 323 memset(chap->c2, 0, chap->hash_len); 324 chap->s2 = 0; 325 } 326 data->seqnum = cpu_to_le32(chap->s2); 327 if (chap->host_key_len) { 328 dev_dbg(ctrl->device, "%s: qid %d host public key %*ph\n", 329 __func__, chap->qid, 330 chap->host_key_len, chap->host_key); 331 memcpy(data->rval + 2 * chap->hash_len, chap->host_key, 332 chap->host_key_len); 333 } 334 335 return size; 336 } 337 338 static int nvme_auth_process_dhchap_success1(struct nvme_ctrl *ctrl, 339 struct nvme_dhchap_queue_context *chap) 340 { 341 struct nvmf_auth_dhchap_success1_data *data = chap->buf; 342 size_t size = sizeof(*data); 343 344 if (chap->ctrl_key) 345 size += chap->hash_len; 346 347 if (size > CHAP_BUF_SIZE) { 348 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 349 return -EINVAL; 350 } 351 352 if (data->hl != chap->hash_len) { 353 dev_warn(ctrl->device, 354 "qid %d: invalid hash length %u\n", 355 chap->qid, data->hl); 356 chap->status = NVME_AUTH_DHCHAP_FAILURE_HASH_UNUSABLE; 357 return -EPROTO; 358 } 359 360 /* Just print out information for the admin queue */ 361 if (chap->qid == 0) 362 dev_info(ctrl->device, 363 "qid 0: authenticated with hash %s dhgroup %s\n", 364 nvme_auth_hmac_name(chap->hash_id), 365 nvme_auth_dhgroup_name(chap->dhgroup_id)); 366 367 if (!data->rvalid) 368 return 0; 369 370 /* Validate controller response */ 371 if (memcmp(chap->response, data->rval, data->hl)) { 372 dev_dbg(ctrl->device, "%s: qid %d ctrl response %*ph\n", 373 __func__, chap->qid, (int)chap->hash_len, data->rval); 374 dev_dbg(ctrl->device, "%s: qid %d host response %*ph\n", 375 __func__, chap->qid, (int)chap->hash_len, 376 chap->response); 377 dev_warn(ctrl->device, 378 "qid %d: controller authentication failed\n", 379 chap->qid); 380 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 381 return -ECONNREFUSED; 382 } 383 384 /* Just print out information for the admin queue */ 385 if (chap->qid == 0) 386 dev_info(ctrl->device, 387 "qid 0: controller authenticated\n"); 388 return 0; 389 } 390 391 static int nvme_auth_set_dhchap_success2_data(struct nvme_ctrl *ctrl, 392 struct nvme_dhchap_queue_context *chap) 393 { 394 struct nvmf_auth_dhchap_success2_data *data = chap->buf; 395 size_t size = sizeof(*data); 396 397 memset(chap->buf, 0, size); 398 data->auth_type = NVME_AUTH_DHCHAP_MESSAGES; 399 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_SUCCESS2; 400 data->t_id = cpu_to_le16(chap->transaction); 401 402 return size; 403 } 404 405 static int nvme_auth_set_dhchap_failure2_data(struct nvme_ctrl *ctrl, 406 struct nvme_dhchap_queue_context *chap) 407 { 408 struct nvmf_auth_dhchap_failure_data *data = chap->buf; 409 size_t size = sizeof(*data); 410 411 memset(chap->buf, 0, size); 412 data->auth_type = NVME_AUTH_COMMON_MESSAGES; 413 data->auth_id = NVME_AUTH_DHCHAP_MESSAGE_FAILURE2; 414 data->t_id = cpu_to_le16(chap->transaction); 415 data->rescode = NVME_AUTH_DHCHAP_FAILURE_REASON_FAILED; 416 data->rescode_exp = chap->status; 417 418 return size; 419 } 420 421 static int nvme_auth_dhchap_setup_host_response(struct nvme_ctrl *ctrl, 422 struct nvme_dhchap_queue_context *chap) 423 { 424 SHASH_DESC_ON_STACK(shash, chap->shash_tfm); 425 u8 buf[4], *challenge = chap->c1; 426 int ret; 427 428 dev_dbg(ctrl->device, "%s: qid %d host response seq %u transaction %d\n", 429 __func__, chap->qid, chap->s1, chap->transaction); 430 431 if (!chap->host_response) { 432 chap->host_response = nvme_auth_transform_key(ctrl->host_key, 433 ctrl->opts->host->nqn); 434 if (IS_ERR(chap->host_response)) { 435 ret = PTR_ERR(chap->host_response); 436 chap->host_response = NULL; 437 return ret; 438 } 439 } else { 440 dev_dbg(ctrl->device, "%s: qid %d re-using host response\n", 441 __func__, chap->qid); 442 } 443 444 ret = crypto_shash_setkey(chap->shash_tfm, 445 chap->host_response, ctrl->host_key->len); 446 if (ret) { 447 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n", 448 chap->qid, ret); 449 goto out; 450 } 451 452 if (chap->dh_tfm) { 453 challenge = kmalloc(chap->hash_len, GFP_KERNEL); 454 if (!challenge) { 455 ret = -ENOMEM; 456 goto out; 457 } 458 ret = nvme_auth_augmented_challenge(chap->hash_id, 459 chap->sess_key, 460 chap->sess_key_len, 461 chap->c1, challenge, 462 chap->hash_len); 463 if (ret) 464 goto out; 465 } 466 467 shash->tfm = chap->shash_tfm; 468 ret = crypto_shash_init(shash); 469 if (ret) 470 goto out; 471 ret = crypto_shash_update(shash, challenge, chap->hash_len); 472 if (ret) 473 goto out; 474 put_unaligned_le32(chap->s1, buf); 475 ret = crypto_shash_update(shash, buf, 4); 476 if (ret) 477 goto out; 478 put_unaligned_le16(chap->transaction, buf); 479 ret = crypto_shash_update(shash, buf, 2); 480 if (ret) 481 goto out; 482 memset(buf, 0, sizeof(buf)); 483 ret = crypto_shash_update(shash, buf, 1); 484 if (ret) 485 goto out; 486 ret = crypto_shash_update(shash, "HostHost", 8); 487 if (ret) 488 goto out; 489 ret = crypto_shash_update(shash, ctrl->opts->host->nqn, 490 strlen(ctrl->opts->host->nqn)); 491 if (ret) 492 goto out; 493 ret = crypto_shash_update(shash, buf, 1); 494 if (ret) 495 goto out; 496 ret = crypto_shash_update(shash, ctrl->opts->subsysnqn, 497 strlen(ctrl->opts->subsysnqn)); 498 if (ret) 499 goto out; 500 ret = crypto_shash_final(shash, chap->response); 501 out: 502 if (challenge != chap->c1) 503 kfree(challenge); 504 return ret; 505 } 506 507 static int nvme_auth_dhchap_setup_ctrl_response(struct nvme_ctrl *ctrl, 508 struct nvme_dhchap_queue_context *chap) 509 { 510 SHASH_DESC_ON_STACK(shash, chap->shash_tfm); 511 u8 *ctrl_response; 512 u8 buf[4], *challenge = chap->c2; 513 int ret; 514 515 ctrl_response = nvme_auth_transform_key(ctrl->ctrl_key, 516 ctrl->opts->subsysnqn); 517 if (IS_ERR(ctrl_response)) { 518 ret = PTR_ERR(ctrl_response); 519 return ret; 520 } 521 522 ret = crypto_shash_setkey(chap->shash_tfm, 523 ctrl_response, ctrl->ctrl_key->len); 524 if (ret) { 525 dev_warn(ctrl->device, "qid %d: failed to set key, error %d\n", 526 chap->qid, ret); 527 goto out; 528 } 529 530 if (chap->dh_tfm) { 531 challenge = kmalloc(chap->hash_len, GFP_KERNEL); 532 if (!challenge) { 533 ret = -ENOMEM; 534 goto out; 535 } 536 ret = nvme_auth_augmented_challenge(chap->hash_id, 537 chap->sess_key, 538 chap->sess_key_len, 539 chap->c2, challenge, 540 chap->hash_len); 541 if (ret) 542 goto out; 543 } 544 dev_dbg(ctrl->device, "%s: qid %d ctrl response seq %u transaction %d\n", 545 __func__, chap->qid, chap->s2, chap->transaction); 546 dev_dbg(ctrl->device, "%s: qid %d challenge %*ph\n", 547 __func__, chap->qid, (int)chap->hash_len, challenge); 548 dev_dbg(ctrl->device, "%s: qid %d subsysnqn %s\n", 549 __func__, chap->qid, ctrl->opts->subsysnqn); 550 dev_dbg(ctrl->device, "%s: qid %d hostnqn %s\n", 551 __func__, chap->qid, ctrl->opts->host->nqn); 552 shash->tfm = chap->shash_tfm; 553 ret = crypto_shash_init(shash); 554 if (ret) 555 goto out; 556 ret = crypto_shash_update(shash, challenge, chap->hash_len); 557 if (ret) 558 goto out; 559 put_unaligned_le32(chap->s2, buf); 560 ret = crypto_shash_update(shash, buf, 4); 561 if (ret) 562 goto out; 563 put_unaligned_le16(chap->transaction, buf); 564 ret = crypto_shash_update(shash, buf, 2); 565 if (ret) 566 goto out; 567 memset(buf, 0, 4); 568 ret = crypto_shash_update(shash, buf, 1); 569 if (ret) 570 goto out; 571 ret = crypto_shash_update(shash, "Controller", 10); 572 if (ret) 573 goto out; 574 ret = crypto_shash_update(shash, ctrl->opts->subsysnqn, 575 strlen(ctrl->opts->subsysnqn)); 576 if (ret) 577 goto out; 578 ret = crypto_shash_update(shash, buf, 1); 579 if (ret) 580 goto out; 581 ret = crypto_shash_update(shash, ctrl->opts->host->nqn, 582 strlen(ctrl->opts->host->nqn)); 583 if (ret) 584 goto out; 585 ret = crypto_shash_final(shash, chap->response); 586 out: 587 if (challenge != chap->c2) 588 kfree(challenge); 589 kfree(ctrl_response); 590 return ret; 591 } 592 593 static int nvme_auth_dhchap_exponential(struct nvme_ctrl *ctrl, 594 struct nvme_dhchap_queue_context *chap) 595 { 596 int ret; 597 598 if (chap->host_key && chap->host_key_len) { 599 dev_dbg(ctrl->device, 600 "qid %d: reusing host key\n", chap->qid); 601 goto gen_sesskey; 602 } 603 ret = nvme_auth_gen_privkey(chap->dh_tfm, chap->dhgroup_id); 604 if (ret < 0) { 605 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 606 return ret; 607 } 608 609 chap->host_key_len = crypto_kpp_maxsize(chap->dh_tfm); 610 611 chap->host_key = kzalloc(chap->host_key_len, GFP_KERNEL); 612 if (!chap->host_key) { 613 chap->host_key_len = 0; 614 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 615 return -ENOMEM; 616 } 617 ret = nvme_auth_gen_pubkey(chap->dh_tfm, 618 chap->host_key, chap->host_key_len); 619 if (ret) { 620 dev_dbg(ctrl->device, 621 "failed to generate public key, error %d\n", ret); 622 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 623 return ret; 624 } 625 626 gen_sesskey: 627 chap->sess_key_len = chap->host_key_len; 628 chap->sess_key = kmalloc(chap->sess_key_len, GFP_KERNEL); 629 if (!chap->sess_key) { 630 chap->sess_key_len = 0; 631 chap->status = NVME_AUTH_DHCHAP_FAILURE_FAILED; 632 return -ENOMEM; 633 } 634 635 ret = nvme_auth_gen_shared_secret(chap->dh_tfm, 636 chap->ctrl_key, chap->ctrl_key_len, 637 chap->sess_key, chap->sess_key_len); 638 if (ret) { 639 dev_dbg(ctrl->device, 640 "failed to generate shared secret, error %d\n", ret); 641 chap->status = NVME_AUTH_DHCHAP_FAILURE_INCORRECT_PAYLOAD; 642 return ret; 643 } 644 dev_dbg(ctrl->device, "shared secret %*ph\n", 645 (int)chap->sess_key_len, chap->sess_key); 646 return 0; 647 } 648 649 static void nvme_auth_reset_dhchap(struct nvme_dhchap_queue_context *chap) 650 { 651 kfree_sensitive(chap->host_response); 652 chap->host_response = NULL; 653 kfree_sensitive(chap->host_key); 654 chap->host_key = NULL; 655 chap->host_key_len = 0; 656 kfree_sensitive(chap->ctrl_key); 657 chap->ctrl_key = NULL; 658 chap->ctrl_key_len = 0; 659 kfree_sensitive(chap->sess_key); 660 chap->sess_key = NULL; 661 chap->sess_key_len = 0; 662 chap->status = 0; 663 chap->error = 0; 664 chap->s1 = 0; 665 chap->s2 = 0; 666 chap->transaction = 0; 667 memset(chap->c1, 0, sizeof(chap->c1)); 668 memset(chap->c2, 0, sizeof(chap->c2)); 669 mempool_free(chap->buf, nvme_chap_buf_pool); 670 chap->buf = NULL; 671 } 672 673 static void nvme_auth_free_dhchap(struct nvme_dhchap_queue_context *chap) 674 { 675 nvme_auth_reset_dhchap(chap); 676 if (chap->shash_tfm) 677 crypto_free_shash(chap->shash_tfm); 678 if (chap->dh_tfm) 679 crypto_free_kpp(chap->dh_tfm); 680 } 681 682 static void nvme_queue_auth_work(struct work_struct *work) 683 { 684 struct nvme_dhchap_queue_context *chap = 685 container_of(work, struct nvme_dhchap_queue_context, auth_work); 686 struct nvme_ctrl *ctrl = chap->ctrl; 687 size_t tl; 688 int ret = 0; 689 690 /* 691 * Allocate a large enough buffer for the entire negotiation: 692 * 4k is enough to ffdhe8192. 693 */ 694 chap->buf = mempool_alloc(nvme_chap_buf_pool, GFP_KERNEL); 695 if (!chap->buf) { 696 chap->error = -ENOMEM; 697 return; 698 } 699 700 chap->transaction = ctrl->transaction++; 701 702 /* DH-HMAC-CHAP Step 1: send negotiate */ 703 dev_dbg(ctrl->device, "%s: qid %d send negotiate\n", 704 __func__, chap->qid); 705 ret = nvme_auth_set_dhchap_negotiate_data(ctrl, chap); 706 if (ret < 0) { 707 chap->error = ret; 708 return; 709 } 710 tl = ret; 711 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 712 if (ret) { 713 chap->error = ret; 714 return; 715 } 716 717 /* DH-HMAC-CHAP Step 2: receive challenge */ 718 dev_dbg(ctrl->device, "%s: qid %d receive challenge\n", 719 __func__, chap->qid); 720 721 memset(chap->buf, 0, CHAP_BUF_SIZE); 722 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE, 723 false); 724 if (ret) { 725 dev_warn(ctrl->device, 726 "qid %d failed to receive challenge, %s %d\n", 727 chap->qid, ret < 0 ? "error" : "nvme status", ret); 728 chap->error = ret; 729 return; 730 } 731 ret = nvme_auth_receive_validate(ctrl, chap->qid, chap->buf, chap->transaction, 732 NVME_AUTH_DHCHAP_MESSAGE_CHALLENGE); 733 if (ret) { 734 chap->status = ret; 735 chap->error = -ECONNREFUSED; 736 return; 737 } 738 739 ret = nvme_auth_process_dhchap_challenge(ctrl, chap); 740 if (ret) { 741 /* Invalid challenge parameters */ 742 chap->error = ret; 743 goto fail2; 744 } 745 746 if (chap->ctrl_key_len) { 747 dev_dbg(ctrl->device, 748 "%s: qid %d DH exponential\n", 749 __func__, chap->qid); 750 ret = nvme_auth_dhchap_exponential(ctrl, chap); 751 if (ret) { 752 chap->error = ret; 753 goto fail2; 754 } 755 } 756 757 dev_dbg(ctrl->device, "%s: qid %d host response\n", 758 __func__, chap->qid); 759 mutex_lock(&ctrl->dhchap_auth_mutex); 760 ret = nvme_auth_dhchap_setup_host_response(ctrl, chap); 761 if (ret) { 762 mutex_unlock(&ctrl->dhchap_auth_mutex); 763 chap->error = ret; 764 goto fail2; 765 } 766 mutex_unlock(&ctrl->dhchap_auth_mutex); 767 768 /* DH-HMAC-CHAP Step 3: send reply */ 769 dev_dbg(ctrl->device, "%s: qid %d send reply\n", 770 __func__, chap->qid); 771 ret = nvme_auth_set_dhchap_reply_data(ctrl, chap); 772 if (ret < 0) { 773 chap->error = ret; 774 goto fail2; 775 } 776 777 tl = ret; 778 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 779 if (ret) { 780 chap->error = ret; 781 goto fail2; 782 } 783 784 /* DH-HMAC-CHAP Step 4: receive success1 */ 785 dev_dbg(ctrl->device, "%s: qid %d receive success1\n", 786 __func__, chap->qid); 787 788 memset(chap->buf, 0, CHAP_BUF_SIZE); 789 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, CHAP_BUF_SIZE, 790 false); 791 if (ret) { 792 dev_warn(ctrl->device, 793 "qid %d failed to receive success1, %s %d\n", 794 chap->qid, ret < 0 ? "error" : "nvme status", ret); 795 chap->error = ret; 796 return; 797 } 798 ret = nvme_auth_receive_validate(ctrl, chap->qid, 799 chap->buf, chap->transaction, 800 NVME_AUTH_DHCHAP_MESSAGE_SUCCESS1); 801 if (ret) { 802 chap->status = ret; 803 chap->error = -ECONNREFUSED; 804 return; 805 } 806 807 mutex_lock(&ctrl->dhchap_auth_mutex); 808 if (ctrl->ctrl_key) { 809 dev_dbg(ctrl->device, 810 "%s: qid %d controller response\n", 811 __func__, chap->qid); 812 ret = nvme_auth_dhchap_setup_ctrl_response(ctrl, chap); 813 if (ret) { 814 mutex_unlock(&ctrl->dhchap_auth_mutex); 815 chap->error = ret; 816 goto fail2; 817 } 818 } 819 mutex_unlock(&ctrl->dhchap_auth_mutex); 820 821 ret = nvme_auth_process_dhchap_success1(ctrl, chap); 822 if (ret) { 823 /* Controller authentication failed */ 824 chap->error = -ECONNREFUSED; 825 goto fail2; 826 } 827 828 if (chap->ctrl_key) { 829 /* DH-HMAC-CHAP Step 5: send success2 */ 830 dev_dbg(ctrl->device, "%s: qid %d send success2\n", 831 __func__, chap->qid); 832 tl = nvme_auth_set_dhchap_success2_data(ctrl, chap); 833 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 834 if (ret) 835 chap->error = ret; 836 } 837 if (!ret) { 838 chap->error = 0; 839 return; 840 } 841 842 fail2: 843 dev_dbg(ctrl->device, "%s: qid %d send failure2, status %x\n", 844 __func__, chap->qid, chap->status); 845 tl = nvme_auth_set_dhchap_failure2_data(ctrl, chap); 846 ret = nvme_auth_submit(ctrl, chap->qid, chap->buf, tl, true); 847 /* 848 * only update error if send failure2 failed and no other 849 * error had been set during authentication. 850 */ 851 if (ret && !chap->error) 852 chap->error = ret; 853 } 854 855 int nvme_auth_negotiate(struct nvme_ctrl *ctrl, int qid) 856 { 857 struct nvme_dhchap_queue_context *chap; 858 859 if (!ctrl->host_key) { 860 dev_warn(ctrl->device, "qid %d: no key\n", qid); 861 return -ENOKEY; 862 } 863 864 if (ctrl->opts->dhchap_ctrl_secret && !ctrl->ctrl_key) { 865 dev_warn(ctrl->device, "qid %d: invalid ctrl key\n", qid); 866 return -ENOKEY; 867 } 868 869 chap = &ctrl->dhchap_ctxs[qid]; 870 cancel_work_sync(&chap->auth_work); 871 queue_work(nvme_auth_wq, &chap->auth_work); 872 return 0; 873 } 874 EXPORT_SYMBOL_GPL(nvme_auth_negotiate); 875 876 int nvme_auth_wait(struct nvme_ctrl *ctrl, int qid) 877 { 878 struct nvme_dhchap_queue_context *chap; 879 int ret; 880 881 chap = &ctrl->dhchap_ctxs[qid]; 882 flush_work(&chap->auth_work); 883 ret = chap->error; 884 /* clear sensitive info */ 885 nvme_auth_reset_dhchap(chap); 886 return ret; 887 } 888 EXPORT_SYMBOL_GPL(nvme_auth_wait); 889 890 static void nvme_ctrl_auth_work(struct work_struct *work) 891 { 892 struct nvme_ctrl *ctrl = 893 container_of(work, struct nvme_ctrl, dhchap_auth_work); 894 int ret, q; 895 896 /* 897 * If the ctrl is no connected, bail as reconnect will handle 898 * authentication. 899 */ 900 if (ctrl->state != NVME_CTRL_LIVE) 901 return; 902 903 /* Authenticate admin queue first */ 904 ret = nvme_auth_negotiate(ctrl, 0); 905 if (ret) { 906 dev_warn(ctrl->device, 907 "qid 0: error %d setting up authentication\n", ret); 908 return; 909 } 910 ret = nvme_auth_wait(ctrl, 0); 911 if (ret) { 912 dev_warn(ctrl->device, 913 "qid 0: authentication failed\n"); 914 return; 915 } 916 917 for (q = 1; q < ctrl->queue_count; q++) { 918 ret = nvme_auth_negotiate(ctrl, q); 919 if (ret) { 920 dev_warn(ctrl->device, 921 "qid %d: error %d setting up authentication\n", 922 q, ret); 923 break; 924 } 925 } 926 927 /* 928 * Failure is a soft-state; credentials remain valid until 929 * the controller terminates the connection. 930 */ 931 for (q = 1; q < ctrl->queue_count; q++) { 932 ret = nvme_auth_wait(ctrl, q); 933 if (ret) 934 dev_warn(ctrl->device, 935 "qid %d: authentication failed\n", q); 936 } 937 } 938 939 int nvme_auth_init_ctrl(struct nvme_ctrl *ctrl) 940 { 941 struct nvme_dhchap_queue_context *chap; 942 int i, ret; 943 944 mutex_init(&ctrl->dhchap_auth_mutex); 945 INIT_WORK(&ctrl->dhchap_auth_work, nvme_ctrl_auth_work); 946 if (!ctrl->opts) 947 return 0; 948 ret = nvme_auth_generate_key(ctrl->opts->dhchap_secret, 949 &ctrl->host_key); 950 if (ret) 951 return ret; 952 ret = nvme_auth_generate_key(ctrl->opts->dhchap_ctrl_secret, 953 &ctrl->ctrl_key); 954 if (ret) 955 goto err_free_dhchap_secret; 956 957 if (!ctrl->opts->dhchap_secret && !ctrl->opts->dhchap_ctrl_secret) 958 return 0; 959 960 ctrl->dhchap_ctxs = kvcalloc(ctrl_max_dhchaps(ctrl), 961 sizeof(*chap), GFP_KERNEL); 962 if (!ctrl->dhchap_ctxs) { 963 ret = -ENOMEM; 964 goto err_free_dhchap_ctrl_secret; 965 } 966 967 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) { 968 chap = &ctrl->dhchap_ctxs[i]; 969 chap->qid = i; 970 chap->ctrl = ctrl; 971 INIT_WORK(&chap->auth_work, nvme_queue_auth_work); 972 } 973 974 return 0; 975 err_free_dhchap_ctrl_secret: 976 nvme_auth_free_key(ctrl->ctrl_key); 977 ctrl->ctrl_key = NULL; 978 err_free_dhchap_secret: 979 nvme_auth_free_key(ctrl->host_key); 980 ctrl->host_key = NULL; 981 return ret; 982 } 983 EXPORT_SYMBOL_GPL(nvme_auth_init_ctrl); 984 985 void nvme_auth_stop(struct nvme_ctrl *ctrl) 986 { 987 cancel_work_sync(&ctrl->dhchap_auth_work); 988 } 989 EXPORT_SYMBOL_GPL(nvme_auth_stop); 990 991 void nvme_auth_free(struct nvme_ctrl *ctrl) 992 { 993 int i; 994 995 if (ctrl->dhchap_ctxs) { 996 for (i = 0; i < ctrl_max_dhchaps(ctrl); i++) 997 nvme_auth_free_dhchap(&ctrl->dhchap_ctxs[i]); 998 kfree(ctrl->dhchap_ctxs); 999 } 1000 if (ctrl->host_key) { 1001 nvme_auth_free_key(ctrl->host_key); 1002 ctrl->host_key = NULL; 1003 } 1004 if (ctrl->ctrl_key) { 1005 nvme_auth_free_key(ctrl->ctrl_key); 1006 ctrl->ctrl_key = NULL; 1007 } 1008 } 1009 EXPORT_SYMBOL_GPL(nvme_auth_free); 1010 1011 int __init nvme_init_auth(void) 1012 { 1013 nvme_auth_wq = alloc_workqueue("nvme-auth-wq", 1014 WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0); 1015 if (!nvme_auth_wq) 1016 return -ENOMEM; 1017 1018 nvme_chap_buf_cache = kmem_cache_create("nvme-chap-buf-cache", 1019 CHAP_BUF_SIZE, 0, SLAB_HWCACHE_ALIGN, NULL); 1020 if (!nvme_chap_buf_cache) 1021 goto err_destroy_workqueue; 1022 1023 nvme_chap_buf_pool = mempool_create(16, mempool_alloc_slab, 1024 mempool_free_slab, nvme_chap_buf_cache); 1025 if (!nvme_chap_buf_pool) 1026 goto err_destroy_chap_buf_cache; 1027 1028 return 0; 1029 err_destroy_chap_buf_cache: 1030 kmem_cache_destroy(nvme_chap_buf_cache); 1031 err_destroy_workqueue: 1032 destroy_workqueue(nvme_auth_wq); 1033 return -ENOMEM; 1034 } 1035 1036 void __exit nvme_exit_auth(void) 1037 { 1038 mempool_destroy(nvme_chap_buf_pool); 1039 kmem_cache_destroy(nvme_chap_buf_cache); 1040 destroy_workqueue(nvme_auth_wq); 1041 } 1042