1 /* 2 * Virtio crypto Support 3 * 4 * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD. 5 * 6 * Authors: 7 * Gonglei <arei.gonglei@huawei.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or 10 * (at your option) any later version. See the COPYING file in the 11 * top-level directory. 12 */ 13 #include "qemu/osdep.h" 14 #include "qemu/iov.h" 15 #include "hw/qdev.h" 16 #include "qapi/error.h" 17 #include "qemu/error-report.h" 18 19 #include "hw/virtio/virtio.h" 20 #include "hw/virtio/virtio-crypto.h" 21 #include "hw/virtio/virtio-access.h" 22 #include "standard-headers/linux/virtio_ids.h" 23 #include "sysemu/cryptodev-vhost.h" 24 25 #define VIRTIO_CRYPTO_VM_VERSION 1 26 27 /* 28 * Transfer virtqueue index to crypto queue index. 29 * The control virtqueue is after the data virtqueues 30 * so the input value doesn't need to be adjusted 31 */ 32 static inline int virtio_crypto_vq2q(int queue_index) 33 { 34 return queue_index; 35 } 36 37 static int 38 virtio_crypto_cipher_session_helper(VirtIODevice *vdev, 39 CryptoDevBackendSymSessionInfo *info, 40 struct virtio_crypto_cipher_session_para *cipher_para, 41 struct iovec **iov, unsigned int *out_num) 42 { 43 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 44 unsigned int num = *out_num; 45 46 info->cipher_alg = ldl_le_p(&cipher_para->algo); 47 info->key_len = ldl_le_p(&cipher_para->keylen); 48 info->direction = ldl_le_p(&cipher_para->op); 49 DPRINTF("cipher_alg=%" PRIu32 ", info->direction=%" PRIu32 "\n", 50 info->cipher_alg, info->direction); 51 52 if (info->key_len > vcrypto->conf.max_cipher_key_len) { 53 error_report("virtio-crypto length of cipher key is too big: %u", 54 info->key_len); 55 return -VIRTIO_CRYPTO_ERR; 56 } 57 /* Get cipher key */ 58 if (info->key_len > 0) { 59 size_t s; 60 DPRINTF("keylen=%" PRIu32 "\n", info->key_len); 61 62 info->cipher_key = g_malloc(info->key_len); 63 s = iov_to_buf(*iov, num, 0, info->cipher_key, info->key_len); 64 if (unlikely(s != info->key_len)) { 65 virtio_error(vdev, "virtio-crypto cipher key incorrect"); 66 return -EFAULT; 67 } 68 iov_discard_front(iov, &num, info->key_len); 69 *out_num = num; 70 } 71 72 return 0; 73 } 74 75 static int64_t 76 virtio_crypto_create_sym_session(VirtIOCrypto *vcrypto, 77 struct virtio_crypto_sym_create_session_req *sess_req, 78 uint32_t queue_id, 79 uint32_t opcode, 80 struct iovec *iov, unsigned int out_num) 81 { 82 VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); 83 CryptoDevBackendSymSessionInfo info; 84 int64_t session_id; 85 int queue_index; 86 uint32_t op_type; 87 Error *local_err = NULL; 88 int ret; 89 90 memset(&info, 0, sizeof(info)); 91 op_type = ldl_le_p(&sess_req->op_type); 92 info.op_type = op_type; 93 info.op_code = opcode; 94 95 if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) { 96 ret = virtio_crypto_cipher_session_helper(vdev, &info, 97 &sess_req->u.cipher.para, 98 &iov, &out_num); 99 if (ret < 0) { 100 goto err; 101 } 102 } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) { 103 size_t s; 104 /* cipher part */ 105 ret = virtio_crypto_cipher_session_helper(vdev, &info, 106 &sess_req->u.chain.para.cipher_param, 107 &iov, &out_num); 108 if (ret < 0) { 109 goto err; 110 } 111 /* hash part */ 112 info.alg_chain_order = ldl_le_p( 113 &sess_req->u.chain.para.alg_chain_order); 114 info.add_len = ldl_le_p(&sess_req->u.chain.para.aad_len); 115 info.hash_mode = ldl_le_p(&sess_req->u.chain.para.hash_mode); 116 if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH) { 117 info.hash_alg = ldl_le_p(&sess_req->u.chain.para.u.mac_param.algo); 118 info.auth_key_len = ldl_le_p( 119 &sess_req->u.chain.para.u.mac_param.auth_key_len); 120 info.hash_result_len = ldl_le_p( 121 &sess_req->u.chain.para.u.mac_param.hash_result_len); 122 if (info.auth_key_len > vcrypto->conf.max_auth_key_len) { 123 error_report("virtio-crypto length of auth key is too big: %u", 124 info.auth_key_len); 125 ret = -VIRTIO_CRYPTO_ERR; 126 goto err; 127 } 128 /* get auth key */ 129 if (info.auth_key_len > 0) { 130 DPRINTF("auth_keylen=%" PRIu32 "\n", info.auth_key_len); 131 info.auth_key = g_malloc(info.auth_key_len); 132 s = iov_to_buf(iov, out_num, 0, info.auth_key, 133 info.auth_key_len); 134 if (unlikely(s != info.auth_key_len)) { 135 virtio_error(vdev, 136 "virtio-crypto authenticated key incorrect"); 137 ret = -EFAULT; 138 goto err; 139 } 140 iov_discard_front(&iov, &out_num, info.auth_key_len); 141 } 142 } else if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN) { 143 info.hash_alg = ldl_le_p( 144 &sess_req->u.chain.para.u.hash_param.algo); 145 info.hash_result_len = ldl_le_p( 146 &sess_req->u.chain.para.u.hash_param.hash_result_len); 147 } else { 148 /* VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */ 149 error_report("unsupported hash mode"); 150 ret = -VIRTIO_CRYPTO_NOTSUPP; 151 goto err; 152 } 153 } else { 154 /* VIRTIO_CRYPTO_SYM_OP_NONE */ 155 error_report("unsupported cipher op_type: VIRTIO_CRYPTO_SYM_OP_NONE"); 156 ret = -VIRTIO_CRYPTO_NOTSUPP; 157 goto err; 158 } 159 160 queue_index = virtio_crypto_vq2q(queue_id); 161 session_id = cryptodev_backend_sym_create_session( 162 vcrypto->cryptodev, 163 &info, queue_index, &local_err); 164 if (session_id >= 0) { 165 DPRINTF("create session_id=%" PRIu64 " successfully\n", 166 session_id); 167 168 ret = session_id; 169 } else { 170 if (local_err) { 171 error_report_err(local_err); 172 } 173 ret = -VIRTIO_CRYPTO_ERR; 174 } 175 176 err: 177 g_free(info.cipher_key); 178 g_free(info.auth_key); 179 return ret; 180 } 181 182 static uint8_t 183 virtio_crypto_handle_close_session(VirtIOCrypto *vcrypto, 184 struct virtio_crypto_destroy_session_req *close_sess_req, 185 uint32_t queue_id) 186 { 187 int ret; 188 uint64_t session_id; 189 uint32_t status; 190 Error *local_err = NULL; 191 192 session_id = ldq_le_p(&close_sess_req->session_id); 193 DPRINTF("close session, id=%" PRIu64 "\n", session_id); 194 195 ret = cryptodev_backend_sym_close_session( 196 vcrypto->cryptodev, session_id, queue_id, &local_err); 197 if (ret == 0) { 198 status = VIRTIO_CRYPTO_OK; 199 } else { 200 if (local_err) { 201 error_report_err(local_err); 202 } else { 203 error_report("destroy session failed"); 204 } 205 status = VIRTIO_CRYPTO_ERR; 206 } 207 208 return status; 209 } 210 211 static void virtio_crypto_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq) 212 { 213 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 214 struct virtio_crypto_op_ctrl_req ctrl; 215 VirtQueueElement *elem; 216 struct iovec *in_iov; 217 struct iovec *out_iov; 218 unsigned in_num; 219 unsigned out_num; 220 uint32_t queue_id; 221 uint32_t opcode; 222 struct virtio_crypto_session_input input; 223 int64_t session_id; 224 uint8_t status; 225 size_t s; 226 227 for (;;) { 228 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 229 if (!elem) { 230 break; 231 } 232 if (elem->out_num < 1 || elem->in_num < 1) { 233 virtio_error(vdev, "virtio-crypto ctrl missing headers"); 234 virtqueue_detach_element(vq, elem, 0); 235 g_free(elem); 236 break; 237 } 238 239 out_num = elem->out_num; 240 out_iov = elem->out_sg; 241 in_num = elem->in_num; 242 in_iov = elem->in_sg; 243 if (unlikely(iov_to_buf(out_iov, out_num, 0, &ctrl, sizeof(ctrl)) 244 != sizeof(ctrl))) { 245 virtio_error(vdev, "virtio-crypto request ctrl_hdr too short"); 246 virtqueue_detach_element(vq, elem, 0); 247 g_free(elem); 248 break; 249 } 250 iov_discard_front(&out_iov, &out_num, sizeof(ctrl)); 251 252 opcode = ldl_le_p(&ctrl.header.opcode); 253 queue_id = ldl_le_p(&ctrl.header.queue_id); 254 255 switch (opcode) { 256 case VIRTIO_CRYPTO_CIPHER_CREATE_SESSION: 257 memset(&input, 0, sizeof(input)); 258 session_id = virtio_crypto_create_sym_session(vcrypto, 259 &ctrl.u.sym_create_session, 260 queue_id, opcode, 261 out_iov, out_num); 262 /* Serious errors, need to reset virtio crypto device */ 263 if (session_id == -EFAULT) { 264 virtqueue_detach_element(vq, elem, 0); 265 break; 266 } else if (session_id == -VIRTIO_CRYPTO_NOTSUPP) { 267 stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP); 268 } else if (session_id == -VIRTIO_CRYPTO_ERR) { 269 stl_le_p(&input.status, VIRTIO_CRYPTO_ERR); 270 } else { 271 /* Set the session id */ 272 stq_le_p(&input.session_id, session_id); 273 stl_le_p(&input.status, VIRTIO_CRYPTO_OK); 274 } 275 276 s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input)); 277 if (unlikely(s != sizeof(input))) { 278 virtio_error(vdev, "virtio-crypto input incorrect"); 279 virtqueue_detach_element(vq, elem, 0); 280 break; 281 } 282 virtqueue_push(vq, elem, sizeof(input)); 283 virtio_notify(vdev, vq); 284 break; 285 case VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION: 286 case VIRTIO_CRYPTO_HASH_DESTROY_SESSION: 287 case VIRTIO_CRYPTO_MAC_DESTROY_SESSION: 288 case VIRTIO_CRYPTO_AEAD_DESTROY_SESSION: 289 status = virtio_crypto_handle_close_session(vcrypto, 290 &ctrl.u.destroy_session, queue_id); 291 /* The status only occupy one byte, we can directly use it */ 292 s = iov_from_buf(in_iov, in_num, 0, &status, sizeof(status)); 293 if (unlikely(s != sizeof(status))) { 294 virtio_error(vdev, "virtio-crypto status incorrect"); 295 virtqueue_detach_element(vq, elem, 0); 296 break; 297 } 298 virtqueue_push(vq, elem, sizeof(status)); 299 virtio_notify(vdev, vq); 300 break; 301 case VIRTIO_CRYPTO_HASH_CREATE_SESSION: 302 case VIRTIO_CRYPTO_MAC_CREATE_SESSION: 303 case VIRTIO_CRYPTO_AEAD_CREATE_SESSION: 304 default: 305 error_report("virtio-crypto unsupported ctrl opcode: %d", opcode); 306 memset(&input, 0, sizeof(input)); 307 stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP); 308 s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input)); 309 if (unlikely(s != sizeof(input))) { 310 virtio_error(vdev, "virtio-crypto input incorrect"); 311 virtqueue_detach_element(vq, elem, 0); 312 break; 313 } 314 virtqueue_push(vq, elem, sizeof(input)); 315 virtio_notify(vdev, vq); 316 317 break; 318 } /* end switch case */ 319 320 g_free(elem); 321 } /* end for loop */ 322 } 323 324 static void virtio_crypto_init_request(VirtIOCrypto *vcrypto, VirtQueue *vq, 325 VirtIOCryptoReq *req) 326 { 327 req->vcrypto = vcrypto; 328 req->vq = vq; 329 req->in = NULL; 330 req->in_iov = NULL; 331 req->in_num = 0; 332 req->in_len = 0; 333 req->flags = CRYPTODEV_BACKEND_ALG__MAX; 334 req->u.sym_op_info = NULL; 335 } 336 337 static void virtio_crypto_free_request(VirtIOCryptoReq *req) 338 { 339 if (req) { 340 if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) { 341 size_t max_len; 342 CryptoDevBackendSymOpInfo *op_info = req->u.sym_op_info; 343 344 max_len = op_info->iv_len + 345 op_info->aad_len + 346 op_info->src_len + 347 op_info->dst_len + 348 op_info->digest_result_len; 349 350 /* Zeroize and free request data structure */ 351 memset(op_info, 0, sizeof(*op_info) + max_len); 352 g_free(op_info); 353 } 354 g_free(req); 355 } 356 } 357 358 static void 359 virtio_crypto_sym_input_data_helper(VirtIODevice *vdev, 360 VirtIOCryptoReq *req, 361 uint32_t status, 362 CryptoDevBackendSymOpInfo *sym_op_info) 363 { 364 size_t s, len; 365 366 if (status != VIRTIO_CRYPTO_OK) { 367 return; 368 } 369 370 len = sym_op_info->src_len; 371 /* Save the cipher result */ 372 s = iov_from_buf(req->in_iov, req->in_num, 0, sym_op_info->dst, len); 373 if (s != len) { 374 virtio_error(vdev, "virtio-crypto dest data incorrect"); 375 return; 376 } 377 378 iov_discard_front(&req->in_iov, &req->in_num, len); 379 380 if (sym_op_info->op_type == 381 VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) { 382 /* Save the digest result */ 383 s = iov_from_buf(req->in_iov, req->in_num, 0, 384 sym_op_info->digest_result, 385 sym_op_info->digest_result_len); 386 if (s != sym_op_info->digest_result_len) { 387 virtio_error(vdev, "virtio-crypto digest result incorrect"); 388 } 389 } 390 } 391 392 static void virtio_crypto_req_complete(VirtIOCryptoReq *req, uint8_t status) 393 { 394 VirtIOCrypto *vcrypto = req->vcrypto; 395 VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); 396 397 if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) { 398 virtio_crypto_sym_input_data_helper(vdev, req, status, 399 req->u.sym_op_info); 400 } 401 stb_p(&req->in->status, status); 402 virtqueue_push(req->vq, &req->elem, req->in_len); 403 virtio_notify(vdev, req->vq); 404 } 405 406 static VirtIOCryptoReq * 407 virtio_crypto_get_request(VirtIOCrypto *s, VirtQueue *vq) 408 { 409 VirtIOCryptoReq *req = virtqueue_pop(vq, sizeof(VirtIOCryptoReq)); 410 411 if (req) { 412 virtio_crypto_init_request(s, vq, req); 413 } 414 return req; 415 } 416 417 static CryptoDevBackendSymOpInfo * 418 virtio_crypto_sym_op_helper(VirtIODevice *vdev, 419 struct virtio_crypto_cipher_para *cipher_para, 420 struct virtio_crypto_alg_chain_data_para *alg_chain_para, 421 struct iovec *iov, unsigned int out_num) 422 { 423 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 424 CryptoDevBackendSymOpInfo *op_info; 425 uint32_t src_len = 0, dst_len = 0; 426 uint32_t iv_len = 0; 427 uint32_t aad_len = 0, hash_result_len = 0; 428 uint32_t hash_start_src_offset = 0, len_to_hash = 0; 429 uint32_t cipher_start_src_offset = 0, len_to_cipher = 0; 430 431 uint64_t max_len, curr_size = 0; 432 size_t s; 433 434 /* Plain cipher */ 435 if (cipher_para) { 436 iv_len = ldl_le_p(&cipher_para->iv_len); 437 src_len = ldl_le_p(&cipher_para->src_data_len); 438 dst_len = ldl_le_p(&cipher_para->dst_data_len); 439 } else if (alg_chain_para) { /* Algorithm chain */ 440 iv_len = ldl_le_p(&alg_chain_para->iv_len); 441 src_len = ldl_le_p(&alg_chain_para->src_data_len); 442 dst_len = ldl_le_p(&alg_chain_para->dst_data_len); 443 444 aad_len = ldl_le_p(&alg_chain_para->aad_len); 445 hash_result_len = ldl_le_p(&alg_chain_para->hash_result_len); 446 hash_start_src_offset = ldl_le_p( 447 &alg_chain_para->hash_start_src_offset); 448 cipher_start_src_offset = ldl_le_p( 449 &alg_chain_para->cipher_start_src_offset); 450 len_to_cipher = ldl_le_p(&alg_chain_para->len_to_cipher); 451 len_to_hash = ldl_le_p(&alg_chain_para->len_to_hash); 452 } else { 453 return NULL; 454 } 455 456 max_len = (uint64_t)iv_len + aad_len + src_len + dst_len + hash_result_len; 457 if (unlikely(max_len > vcrypto->conf.max_size)) { 458 virtio_error(vdev, "virtio-crypto too big length"); 459 return NULL; 460 } 461 462 op_info = g_malloc0(sizeof(CryptoDevBackendSymOpInfo) + max_len); 463 op_info->iv_len = iv_len; 464 op_info->src_len = src_len; 465 op_info->dst_len = dst_len; 466 op_info->aad_len = aad_len; 467 op_info->digest_result_len = hash_result_len; 468 op_info->hash_start_src_offset = hash_start_src_offset; 469 op_info->len_to_hash = len_to_hash; 470 op_info->cipher_start_src_offset = cipher_start_src_offset; 471 op_info->len_to_cipher = len_to_cipher; 472 /* Handle the initilization vector */ 473 if (op_info->iv_len > 0) { 474 DPRINTF("iv_len=%" PRIu32 "\n", op_info->iv_len); 475 op_info->iv = op_info->data + curr_size; 476 477 s = iov_to_buf(iov, out_num, 0, op_info->iv, op_info->iv_len); 478 if (unlikely(s != op_info->iv_len)) { 479 virtio_error(vdev, "virtio-crypto iv incorrect"); 480 goto err; 481 } 482 iov_discard_front(&iov, &out_num, op_info->iv_len); 483 curr_size += op_info->iv_len; 484 } 485 486 /* Handle additional authentication data if exists */ 487 if (op_info->aad_len > 0) { 488 DPRINTF("aad_len=%" PRIu32 "\n", op_info->aad_len); 489 op_info->aad_data = op_info->data + curr_size; 490 491 s = iov_to_buf(iov, out_num, 0, op_info->aad_data, op_info->aad_len); 492 if (unlikely(s != op_info->aad_len)) { 493 virtio_error(vdev, "virtio-crypto additional auth data incorrect"); 494 goto err; 495 } 496 iov_discard_front(&iov, &out_num, op_info->aad_len); 497 498 curr_size += op_info->aad_len; 499 } 500 501 /* Handle the source data */ 502 if (op_info->src_len > 0) { 503 DPRINTF("src_len=%" PRIu32 "\n", op_info->src_len); 504 op_info->src = op_info->data + curr_size; 505 506 s = iov_to_buf(iov, out_num, 0, op_info->src, op_info->src_len); 507 if (unlikely(s != op_info->src_len)) { 508 virtio_error(vdev, "virtio-crypto source data incorrect"); 509 goto err; 510 } 511 iov_discard_front(&iov, &out_num, op_info->src_len); 512 513 curr_size += op_info->src_len; 514 } 515 516 /* Handle the destination data */ 517 op_info->dst = op_info->data + curr_size; 518 curr_size += op_info->dst_len; 519 520 DPRINTF("dst_len=%" PRIu32 "\n", op_info->dst_len); 521 522 /* Handle the hash digest result */ 523 if (hash_result_len > 0) { 524 DPRINTF("hash_result_len=%" PRIu32 "\n", hash_result_len); 525 op_info->digest_result = op_info->data + curr_size; 526 } 527 528 return op_info; 529 530 err: 531 g_free(op_info); 532 return NULL; 533 } 534 535 static int 536 virtio_crypto_handle_sym_req(VirtIOCrypto *vcrypto, 537 struct virtio_crypto_sym_data_req *req, 538 CryptoDevBackendSymOpInfo **sym_op_info, 539 struct iovec *iov, unsigned int out_num) 540 { 541 VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); 542 uint32_t op_type; 543 CryptoDevBackendSymOpInfo *op_info; 544 545 op_type = ldl_le_p(&req->op_type); 546 547 if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) { 548 op_info = virtio_crypto_sym_op_helper(vdev, &req->u.cipher.para, 549 NULL, iov, out_num); 550 if (!op_info) { 551 return -EFAULT; 552 } 553 op_info->op_type = op_type; 554 } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) { 555 op_info = virtio_crypto_sym_op_helper(vdev, NULL, 556 &req->u.chain.para, 557 iov, out_num); 558 if (!op_info) { 559 return -EFAULT; 560 } 561 op_info->op_type = op_type; 562 } else { 563 /* VIRTIO_CRYPTO_SYM_OP_NONE */ 564 error_report("virtio-crypto unsupported cipher type"); 565 return -VIRTIO_CRYPTO_NOTSUPP; 566 } 567 568 *sym_op_info = op_info; 569 570 return 0; 571 } 572 573 static int 574 virtio_crypto_handle_request(VirtIOCryptoReq *request) 575 { 576 VirtIOCrypto *vcrypto = request->vcrypto; 577 VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); 578 VirtQueueElement *elem = &request->elem; 579 int queue_index = virtio_crypto_vq2q(virtio_get_queue_index(request->vq)); 580 struct virtio_crypto_op_data_req req; 581 int ret; 582 struct iovec *in_iov; 583 struct iovec *out_iov; 584 unsigned in_num; 585 unsigned out_num; 586 uint32_t opcode; 587 uint8_t status = VIRTIO_CRYPTO_ERR; 588 uint64_t session_id; 589 CryptoDevBackendSymOpInfo *sym_op_info = NULL; 590 Error *local_err = NULL; 591 592 if (elem->out_num < 1 || elem->in_num < 1) { 593 virtio_error(vdev, "virtio-crypto dataq missing headers"); 594 return -1; 595 } 596 597 out_num = elem->out_num; 598 out_iov = elem->out_sg; 599 in_num = elem->in_num; 600 in_iov = elem->in_sg; 601 if (unlikely(iov_to_buf(out_iov, out_num, 0, &req, sizeof(req)) 602 != sizeof(req))) { 603 virtio_error(vdev, "virtio-crypto request outhdr too short"); 604 return -1; 605 } 606 iov_discard_front(&out_iov, &out_num, sizeof(req)); 607 608 if (in_iov[in_num - 1].iov_len < 609 sizeof(struct virtio_crypto_inhdr)) { 610 virtio_error(vdev, "virtio-crypto request inhdr too short"); 611 return -1; 612 } 613 /* We always touch the last byte, so just see how big in_iov is. */ 614 request->in_len = iov_size(in_iov, in_num); 615 request->in = (void *)in_iov[in_num - 1].iov_base 616 + in_iov[in_num - 1].iov_len 617 - sizeof(struct virtio_crypto_inhdr); 618 iov_discard_back(in_iov, &in_num, sizeof(struct virtio_crypto_inhdr)); 619 620 /* 621 * The length of operation result, including dest_data 622 * and digest_result if exists. 623 */ 624 request->in_num = in_num; 625 request->in_iov = in_iov; 626 627 opcode = ldl_le_p(&req.header.opcode); 628 session_id = ldq_le_p(&req.header.session_id); 629 630 switch (opcode) { 631 case VIRTIO_CRYPTO_CIPHER_ENCRYPT: 632 case VIRTIO_CRYPTO_CIPHER_DECRYPT: 633 ret = virtio_crypto_handle_sym_req(vcrypto, 634 &req.u.sym_req, 635 &sym_op_info, 636 out_iov, out_num); 637 /* Serious errors, need to reset virtio crypto device */ 638 if (ret == -EFAULT) { 639 return -1; 640 } else if (ret == -VIRTIO_CRYPTO_NOTSUPP) { 641 virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP); 642 virtio_crypto_free_request(request); 643 } else { 644 sym_op_info->session_id = session_id; 645 646 /* Set request's parameter */ 647 request->flags = CRYPTODEV_BACKEND_ALG_SYM; 648 request->u.sym_op_info = sym_op_info; 649 ret = cryptodev_backend_crypto_operation(vcrypto->cryptodev, 650 request, queue_index, &local_err); 651 if (ret < 0) { 652 status = -ret; 653 if (local_err) { 654 error_report_err(local_err); 655 } 656 } else { /* ret == VIRTIO_CRYPTO_OK */ 657 status = ret; 658 } 659 virtio_crypto_req_complete(request, status); 660 virtio_crypto_free_request(request); 661 } 662 break; 663 case VIRTIO_CRYPTO_HASH: 664 case VIRTIO_CRYPTO_MAC: 665 case VIRTIO_CRYPTO_AEAD_ENCRYPT: 666 case VIRTIO_CRYPTO_AEAD_DECRYPT: 667 default: 668 error_report("virtio-crypto unsupported dataq opcode: %u", 669 opcode); 670 virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP); 671 virtio_crypto_free_request(request); 672 } 673 674 return 0; 675 } 676 677 static void virtio_crypto_handle_dataq(VirtIODevice *vdev, VirtQueue *vq) 678 { 679 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 680 VirtIOCryptoReq *req; 681 682 while ((req = virtio_crypto_get_request(vcrypto, vq))) { 683 if (virtio_crypto_handle_request(req) < 0) { 684 virtqueue_detach_element(req->vq, &req->elem, 0); 685 virtio_crypto_free_request(req); 686 break; 687 } 688 } 689 } 690 691 static void virtio_crypto_dataq_bh(void *opaque) 692 { 693 VirtIOCryptoQueue *q = opaque; 694 VirtIOCrypto *vcrypto = q->vcrypto; 695 VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); 696 697 /* This happens when device was stopped but BH wasn't. */ 698 if (!vdev->vm_running) { 699 return; 700 } 701 702 /* Just in case the driver is not ready on more */ 703 if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) { 704 return; 705 } 706 707 for (;;) { 708 virtio_crypto_handle_dataq(vdev, q->dataq); 709 virtio_queue_set_notification(q->dataq, 1); 710 711 /* Are we done or did the guest add more buffers? */ 712 if (virtio_queue_empty(q->dataq)) { 713 break; 714 } 715 716 virtio_queue_set_notification(q->dataq, 0); 717 } 718 } 719 720 static void 721 virtio_crypto_handle_dataq_bh(VirtIODevice *vdev, VirtQueue *vq) 722 { 723 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 724 VirtIOCryptoQueue *q = 725 &vcrypto->vqs[virtio_crypto_vq2q(virtio_get_queue_index(vq))]; 726 727 /* This happens when device was stopped but VCPU wasn't. */ 728 if (!vdev->vm_running) { 729 return; 730 } 731 virtio_queue_set_notification(vq, 0); 732 qemu_bh_schedule(q->dataq_bh); 733 } 734 735 static uint64_t virtio_crypto_get_features(VirtIODevice *vdev, 736 uint64_t features, 737 Error **errp) 738 { 739 return features; 740 } 741 742 static void virtio_crypto_reset(VirtIODevice *vdev) 743 { 744 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 745 /* multiqueue is disabled by default */ 746 vcrypto->curr_queues = 1; 747 if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) { 748 vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY; 749 } else { 750 vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY; 751 } 752 } 753 754 static void virtio_crypto_init_config(VirtIODevice *vdev) 755 { 756 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 757 758 vcrypto->conf.crypto_services = 759 vcrypto->conf.cryptodev->conf.crypto_services; 760 vcrypto->conf.cipher_algo_l = 761 vcrypto->conf.cryptodev->conf.cipher_algo_l; 762 vcrypto->conf.cipher_algo_h = 763 vcrypto->conf.cryptodev->conf.cipher_algo_h; 764 vcrypto->conf.hash_algo = vcrypto->conf.cryptodev->conf.hash_algo; 765 vcrypto->conf.mac_algo_l = vcrypto->conf.cryptodev->conf.mac_algo_l; 766 vcrypto->conf.mac_algo_h = vcrypto->conf.cryptodev->conf.mac_algo_h; 767 vcrypto->conf.aead_algo = vcrypto->conf.cryptodev->conf.aead_algo; 768 vcrypto->conf.max_cipher_key_len = 769 vcrypto->conf.cryptodev->conf.max_cipher_key_len; 770 vcrypto->conf.max_auth_key_len = 771 vcrypto->conf.cryptodev->conf.max_auth_key_len; 772 vcrypto->conf.max_size = vcrypto->conf.cryptodev->conf.max_size; 773 } 774 775 static void virtio_crypto_device_realize(DeviceState *dev, Error **errp) 776 { 777 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 778 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev); 779 int i; 780 781 vcrypto->cryptodev = vcrypto->conf.cryptodev; 782 if (vcrypto->cryptodev == NULL) { 783 error_setg(errp, "'cryptodev' parameter expects a valid object"); 784 return; 785 } else if (cryptodev_backend_is_used(vcrypto->cryptodev)) { 786 char *path = object_get_canonical_path_component(OBJECT(vcrypto->conf.cryptodev)); 787 error_setg(errp, "can't use already used cryptodev backend: %s", path); 788 g_free(path); 789 return; 790 } 791 792 vcrypto->max_queues = MAX(vcrypto->cryptodev->conf.peers.queues, 1); 793 if (vcrypto->max_queues + 1 > VIRTIO_QUEUE_MAX) { 794 error_setg(errp, "Invalid number of queues (= %" PRIu32 "), " 795 "must be a positive integer less than %d.", 796 vcrypto->max_queues, VIRTIO_QUEUE_MAX); 797 return; 798 } 799 800 virtio_init(vdev, "virtio-crypto", VIRTIO_ID_CRYPTO, vcrypto->config_size); 801 vcrypto->curr_queues = 1; 802 vcrypto->vqs = g_malloc0(sizeof(VirtIOCryptoQueue) * vcrypto->max_queues); 803 for (i = 0; i < vcrypto->max_queues; i++) { 804 vcrypto->vqs[i].dataq = 805 virtio_add_queue(vdev, 1024, virtio_crypto_handle_dataq_bh); 806 vcrypto->vqs[i].dataq_bh = 807 qemu_bh_new(virtio_crypto_dataq_bh, &vcrypto->vqs[i]); 808 vcrypto->vqs[i].vcrypto = vcrypto; 809 } 810 811 vcrypto->ctrl_vq = virtio_add_queue(vdev, 64, virtio_crypto_handle_ctrl); 812 if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) { 813 vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY; 814 } else { 815 vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY; 816 } 817 818 virtio_crypto_init_config(vdev); 819 cryptodev_backend_set_used(vcrypto->cryptodev, true); 820 } 821 822 static void virtio_crypto_device_unrealize(DeviceState *dev, Error **errp) 823 { 824 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 825 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev); 826 VirtIOCryptoQueue *q; 827 int i, max_queues; 828 829 max_queues = vcrypto->multiqueue ? vcrypto->max_queues : 1; 830 for (i = 0; i < max_queues; i++) { 831 virtio_del_queue(vdev, i); 832 q = &vcrypto->vqs[i]; 833 qemu_bh_delete(q->dataq_bh); 834 } 835 836 g_free(vcrypto->vqs); 837 838 virtio_cleanup(vdev); 839 cryptodev_backend_set_used(vcrypto->cryptodev, false); 840 } 841 842 static const VMStateDescription vmstate_virtio_crypto = { 843 .name = "virtio-crypto", 844 .unmigratable = 1, 845 .minimum_version_id = VIRTIO_CRYPTO_VM_VERSION, 846 .version_id = VIRTIO_CRYPTO_VM_VERSION, 847 .fields = (VMStateField[]) { 848 VMSTATE_VIRTIO_DEVICE, 849 VMSTATE_END_OF_LIST() 850 }, 851 }; 852 853 static Property virtio_crypto_properties[] = { 854 DEFINE_PROP_LINK("cryptodev", VirtIOCrypto, conf.cryptodev, 855 TYPE_CRYPTODEV_BACKEND, CryptoDevBackend *), 856 DEFINE_PROP_END_OF_LIST(), 857 }; 858 859 static void virtio_crypto_get_config(VirtIODevice *vdev, uint8_t *config) 860 { 861 VirtIOCrypto *c = VIRTIO_CRYPTO(vdev); 862 struct virtio_crypto_config crypto_cfg = {}; 863 864 /* 865 * Virtio-crypto device conforms to VIRTIO 1.0 which is always LE, 866 * so we can use LE accessors directly. 867 */ 868 stl_le_p(&crypto_cfg.status, c->status); 869 stl_le_p(&crypto_cfg.max_dataqueues, c->max_queues); 870 stl_le_p(&crypto_cfg.crypto_services, c->conf.crypto_services); 871 stl_le_p(&crypto_cfg.cipher_algo_l, c->conf.cipher_algo_l); 872 stl_le_p(&crypto_cfg.cipher_algo_h, c->conf.cipher_algo_h); 873 stl_le_p(&crypto_cfg.hash_algo, c->conf.hash_algo); 874 stl_le_p(&crypto_cfg.mac_algo_l, c->conf.mac_algo_l); 875 stl_le_p(&crypto_cfg.mac_algo_h, c->conf.mac_algo_h); 876 stl_le_p(&crypto_cfg.aead_algo, c->conf.aead_algo); 877 stl_le_p(&crypto_cfg.max_cipher_key_len, c->conf.max_cipher_key_len); 878 stl_le_p(&crypto_cfg.max_auth_key_len, c->conf.max_auth_key_len); 879 stq_le_p(&crypto_cfg.max_size, c->conf.max_size); 880 881 memcpy(config, &crypto_cfg, c->config_size); 882 } 883 884 static bool virtio_crypto_started(VirtIOCrypto *c, uint8_t status) 885 { 886 VirtIODevice *vdev = VIRTIO_DEVICE(c); 887 return (status & VIRTIO_CONFIG_S_DRIVER_OK) && 888 (c->status & VIRTIO_CRYPTO_S_HW_READY) && vdev->vm_running; 889 } 890 891 static void virtio_crypto_vhost_status(VirtIOCrypto *c, uint8_t status) 892 { 893 VirtIODevice *vdev = VIRTIO_DEVICE(c); 894 int queues = c->multiqueue ? c->max_queues : 1; 895 CryptoDevBackend *b = c->cryptodev; 896 CryptoDevBackendClient *cc = b->conf.peers.ccs[0]; 897 898 if (!cryptodev_get_vhost(cc, b, 0)) { 899 return; 900 } 901 902 if ((virtio_crypto_started(c, status)) == !!c->vhost_started) { 903 return; 904 } 905 906 if (!c->vhost_started) { 907 int r; 908 909 c->vhost_started = 1; 910 r = cryptodev_vhost_start(vdev, queues); 911 if (r < 0) { 912 error_report("unable to start vhost crypto: %d: " 913 "falling back on userspace virtio", -r); 914 c->vhost_started = 0; 915 } 916 } else { 917 cryptodev_vhost_stop(vdev, queues); 918 c->vhost_started = 0; 919 } 920 } 921 922 static void virtio_crypto_set_status(VirtIODevice *vdev, uint8_t status) 923 { 924 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 925 926 virtio_crypto_vhost_status(vcrypto, status); 927 } 928 929 static void virtio_crypto_guest_notifier_mask(VirtIODevice *vdev, int idx, 930 bool mask) 931 { 932 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 933 int queue = virtio_crypto_vq2q(idx); 934 935 assert(vcrypto->vhost_started); 936 937 cryptodev_vhost_virtqueue_mask(vdev, queue, idx, mask); 938 } 939 940 static bool virtio_crypto_guest_notifier_pending(VirtIODevice *vdev, int idx) 941 { 942 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 943 int queue = virtio_crypto_vq2q(idx); 944 945 assert(vcrypto->vhost_started); 946 947 return cryptodev_vhost_virtqueue_pending(vdev, queue, idx); 948 } 949 950 static void virtio_crypto_class_init(ObjectClass *klass, void *data) 951 { 952 DeviceClass *dc = DEVICE_CLASS(klass); 953 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 954 955 dc->props = virtio_crypto_properties; 956 dc->vmsd = &vmstate_virtio_crypto; 957 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 958 vdc->realize = virtio_crypto_device_realize; 959 vdc->unrealize = virtio_crypto_device_unrealize; 960 vdc->get_config = virtio_crypto_get_config; 961 vdc->get_features = virtio_crypto_get_features; 962 vdc->reset = virtio_crypto_reset; 963 vdc->set_status = virtio_crypto_set_status; 964 vdc->guest_notifier_mask = virtio_crypto_guest_notifier_mask; 965 vdc->guest_notifier_pending = virtio_crypto_guest_notifier_pending; 966 } 967 968 static void virtio_crypto_instance_init(Object *obj) 969 { 970 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(obj); 971 972 /* 973 * The default config_size is sizeof(struct virtio_crypto_config). 974 * Can be overriden with virtio_crypto_set_config_size. 975 */ 976 vcrypto->config_size = sizeof(struct virtio_crypto_config); 977 } 978 979 static const TypeInfo virtio_crypto_info = { 980 .name = TYPE_VIRTIO_CRYPTO, 981 .parent = TYPE_VIRTIO_DEVICE, 982 .instance_size = sizeof(VirtIOCrypto), 983 .instance_init = virtio_crypto_instance_init, 984 .class_init = virtio_crypto_class_init, 985 }; 986 987 static void virtio_register_types(void) 988 { 989 type_register_static(&virtio_crypto_info); 990 } 991 992 type_init(virtio_register_types) 993