1 /* 2 * Virtio crypto Support 3 * 4 * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD. 5 * 6 * Authors: 7 * Gonglei <arei.gonglei@huawei.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or 10 * (at your option) any later version. See the COPYING file in the 11 * top-level directory. 12 */ 13 14 #include "qemu/osdep.h" 15 #include "qemu/iov.h" 16 #include "qemu/main-loop.h" 17 #include "qemu/module.h" 18 #include "qapi/error.h" 19 #include "qemu/error-report.h" 20 21 #include "hw/virtio/virtio.h" 22 #include "hw/virtio/virtio-crypto.h" 23 #include "hw/qdev-properties.h" 24 #include "hw/virtio/virtio-access.h" 25 #include "standard-headers/linux/virtio_ids.h" 26 #include "sysemu/cryptodev-vhost.h" 27 28 #define VIRTIO_CRYPTO_VM_VERSION 1 29 30 /* 31 * Transfer virtqueue index to crypto queue index. 32 * The control virtqueue is after the data virtqueues 33 * so the input value doesn't need to be adjusted 34 */ 35 static inline int virtio_crypto_vq2q(int queue_index) 36 { 37 return queue_index; 38 } 39 40 static int 41 virtio_crypto_cipher_session_helper(VirtIODevice *vdev, 42 CryptoDevBackendSymSessionInfo *info, 43 struct virtio_crypto_cipher_session_para *cipher_para, 44 struct iovec **iov, unsigned int *out_num) 45 { 46 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 47 unsigned int num = *out_num; 48 49 info->cipher_alg = ldl_le_p(&cipher_para->algo); 50 info->key_len = ldl_le_p(&cipher_para->keylen); 51 info->direction = ldl_le_p(&cipher_para->op); 52 DPRINTF("cipher_alg=%" PRIu32 ", info->direction=%" PRIu32 "\n", 53 info->cipher_alg, info->direction); 54 55 if (info->key_len > vcrypto->conf.max_cipher_key_len) { 56 error_report("virtio-crypto length of cipher key is too big: %u", 57 info->key_len); 58 return -VIRTIO_CRYPTO_ERR; 59 } 60 /* Get cipher key */ 61 if (info->key_len > 0) { 62 size_t s; 63 DPRINTF("keylen=%" PRIu32 "\n", info->key_len); 64 65 info->cipher_key = g_malloc(info->key_len); 66 s = iov_to_buf(*iov, num, 0, info->cipher_key, info->key_len); 67 if (unlikely(s != info->key_len)) { 68 virtio_error(vdev, "virtio-crypto cipher key incorrect"); 69 return -EFAULT; 70 } 71 iov_discard_front(iov, &num, info->key_len); 72 *out_num = num; 73 } 74 75 return 0; 76 } 77 78 static int64_t 79 virtio_crypto_create_sym_session(VirtIOCrypto *vcrypto, 80 struct virtio_crypto_sym_create_session_req *sess_req, 81 uint32_t queue_id, 82 uint32_t opcode, 83 struct iovec *iov, unsigned int out_num) 84 { 85 VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); 86 CryptoDevBackendSymSessionInfo info; 87 int64_t session_id; 88 int queue_index; 89 uint32_t op_type; 90 Error *local_err = NULL; 91 int ret; 92 93 memset(&info, 0, sizeof(info)); 94 op_type = ldl_le_p(&sess_req->op_type); 95 info.op_type = op_type; 96 info.op_code = opcode; 97 98 if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) { 99 ret = virtio_crypto_cipher_session_helper(vdev, &info, 100 &sess_req->u.cipher.para, 101 &iov, &out_num); 102 if (ret < 0) { 103 goto err; 104 } 105 } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) { 106 size_t s; 107 /* cipher part */ 108 ret = virtio_crypto_cipher_session_helper(vdev, &info, 109 &sess_req->u.chain.para.cipher_param, 110 &iov, &out_num); 111 if (ret < 0) { 112 goto err; 113 } 114 /* hash part */ 115 info.alg_chain_order = ldl_le_p( 116 &sess_req->u.chain.para.alg_chain_order); 117 info.add_len = ldl_le_p(&sess_req->u.chain.para.aad_len); 118 info.hash_mode = ldl_le_p(&sess_req->u.chain.para.hash_mode); 119 if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH) { 120 info.hash_alg = ldl_le_p(&sess_req->u.chain.para.u.mac_param.algo); 121 info.auth_key_len = ldl_le_p( 122 &sess_req->u.chain.para.u.mac_param.auth_key_len); 123 info.hash_result_len = ldl_le_p( 124 &sess_req->u.chain.para.u.mac_param.hash_result_len); 125 if (info.auth_key_len > vcrypto->conf.max_auth_key_len) { 126 error_report("virtio-crypto length of auth key is too big: %u", 127 info.auth_key_len); 128 ret = -VIRTIO_CRYPTO_ERR; 129 goto err; 130 } 131 /* get auth key */ 132 if (info.auth_key_len > 0) { 133 DPRINTF("auth_keylen=%" PRIu32 "\n", info.auth_key_len); 134 info.auth_key = g_malloc(info.auth_key_len); 135 s = iov_to_buf(iov, out_num, 0, info.auth_key, 136 info.auth_key_len); 137 if (unlikely(s != info.auth_key_len)) { 138 virtio_error(vdev, 139 "virtio-crypto authenticated key incorrect"); 140 ret = -EFAULT; 141 goto err; 142 } 143 iov_discard_front(&iov, &out_num, info.auth_key_len); 144 } 145 } else if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN) { 146 info.hash_alg = ldl_le_p( 147 &sess_req->u.chain.para.u.hash_param.algo); 148 info.hash_result_len = ldl_le_p( 149 &sess_req->u.chain.para.u.hash_param.hash_result_len); 150 } else { 151 /* VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */ 152 error_report("unsupported hash mode"); 153 ret = -VIRTIO_CRYPTO_NOTSUPP; 154 goto err; 155 } 156 } else { 157 /* VIRTIO_CRYPTO_SYM_OP_NONE */ 158 error_report("unsupported cipher op_type: VIRTIO_CRYPTO_SYM_OP_NONE"); 159 ret = -VIRTIO_CRYPTO_NOTSUPP; 160 goto err; 161 } 162 163 queue_index = virtio_crypto_vq2q(queue_id); 164 session_id = cryptodev_backend_sym_create_session( 165 vcrypto->cryptodev, 166 &info, queue_index, &local_err); 167 if (session_id >= 0) { 168 DPRINTF("create session_id=%" PRIu64 " successfully\n", 169 session_id); 170 171 ret = session_id; 172 } else { 173 if (local_err) { 174 error_report_err(local_err); 175 } 176 ret = -VIRTIO_CRYPTO_ERR; 177 } 178 179 err: 180 g_free(info.cipher_key); 181 g_free(info.auth_key); 182 return ret; 183 } 184 185 static uint8_t 186 virtio_crypto_handle_close_session(VirtIOCrypto *vcrypto, 187 struct virtio_crypto_destroy_session_req *close_sess_req, 188 uint32_t queue_id) 189 { 190 int ret; 191 uint64_t session_id; 192 uint32_t status; 193 Error *local_err = NULL; 194 195 session_id = ldq_le_p(&close_sess_req->session_id); 196 DPRINTF("close session, id=%" PRIu64 "\n", session_id); 197 198 ret = cryptodev_backend_sym_close_session( 199 vcrypto->cryptodev, session_id, queue_id, &local_err); 200 if (ret == 0) { 201 status = VIRTIO_CRYPTO_OK; 202 } else { 203 if (local_err) { 204 error_report_err(local_err); 205 } else { 206 error_report("destroy session failed"); 207 } 208 status = VIRTIO_CRYPTO_ERR; 209 } 210 211 return status; 212 } 213 214 static void virtio_crypto_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq) 215 { 216 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 217 struct virtio_crypto_op_ctrl_req ctrl; 218 VirtQueueElement *elem; 219 struct iovec *in_iov; 220 struct iovec *out_iov; 221 unsigned in_num; 222 unsigned out_num; 223 uint32_t queue_id; 224 uint32_t opcode; 225 struct virtio_crypto_session_input input; 226 int64_t session_id; 227 uint8_t status; 228 size_t s; 229 230 for (;;) { 231 g_autofree struct iovec *out_iov_copy = NULL; 232 233 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 234 if (!elem) { 235 break; 236 } 237 if (elem->out_num < 1 || elem->in_num < 1) { 238 virtio_error(vdev, "virtio-crypto ctrl missing headers"); 239 virtqueue_detach_element(vq, elem, 0); 240 g_free(elem); 241 break; 242 } 243 244 out_num = elem->out_num; 245 out_iov_copy = g_memdup(elem->out_sg, sizeof(out_iov[0]) * out_num); 246 out_iov = out_iov_copy; 247 248 in_num = elem->in_num; 249 in_iov = elem->in_sg; 250 251 if (unlikely(iov_to_buf(out_iov, out_num, 0, &ctrl, sizeof(ctrl)) 252 != sizeof(ctrl))) { 253 virtio_error(vdev, "virtio-crypto request ctrl_hdr too short"); 254 virtqueue_detach_element(vq, elem, 0); 255 g_free(elem); 256 break; 257 } 258 iov_discard_front(&out_iov, &out_num, sizeof(ctrl)); 259 260 opcode = ldl_le_p(&ctrl.header.opcode); 261 queue_id = ldl_le_p(&ctrl.header.queue_id); 262 263 switch (opcode) { 264 case VIRTIO_CRYPTO_CIPHER_CREATE_SESSION: 265 memset(&input, 0, sizeof(input)); 266 session_id = virtio_crypto_create_sym_session(vcrypto, 267 &ctrl.u.sym_create_session, 268 queue_id, opcode, 269 out_iov, out_num); 270 /* Serious errors, need to reset virtio crypto device */ 271 if (session_id == -EFAULT) { 272 virtqueue_detach_element(vq, elem, 0); 273 break; 274 } else if (session_id == -VIRTIO_CRYPTO_NOTSUPP) { 275 stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP); 276 } else if (session_id == -VIRTIO_CRYPTO_ERR) { 277 stl_le_p(&input.status, VIRTIO_CRYPTO_ERR); 278 } else { 279 /* Set the session id */ 280 stq_le_p(&input.session_id, session_id); 281 stl_le_p(&input.status, VIRTIO_CRYPTO_OK); 282 } 283 284 s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input)); 285 if (unlikely(s != sizeof(input))) { 286 virtio_error(vdev, "virtio-crypto input incorrect"); 287 virtqueue_detach_element(vq, elem, 0); 288 break; 289 } 290 virtqueue_push(vq, elem, sizeof(input)); 291 virtio_notify(vdev, vq); 292 break; 293 case VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION: 294 case VIRTIO_CRYPTO_HASH_DESTROY_SESSION: 295 case VIRTIO_CRYPTO_MAC_DESTROY_SESSION: 296 case VIRTIO_CRYPTO_AEAD_DESTROY_SESSION: 297 status = virtio_crypto_handle_close_session(vcrypto, 298 &ctrl.u.destroy_session, queue_id); 299 /* The status only occupy one byte, we can directly use it */ 300 s = iov_from_buf(in_iov, in_num, 0, &status, sizeof(status)); 301 if (unlikely(s != sizeof(status))) { 302 virtio_error(vdev, "virtio-crypto status incorrect"); 303 virtqueue_detach_element(vq, elem, 0); 304 break; 305 } 306 virtqueue_push(vq, elem, sizeof(status)); 307 virtio_notify(vdev, vq); 308 break; 309 case VIRTIO_CRYPTO_HASH_CREATE_SESSION: 310 case VIRTIO_CRYPTO_MAC_CREATE_SESSION: 311 case VIRTIO_CRYPTO_AEAD_CREATE_SESSION: 312 default: 313 error_report("virtio-crypto unsupported ctrl opcode: %d", opcode); 314 memset(&input, 0, sizeof(input)); 315 stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP); 316 s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input)); 317 if (unlikely(s != sizeof(input))) { 318 virtio_error(vdev, "virtio-crypto input incorrect"); 319 virtqueue_detach_element(vq, elem, 0); 320 break; 321 } 322 virtqueue_push(vq, elem, sizeof(input)); 323 virtio_notify(vdev, vq); 324 325 break; 326 } /* end switch case */ 327 328 g_free(elem); 329 } /* end for loop */ 330 } 331 332 static void virtio_crypto_init_request(VirtIOCrypto *vcrypto, VirtQueue *vq, 333 VirtIOCryptoReq *req) 334 { 335 req->vcrypto = vcrypto; 336 req->vq = vq; 337 req->in = NULL; 338 req->in_iov = NULL; 339 req->in_num = 0; 340 req->in_len = 0; 341 req->flags = CRYPTODEV_BACKEND_ALG__MAX; 342 req->u.sym_op_info = NULL; 343 } 344 345 static void virtio_crypto_free_request(VirtIOCryptoReq *req) 346 { 347 if (req) { 348 if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) { 349 size_t max_len; 350 CryptoDevBackendSymOpInfo *op_info = req->u.sym_op_info; 351 352 max_len = op_info->iv_len + 353 op_info->aad_len + 354 op_info->src_len + 355 op_info->dst_len + 356 op_info->digest_result_len; 357 358 /* Zeroize and free request data structure */ 359 memset(op_info, 0, sizeof(*op_info) + max_len); 360 g_free(op_info); 361 } 362 g_free(req); 363 } 364 } 365 366 static void 367 virtio_crypto_sym_input_data_helper(VirtIODevice *vdev, 368 VirtIOCryptoReq *req, 369 uint32_t status, 370 CryptoDevBackendSymOpInfo *sym_op_info) 371 { 372 size_t s, len; 373 374 if (status != VIRTIO_CRYPTO_OK) { 375 return; 376 } 377 378 len = sym_op_info->src_len; 379 /* Save the cipher result */ 380 s = iov_from_buf(req->in_iov, req->in_num, 0, sym_op_info->dst, len); 381 if (s != len) { 382 virtio_error(vdev, "virtio-crypto dest data incorrect"); 383 return; 384 } 385 386 iov_discard_front(&req->in_iov, &req->in_num, len); 387 388 if (sym_op_info->op_type == 389 VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) { 390 /* Save the digest result */ 391 s = iov_from_buf(req->in_iov, req->in_num, 0, 392 sym_op_info->digest_result, 393 sym_op_info->digest_result_len); 394 if (s != sym_op_info->digest_result_len) { 395 virtio_error(vdev, "virtio-crypto digest result incorrect"); 396 } 397 } 398 } 399 400 static void virtio_crypto_req_complete(VirtIOCryptoReq *req, uint8_t status) 401 { 402 VirtIOCrypto *vcrypto = req->vcrypto; 403 VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); 404 405 if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) { 406 virtio_crypto_sym_input_data_helper(vdev, req, status, 407 req->u.sym_op_info); 408 } 409 stb_p(&req->in->status, status); 410 virtqueue_push(req->vq, &req->elem, req->in_len); 411 virtio_notify(vdev, req->vq); 412 } 413 414 static VirtIOCryptoReq * 415 virtio_crypto_get_request(VirtIOCrypto *s, VirtQueue *vq) 416 { 417 VirtIOCryptoReq *req = virtqueue_pop(vq, sizeof(VirtIOCryptoReq)); 418 419 if (req) { 420 virtio_crypto_init_request(s, vq, req); 421 } 422 return req; 423 } 424 425 static CryptoDevBackendSymOpInfo * 426 virtio_crypto_sym_op_helper(VirtIODevice *vdev, 427 struct virtio_crypto_cipher_para *cipher_para, 428 struct virtio_crypto_alg_chain_data_para *alg_chain_para, 429 struct iovec *iov, unsigned int out_num) 430 { 431 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 432 CryptoDevBackendSymOpInfo *op_info; 433 uint32_t src_len = 0, dst_len = 0; 434 uint32_t iv_len = 0; 435 uint32_t aad_len = 0, hash_result_len = 0; 436 uint32_t hash_start_src_offset = 0, len_to_hash = 0; 437 uint32_t cipher_start_src_offset = 0, len_to_cipher = 0; 438 439 uint64_t max_len, curr_size = 0; 440 size_t s; 441 442 /* Plain cipher */ 443 if (cipher_para) { 444 iv_len = ldl_le_p(&cipher_para->iv_len); 445 src_len = ldl_le_p(&cipher_para->src_data_len); 446 dst_len = ldl_le_p(&cipher_para->dst_data_len); 447 } else if (alg_chain_para) { /* Algorithm chain */ 448 iv_len = ldl_le_p(&alg_chain_para->iv_len); 449 src_len = ldl_le_p(&alg_chain_para->src_data_len); 450 dst_len = ldl_le_p(&alg_chain_para->dst_data_len); 451 452 aad_len = ldl_le_p(&alg_chain_para->aad_len); 453 hash_result_len = ldl_le_p(&alg_chain_para->hash_result_len); 454 hash_start_src_offset = ldl_le_p( 455 &alg_chain_para->hash_start_src_offset); 456 cipher_start_src_offset = ldl_le_p( 457 &alg_chain_para->cipher_start_src_offset); 458 len_to_cipher = ldl_le_p(&alg_chain_para->len_to_cipher); 459 len_to_hash = ldl_le_p(&alg_chain_para->len_to_hash); 460 } else { 461 return NULL; 462 } 463 464 max_len = (uint64_t)iv_len + aad_len + src_len + dst_len + hash_result_len; 465 if (unlikely(max_len > vcrypto->conf.max_size)) { 466 virtio_error(vdev, "virtio-crypto too big length"); 467 return NULL; 468 } 469 470 op_info = g_malloc0(sizeof(CryptoDevBackendSymOpInfo) + max_len); 471 op_info->iv_len = iv_len; 472 op_info->src_len = src_len; 473 op_info->dst_len = dst_len; 474 op_info->aad_len = aad_len; 475 op_info->digest_result_len = hash_result_len; 476 op_info->hash_start_src_offset = hash_start_src_offset; 477 op_info->len_to_hash = len_to_hash; 478 op_info->cipher_start_src_offset = cipher_start_src_offset; 479 op_info->len_to_cipher = len_to_cipher; 480 /* Handle the initilization vector */ 481 if (op_info->iv_len > 0) { 482 DPRINTF("iv_len=%" PRIu32 "\n", op_info->iv_len); 483 op_info->iv = op_info->data + curr_size; 484 485 s = iov_to_buf(iov, out_num, 0, op_info->iv, op_info->iv_len); 486 if (unlikely(s != op_info->iv_len)) { 487 virtio_error(vdev, "virtio-crypto iv incorrect"); 488 goto err; 489 } 490 iov_discard_front(&iov, &out_num, op_info->iv_len); 491 curr_size += op_info->iv_len; 492 } 493 494 /* Handle additional authentication data if exists */ 495 if (op_info->aad_len > 0) { 496 DPRINTF("aad_len=%" PRIu32 "\n", op_info->aad_len); 497 op_info->aad_data = op_info->data + curr_size; 498 499 s = iov_to_buf(iov, out_num, 0, op_info->aad_data, op_info->aad_len); 500 if (unlikely(s != op_info->aad_len)) { 501 virtio_error(vdev, "virtio-crypto additional auth data incorrect"); 502 goto err; 503 } 504 iov_discard_front(&iov, &out_num, op_info->aad_len); 505 506 curr_size += op_info->aad_len; 507 } 508 509 /* Handle the source data */ 510 if (op_info->src_len > 0) { 511 DPRINTF("src_len=%" PRIu32 "\n", op_info->src_len); 512 op_info->src = op_info->data + curr_size; 513 514 s = iov_to_buf(iov, out_num, 0, op_info->src, op_info->src_len); 515 if (unlikely(s != op_info->src_len)) { 516 virtio_error(vdev, "virtio-crypto source data incorrect"); 517 goto err; 518 } 519 iov_discard_front(&iov, &out_num, op_info->src_len); 520 521 curr_size += op_info->src_len; 522 } 523 524 /* Handle the destination data */ 525 op_info->dst = op_info->data + curr_size; 526 curr_size += op_info->dst_len; 527 528 DPRINTF("dst_len=%" PRIu32 "\n", op_info->dst_len); 529 530 /* Handle the hash digest result */ 531 if (hash_result_len > 0) { 532 DPRINTF("hash_result_len=%" PRIu32 "\n", hash_result_len); 533 op_info->digest_result = op_info->data + curr_size; 534 } 535 536 return op_info; 537 538 err: 539 g_free(op_info); 540 return NULL; 541 } 542 543 static int 544 virtio_crypto_handle_sym_req(VirtIOCrypto *vcrypto, 545 struct virtio_crypto_sym_data_req *req, 546 CryptoDevBackendSymOpInfo **sym_op_info, 547 struct iovec *iov, unsigned int out_num) 548 { 549 VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); 550 uint32_t op_type; 551 CryptoDevBackendSymOpInfo *op_info; 552 553 op_type = ldl_le_p(&req->op_type); 554 555 if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) { 556 op_info = virtio_crypto_sym_op_helper(vdev, &req->u.cipher.para, 557 NULL, iov, out_num); 558 if (!op_info) { 559 return -EFAULT; 560 } 561 op_info->op_type = op_type; 562 } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) { 563 op_info = virtio_crypto_sym_op_helper(vdev, NULL, 564 &req->u.chain.para, 565 iov, out_num); 566 if (!op_info) { 567 return -EFAULT; 568 } 569 op_info->op_type = op_type; 570 } else { 571 /* VIRTIO_CRYPTO_SYM_OP_NONE */ 572 error_report("virtio-crypto unsupported cipher type"); 573 return -VIRTIO_CRYPTO_NOTSUPP; 574 } 575 576 *sym_op_info = op_info; 577 578 return 0; 579 } 580 581 static int 582 virtio_crypto_handle_request(VirtIOCryptoReq *request) 583 { 584 VirtIOCrypto *vcrypto = request->vcrypto; 585 VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); 586 VirtQueueElement *elem = &request->elem; 587 int queue_index = virtio_crypto_vq2q(virtio_get_queue_index(request->vq)); 588 struct virtio_crypto_op_data_req req; 589 int ret; 590 g_autofree struct iovec *in_iov_copy = NULL; 591 g_autofree struct iovec *out_iov_copy = NULL; 592 struct iovec *in_iov; 593 struct iovec *out_iov; 594 unsigned in_num; 595 unsigned out_num; 596 uint32_t opcode; 597 uint8_t status = VIRTIO_CRYPTO_ERR; 598 uint64_t session_id; 599 CryptoDevBackendSymOpInfo *sym_op_info = NULL; 600 Error *local_err = NULL; 601 602 if (elem->out_num < 1 || elem->in_num < 1) { 603 virtio_error(vdev, "virtio-crypto dataq missing headers"); 604 return -1; 605 } 606 607 out_num = elem->out_num; 608 out_iov_copy = g_memdup(elem->out_sg, sizeof(out_iov[0]) * out_num); 609 out_iov = out_iov_copy; 610 611 in_num = elem->in_num; 612 in_iov_copy = g_memdup(elem->in_sg, sizeof(in_iov[0]) * in_num); 613 in_iov = in_iov_copy; 614 615 if (unlikely(iov_to_buf(out_iov, out_num, 0, &req, sizeof(req)) 616 != sizeof(req))) { 617 virtio_error(vdev, "virtio-crypto request outhdr too short"); 618 return -1; 619 } 620 iov_discard_front(&out_iov, &out_num, sizeof(req)); 621 622 if (in_iov[in_num - 1].iov_len < 623 sizeof(struct virtio_crypto_inhdr)) { 624 virtio_error(vdev, "virtio-crypto request inhdr too short"); 625 return -1; 626 } 627 /* We always touch the last byte, so just see how big in_iov is. */ 628 request->in_len = iov_size(in_iov, in_num); 629 request->in = (void *)in_iov[in_num - 1].iov_base 630 + in_iov[in_num - 1].iov_len 631 - sizeof(struct virtio_crypto_inhdr); 632 iov_discard_back(in_iov, &in_num, sizeof(struct virtio_crypto_inhdr)); 633 634 /* 635 * The length of operation result, including dest_data 636 * and digest_result if exists. 637 */ 638 request->in_num = in_num; 639 request->in_iov = in_iov; 640 641 opcode = ldl_le_p(&req.header.opcode); 642 session_id = ldq_le_p(&req.header.session_id); 643 644 switch (opcode) { 645 case VIRTIO_CRYPTO_CIPHER_ENCRYPT: 646 case VIRTIO_CRYPTO_CIPHER_DECRYPT: 647 ret = virtio_crypto_handle_sym_req(vcrypto, 648 &req.u.sym_req, 649 &sym_op_info, 650 out_iov, out_num); 651 /* Serious errors, need to reset virtio crypto device */ 652 if (ret == -EFAULT) { 653 return -1; 654 } else if (ret == -VIRTIO_CRYPTO_NOTSUPP) { 655 virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP); 656 virtio_crypto_free_request(request); 657 } else { 658 sym_op_info->session_id = session_id; 659 660 /* Set request's parameter */ 661 request->flags = CRYPTODEV_BACKEND_ALG_SYM; 662 request->u.sym_op_info = sym_op_info; 663 ret = cryptodev_backend_crypto_operation(vcrypto->cryptodev, 664 request, queue_index, &local_err); 665 if (ret < 0) { 666 status = -ret; 667 if (local_err) { 668 error_report_err(local_err); 669 } 670 } else { /* ret == VIRTIO_CRYPTO_OK */ 671 status = ret; 672 } 673 virtio_crypto_req_complete(request, status); 674 virtio_crypto_free_request(request); 675 } 676 break; 677 case VIRTIO_CRYPTO_HASH: 678 case VIRTIO_CRYPTO_MAC: 679 case VIRTIO_CRYPTO_AEAD_ENCRYPT: 680 case VIRTIO_CRYPTO_AEAD_DECRYPT: 681 default: 682 error_report("virtio-crypto unsupported dataq opcode: %u", 683 opcode); 684 virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP); 685 virtio_crypto_free_request(request); 686 } 687 688 return 0; 689 } 690 691 static void virtio_crypto_handle_dataq(VirtIODevice *vdev, VirtQueue *vq) 692 { 693 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 694 VirtIOCryptoReq *req; 695 696 while ((req = virtio_crypto_get_request(vcrypto, vq))) { 697 if (virtio_crypto_handle_request(req) < 0) { 698 virtqueue_detach_element(req->vq, &req->elem, 0); 699 virtio_crypto_free_request(req); 700 break; 701 } 702 } 703 } 704 705 static void virtio_crypto_dataq_bh(void *opaque) 706 { 707 VirtIOCryptoQueue *q = opaque; 708 VirtIOCrypto *vcrypto = q->vcrypto; 709 VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); 710 711 /* This happens when device was stopped but BH wasn't. */ 712 if (!vdev->vm_running) { 713 return; 714 } 715 716 /* Just in case the driver is not ready on more */ 717 if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) { 718 return; 719 } 720 721 for (;;) { 722 virtio_crypto_handle_dataq(vdev, q->dataq); 723 virtio_queue_set_notification(q->dataq, 1); 724 725 /* Are we done or did the guest add more buffers? */ 726 if (virtio_queue_empty(q->dataq)) { 727 break; 728 } 729 730 virtio_queue_set_notification(q->dataq, 0); 731 } 732 } 733 734 static void 735 virtio_crypto_handle_dataq_bh(VirtIODevice *vdev, VirtQueue *vq) 736 { 737 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 738 VirtIOCryptoQueue *q = 739 &vcrypto->vqs[virtio_crypto_vq2q(virtio_get_queue_index(vq))]; 740 741 /* This happens when device was stopped but VCPU wasn't. */ 742 if (!vdev->vm_running) { 743 return; 744 } 745 virtio_queue_set_notification(vq, 0); 746 qemu_bh_schedule(q->dataq_bh); 747 } 748 749 static uint64_t virtio_crypto_get_features(VirtIODevice *vdev, 750 uint64_t features, 751 Error **errp) 752 { 753 return features; 754 } 755 756 static void virtio_crypto_reset(VirtIODevice *vdev) 757 { 758 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 759 /* multiqueue is disabled by default */ 760 vcrypto->curr_queues = 1; 761 if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) { 762 vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY; 763 } else { 764 vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY; 765 } 766 } 767 768 static void virtio_crypto_init_config(VirtIODevice *vdev) 769 { 770 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 771 772 vcrypto->conf.crypto_services = 773 vcrypto->conf.cryptodev->conf.crypto_services; 774 vcrypto->conf.cipher_algo_l = 775 vcrypto->conf.cryptodev->conf.cipher_algo_l; 776 vcrypto->conf.cipher_algo_h = 777 vcrypto->conf.cryptodev->conf.cipher_algo_h; 778 vcrypto->conf.hash_algo = vcrypto->conf.cryptodev->conf.hash_algo; 779 vcrypto->conf.mac_algo_l = vcrypto->conf.cryptodev->conf.mac_algo_l; 780 vcrypto->conf.mac_algo_h = vcrypto->conf.cryptodev->conf.mac_algo_h; 781 vcrypto->conf.aead_algo = vcrypto->conf.cryptodev->conf.aead_algo; 782 vcrypto->conf.max_cipher_key_len = 783 vcrypto->conf.cryptodev->conf.max_cipher_key_len; 784 vcrypto->conf.max_auth_key_len = 785 vcrypto->conf.cryptodev->conf.max_auth_key_len; 786 vcrypto->conf.max_size = vcrypto->conf.cryptodev->conf.max_size; 787 } 788 789 static void virtio_crypto_device_realize(DeviceState *dev, Error **errp) 790 { 791 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 792 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev); 793 int i; 794 795 vcrypto->cryptodev = vcrypto->conf.cryptodev; 796 if (vcrypto->cryptodev == NULL) { 797 error_setg(errp, "'cryptodev' parameter expects a valid object"); 798 return; 799 } else if (cryptodev_backend_is_used(vcrypto->cryptodev)) { 800 error_setg(errp, "can't use already used cryptodev backend: %s", 801 object_get_canonical_path_component(OBJECT(vcrypto->conf.cryptodev))); 802 return; 803 } 804 805 vcrypto->max_queues = MAX(vcrypto->cryptodev->conf.peers.queues, 1); 806 if (vcrypto->max_queues + 1 > VIRTIO_QUEUE_MAX) { 807 error_setg(errp, "Invalid number of queues (= %" PRIu32 "), " 808 "must be a positive integer less than %d.", 809 vcrypto->max_queues, VIRTIO_QUEUE_MAX); 810 return; 811 } 812 813 virtio_init(vdev, "virtio-crypto", VIRTIO_ID_CRYPTO, vcrypto->config_size); 814 vcrypto->curr_queues = 1; 815 vcrypto->vqs = g_new0(VirtIOCryptoQueue, vcrypto->max_queues); 816 for (i = 0; i < vcrypto->max_queues; i++) { 817 vcrypto->vqs[i].dataq = 818 virtio_add_queue(vdev, 1024, virtio_crypto_handle_dataq_bh); 819 vcrypto->vqs[i].dataq_bh = 820 qemu_bh_new(virtio_crypto_dataq_bh, &vcrypto->vqs[i]); 821 vcrypto->vqs[i].vcrypto = vcrypto; 822 } 823 824 vcrypto->ctrl_vq = virtio_add_queue(vdev, 64, virtio_crypto_handle_ctrl); 825 if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) { 826 vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY; 827 } else { 828 vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY; 829 } 830 831 virtio_crypto_init_config(vdev); 832 cryptodev_backend_set_used(vcrypto->cryptodev, true); 833 } 834 835 static void virtio_crypto_device_unrealize(DeviceState *dev) 836 { 837 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 838 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev); 839 VirtIOCryptoQueue *q; 840 int i, max_queues; 841 842 max_queues = vcrypto->multiqueue ? vcrypto->max_queues : 1; 843 for (i = 0; i < max_queues; i++) { 844 virtio_delete_queue(vcrypto->vqs[i].dataq); 845 q = &vcrypto->vqs[i]; 846 qemu_bh_delete(q->dataq_bh); 847 } 848 849 g_free(vcrypto->vqs); 850 virtio_delete_queue(vcrypto->ctrl_vq); 851 852 virtio_cleanup(vdev); 853 cryptodev_backend_set_used(vcrypto->cryptodev, false); 854 } 855 856 static const VMStateDescription vmstate_virtio_crypto = { 857 .name = "virtio-crypto", 858 .unmigratable = 1, 859 .minimum_version_id = VIRTIO_CRYPTO_VM_VERSION, 860 .version_id = VIRTIO_CRYPTO_VM_VERSION, 861 .fields = (VMStateField[]) { 862 VMSTATE_VIRTIO_DEVICE, 863 VMSTATE_END_OF_LIST() 864 }, 865 }; 866 867 static Property virtio_crypto_properties[] = { 868 DEFINE_PROP_LINK("cryptodev", VirtIOCrypto, conf.cryptodev, 869 TYPE_CRYPTODEV_BACKEND, CryptoDevBackend *), 870 DEFINE_PROP_END_OF_LIST(), 871 }; 872 873 static void virtio_crypto_get_config(VirtIODevice *vdev, uint8_t *config) 874 { 875 VirtIOCrypto *c = VIRTIO_CRYPTO(vdev); 876 struct virtio_crypto_config crypto_cfg = {}; 877 878 /* 879 * Virtio-crypto device conforms to VIRTIO 1.0 which is always LE, 880 * so we can use LE accessors directly. 881 */ 882 stl_le_p(&crypto_cfg.status, c->status); 883 stl_le_p(&crypto_cfg.max_dataqueues, c->max_queues); 884 stl_le_p(&crypto_cfg.crypto_services, c->conf.crypto_services); 885 stl_le_p(&crypto_cfg.cipher_algo_l, c->conf.cipher_algo_l); 886 stl_le_p(&crypto_cfg.cipher_algo_h, c->conf.cipher_algo_h); 887 stl_le_p(&crypto_cfg.hash_algo, c->conf.hash_algo); 888 stl_le_p(&crypto_cfg.mac_algo_l, c->conf.mac_algo_l); 889 stl_le_p(&crypto_cfg.mac_algo_h, c->conf.mac_algo_h); 890 stl_le_p(&crypto_cfg.aead_algo, c->conf.aead_algo); 891 stl_le_p(&crypto_cfg.max_cipher_key_len, c->conf.max_cipher_key_len); 892 stl_le_p(&crypto_cfg.max_auth_key_len, c->conf.max_auth_key_len); 893 stq_le_p(&crypto_cfg.max_size, c->conf.max_size); 894 895 memcpy(config, &crypto_cfg, c->config_size); 896 } 897 898 static bool virtio_crypto_started(VirtIOCrypto *c, uint8_t status) 899 { 900 VirtIODevice *vdev = VIRTIO_DEVICE(c); 901 return (status & VIRTIO_CONFIG_S_DRIVER_OK) && 902 (c->status & VIRTIO_CRYPTO_S_HW_READY) && vdev->vm_running; 903 } 904 905 static void virtio_crypto_vhost_status(VirtIOCrypto *c, uint8_t status) 906 { 907 VirtIODevice *vdev = VIRTIO_DEVICE(c); 908 int queues = c->multiqueue ? c->max_queues : 1; 909 CryptoDevBackend *b = c->cryptodev; 910 CryptoDevBackendClient *cc = b->conf.peers.ccs[0]; 911 912 if (!cryptodev_get_vhost(cc, b, 0)) { 913 return; 914 } 915 916 if ((virtio_crypto_started(c, status)) == !!c->vhost_started) { 917 return; 918 } 919 920 if (!c->vhost_started) { 921 int r; 922 923 c->vhost_started = 1; 924 r = cryptodev_vhost_start(vdev, queues); 925 if (r < 0) { 926 error_report("unable to start vhost crypto: %d: " 927 "falling back on userspace virtio", -r); 928 c->vhost_started = 0; 929 } 930 } else { 931 cryptodev_vhost_stop(vdev, queues); 932 c->vhost_started = 0; 933 } 934 } 935 936 static void virtio_crypto_set_status(VirtIODevice *vdev, uint8_t status) 937 { 938 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 939 940 virtio_crypto_vhost_status(vcrypto, status); 941 } 942 943 static void virtio_crypto_guest_notifier_mask(VirtIODevice *vdev, int idx, 944 bool mask) 945 { 946 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 947 int queue = virtio_crypto_vq2q(idx); 948 949 assert(vcrypto->vhost_started); 950 951 cryptodev_vhost_virtqueue_mask(vdev, queue, idx, mask); 952 } 953 954 static bool virtio_crypto_guest_notifier_pending(VirtIODevice *vdev, int idx) 955 { 956 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 957 int queue = virtio_crypto_vq2q(idx); 958 959 assert(vcrypto->vhost_started); 960 961 return cryptodev_vhost_virtqueue_pending(vdev, queue, idx); 962 } 963 964 static void virtio_crypto_class_init(ObjectClass *klass, void *data) 965 { 966 DeviceClass *dc = DEVICE_CLASS(klass); 967 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 968 969 device_class_set_props(dc, virtio_crypto_properties); 970 dc->vmsd = &vmstate_virtio_crypto; 971 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 972 vdc->realize = virtio_crypto_device_realize; 973 vdc->unrealize = virtio_crypto_device_unrealize; 974 vdc->get_config = virtio_crypto_get_config; 975 vdc->get_features = virtio_crypto_get_features; 976 vdc->reset = virtio_crypto_reset; 977 vdc->set_status = virtio_crypto_set_status; 978 vdc->guest_notifier_mask = virtio_crypto_guest_notifier_mask; 979 vdc->guest_notifier_pending = virtio_crypto_guest_notifier_pending; 980 } 981 982 static void virtio_crypto_instance_init(Object *obj) 983 { 984 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(obj); 985 986 /* 987 * The default config_size is sizeof(struct virtio_crypto_config). 988 * Can be overriden with virtio_crypto_set_config_size. 989 */ 990 vcrypto->config_size = sizeof(struct virtio_crypto_config); 991 } 992 993 static const TypeInfo virtio_crypto_info = { 994 .name = TYPE_VIRTIO_CRYPTO, 995 .parent = TYPE_VIRTIO_DEVICE, 996 .instance_size = sizeof(VirtIOCrypto), 997 .instance_init = virtio_crypto_instance_init, 998 .class_init = virtio_crypto_class_init, 999 }; 1000 1001 static void virtio_register_types(void) 1002 { 1003 type_register_static(&virtio_crypto_info); 1004 } 1005 1006 type_init(virtio_register_types) 1007