1 /* 2 * Virtio crypto Support 3 * 4 * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD. 5 * 6 * Authors: 7 * Gonglei <arei.gonglei@huawei.com> 8 * 9 * This work is licensed under the terms of the GNU GPL, version 2 or 10 * (at your option) any later version. See the COPYING file in the 11 * top-level directory. 12 */ 13 14 #include "qemu/osdep.h" 15 #include "qemu/iov.h" 16 #include "qemu/module.h" 17 #include "hw/qdev.h" 18 #include "qapi/error.h" 19 #include "qemu/error-report.h" 20 21 #include "hw/virtio/virtio.h" 22 #include "hw/virtio/virtio-crypto.h" 23 #include "hw/virtio/virtio-access.h" 24 #include "standard-headers/linux/virtio_ids.h" 25 #include "sysemu/cryptodev-vhost.h" 26 27 #define VIRTIO_CRYPTO_VM_VERSION 1 28 29 /* 30 * Transfer virtqueue index to crypto queue index. 31 * The control virtqueue is after the data virtqueues 32 * so the input value doesn't need to be adjusted 33 */ 34 static inline int virtio_crypto_vq2q(int queue_index) 35 { 36 return queue_index; 37 } 38 39 static int 40 virtio_crypto_cipher_session_helper(VirtIODevice *vdev, 41 CryptoDevBackendSymSessionInfo *info, 42 struct virtio_crypto_cipher_session_para *cipher_para, 43 struct iovec **iov, unsigned int *out_num) 44 { 45 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 46 unsigned int num = *out_num; 47 48 info->cipher_alg = ldl_le_p(&cipher_para->algo); 49 info->key_len = ldl_le_p(&cipher_para->keylen); 50 info->direction = ldl_le_p(&cipher_para->op); 51 DPRINTF("cipher_alg=%" PRIu32 ", info->direction=%" PRIu32 "\n", 52 info->cipher_alg, info->direction); 53 54 if (info->key_len > vcrypto->conf.max_cipher_key_len) { 55 error_report("virtio-crypto length of cipher key is too big: %u", 56 info->key_len); 57 return -VIRTIO_CRYPTO_ERR; 58 } 59 /* Get cipher key */ 60 if (info->key_len > 0) { 61 size_t s; 62 DPRINTF("keylen=%" PRIu32 "\n", info->key_len); 63 64 info->cipher_key = g_malloc(info->key_len); 65 s = iov_to_buf(*iov, num, 0, info->cipher_key, info->key_len); 66 if (unlikely(s != info->key_len)) { 67 virtio_error(vdev, "virtio-crypto cipher key incorrect"); 68 return -EFAULT; 69 } 70 iov_discard_front(iov, &num, info->key_len); 71 *out_num = num; 72 } 73 74 return 0; 75 } 76 77 static int64_t 78 virtio_crypto_create_sym_session(VirtIOCrypto *vcrypto, 79 struct virtio_crypto_sym_create_session_req *sess_req, 80 uint32_t queue_id, 81 uint32_t opcode, 82 struct iovec *iov, unsigned int out_num) 83 { 84 VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); 85 CryptoDevBackendSymSessionInfo info; 86 int64_t session_id; 87 int queue_index; 88 uint32_t op_type; 89 Error *local_err = NULL; 90 int ret; 91 92 memset(&info, 0, sizeof(info)); 93 op_type = ldl_le_p(&sess_req->op_type); 94 info.op_type = op_type; 95 info.op_code = opcode; 96 97 if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) { 98 ret = virtio_crypto_cipher_session_helper(vdev, &info, 99 &sess_req->u.cipher.para, 100 &iov, &out_num); 101 if (ret < 0) { 102 goto err; 103 } 104 } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) { 105 size_t s; 106 /* cipher part */ 107 ret = virtio_crypto_cipher_session_helper(vdev, &info, 108 &sess_req->u.chain.para.cipher_param, 109 &iov, &out_num); 110 if (ret < 0) { 111 goto err; 112 } 113 /* hash part */ 114 info.alg_chain_order = ldl_le_p( 115 &sess_req->u.chain.para.alg_chain_order); 116 info.add_len = ldl_le_p(&sess_req->u.chain.para.aad_len); 117 info.hash_mode = ldl_le_p(&sess_req->u.chain.para.hash_mode); 118 if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH) { 119 info.hash_alg = ldl_le_p(&sess_req->u.chain.para.u.mac_param.algo); 120 info.auth_key_len = ldl_le_p( 121 &sess_req->u.chain.para.u.mac_param.auth_key_len); 122 info.hash_result_len = ldl_le_p( 123 &sess_req->u.chain.para.u.mac_param.hash_result_len); 124 if (info.auth_key_len > vcrypto->conf.max_auth_key_len) { 125 error_report("virtio-crypto length of auth key is too big: %u", 126 info.auth_key_len); 127 ret = -VIRTIO_CRYPTO_ERR; 128 goto err; 129 } 130 /* get auth key */ 131 if (info.auth_key_len > 0) { 132 DPRINTF("auth_keylen=%" PRIu32 "\n", info.auth_key_len); 133 info.auth_key = g_malloc(info.auth_key_len); 134 s = iov_to_buf(iov, out_num, 0, info.auth_key, 135 info.auth_key_len); 136 if (unlikely(s != info.auth_key_len)) { 137 virtio_error(vdev, 138 "virtio-crypto authenticated key incorrect"); 139 ret = -EFAULT; 140 goto err; 141 } 142 iov_discard_front(&iov, &out_num, info.auth_key_len); 143 } 144 } else if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN) { 145 info.hash_alg = ldl_le_p( 146 &sess_req->u.chain.para.u.hash_param.algo); 147 info.hash_result_len = ldl_le_p( 148 &sess_req->u.chain.para.u.hash_param.hash_result_len); 149 } else { 150 /* VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */ 151 error_report("unsupported hash mode"); 152 ret = -VIRTIO_CRYPTO_NOTSUPP; 153 goto err; 154 } 155 } else { 156 /* VIRTIO_CRYPTO_SYM_OP_NONE */ 157 error_report("unsupported cipher op_type: VIRTIO_CRYPTO_SYM_OP_NONE"); 158 ret = -VIRTIO_CRYPTO_NOTSUPP; 159 goto err; 160 } 161 162 queue_index = virtio_crypto_vq2q(queue_id); 163 session_id = cryptodev_backend_sym_create_session( 164 vcrypto->cryptodev, 165 &info, queue_index, &local_err); 166 if (session_id >= 0) { 167 DPRINTF("create session_id=%" PRIu64 " successfully\n", 168 session_id); 169 170 ret = session_id; 171 } else { 172 if (local_err) { 173 error_report_err(local_err); 174 } 175 ret = -VIRTIO_CRYPTO_ERR; 176 } 177 178 err: 179 g_free(info.cipher_key); 180 g_free(info.auth_key); 181 return ret; 182 } 183 184 static uint8_t 185 virtio_crypto_handle_close_session(VirtIOCrypto *vcrypto, 186 struct virtio_crypto_destroy_session_req *close_sess_req, 187 uint32_t queue_id) 188 { 189 int ret; 190 uint64_t session_id; 191 uint32_t status; 192 Error *local_err = NULL; 193 194 session_id = ldq_le_p(&close_sess_req->session_id); 195 DPRINTF("close session, id=%" PRIu64 "\n", session_id); 196 197 ret = cryptodev_backend_sym_close_session( 198 vcrypto->cryptodev, session_id, queue_id, &local_err); 199 if (ret == 0) { 200 status = VIRTIO_CRYPTO_OK; 201 } else { 202 if (local_err) { 203 error_report_err(local_err); 204 } else { 205 error_report("destroy session failed"); 206 } 207 status = VIRTIO_CRYPTO_ERR; 208 } 209 210 return status; 211 } 212 213 static void virtio_crypto_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq) 214 { 215 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 216 struct virtio_crypto_op_ctrl_req ctrl; 217 VirtQueueElement *elem; 218 struct iovec *in_iov; 219 struct iovec *out_iov; 220 unsigned in_num; 221 unsigned out_num; 222 uint32_t queue_id; 223 uint32_t opcode; 224 struct virtio_crypto_session_input input; 225 int64_t session_id; 226 uint8_t status; 227 size_t s; 228 229 for (;;) { 230 elem = virtqueue_pop(vq, sizeof(VirtQueueElement)); 231 if (!elem) { 232 break; 233 } 234 if (elem->out_num < 1 || elem->in_num < 1) { 235 virtio_error(vdev, "virtio-crypto ctrl missing headers"); 236 virtqueue_detach_element(vq, elem, 0); 237 g_free(elem); 238 break; 239 } 240 241 out_num = elem->out_num; 242 out_iov = elem->out_sg; 243 in_num = elem->in_num; 244 in_iov = elem->in_sg; 245 if (unlikely(iov_to_buf(out_iov, out_num, 0, &ctrl, sizeof(ctrl)) 246 != sizeof(ctrl))) { 247 virtio_error(vdev, "virtio-crypto request ctrl_hdr too short"); 248 virtqueue_detach_element(vq, elem, 0); 249 g_free(elem); 250 break; 251 } 252 iov_discard_front(&out_iov, &out_num, sizeof(ctrl)); 253 254 opcode = ldl_le_p(&ctrl.header.opcode); 255 queue_id = ldl_le_p(&ctrl.header.queue_id); 256 257 switch (opcode) { 258 case VIRTIO_CRYPTO_CIPHER_CREATE_SESSION: 259 memset(&input, 0, sizeof(input)); 260 session_id = virtio_crypto_create_sym_session(vcrypto, 261 &ctrl.u.sym_create_session, 262 queue_id, opcode, 263 out_iov, out_num); 264 /* Serious errors, need to reset virtio crypto device */ 265 if (session_id == -EFAULT) { 266 virtqueue_detach_element(vq, elem, 0); 267 break; 268 } else if (session_id == -VIRTIO_CRYPTO_NOTSUPP) { 269 stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP); 270 } else if (session_id == -VIRTIO_CRYPTO_ERR) { 271 stl_le_p(&input.status, VIRTIO_CRYPTO_ERR); 272 } else { 273 /* Set the session id */ 274 stq_le_p(&input.session_id, session_id); 275 stl_le_p(&input.status, VIRTIO_CRYPTO_OK); 276 } 277 278 s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input)); 279 if (unlikely(s != sizeof(input))) { 280 virtio_error(vdev, "virtio-crypto input incorrect"); 281 virtqueue_detach_element(vq, elem, 0); 282 break; 283 } 284 virtqueue_push(vq, elem, sizeof(input)); 285 virtio_notify(vdev, vq); 286 break; 287 case VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION: 288 case VIRTIO_CRYPTO_HASH_DESTROY_SESSION: 289 case VIRTIO_CRYPTO_MAC_DESTROY_SESSION: 290 case VIRTIO_CRYPTO_AEAD_DESTROY_SESSION: 291 status = virtio_crypto_handle_close_session(vcrypto, 292 &ctrl.u.destroy_session, queue_id); 293 /* The status only occupy one byte, we can directly use it */ 294 s = iov_from_buf(in_iov, in_num, 0, &status, sizeof(status)); 295 if (unlikely(s != sizeof(status))) { 296 virtio_error(vdev, "virtio-crypto status incorrect"); 297 virtqueue_detach_element(vq, elem, 0); 298 break; 299 } 300 virtqueue_push(vq, elem, sizeof(status)); 301 virtio_notify(vdev, vq); 302 break; 303 case VIRTIO_CRYPTO_HASH_CREATE_SESSION: 304 case VIRTIO_CRYPTO_MAC_CREATE_SESSION: 305 case VIRTIO_CRYPTO_AEAD_CREATE_SESSION: 306 default: 307 error_report("virtio-crypto unsupported ctrl opcode: %d", opcode); 308 memset(&input, 0, sizeof(input)); 309 stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP); 310 s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input)); 311 if (unlikely(s != sizeof(input))) { 312 virtio_error(vdev, "virtio-crypto input incorrect"); 313 virtqueue_detach_element(vq, elem, 0); 314 break; 315 } 316 virtqueue_push(vq, elem, sizeof(input)); 317 virtio_notify(vdev, vq); 318 319 break; 320 } /* end switch case */ 321 322 g_free(elem); 323 } /* end for loop */ 324 } 325 326 static void virtio_crypto_init_request(VirtIOCrypto *vcrypto, VirtQueue *vq, 327 VirtIOCryptoReq *req) 328 { 329 req->vcrypto = vcrypto; 330 req->vq = vq; 331 req->in = NULL; 332 req->in_iov = NULL; 333 req->in_num = 0; 334 req->in_len = 0; 335 req->flags = CRYPTODEV_BACKEND_ALG__MAX; 336 req->u.sym_op_info = NULL; 337 } 338 339 static void virtio_crypto_free_request(VirtIOCryptoReq *req) 340 { 341 if (req) { 342 if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) { 343 size_t max_len; 344 CryptoDevBackendSymOpInfo *op_info = req->u.sym_op_info; 345 346 max_len = op_info->iv_len + 347 op_info->aad_len + 348 op_info->src_len + 349 op_info->dst_len + 350 op_info->digest_result_len; 351 352 /* Zeroize and free request data structure */ 353 memset(op_info, 0, sizeof(*op_info) + max_len); 354 g_free(op_info); 355 } 356 g_free(req); 357 } 358 } 359 360 static void 361 virtio_crypto_sym_input_data_helper(VirtIODevice *vdev, 362 VirtIOCryptoReq *req, 363 uint32_t status, 364 CryptoDevBackendSymOpInfo *sym_op_info) 365 { 366 size_t s, len; 367 368 if (status != VIRTIO_CRYPTO_OK) { 369 return; 370 } 371 372 len = sym_op_info->src_len; 373 /* Save the cipher result */ 374 s = iov_from_buf(req->in_iov, req->in_num, 0, sym_op_info->dst, len); 375 if (s != len) { 376 virtio_error(vdev, "virtio-crypto dest data incorrect"); 377 return; 378 } 379 380 iov_discard_front(&req->in_iov, &req->in_num, len); 381 382 if (sym_op_info->op_type == 383 VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) { 384 /* Save the digest result */ 385 s = iov_from_buf(req->in_iov, req->in_num, 0, 386 sym_op_info->digest_result, 387 sym_op_info->digest_result_len); 388 if (s != sym_op_info->digest_result_len) { 389 virtio_error(vdev, "virtio-crypto digest result incorrect"); 390 } 391 } 392 } 393 394 static void virtio_crypto_req_complete(VirtIOCryptoReq *req, uint8_t status) 395 { 396 VirtIOCrypto *vcrypto = req->vcrypto; 397 VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); 398 399 if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) { 400 virtio_crypto_sym_input_data_helper(vdev, req, status, 401 req->u.sym_op_info); 402 } 403 stb_p(&req->in->status, status); 404 virtqueue_push(req->vq, &req->elem, req->in_len); 405 virtio_notify(vdev, req->vq); 406 } 407 408 static VirtIOCryptoReq * 409 virtio_crypto_get_request(VirtIOCrypto *s, VirtQueue *vq) 410 { 411 VirtIOCryptoReq *req = virtqueue_pop(vq, sizeof(VirtIOCryptoReq)); 412 413 if (req) { 414 virtio_crypto_init_request(s, vq, req); 415 } 416 return req; 417 } 418 419 static CryptoDevBackendSymOpInfo * 420 virtio_crypto_sym_op_helper(VirtIODevice *vdev, 421 struct virtio_crypto_cipher_para *cipher_para, 422 struct virtio_crypto_alg_chain_data_para *alg_chain_para, 423 struct iovec *iov, unsigned int out_num) 424 { 425 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 426 CryptoDevBackendSymOpInfo *op_info; 427 uint32_t src_len = 0, dst_len = 0; 428 uint32_t iv_len = 0; 429 uint32_t aad_len = 0, hash_result_len = 0; 430 uint32_t hash_start_src_offset = 0, len_to_hash = 0; 431 uint32_t cipher_start_src_offset = 0, len_to_cipher = 0; 432 433 uint64_t max_len, curr_size = 0; 434 size_t s; 435 436 /* Plain cipher */ 437 if (cipher_para) { 438 iv_len = ldl_le_p(&cipher_para->iv_len); 439 src_len = ldl_le_p(&cipher_para->src_data_len); 440 dst_len = ldl_le_p(&cipher_para->dst_data_len); 441 } else if (alg_chain_para) { /* Algorithm chain */ 442 iv_len = ldl_le_p(&alg_chain_para->iv_len); 443 src_len = ldl_le_p(&alg_chain_para->src_data_len); 444 dst_len = ldl_le_p(&alg_chain_para->dst_data_len); 445 446 aad_len = ldl_le_p(&alg_chain_para->aad_len); 447 hash_result_len = ldl_le_p(&alg_chain_para->hash_result_len); 448 hash_start_src_offset = ldl_le_p( 449 &alg_chain_para->hash_start_src_offset); 450 cipher_start_src_offset = ldl_le_p( 451 &alg_chain_para->cipher_start_src_offset); 452 len_to_cipher = ldl_le_p(&alg_chain_para->len_to_cipher); 453 len_to_hash = ldl_le_p(&alg_chain_para->len_to_hash); 454 } else { 455 return NULL; 456 } 457 458 max_len = (uint64_t)iv_len + aad_len + src_len + dst_len + hash_result_len; 459 if (unlikely(max_len > vcrypto->conf.max_size)) { 460 virtio_error(vdev, "virtio-crypto too big length"); 461 return NULL; 462 } 463 464 op_info = g_malloc0(sizeof(CryptoDevBackendSymOpInfo) + max_len); 465 op_info->iv_len = iv_len; 466 op_info->src_len = src_len; 467 op_info->dst_len = dst_len; 468 op_info->aad_len = aad_len; 469 op_info->digest_result_len = hash_result_len; 470 op_info->hash_start_src_offset = hash_start_src_offset; 471 op_info->len_to_hash = len_to_hash; 472 op_info->cipher_start_src_offset = cipher_start_src_offset; 473 op_info->len_to_cipher = len_to_cipher; 474 /* Handle the initilization vector */ 475 if (op_info->iv_len > 0) { 476 DPRINTF("iv_len=%" PRIu32 "\n", op_info->iv_len); 477 op_info->iv = op_info->data + curr_size; 478 479 s = iov_to_buf(iov, out_num, 0, op_info->iv, op_info->iv_len); 480 if (unlikely(s != op_info->iv_len)) { 481 virtio_error(vdev, "virtio-crypto iv incorrect"); 482 goto err; 483 } 484 iov_discard_front(&iov, &out_num, op_info->iv_len); 485 curr_size += op_info->iv_len; 486 } 487 488 /* Handle additional authentication data if exists */ 489 if (op_info->aad_len > 0) { 490 DPRINTF("aad_len=%" PRIu32 "\n", op_info->aad_len); 491 op_info->aad_data = op_info->data + curr_size; 492 493 s = iov_to_buf(iov, out_num, 0, op_info->aad_data, op_info->aad_len); 494 if (unlikely(s != op_info->aad_len)) { 495 virtio_error(vdev, "virtio-crypto additional auth data incorrect"); 496 goto err; 497 } 498 iov_discard_front(&iov, &out_num, op_info->aad_len); 499 500 curr_size += op_info->aad_len; 501 } 502 503 /* Handle the source data */ 504 if (op_info->src_len > 0) { 505 DPRINTF("src_len=%" PRIu32 "\n", op_info->src_len); 506 op_info->src = op_info->data + curr_size; 507 508 s = iov_to_buf(iov, out_num, 0, op_info->src, op_info->src_len); 509 if (unlikely(s != op_info->src_len)) { 510 virtio_error(vdev, "virtio-crypto source data incorrect"); 511 goto err; 512 } 513 iov_discard_front(&iov, &out_num, op_info->src_len); 514 515 curr_size += op_info->src_len; 516 } 517 518 /* Handle the destination data */ 519 op_info->dst = op_info->data + curr_size; 520 curr_size += op_info->dst_len; 521 522 DPRINTF("dst_len=%" PRIu32 "\n", op_info->dst_len); 523 524 /* Handle the hash digest result */ 525 if (hash_result_len > 0) { 526 DPRINTF("hash_result_len=%" PRIu32 "\n", hash_result_len); 527 op_info->digest_result = op_info->data + curr_size; 528 } 529 530 return op_info; 531 532 err: 533 g_free(op_info); 534 return NULL; 535 } 536 537 static int 538 virtio_crypto_handle_sym_req(VirtIOCrypto *vcrypto, 539 struct virtio_crypto_sym_data_req *req, 540 CryptoDevBackendSymOpInfo **sym_op_info, 541 struct iovec *iov, unsigned int out_num) 542 { 543 VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); 544 uint32_t op_type; 545 CryptoDevBackendSymOpInfo *op_info; 546 547 op_type = ldl_le_p(&req->op_type); 548 549 if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) { 550 op_info = virtio_crypto_sym_op_helper(vdev, &req->u.cipher.para, 551 NULL, iov, out_num); 552 if (!op_info) { 553 return -EFAULT; 554 } 555 op_info->op_type = op_type; 556 } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) { 557 op_info = virtio_crypto_sym_op_helper(vdev, NULL, 558 &req->u.chain.para, 559 iov, out_num); 560 if (!op_info) { 561 return -EFAULT; 562 } 563 op_info->op_type = op_type; 564 } else { 565 /* VIRTIO_CRYPTO_SYM_OP_NONE */ 566 error_report("virtio-crypto unsupported cipher type"); 567 return -VIRTIO_CRYPTO_NOTSUPP; 568 } 569 570 *sym_op_info = op_info; 571 572 return 0; 573 } 574 575 static int 576 virtio_crypto_handle_request(VirtIOCryptoReq *request) 577 { 578 VirtIOCrypto *vcrypto = request->vcrypto; 579 VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); 580 VirtQueueElement *elem = &request->elem; 581 int queue_index = virtio_crypto_vq2q(virtio_get_queue_index(request->vq)); 582 struct virtio_crypto_op_data_req req; 583 int ret; 584 struct iovec *in_iov; 585 struct iovec *out_iov; 586 unsigned in_num; 587 unsigned out_num; 588 uint32_t opcode; 589 uint8_t status = VIRTIO_CRYPTO_ERR; 590 uint64_t session_id; 591 CryptoDevBackendSymOpInfo *sym_op_info = NULL; 592 Error *local_err = NULL; 593 594 if (elem->out_num < 1 || elem->in_num < 1) { 595 virtio_error(vdev, "virtio-crypto dataq missing headers"); 596 return -1; 597 } 598 599 out_num = elem->out_num; 600 out_iov = elem->out_sg; 601 in_num = elem->in_num; 602 in_iov = elem->in_sg; 603 if (unlikely(iov_to_buf(out_iov, out_num, 0, &req, sizeof(req)) 604 != sizeof(req))) { 605 virtio_error(vdev, "virtio-crypto request outhdr too short"); 606 return -1; 607 } 608 iov_discard_front(&out_iov, &out_num, sizeof(req)); 609 610 if (in_iov[in_num - 1].iov_len < 611 sizeof(struct virtio_crypto_inhdr)) { 612 virtio_error(vdev, "virtio-crypto request inhdr too short"); 613 return -1; 614 } 615 /* We always touch the last byte, so just see how big in_iov is. */ 616 request->in_len = iov_size(in_iov, in_num); 617 request->in = (void *)in_iov[in_num - 1].iov_base 618 + in_iov[in_num - 1].iov_len 619 - sizeof(struct virtio_crypto_inhdr); 620 iov_discard_back(in_iov, &in_num, sizeof(struct virtio_crypto_inhdr)); 621 622 /* 623 * The length of operation result, including dest_data 624 * and digest_result if exists. 625 */ 626 request->in_num = in_num; 627 request->in_iov = in_iov; 628 629 opcode = ldl_le_p(&req.header.opcode); 630 session_id = ldq_le_p(&req.header.session_id); 631 632 switch (opcode) { 633 case VIRTIO_CRYPTO_CIPHER_ENCRYPT: 634 case VIRTIO_CRYPTO_CIPHER_DECRYPT: 635 ret = virtio_crypto_handle_sym_req(vcrypto, 636 &req.u.sym_req, 637 &sym_op_info, 638 out_iov, out_num); 639 /* Serious errors, need to reset virtio crypto device */ 640 if (ret == -EFAULT) { 641 return -1; 642 } else if (ret == -VIRTIO_CRYPTO_NOTSUPP) { 643 virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP); 644 virtio_crypto_free_request(request); 645 } else { 646 sym_op_info->session_id = session_id; 647 648 /* Set request's parameter */ 649 request->flags = CRYPTODEV_BACKEND_ALG_SYM; 650 request->u.sym_op_info = sym_op_info; 651 ret = cryptodev_backend_crypto_operation(vcrypto->cryptodev, 652 request, queue_index, &local_err); 653 if (ret < 0) { 654 status = -ret; 655 if (local_err) { 656 error_report_err(local_err); 657 } 658 } else { /* ret == VIRTIO_CRYPTO_OK */ 659 status = ret; 660 } 661 virtio_crypto_req_complete(request, status); 662 virtio_crypto_free_request(request); 663 } 664 break; 665 case VIRTIO_CRYPTO_HASH: 666 case VIRTIO_CRYPTO_MAC: 667 case VIRTIO_CRYPTO_AEAD_ENCRYPT: 668 case VIRTIO_CRYPTO_AEAD_DECRYPT: 669 default: 670 error_report("virtio-crypto unsupported dataq opcode: %u", 671 opcode); 672 virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP); 673 virtio_crypto_free_request(request); 674 } 675 676 return 0; 677 } 678 679 static void virtio_crypto_handle_dataq(VirtIODevice *vdev, VirtQueue *vq) 680 { 681 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 682 VirtIOCryptoReq *req; 683 684 while ((req = virtio_crypto_get_request(vcrypto, vq))) { 685 if (virtio_crypto_handle_request(req) < 0) { 686 virtqueue_detach_element(req->vq, &req->elem, 0); 687 virtio_crypto_free_request(req); 688 break; 689 } 690 } 691 } 692 693 static void virtio_crypto_dataq_bh(void *opaque) 694 { 695 VirtIOCryptoQueue *q = opaque; 696 VirtIOCrypto *vcrypto = q->vcrypto; 697 VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto); 698 699 /* This happens when device was stopped but BH wasn't. */ 700 if (!vdev->vm_running) { 701 return; 702 } 703 704 /* Just in case the driver is not ready on more */ 705 if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) { 706 return; 707 } 708 709 for (;;) { 710 virtio_crypto_handle_dataq(vdev, q->dataq); 711 virtio_queue_set_notification(q->dataq, 1); 712 713 /* Are we done or did the guest add more buffers? */ 714 if (virtio_queue_empty(q->dataq)) { 715 break; 716 } 717 718 virtio_queue_set_notification(q->dataq, 0); 719 } 720 } 721 722 static void 723 virtio_crypto_handle_dataq_bh(VirtIODevice *vdev, VirtQueue *vq) 724 { 725 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 726 VirtIOCryptoQueue *q = 727 &vcrypto->vqs[virtio_crypto_vq2q(virtio_get_queue_index(vq))]; 728 729 /* This happens when device was stopped but VCPU wasn't. */ 730 if (!vdev->vm_running) { 731 return; 732 } 733 virtio_queue_set_notification(vq, 0); 734 qemu_bh_schedule(q->dataq_bh); 735 } 736 737 static uint64_t virtio_crypto_get_features(VirtIODevice *vdev, 738 uint64_t features, 739 Error **errp) 740 { 741 return features; 742 } 743 744 static void virtio_crypto_reset(VirtIODevice *vdev) 745 { 746 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 747 /* multiqueue is disabled by default */ 748 vcrypto->curr_queues = 1; 749 if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) { 750 vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY; 751 } else { 752 vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY; 753 } 754 } 755 756 static void virtio_crypto_init_config(VirtIODevice *vdev) 757 { 758 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 759 760 vcrypto->conf.crypto_services = 761 vcrypto->conf.cryptodev->conf.crypto_services; 762 vcrypto->conf.cipher_algo_l = 763 vcrypto->conf.cryptodev->conf.cipher_algo_l; 764 vcrypto->conf.cipher_algo_h = 765 vcrypto->conf.cryptodev->conf.cipher_algo_h; 766 vcrypto->conf.hash_algo = vcrypto->conf.cryptodev->conf.hash_algo; 767 vcrypto->conf.mac_algo_l = vcrypto->conf.cryptodev->conf.mac_algo_l; 768 vcrypto->conf.mac_algo_h = vcrypto->conf.cryptodev->conf.mac_algo_h; 769 vcrypto->conf.aead_algo = vcrypto->conf.cryptodev->conf.aead_algo; 770 vcrypto->conf.max_cipher_key_len = 771 vcrypto->conf.cryptodev->conf.max_cipher_key_len; 772 vcrypto->conf.max_auth_key_len = 773 vcrypto->conf.cryptodev->conf.max_auth_key_len; 774 vcrypto->conf.max_size = vcrypto->conf.cryptodev->conf.max_size; 775 } 776 777 static void virtio_crypto_device_realize(DeviceState *dev, Error **errp) 778 { 779 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 780 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev); 781 int i; 782 783 vcrypto->cryptodev = vcrypto->conf.cryptodev; 784 if (vcrypto->cryptodev == NULL) { 785 error_setg(errp, "'cryptodev' parameter expects a valid object"); 786 return; 787 } else if (cryptodev_backend_is_used(vcrypto->cryptodev)) { 788 char *path = object_get_canonical_path_component(OBJECT(vcrypto->conf.cryptodev)); 789 error_setg(errp, "can't use already used cryptodev backend: %s", path); 790 g_free(path); 791 return; 792 } 793 794 vcrypto->max_queues = MAX(vcrypto->cryptodev->conf.peers.queues, 1); 795 if (vcrypto->max_queues + 1 > VIRTIO_QUEUE_MAX) { 796 error_setg(errp, "Invalid number of queues (= %" PRIu32 "), " 797 "must be a positive integer less than %d.", 798 vcrypto->max_queues, VIRTIO_QUEUE_MAX); 799 return; 800 } 801 802 virtio_init(vdev, "virtio-crypto", VIRTIO_ID_CRYPTO, vcrypto->config_size); 803 vcrypto->curr_queues = 1; 804 vcrypto->vqs = g_malloc0(sizeof(VirtIOCryptoQueue) * vcrypto->max_queues); 805 for (i = 0; i < vcrypto->max_queues; i++) { 806 vcrypto->vqs[i].dataq = 807 virtio_add_queue(vdev, 1024, virtio_crypto_handle_dataq_bh); 808 vcrypto->vqs[i].dataq_bh = 809 qemu_bh_new(virtio_crypto_dataq_bh, &vcrypto->vqs[i]); 810 vcrypto->vqs[i].vcrypto = vcrypto; 811 } 812 813 vcrypto->ctrl_vq = virtio_add_queue(vdev, 64, virtio_crypto_handle_ctrl); 814 if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) { 815 vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY; 816 } else { 817 vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY; 818 } 819 820 virtio_crypto_init_config(vdev); 821 cryptodev_backend_set_used(vcrypto->cryptodev, true); 822 } 823 824 static void virtio_crypto_device_unrealize(DeviceState *dev, Error **errp) 825 { 826 VirtIODevice *vdev = VIRTIO_DEVICE(dev); 827 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev); 828 VirtIOCryptoQueue *q; 829 int i, max_queues; 830 831 max_queues = vcrypto->multiqueue ? vcrypto->max_queues : 1; 832 for (i = 0; i < max_queues; i++) { 833 virtio_del_queue(vdev, i); 834 q = &vcrypto->vqs[i]; 835 qemu_bh_delete(q->dataq_bh); 836 } 837 838 g_free(vcrypto->vqs); 839 840 virtio_cleanup(vdev); 841 cryptodev_backend_set_used(vcrypto->cryptodev, false); 842 } 843 844 static const VMStateDescription vmstate_virtio_crypto = { 845 .name = "virtio-crypto", 846 .unmigratable = 1, 847 .minimum_version_id = VIRTIO_CRYPTO_VM_VERSION, 848 .version_id = VIRTIO_CRYPTO_VM_VERSION, 849 .fields = (VMStateField[]) { 850 VMSTATE_VIRTIO_DEVICE, 851 VMSTATE_END_OF_LIST() 852 }, 853 }; 854 855 static Property virtio_crypto_properties[] = { 856 DEFINE_PROP_LINK("cryptodev", VirtIOCrypto, conf.cryptodev, 857 TYPE_CRYPTODEV_BACKEND, CryptoDevBackend *), 858 DEFINE_PROP_END_OF_LIST(), 859 }; 860 861 static void virtio_crypto_get_config(VirtIODevice *vdev, uint8_t *config) 862 { 863 VirtIOCrypto *c = VIRTIO_CRYPTO(vdev); 864 struct virtio_crypto_config crypto_cfg = {}; 865 866 /* 867 * Virtio-crypto device conforms to VIRTIO 1.0 which is always LE, 868 * so we can use LE accessors directly. 869 */ 870 stl_le_p(&crypto_cfg.status, c->status); 871 stl_le_p(&crypto_cfg.max_dataqueues, c->max_queues); 872 stl_le_p(&crypto_cfg.crypto_services, c->conf.crypto_services); 873 stl_le_p(&crypto_cfg.cipher_algo_l, c->conf.cipher_algo_l); 874 stl_le_p(&crypto_cfg.cipher_algo_h, c->conf.cipher_algo_h); 875 stl_le_p(&crypto_cfg.hash_algo, c->conf.hash_algo); 876 stl_le_p(&crypto_cfg.mac_algo_l, c->conf.mac_algo_l); 877 stl_le_p(&crypto_cfg.mac_algo_h, c->conf.mac_algo_h); 878 stl_le_p(&crypto_cfg.aead_algo, c->conf.aead_algo); 879 stl_le_p(&crypto_cfg.max_cipher_key_len, c->conf.max_cipher_key_len); 880 stl_le_p(&crypto_cfg.max_auth_key_len, c->conf.max_auth_key_len); 881 stq_le_p(&crypto_cfg.max_size, c->conf.max_size); 882 883 memcpy(config, &crypto_cfg, c->config_size); 884 } 885 886 static bool virtio_crypto_started(VirtIOCrypto *c, uint8_t status) 887 { 888 VirtIODevice *vdev = VIRTIO_DEVICE(c); 889 return (status & VIRTIO_CONFIG_S_DRIVER_OK) && 890 (c->status & VIRTIO_CRYPTO_S_HW_READY) && vdev->vm_running; 891 } 892 893 static void virtio_crypto_vhost_status(VirtIOCrypto *c, uint8_t status) 894 { 895 VirtIODevice *vdev = VIRTIO_DEVICE(c); 896 int queues = c->multiqueue ? c->max_queues : 1; 897 CryptoDevBackend *b = c->cryptodev; 898 CryptoDevBackendClient *cc = b->conf.peers.ccs[0]; 899 900 if (!cryptodev_get_vhost(cc, b, 0)) { 901 return; 902 } 903 904 if ((virtio_crypto_started(c, status)) == !!c->vhost_started) { 905 return; 906 } 907 908 if (!c->vhost_started) { 909 int r; 910 911 c->vhost_started = 1; 912 r = cryptodev_vhost_start(vdev, queues); 913 if (r < 0) { 914 error_report("unable to start vhost crypto: %d: " 915 "falling back on userspace virtio", -r); 916 c->vhost_started = 0; 917 } 918 } else { 919 cryptodev_vhost_stop(vdev, queues); 920 c->vhost_started = 0; 921 } 922 } 923 924 static void virtio_crypto_set_status(VirtIODevice *vdev, uint8_t status) 925 { 926 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 927 928 virtio_crypto_vhost_status(vcrypto, status); 929 } 930 931 static void virtio_crypto_guest_notifier_mask(VirtIODevice *vdev, int idx, 932 bool mask) 933 { 934 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 935 int queue = virtio_crypto_vq2q(idx); 936 937 assert(vcrypto->vhost_started); 938 939 cryptodev_vhost_virtqueue_mask(vdev, queue, idx, mask); 940 } 941 942 static bool virtio_crypto_guest_notifier_pending(VirtIODevice *vdev, int idx) 943 { 944 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev); 945 int queue = virtio_crypto_vq2q(idx); 946 947 assert(vcrypto->vhost_started); 948 949 return cryptodev_vhost_virtqueue_pending(vdev, queue, idx); 950 } 951 952 static void virtio_crypto_class_init(ObjectClass *klass, void *data) 953 { 954 DeviceClass *dc = DEVICE_CLASS(klass); 955 VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass); 956 957 dc->props = virtio_crypto_properties; 958 dc->vmsd = &vmstate_virtio_crypto; 959 set_bit(DEVICE_CATEGORY_MISC, dc->categories); 960 vdc->realize = virtio_crypto_device_realize; 961 vdc->unrealize = virtio_crypto_device_unrealize; 962 vdc->get_config = virtio_crypto_get_config; 963 vdc->get_features = virtio_crypto_get_features; 964 vdc->reset = virtio_crypto_reset; 965 vdc->set_status = virtio_crypto_set_status; 966 vdc->guest_notifier_mask = virtio_crypto_guest_notifier_mask; 967 vdc->guest_notifier_pending = virtio_crypto_guest_notifier_pending; 968 } 969 970 static void virtio_crypto_instance_init(Object *obj) 971 { 972 VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(obj); 973 974 /* 975 * The default config_size is sizeof(struct virtio_crypto_config). 976 * Can be overriden with virtio_crypto_set_config_size. 977 */ 978 vcrypto->config_size = sizeof(struct virtio_crypto_config); 979 } 980 981 static const TypeInfo virtio_crypto_info = { 982 .name = TYPE_VIRTIO_CRYPTO, 983 .parent = TYPE_VIRTIO_DEVICE, 984 .instance_size = sizeof(VirtIOCrypto), 985 .instance_init = virtio_crypto_instance_init, 986 .class_init = virtio_crypto_class_init, 987 }; 988 989 static void virtio_register_types(void) 990 { 991 type_register_static(&virtio_crypto_info); 992 } 993 994 type_init(virtio_register_types) 995