xref: /openbmc/qemu/hw/virtio/virtio-crypto.c (revision 979a8902)
1 /*
2  * Virtio crypto Support
3  *
4  * Copyright (c) 2016 HUAWEI TECHNOLOGIES CO., LTD.
5  *
6  * Authors:
7  *    Gonglei <arei.gonglei@huawei.com>
8  *
9  * This work is licensed under the terms of the GNU GPL, version 2 or
10  * (at your option) any later version.  See the COPYING file in the
11  * top-level directory.
12  */
13 
14 #include "qemu/osdep.h"
15 #include "qemu/iov.h"
16 #include "qemu/main-loop.h"
17 #include "qemu/module.h"
18 #include "qapi/error.h"
19 #include "qemu/error-report.h"
20 
21 #include "hw/virtio/virtio.h"
22 #include "hw/virtio/virtio-crypto.h"
23 #include "hw/qdev-properties.h"
24 #include "hw/virtio/virtio-access.h"
25 #include "standard-headers/linux/virtio_ids.h"
26 #include "sysemu/cryptodev-vhost.h"
27 
28 #define VIRTIO_CRYPTO_VM_VERSION 1
29 
30 /*
31  * Transfer virtqueue index to crypto queue index.
32  * The control virtqueue is after the data virtqueues
33  * so the input value doesn't need to be adjusted
34  */
35 static inline int virtio_crypto_vq2q(int queue_index)
36 {
37     return queue_index;
38 }
39 
40 static int
41 virtio_crypto_cipher_session_helper(VirtIODevice *vdev,
42            CryptoDevBackendSymSessionInfo *info,
43            struct virtio_crypto_cipher_session_para *cipher_para,
44            struct iovec **iov, unsigned int *out_num)
45 {
46     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
47     unsigned int num = *out_num;
48 
49     info->cipher_alg = ldl_le_p(&cipher_para->algo);
50     info->key_len = ldl_le_p(&cipher_para->keylen);
51     info->direction = ldl_le_p(&cipher_para->op);
52     DPRINTF("cipher_alg=%" PRIu32 ", info->direction=%" PRIu32 "\n",
53              info->cipher_alg, info->direction);
54 
55     if (info->key_len > vcrypto->conf.max_cipher_key_len) {
56         error_report("virtio-crypto length of cipher key is too big: %u",
57                      info->key_len);
58         return -VIRTIO_CRYPTO_ERR;
59     }
60     /* Get cipher key */
61     if (info->key_len > 0) {
62         size_t s;
63         DPRINTF("keylen=%" PRIu32 "\n", info->key_len);
64 
65         info->cipher_key = g_malloc(info->key_len);
66         s = iov_to_buf(*iov, num, 0, info->cipher_key, info->key_len);
67         if (unlikely(s != info->key_len)) {
68             virtio_error(vdev, "virtio-crypto cipher key incorrect");
69             return -EFAULT;
70         }
71         iov_discard_front(iov, &num, info->key_len);
72         *out_num = num;
73     }
74 
75     return 0;
76 }
77 
78 static int64_t
79 virtio_crypto_create_sym_session(VirtIOCrypto *vcrypto,
80                struct virtio_crypto_sym_create_session_req *sess_req,
81                uint32_t queue_id,
82                uint32_t opcode,
83                struct iovec *iov, unsigned int out_num)
84 {
85     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
86     CryptoDevBackendSymSessionInfo info;
87     int64_t session_id;
88     int queue_index;
89     uint32_t op_type;
90     Error *local_err = NULL;
91     int ret;
92 
93     memset(&info, 0, sizeof(info));
94     op_type = ldl_le_p(&sess_req->op_type);
95     info.op_type = op_type;
96     info.op_code = opcode;
97 
98     if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
99         ret = virtio_crypto_cipher_session_helper(vdev, &info,
100                            &sess_req->u.cipher.para,
101                            &iov, &out_num);
102         if (ret < 0) {
103             goto err;
104         }
105     } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
106         size_t s;
107         /* cipher part */
108         ret = virtio_crypto_cipher_session_helper(vdev, &info,
109                            &sess_req->u.chain.para.cipher_param,
110                            &iov, &out_num);
111         if (ret < 0) {
112             goto err;
113         }
114         /* hash part */
115         info.alg_chain_order = ldl_le_p(
116                                      &sess_req->u.chain.para.alg_chain_order);
117         info.add_len = ldl_le_p(&sess_req->u.chain.para.aad_len);
118         info.hash_mode = ldl_le_p(&sess_req->u.chain.para.hash_mode);
119         if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_AUTH) {
120             info.hash_alg = ldl_le_p(&sess_req->u.chain.para.u.mac_param.algo);
121             info.auth_key_len = ldl_le_p(
122                              &sess_req->u.chain.para.u.mac_param.auth_key_len);
123             info.hash_result_len = ldl_le_p(
124                            &sess_req->u.chain.para.u.mac_param.hash_result_len);
125             if (info.auth_key_len > vcrypto->conf.max_auth_key_len) {
126                 error_report("virtio-crypto length of auth key is too big: %u",
127                              info.auth_key_len);
128                 ret = -VIRTIO_CRYPTO_ERR;
129                 goto err;
130             }
131             /* get auth key */
132             if (info.auth_key_len > 0) {
133                 DPRINTF("auth_keylen=%" PRIu32 "\n", info.auth_key_len);
134                 info.auth_key = g_malloc(info.auth_key_len);
135                 s = iov_to_buf(iov, out_num, 0, info.auth_key,
136                                info.auth_key_len);
137                 if (unlikely(s != info.auth_key_len)) {
138                     virtio_error(vdev,
139                           "virtio-crypto authenticated key incorrect");
140                     ret = -EFAULT;
141                     goto err;
142                 }
143                 iov_discard_front(&iov, &out_num, info.auth_key_len);
144             }
145         } else if (info.hash_mode == VIRTIO_CRYPTO_SYM_HASH_MODE_PLAIN) {
146             info.hash_alg = ldl_le_p(
147                              &sess_req->u.chain.para.u.hash_param.algo);
148             info.hash_result_len = ldl_le_p(
149                         &sess_req->u.chain.para.u.hash_param.hash_result_len);
150         } else {
151             /* VIRTIO_CRYPTO_SYM_HASH_MODE_NESTED */
152             error_report("unsupported hash mode");
153             ret = -VIRTIO_CRYPTO_NOTSUPP;
154             goto err;
155         }
156     } else {
157         /* VIRTIO_CRYPTO_SYM_OP_NONE */
158         error_report("unsupported cipher op_type: VIRTIO_CRYPTO_SYM_OP_NONE");
159         ret = -VIRTIO_CRYPTO_NOTSUPP;
160         goto err;
161     }
162 
163     queue_index = virtio_crypto_vq2q(queue_id);
164     session_id = cryptodev_backend_sym_create_session(
165                                      vcrypto->cryptodev,
166                                      &info, queue_index, &local_err);
167     if (session_id >= 0) {
168         DPRINTF("create session_id=%" PRIu64 " successfully\n",
169                 session_id);
170 
171         ret = session_id;
172     } else {
173         if (local_err) {
174             error_report_err(local_err);
175         }
176         ret = -VIRTIO_CRYPTO_ERR;
177     }
178 
179 err:
180     g_free(info.cipher_key);
181     g_free(info.auth_key);
182     return ret;
183 }
184 
185 static uint8_t
186 virtio_crypto_handle_close_session(VirtIOCrypto *vcrypto,
187          struct virtio_crypto_destroy_session_req *close_sess_req,
188          uint32_t queue_id)
189 {
190     int ret;
191     uint64_t session_id;
192     uint32_t status;
193     Error *local_err = NULL;
194 
195     session_id = ldq_le_p(&close_sess_req->session_id);
196     DPRINTF("close session, id=%" PRIu64 "\n", session_id);
197 
198     ret = cryptodev_backend_sym_close_session(
199               vcrypto->cryptodev, session_id, queue_id, &local_err);
200     if (ret == 0) {
201         status = VIRTIO_CRYPTO_OK;
202     } else {
203         if (local_err) {
204             error_report_err(local_err);
205         } else {
206             error_report("destroy session failed");
207         }
208         status = VIRTIO_CRYPTO_ERR;
209     }
210 
211     return status;
212 }
213 
214 static void virtio_crypto_handle_ctrl(VirtIODevice *vdev, VirtQueue *vq)
215 {
216     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
217     struct virtio_crypto_op_ctrl_req ctrl;
218     VirtQueueElement *elem;
219     struct iovec *in_iov;
220     struct iovec *out_iov;
221     unsigned in_num;
222     unsigned out_num;
223     uint32_t queue_id;
224     uint32_t opcode;
225     struct virtio_crypto_session_input input;
226     int64_t session_id;
227     uint8_t status;
228     size_t s;
229 
230     for (;;) {
231         elem = virtqueue_pop(vq, sizeof(VirtQueueElement));
232         if (!elem) {
233             break;
234         }
235         if (elem->out_num < 1 || elem->in_num < 1) {
236             virtio_error(vdev, "virtio-crypto ctrl missing headers");
237             virtqueue_detach_element(vq, elem, 0);
238             g_free(elem);
239             break;
240         }
241 
242         out_num = elem->out_num;
243         out_iov = elem->out_sg;
244         in_num = elem->in_num;
245         in_iov = elem->in_sg;
246         if (unlikely(iov_to_buf(out_iov, out_num, 0, &ctrl, sizeof(ctrl))
247                     != sizeof(ctrl))) {
248             virtio_error(vdev, "virtio-crypto request ctrl_hdr too short");
249             virtqueue_detach_element(vq, elem, 0);
250             g_free(elem);
251             break;
252         }
253         iov_discard_front(&out_iov, &out_num, sizeof(ctrl));
254 
255         opcode = ldl_le_p(&ctrl.header.opcode);
256         queue_id = ldl_le_p(&ctrl.header.queue_id);
257 
258         switch (opcode) {
259         case VIRTIO_CRYPTO_CIPHER_CREATE_SESSION:
260             memset(&input, 0, sizeof(input));
261             session_id = virtio_crypto_create_sym_session(vcrypto,
262                              &ctrl.u.sym_create_session,
263                              queue_id, opcode,
264                              out_iov, out_num);
265             /* Serious errors, need to reset virtio crypto device */
266             if (session_id == -EFAULT) {
267                 virtqueue_detach_element(vq, elem, 0);
268                 break;
269             } else if (session_id == -VIRTIO_CRYPTO_NOTSUPP) {
270                 stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP);
271             } else if (session_id == -VIRTIO_CRYPTO_ERR) {
272                 stl_le_p(&input.status, VIRTIO_CRYPTO_ERR);
273             } else {
274                 /* Set the session id */
275                 stq_le_p(&input.session_id, session_id);
276                 stl_le_p(&input.status, VIRTIO_CRYPTO_OK);
277             }
278 
279             s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input));
280             if (unlikely(s != sizeof(input))) {
281                 virtio_error(vdev, "virtio-crypto input incorrect");
282                 virtqueue_detach_element(vq, elem, 0);
283                 break;
284             }
285             virtqueue_push(vq, elem, sizeof(input));
286             virtio_notify(vdev, vq);
287             break;
288         case VIRTIO_CRYPTO_CIPHER_DESTROY_SESSION:
289         case VIRTIO_CRYPTO_HASH_DESTROY_SESSION:
290         case VIRTIO_CRYPTO_MAC_DESTROY_SESSION:
291         case VIRTIO_CRYPTO_AEAD_DESTROY_SESSION:
292             status = virtio_crypto_handle_close_session(vcrypto,
293                    &ctrl.u.destroy_session, queue_id);
294             /* The status only occupy one byte, we can directly use it */
295             s = iov_from_buf(in_iov, in_num, 0, &status, sizeof(status));
296             if (unlikely(s != sizeof(status))) {
297                 virtio_error(vdev, "virtio-crypto status incorrect");
298                 virtqueue_detach_element(vq, elem, 0);
299                 break;
300             }
301             virtqueue_push(vq, elem, sizeof(status));
302             virtio_notify(vdev, vq);
303             break;
304         case VIRTIO_CRYPTO_HASH_CREATE_SESSION:
305         case VIRTIO_CRYPTO_MAC_CREATE_SESSION:
306         case VIRTIO_CRYPTO_AEAD_CREATE_SESSION:
307         default:
308             error_report("virtio-crypto unsupported ctrl opcode: %d", opcode);
309             memset(&input, 0, sizeof(input));
310             stl_le_p(&input.status, VIRTIO_CRYPTO_NOTSUPP);
311             s = iov_from_buf(in_iov, in_num, 0, &input, sizeof(input));
312             if (unlikely(s != sizeof(input))) {
313                 virtio_error(vdev, "virtio-crypto input incorrect");
314                 virtqueue_detach_element(vq, elem, 0);
315                 break;
316             }
317             virtqueue_push(vq, elem, sizeof(input));
318             virtio_notify(vdev, vq);
319 
320             break;
321         } /* end switch case */
322 
323         g_free(elem);
324     } /* end for loop */
325 }
326 
327 static void virtio_crypto_init_request(VirtIOCrypto *vcrypto, VirtQueue *vq,
328                                 VirtIOCryptoReq *req)
329 {
330     req->vcrypto = vcrypto;
331     req->vq = vq;
332     req->in = NULL;
333     req->in_iov = NULL;
334     req->in_num = 0;
335     req->in_len = 0;
336     req->flags = CRYPTODEV_BACKEND_ALG__MAX;
337     req->u.sym_op_info = NULL;
338 }
339 
340 static void virtio_crypto_free_request(VirtIOCryptoReq *req)
341 {
342     if (req) {
343         if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) {
344             size_t max_len;
345             CryptoDevBackendSymOpInfo *op_info = req->u.sym_op_info;
346 
347             max_len = op_info->iv_len +
348                       op_info->aad_len +
349                       op_info->src_len +
350                       op_info->dst_len +
351                       op_info->digest_result_len;
352 
353             /* Zeroize and free request data structure */
354             memset(op_info, 0, sizeof(*op_info) + max_len);
355             g_free(op_info);
356         }
357         g_free(req);
358     }
359 }
360 
361 static void
362 virtio_crypto_sym_input_data_helper(VirtIODevice *vdev,
363                 VirtIOCryptoReq *req,
364                 uint32_t status,
365                 CryptoDevBackendSymOpInfo *sym_op_info)
366 {
367     size_t s, len;
368 
369     if (status != VIRTIO_CRYPTO_OK) {
370         return;
371     }
372 
373     len = sym_op_info->src_len;
374     /* Save the cipher result */
375     s = iov_from_buf(req->in_iov, req->in_num, 0, sym_op_info->dst, len);
376     if (s != len) {
377         virtio_error(vdev, "virtio-crypto dest data incorrect");
378         return;
379     }
380 
381     iov_discard_front(&req->in_iov, &req->in_num, len);
382 
383     if (sym_op_info->op_type ==
384                       VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
385         /* Save the digest result */
386         s = iov_from_buf(req->in_iov, req->in_num, 0,
387                          sym_op_info->digest_result,
388                          sym_op_info->digest_result_len);
389         if (s != sym_op_info->digest_result_len) {
390             virtio_error(vdev, "virtio-crypto digest result incorrect");
391         }
392     }
393 }
394 
395 static void virtio_crypto_req_complete(VirtIOCryptoReq *req, uint8_t status)
396 {
397     VirtIOCrypto *vcrypto = req->vcrypto;
398     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
399 
400     if (req->flags == CRYPTODEV_BACKEND_ALG_SYM) {
401         virtio_crypto_sym_input_data_helper(vdev, req, status,
402                                             req->u.sym_op_info);
403     }
404     stb_p(&req->in->status, status);
405     virtqueue_push(req->vq, &req->elem, req->in_len);
406     virtio_notify(vdev, req->vq);
407 }
408 
409 static VirtIOCryptoReq *
410 virtio_crypto_get_request(VirtIOCrypto *s, VirtQueue *vq)
411 {
412     VirtIOCryptoReq *req = virtqueue_pop(vq, sizeof(VirtIOCryptoReq));
413 
414     if (req) {
415         virtio_crypto_init_request(s, vq, req);
416     }
417     return req;
418 }
419 
420 static CryptoDevBackendSymOpInfo *
421 virtio_crypto_sym_op_helper(VirtIODevice *vdev,
422            struct virtio_crypto_cipher_para *cipher_para,
423            struct virtio_crypto_alg_chain_data_para *alg_chain_para,
424            struct iovec *iov, unsigned int out_num)
425 {
426     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
427     CryptoDevBackendSymOpInfo *op_info;
428     uint32_t src_len = 0, dst_len = 0;
429     uint32_t iv_len = 0;
430     uint32_t aad_len = 0, hash_result_len = 0;
431     uint32_t hash_start_src_offset = 0, len_to_hash = 0;
432     uint32_t cipher_start_src_offset = 0, len_to_cipher = 0;
433 
434     uint64_t max_len, curr_size = 0;
435     size_t s;
436 
437     /* Plain cipher */
438     if (cipher_para) {
439         iv_len = ldl_le_p(&cipher_para->iv_len);
440         src_len = ldl_le_p(&cipher_para->src_data_len);
441         dst_len = ldl_le_p(&cipher_para->dst_data_len);
442     } else if (alg_chain_para) { /* Algorithm chain */
443         iv_len = ldl_le_p(&alg_chain_para->iv_len);
444         src_len = ldl_le_p(&alg_chain_para->src_data_len);
445         dst_len = ldl_le_p(&alg_chain_para->dst_data_len);
446 
447         aad_len = ldl_le_p(&alg_chain_para->aad_len);
448         hash_result_len = ldl_le_p(&alg_chain_para->hash_result_len);
449         hash_start_src_offset = ldl_le_p(
450                          &alg_chain_para->hash_start_src_offset);
451         cipher_start_src_offset = ldl_le_p(
452                          &alg_chain_para->cipher_start_src_offset);
453         len_to_cipher = ldl_le_p(&alg_chain_para->len_to_cipher);
454         len_to_hash = ldl_le_p(&alg_chain_para->len_to_hash);
455     } else {
456         return NULL;
457     }
458 
459     max_len = (uint64_t)iv_len + aad_len + src_len + dst_len + hash_result_len;
460     if (unlikely(max_len > vcrypto->conf.max_size)) {
461         virtio_error(vdev, "virtio-crypto too big length");
462         return NULL;
463     }
464 
465     op_info = g_malloc0(sizeof(CryptoDevBackendSymOpInfo) + max_len);
466     op_info->iv_len = iv_len;
467     op_info->src_len = src_len;
468     op_info->dst_len = dst_len;
469     op_info->aad_len = aad_len;
470     op_info->digest_result_len = hash_result_len;
471     op_info->hash_start_src_offset = hash_start_src_offset;
472     op_info->len_to_hash = len_to_hash;
473     op_info->cipher_start_src_offset = cipher_start_src_offset;
474     op_info->len_to_cipher = len_to_cipher;
475     /* Handle the initilization vector */
476     if (op_info->iv_len > 0) {
477         DPRINTF("iv_len=%" PRIu32 "\n", op_info->iv_len);
478         op_info->iv = op_info->data + curr_size;
479 
480         s = iov_to_buf(iov, out_num, 0, op_info->iv, op_info->iv_len);
481         if (unlikely(s != op_info->iv_len)) {
482             virtio_error(vdev, "virtio-crypto iv incorrect");
483             goto err;
484         }
485         iov_discard_front(&iov, &out_num, op_info->iv_len);
486         curr_size += op_info->iv_len;
487     }
488 
489     /* Handle additional authentication data if exists */
490     if (op_info->aad_len > 0) {
491         DPRINTF("aad_len=%" PRIu32 "\n", op_info->aad_len);
492         op_info->aad_data = op_info->data + curr_size;
493 
494         s = iov_to_buf(iov, out_num, 0, op_info->aad_data, op_info->aad_len);
495         if (unlikely(s != op_info->aad_len)) {
496             virtio_error(vdev, "virtio-crypto additional auth data incorrect");
497             goto err;
498         }
499         iov_discard_front(&iov, &out_num, op_info->aad_len);
500 
501         curr_size += op_info->aad_len;
502     }
503 
504     /* Handle the source data */
505     if (op_info->src_len > 0) {
506         DPRINTF("src_len=%" PRIu32 "\n", op_info->src_len);
507         op_info->src = op_info->data + curr_size;
508 
509         s = iov_to_buf(iov, out_num, 0, op_info->src, op_info->src_len);
510         if (unlikely(s != op_info->src_len)) {
511             virtio_error(vdev, "virtio-crypto source data incorrect");
512             goto err;
513         }
514         iov_discard_front(&iov, &out_num, op_info->src_len);
515 
516         curr_size += op_info->src_len;
517     }
518 
519     /* Handle the destination data */
520     op_info->dst = op_info->data + curr_size;
521     curr_size += op_info->dst_len;
522 
523     DPRINTF("dst_len=%" PRIu32 "\n", op_info->dst_len);
524 
525     /* Handle the hash digest result */
526     if (hash_result_len > 0) {
527         DPRINTF("hash_result_len=%" PRIu32 "\n", hash_result_len);
528         op_info->digest_result = op_info->data + curr_size;
529     }
530 
531     return op_info;
532 
533 err:
534     g_free(op_info);
535     return NULL;
536 }
537 
538 static int
539 virtio_crypto_handle_sym_req(VirtIOCrypto *vcrypto,
540                struct virtio_crypto_sym_data_req *req,
541                CryptoDevBackendSymOpInfo **sym_op_info,
542                struct iovec *iov, unsigned int out_num)
543 {
544     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
545     uint32_t op_type;
546     CryptoDevBackendSymOpInfo *op_info;
547 
548     op_type = ldl_le_p(&req->op_type);
549 
550     if (op_type == VIRTIO_CRYPTO_SYM_OP_CIPHER) {
551         op_info = virtio_crypto_sym_op_helper(vdev, &req->u.cipher.para,
552                                               NULL, iov, out_num);
553         if (!op_info) {
554             return -EFAULT;
555         }
556         op_info->op_type = op_type;
557     } else if (op_type == VIRTIO_CRYPTO_SYM_OP_ALGORITHM_CHAINING) {
558         op_info = virtio_crypto_sym_op_helper(vdev, NULL,
559                                               &req->u.chain.para,
560                                               iov, out_num);
561         if (!op_info) {
562             return -EFAULT;
563         }
564         op_info->op_type = op_type;
565     } else {
566         /* VIRTIO_CRYPTO_SYM_OP_NONE */
567         error_report("virtio-crypto unsupported cipher type");
568         return -VIRTIO_CRYPTO_NOTSUPP;
569     }
570 
571     *sym_op_info = op_info;
572 
573     return 0;
574 }
575 
576 static int
577 virtio_crypto_handle_request(VirtIOCryptoReq *request)
578 {
579     VirtIOCrypto *vcrypto = request->vcrypto;
580     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
581     VirtQueueElement *elem = &request->elem;
582     int queue_index = virtio_crypto_vq2q(virtio_get_queue_index(request->vq));
583     struct virtio_crypto_op_data_req req;
584     int ret;
585     struct iovec *in_iov;
586     struct iovec *out_iov;
587     unsigned in_num;
588     unsigned out_num;
589     uint32_t opcode;
590     uint8_t status = VIRTIO_CRYPTO_ERR;
591     uint64_t session_id;
592     CryptoDevBackendSymOpInfo *sym_op_info = NULL;
593     Error *local_err = NULL;
594 
595     if (elem->out_num < 1 || elem->in_num < 1) {
596         virtio_error(vdev, "virtio-crypto dataq missing headers");
597         return -1;
598     }
599 
600     out_num = elem->out_num;
601     out_iov = elem->out_sg;
602     in_num = elem->in_num;
603     in_iov = elem->in_sg;
604     if (unlikely(iov_to_buf(out_iov, out_num, 0, &req, sizeof(req))
605                 != sizeof(req))) {
606         virtio_error(vdev, "virtio-crypto request outhdr too short");
607         return -1;
608     }
609     iov_discard_front(&out_iov, &out_num, sizeof(req));
610 
611     if (in_iov[in_num - 1].iov_len <
612             sizeof(struct virtio_crypto_inhdr)) {
613         virtio_error(vdev, "virtio-crypto request inhdr too short");
614         return -1;
615     }
616     /* We always touch the last byte, so just see how big in_iov is. */
617     request->in_len = iov_size(in_iov, in_num);
618     request->in = (void *)in_iov[in_num - 1].iov_base
619               + in_iov[in_num - 1].iov_len
620               - sizeof(struct virtio_crypto_inhdr);
621     iov_discard_back(in_iov, &in_num, sizeof(struct virtio_crypto_inhdr));
622 
623     /*
624      * The length of operation result, including dest_data
625      * and digest_result if exists.
626      */
627     request->in_num = in_num;
628     request->in_iov = in_iov;
629 
630     opcode = ldl_le_p(&req.header.opcode);
631     session_id = ldq_le_p(&req.header.session_id);
632 
633     switch (opcode) {
634     case VIRTIO_CRYPTO_CIPHER_ENCRYPT:
635     case VIRTIO_CRYPTO_CIPHER_DECRYPT:
636         ret = virtio_crypto_handle_sym_req(vcrypto,
637                          &req.u.sym_req,
638                          &sym_op_info,
639                          out_iov, out_num);
640         /* Serious errors, need to reset virtio crypto device */
641         if (ret == -EFAULT) {
642             return -1;
643         } else if (ret == -VIRTIO_CRYPTO_NOTSUPP) {
644             virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP);
645             virtio_crypto_free_request(request);
646         } else {
647             sym_op_info->session_id = session_id;
648 
649             /* Set request's parameter */
650             request->flags = CRYPTODEV_BACKEND_ALG_SYM;
651             request->u.sym_op_info = sym_op_info;
652             ret = cryptodev_backend_crypto_operation(vcrypto->cryptodev,
653                                     request, queue_index, &local_err);
654             if (ret < 0) {
655                 status = -ret;
656                 if (local_err) {
657                     error_report_err(local_err);
658                 }
659             } else { /* ret == VIRTIO_CRYPTO_OK */
660                 status = ret;
661             }
662             virtio_crypto_req_complete(request, status);
663             virtio_crypto_free_request(request);
664         }
665         break;
666     case VIRTIO_CRYPTO_HASH:
667     case VIRTIO_CRYPTO_MAC:
668     case VIRTIO_CRYPTO_AEAD_ENCRYPT:
669     case VIRTIO_CRYPTO_AEAD_DECRYPT:
670     default:
671         error_report("virtio-crypto unsupported dataq opcode: %u",
672                      opcode);
673         virtio_crypto_req_complete(request, VIRTIO_CRYPTO_NOTSUPP);
674         virtio_crypto_free_request(request);
675     }
676 
677     return 0;
678 }
679 
680 static void virtio_crypto_handle_dataq(VirtIODevice *vdev, VirtQueue *vq)
681 {
682     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
683     VirtIOCryptoReq *req;
684 
685     while ((req = virtio_crypto_get_request(vcrypto, vq))) {
686         if (virtio_crypto_handle_request(req) < 0) {
687             virtqueue_detach_element(req->vq, &req->elem, 0);
688             virtio_crypto_free_request(req);
689             break;
690         }
691     }
692 }
693 
694 static void virtio_crypto_dataq_bh(void *opaque)
695 {
696     VirtIOCryptoQueue *q = opaque;
697     VirtIOCrypto *vcrypto = q->vcrypto;
698     VirtIODevice *vdev = VIRTIO_DEVICE(vcrypto);
699 
700     /* This happens when device was stopped but BH wasn't. */
701     if (!vdev->vm_running) {
702         return;
703     }
704 
705     /* Just in case the driver is not ready on more */
706     if (unlikely(!(vdev->status & VIRTIO_CONFIG_S_DRIVER_OK))) {
707         return;
708     }
709 
710     for (;;) {
711         virtio_crypto_handle_dataq(vdev, q->dataq);
712         virtio_queue_set_notification(q->dataq, 1);
713 
714         /* Are we done or did the guest add more buffers? */
715         if (virtio_queue_empty(q->dataq)) {
716             break;
717         }
718 
719         virtio_queue_set_notification(q->dataq, 0);
720     }
721 }
722 
723 static void
724 virtio_crypto_handle_dataq_bh(VirtIODevice *vdev, VirtQueue *vq)
725 {
726     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
727     VirtIOCryptoQueue *q =
728          &vcrypto->vqs[virtio_crypto_vq2q(virtio_get_queue_index(vq))];
729 
730     /* This happens when device was stopped but VCPU wasn't. */
731     if (!vdev->vm_running) {
732         return;
733     }
734     virtio_queue_set_notification(vq, 0);
735     qemu_bh_schedule(q->dataq_bh);
736 }
737 
738 static uint64_t virtio_crypto_get_features(VirtIODevice *vdev,
739                                            uint64_t features,
740                                            Error **errp)
741 {
742     return features;
743 }
744 
745 static void virtio_crypto_reset(VirtIODevice *vdev)
746 {
747     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
748     /* multiqueue is disabled by default */
749     vcrypto->curr_queues = 1;
750     if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) {
751         vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY;
752     } else {
753         vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY;
754     }
755 }
756 
757 static void virtio_crypto_init_config(VirtIODevice *vdev)
758 {
759     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
760 
761     vcrypto->conf.crypto_services =
762                      vcrypto->conf.cryptodev->conf.crypto_services;
763     vcrypto->conf.cipher_algo_l =
764                      vcrypto->conf.cryptodev->conf.cipher_algo_l;
765     vcrypto->conf.cipher_algo_h =
766                      vcrypto->conf.cryptodev->conf.cipher_algo_h;
767     vcrypto->conf.hash_algo = vcrypto->conf.cryptodev->conf.hash_algo;
768     vcrypto->conf.mac_algo_l = vcrypto->conf.cryptodev->conf.mac_algo_l;
769     vcrypto->conf.mac_algo_h = vcrypto->conf.cryptodev->conf.mac_algo_h;
770     vcrypto->conf.aead_algo = vcrypto->conf.cryptodev->conf.aead_algo;
771     vcrypto->conf.max_cipher_key_len =
772                   vcrypto->conf.cryptodev->conf.max_cipher_key_len;
773     vcrypto->conf.max_auth_key_len =
774                   vcrypto->conf.cryptodev->conf.max_auth_key_len;
775     vcrypto->conf.max_size = vcrypto->conf.cryptodev->conf.max_size;
776 }
777 
778 static void virtio_crypto_device_realize(DeviceState *dev, Error **errp)
779 {
780     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
781     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev);
782     int i;
783 
784     vcrypto->cryptodev = vcrypto->conf.cryptodev;
785     if (vcrypto->cryptodev == NULL) {
786         error_setg(errp, "'cryptodev' parameter expects a valid object");
787         return;
788     } else if (cryptodev_backend_is_used(vcrypto->cryptodev)) {
789         char *path = object_get_canonical_path_component(OBJECT(vcrypto->conf.cryptodev));
790         error_setg(errp, "can't use already used cryptodev backend: %s", path);
791         g_free(path);
792         return;
793     }
794 
795     vcrypto->max_queues = MAX(vcrypto->cryptodev->conf.peers.queues, 1);
796     if (vcrypto->max_queues + 1 > VIRTIO_QUEUE_MAX) {
797         error_setg(errp, "Invalid number of queues (= %" PRIu32 "), "
798                    "must be a positive integer less than %d.",
799                    vcrypto->max_queues, VIRTIO_QUEUE_MAX);
800         return;
801     }
802 
803     virtio_init(vdev, "virtio-crypto", VIRTIO_ID_CRYPTO, vcrypto->config_size);
804     vcrypto->curr_queues = 1;
805     vcrypto->vqs = g_malloc0(sizeof(VirtIOCryptoQueue) * vcrypto->max_queues);
806     for (i = 0; i < vcrypto->max_queues; i++) {
807         vcrypto->vqs[i].dataq =
808                  virtio_add_queue(vdev, 1024, virtio_crypto_handle_dataq_bh);
809         vcrypto->vqs[i].dataq_bh =
810                  qemu_bh_new(virtio_crypto_dataq_bh, &vcrypto->vqs[i]);
811         vcrypto->vqs[i].vcrypto = vcrypto;
812     }
813 
814     vcrypto->ctrl_vq = virtio_add_queue(vdev, 64, virtio_crypto_handle_ctrl);
815     if (!cryptodev_backend_is_ready(vcrypto->cryptodev)) {
816         vcrypto->status &= ~VIRTIO_CRYPTO_S_HW_READY;
817     } else {
818         vcrypto->status |= VIRTIO_CRYPTO_S_HW_READY;
819     }
820 
821     virtio_crypto_init_config(vdev);
822     cryptodev_backend_set_used(vcrypto->cryptodev, true);
823 }
824 
825 static void virtio_crypto_device_unrealize(DeviceState *dev, Error **errp)
826 {
827     VirtIODevice *vdev = VIRTIO_DEVICE(dev);
828     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(dev);
829     VirtIOCryptoQueue *q;
830     int i, max_queues;
831 
832     max_queues = vcrypto->multiqueue ? vcrypto->max_queues : 1;
833     for (i = 0; i < max_queues; i++) {
834         virtio_del_queue(vdev, i);
835         q = &vcrypto->vqs[i];
836         qemu_bh_delete(q->dataq_bh);
837     }
838 
839     g_free(vcrypto->vqs);
840 
841     virtio_cleanup(vdev);
842     cryptodev_backend_set_used(vcrypto->cryptodev, false);
843 }
844 
845 static const VMStateDescription vmstate_virtio_crypto = {
846     .name = "virtio-crypto",
847     .unmigratable = 1,
848     .minimum_version_id = VIRTIO_CRYPTO_VM_VERSION,
849     .version_id = VIRTIO_CRYPTO_VM_VERSION,
850     .fields = (VMStateField[]) {
851         VMSTATE_VIRTIO_DEVICE,
852         VMSTATE_END_OF_LIST()
853     },
854 };
855 
856 static Property virtio_crypto_properties[] = {
857     DEFINE_PROP_LINK("cryptodev", VirtIOCrypto, conf.cryptodev,
858                      TYPE_CRYPTODEV_BACKEND, CryptoDevBackend *),
859     DEFINE_PROP_END_OF_LIST(),
860 };
861 
862 static void virtio_crypto_get_config(VirtIODevice *vdev, uint8_t *config)
863 {
864     VirtIOCrypto *c = VIRTIO_CRYPTO(vdev);
865     struct virtio_crypto_config crypto_cfg = {};
866 
867     /*
868      * Virtio-crypto device conforms to VIRTIO 1.0 which is always LE,
869      * so we can use LE accessors directly.
870      */
871     stl_le_p(&crypto_cfg.status, c->status);
872     stl_le_p(&crypto_cfg.max_dataqueues, c->max_queues);
873     stl_le_p(&crypto_cfg.crypto_services, c->conf.crypto_services);
874     stl_le_p(&crypto_cfg.cipher_algo_l, c->conf.cipher_algo_l);
875     stl_le_p(&crypto_cfg.cipher_algo_h, c->conf.cipher_algo_h);
876     stl_le_p(&crypto_cfg.hash_algo, c->conf.hash_algo);
877     stl_le_p(&crypto_cfg.mac_algo_l, c->conf.mac_algo_l);
878     stl_le_p(&crypto_cfg.mac_algo_h, c->conf.mac_algo_h);
879     stl_le_p(&crypto_cfg.aead_algo, c->conf.aead_algo);
880     stl_le_p(&crypto_cfg.max_cipher_key_len, c->conf.max_cipher_key_len);
881     stl_le_p(&crypto_cfg.max_auth_key_len, c->conf.max_auth_key_len);
882     stq_le_p(&crypto_cfg.max_size, c->conf.max_size);
883 
884     memcpy(config, &crypto_cfg, c->config_size);
885 }
886 
887 static bool virtio_crypto_started(VirtIOCrypto *c, uint8_t status)
888 {
889     VirtIODevice *vdev = VIRTIO_DEVICE(c);
890     return (status & VIRTIO_CONFIG_S_DRIVER_OK) &&
891         (c->status & VIRTIO_CRYPTO_S_HW_READY) && vdev->vm_running;
892 }
893 
894 static void virtio_crypto_vhost_status(VirtIOCrypto *c, uint8_t status)
895 {
896     VirtIODevice *vdev = VIRTIO_DEVICE(c);
897     int queues = c->multiqueue ? c->max_queues : 1;
898     CryptoDevBackend *b = c->cryptodev;
899     CryptoDevBackendClient *cc = b->conf.peers.ccs[0];
900 
901     if (!cryptodev_get_vhost(cc, b, 0)) {
902         return;
903     }
904 
905     if ((virtio_crypto_started(c, status)) == !!c->vhost_started) {
906         return;
907     }
908 
909     if (!c->vhost_started) {
910         int r;
911 
912         c->vhost_started = 1;
913         r = cryptodev_vhost_start(vdev, queues);
914         if (r < 0) {
915             error_report("unable to start vhost crypto: %d: "
916                          "falling back on userspace virtio", -r);
917             c->vhost_started = 0;
918         }
919     } else {
920         cryptodev_vhost_stop(vdev, queues);
921         c->vhost_started = 0;
922     }
923 }
924 
925 static void virtio_crypto_set_status(VirtIODevice *vdev, uint8_t status)
926 {
927     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
928 
929     virtio_crypto_vhost_status(vcrypto, status);
930 }
931 
932 static void virtio_crypto_guest_notifier_mask(VirtIODevice *vdev, int idx,
933                                            bool mask)
934 {
935     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
936     int queue = virtio_crypto_vq2q(idx);
937 
938     assert(vcrypto->vhost_started);
939 
940     cryptodev_vhost_virtqueue_mask(vdev, queue, idx, mask);
941 }
942 
943 static bool virtio_crypto_guest_notifier_pending(VirtIODevice *vdev, int idx)
944 {
945     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(vdev);
946     int queue = virtio_crypto_vq2q(idx);
947 
948     assert(vcrypto->vhost_started);
949 
950     return cryptodev_vhost_virtqueue_pending(vdev, queue, idx);
951 }
952 
953 static void virtio_crypto_class_init(ObjectClass *klass, void *data)
954 {
955     DeviceClass *dc = DEVICE_CLASS(klass);
956     VirtioDeviceClass *vdc = VIRTIO_DEVICE_CLASS(klass);
957 
958     device_class_set_props(dc, virtio_crypto_properties);
959     dc->vmsd = &vmstate_virtio_crypto;
960     set_bit(DEVICE_CATEGORY_MISC, dc->categories);
961     vdc->realize = virtio_crypto_device_realize;
962     vdc->unrealize = virtio_crypto_device_unrealize;
963     vdc->get_config = virtio_crypto_get_config;
964     vdc->get_features = virtio_crypto_get_features;
965     vdc->reset = virtio_crypto_reset;
966     vdc->set_status = virtio_crypto_set_status;
967     vdc->guest_notifier_mask = virtio_crypto_guest_notifier_mask;
968     vdc->guest_notifier_pending = virtio_crypto_guest_notifier_pending;
969 }
970 
971 static void virtio_crypto_instance_init(Object *obj)
972 {
973     VirtIOCrypto *vcrypto = VIRTIO_CRYPTO(obj);
974 
975     /*
976      * The default config_size is sizeof(struct virtio_crypto_config).
977      * Can be overriden with virtio_crypto_set_config_size.
978      */
979     vcrypto->config_size = sizeof(struct virtio_crypto_config);
980 }
981 
982 static const TypeInfo virtio_crypto_info = {
983     .name = TYPE_VIRTIO_CRYPTO,
984     .parent = TYPE_VIRTIO_DEVICE,
985     .instance_size = sizeof(VirtIOCrypto),
986     .instance_init = virtio_crypto_instance_init,
987     .class_init = virtio_crypto_class_init,
988 };
989 
990 static void virtio_register_types(void)
991 {
992     type_register_static(&virtio_crypto_info);
993 }
994 
995 type_init(virtio_register_types)
996