xref: /openbmc/qemu/nbd/server.c (revision 8f75cae2dda8a55eb3a6c712bd22b18a90c0a5ac)
1 /*
2  *  Copyright (C) 2016-2021 Red Hat, Inc.
3  *  Copyright (C) 2005  Anthony Liguori <anthony@codemonkey.ws>
4  *
5  *  Network Block Device Server Side
6  *
7  *  This program is free software; you can redistribute it and/or modify
8  *  it under the terms of the GNU General Public License as published by
9  *  the Free Software Foundation; under version 2 of the License.
10  *
11  *  This program is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  *  GNU General Public License for more details.
15  *
16  *  You should have received a copy of the GNU General Public License
17  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #include "qemu/osdep.h"
21 
22 #include "block/export.h"
23 #include "qapi/error.h"
24 #include "qemu/queue.h"
25 #include "trace.h"
26 #include "nbd-internal.h"
27 #include "qemu/units.h"
28 
29 #define NBD_META_ID_BASE_ALLOCATION 0
30 #define NBD_META_ID_ALLOCATION_DEPTH 1
31 /* Dirty bitmaps use 'NBD_META_ID_DIRTY_BITMAP + i', so keep this id last. */
32 #define NBD_META_ID_DIRTY_BITMAP 2
33 
34 /*
35  * NBD_MAX_BLOCK_STATUS_EXTENTS: 1 MiB of extents data. An empirical
36  * constant. If an increase is needed, note that the NBD protocol
37  * recommends no larger than 32 mb, so that the client won't consider
38  * the reply as a denial of service attack.
39  */
40 #define NBD_MAX_BLOCK_STATUS_EXTENTS (1 * MiB / 8)
41 
42 static int system_errno_to_nbd_errno(int err)
43 {
44     switch (err) {
45     case 0:
46         return NBD_SUCCESS;
47     case EPERM:
48     case EROFS:
49         return NBD_EPERM;
50     case EIO:
51         return NBD_EIO;
52     case ENOMEM:
53         return NBD_ENOMEM;
54 #ifdef EDQUOT
55     case EDQUOT:
56 #endif
57     case EFBIG:
58     case ENOSPC:
59         return NBD_ENOSPC;
60     case EOVERFLOW:
61         return NBD_EOVERFLOW;
62     case ENOTSUP:
63 #if ENOTSUP != EOPNOTSUPP
64     case EOPNOTSUPP:
65 #endif
66         return NBD_ENOTSUP;
67     case ESHUTDOWN:
68         return NBD_ESHUTDOWN;
69     case EINVAL:
70     default:
71         return NBD_EINVAL;
72     }
73 }
74 
75 /* Definitions for opaque data types */
76 
77 typedef struct NBDRequestData NBDRequestData;
78 
79 struct NBDRequestData {
80     QSIMPLEQ_ENTRY(NBDRequestData) entry;
81     NBDClient *client;
82     uint8_t *data;
83     bool complete;
84 };
85 
86 struct NBDExport {
87     BlockExport common;
88 
89     char *name;
90     char *description;
91     uint64_t size;
92     uint16_t nbdflags;
93     QTAILQ_HEAD(, NBDClient) clients;
94     QTAILQ_ENTRY(NBDExport) next;
95 
96     BlockBackend *eject_notifier_blk;
97     Notifier eject_notifier;
98 
99     bool allocation_depth;
100     BdrvDirtyBitmap **export_bitmaps;
101     size_t nr_export_bitmaps;
102 };
103 
104 static QTAILQ_HEAD(, NBDExport) exports = QTAILQ_HEAD_INITIALIZER(exports);
105 
106 /* NBDExportMetaContexts represents a list of contexts to be exported,
107  * as selected by NBD_OPT_SET_META_CONTEXT. Also used for
108  * NBD_OPT_LIST_META_CONTEXT. */
109 typedef struct NBDExportMetaContexts {
110     NBDExport *exp;
111     size_t count; /* number of negotiated contexts */
112     bool base_allocation; /* export base:allocation context (block status) */
113     bool allocation_depth; /* export qemu:allocation-depth */
114     bool *bitmaps; /*
115                     * export qemu:dirty-bitmap:<export bitmap name>,
116                     * sized by exp->nr_export_bitmaps
117                     */
118 } NBDExportMetaContexts;
119 
120 struct NBDClient {
121     int refcount;
122     void (*close_fn)(NBDClient *client, bool negotiated);
123 
124     NBDExport *exp;
125     QCryptoTLSCreds *tlscreds;
126     char *tlsauthz;
127     QIOChannelSocket *sioc; /* The underlying data channel */
128     QIOChannel *ioc; /* The current I/O channel which may differ (eg TLS) */
129 
130     Coroutine *recv_coroutine;
131 
132     CoMutex send_lock;
133     Coroutine *send_coroutine;
134 
135     bool read_yielding;
136     bool quiescing;
137 
138     QTAILQ_ENTRY(NBDClient) next;
139     int nb_requests;
140     bool closing;
141 
142     uint32_t check_align; /* If non-zero, check for aligned client requests */
143 
144     bool structured_reply;
145     NBDExportMetaContexts export_meta;
146 
147     uint32_t opt; /* Current option being negotiated */
148     uint32_t optlen; /* remaining length of data in ioc for the option being
149                         negotiated now */
150 };
151 
152 static void nbd_client_receive_next_request(NBDClient *client);
153 
154 /* Basic flow for negotiation
155 
156    Server         Client
157    Negotiate
158 
159    or
160 
161    Server         Client
162    Negotiate #1
163                   Option
164    Negotiate #2
165 
166    ----
167 
168    followed by
169 
170    Server         Client
171                   Request
172    Response
173                   Request
174    Response
175                   ...
176    ...
177                   Request (type == 2)
178 
179 */
180 
181 static inline void set_be_option_rep(NBDOptionReply *rep, uint32_t option,
182                                      uint32_t type, uint32_t length)
183 {
184     stq_be_p(&rep->magic, NBD_REP_MAGIC);
185     stl_be_p(&rep->option, option);
186     stl_be_p(&rep->type, type);
187     stl_be_p(&rep->length, length);
188 }
189 
190 /* Send a reply header, including length, but no payload.
191  * Return -errno on error, 0 on success. */
192 static int nbd_negotiate_send_rep_len(NBDClient *client, uint32_t type,
193                                       uint32_t len, Error **errp)
194 {
195     NBDOptionReply rep;
196 
197     trace_nbd_negotiate_send_rep_len(client->opt, nbd_opt_lookup(client->opt),
198                                      type, nbd_rep_lookup(type), len);
199 
200     assert(len < NBD_MAX_BUFFER_SIZE);
201 
202     set_be_option_rep(&rep, client->opt, type, len);
203     return nbd_write(client->ioc, &rep, sizeof(rep), errp);
204 }
205 
206 /* Send a reply header with default 0 length.
207  * Return -errno on error, 0 on success. */
208 static int nbd_negotiate_send_rep(NBDClient *client, uint32_t type,
209                                   Error **errp)
210 {
211     return nbd_negotiate_send_rep_len(client, type, 0, errp);
212 }
213 
214 /* Send an error reply.
215  * Return -errno on error, 0 on success. */
216 static int GCC_FMT_ATTR(4, 0)
217 nbd_negotiate_send_rep_verr(NBDClient *client, uint32_t type,
218                             Error **errp, const char *fmt, va_list va)
219 {
220     ERRP_GUARD();
221     g_autofree char *msg = NULL;
222     int ret;
223     size_t len;
224 
225     msg = g_strdup_vprintf(fmt, va);
226     len = strlen(msg);
227     assert(len < NBD_MAX_STRING_SIZE);
228     trace_nbd_negotiate_send_rep_err(msg);
229     ret = nbd_negotiate_send_rep_len(client, type, len, errp);
230     if (ret < 0) {
231         return ret;
232     }
233     if (nbd_write(client->ioc, msg, len, errp) < 0) {
234         error_prepend(errp, "write failed (error message): ");
235         return -EIO;
236     }
237 
238     return 0;
239 }
240 
241 /*
242  * Return a malloc'd copy of @name suitable for use in an error reply.
243  */
244 static char *
245 nbd_sanitize_name(const char *name)
246 {
247     if (strnlen(name, 80) < 80) {
248         return g_strdup(name);
249     }
250     /* XXX Should we also try to sanitize any control characters? */
251     return g_strdup_printf("%.80s...", name);
252 }
253 
254 /* Send an error reply.
255  * Return -errno on error, 0 on success. */
256 static int GCC_FMT_ATTR(4, 5)
257 nbd_negotiate_send_rep_err(NBDClient *client, uint32_t type,
258                            Error **errp, const char *fmt, ...)
259 {
260     va_list va;
261     int ret;
262 
263     va_start(va, fmt);
264     ret = nbd_negotiate_send_rep_verr(client, type, errp, fmt, va);
265     va_end(va);
266     return ret;
267 }
268 
269 /* Drop remainder of the current option, and send a reply with the
270  * given error type and message. Return -errno on read or write
271  * failure; or 0 if connection is still live. */
272 static int GCC_FMT_ATTR(4, 0)
273 nbd_opt_vdrop(NBDClient *client, uint32_t type, Error **errp,
274               const char *fmt, va_list va)
275 {
276     int ret = nbd_drop(client->ioc, client->optlen, errp);
277 
278     client->optlen = 0;
279     if (!ret) {
280         ret = nbd_negotiate_send_rep_verr(client, type, errp, fmt, va);
281     }
282     return ret;
283 }
284 
285 static int GCC_FMT_ATTR(4, 5)
286 nbd_opt_drop(NBDClient *client, uint32_t type, Error **errp,
287              const char *fmt, ...)
288 {
289     int ret;
290     va_list va;
291 
292     va_start(va, fmt);
293     ret = nbd_opt_vdrop(client, type, errp, fmt, va);
294     va_end(va);
295 
296     return ret;
297 }
298 
299 static int GCC_FMT_ATTR(3, 4)
300 nbd_opt_invalid(NBDClient *client, Error **errp, const char *fmt, ...)
301 {
302     int ret;
303     va_list va;
304 
305     va_start(va, fmt);
306     ret = nbd_opt_vdrop(client, NBD_REP_ERR_INVALID, errp, fmt, va);
307     va_end(va);
308 
309     return ret;
310 }
311 
312 /* Read size bytes from the unparsed payload of the current option.
313  * If @check_nul, require that no NUL bytes appear in buffer.
314  * Return -errno on I/O error, 0 if option was completely handled by
315  * sending a reply about inconsistent lengths, or 1 on success. */
316 static int nbd_opt_read(NBDClient *client, void *buffer, size_t size,
317                         bool check_nul, Error **errp)
318 {
319     if (size > client->optlen) {
320         return nbd_opt_invalid(client, errp,
321                                "Inconsistent lengths in option %s",
322                                nbd_opt_lookup(client->opt));
323     }
324     client->optlen -= size;
325     if (qio_channel_read_all(client->ioc, buffer, size, errp) < 0) {
326         return -EIO;
327     }
328 
329     if (check_nul && strnlen(buffer, size) != size) {
330         return nbd_opt_invalid(client, errp,
331                                "Unexpected embedded NUL in option %s",
332                                nbd_opt_lookup(client->opt));
333     }
334     return 1;
335 }
336 
337 /* Drop size bytes from the unparsed payload of the current option.
338  * Return -errno on I/O error, 0 if option was completely handled by
339  * sending a reply about inconsistent lengths, or 1 on success. */
340 static int nbd_opt_skip(NBDClient *client, size_t size, Error **errp)
341 {
342     if (size > client->optlen) {
343         return nbd_opt_invalid(client, errp,
344                                "Inconsistent lengths in option %s",
345                                nbd_opt_lookup(client->opt));
346     }
347     client->optlen -= size;
348     return nbd_drop(client->ioc, size, errp) < 0 ? -EIO : 1;
349 }
350 
351 /* nbd_opt_read_name
352  *
353  * Read a string with the format:
354  *   uint32_t len     (<= NBD_MAX_STRING_SIZE)
355  *   len bytes string (not 0-terminated)
356  *
357  * On success, @name will be allocated.
358  * If @length is non-null, it will be set to the actual string length.
359  *
360  * Return -errno on I/O error, 0 if option was completely handled by
361  * sending a reply about inconsistent lengths, or 1 on success.
362  */
363 static int nbd_opt_read_name(NBDClient *client, char **name, uint32_t *length,
364                              Error **errp)
365 {
366     int ret;
367     uint32_t len;
368     g_autofree char *local_name = NULL;
369 
370     *name = NULL;
371     ret = nbd_opt_read(client, &len, sizeof(len), false, errp);
372     if (ret <= 0) {
373         return ret;
374     }
375     len = cpu_to_be32(len);
376 
377     if (len > NBD_MAX_STRING_SIZE) {
378         return nbd_opt_invalid(client, errp,
379                                "Invalid name length: %" PRIu32, len);
380     }
381 
382     local_name = g_malloc(len + 1);
383     ret = nbd_opt_read(client, local_name, len, true, errp);
384     if (ret <= 0) {
385         return ret;
386     }
387     local_name[len] = '\0';
388 
389     if (length) {
390         *length = len;
391     }
392     *name = g_steal_pointer(&local_name);
393 
394     return 1;
395 }
396 
397 /* Send a single NBD_REP_SERVER reply to NBD_OPT_LIST, including payload.
398  * Return -errno on error, 0 on success. */
399 static int nbd_negotiate_send_rep_list(NBDClient *client, NBDExport *exp,
400                                        Error **errp)
401 {
402     ERRP_GUARD();
403     size_t name_len, desc_len;
404     uint32_t len;
405     const char *name = exp->name ? exp->name : "";
406     const char *desc = exp->description ? exp->description : "";
407     QIOChannel *ioc = client->ioc;
408     int ret;
409 
410     trace_nbd_negotiate_send_rep_list(name, desc);
411     name_len = strlen(name);
412     desc_len = strlen(desc);
413     assert(name_len <= NBD_MAX_STRING_SIZE && desc_len <= NBD_MAX_STRING_SIZE);
414     len = name_len + desc_len + sizeof(len);
415     ret = nbd_negotiate_send_rep_len(client, NBD_REP_SERVER, len, errp);
416     if (ret < 0) {
417         return ret;
418     }
419 
420     len = cpu_to_be32(name_len);
421     if (nbd_write(ioc, &len, sizeof(len), errp) < 0) {
422         error_prepend(errp, "write failed (name length): ");
423         return -EINVAL;
424     }
425 
426     if (nbd_write(ioc, name, name_len, errp) < 0) {
427         error_prepend(errp, "write failed (name buffer): ");
428         return -EINVAL;
429     }
430 
431     if (nbd_write(ioc, desc, desc_len, errp) < 0) {
432         error_prepend(errp, "write failed (description buffer): ");
433         return -EINVAL;
434     }
435 
436     return 0;
437 }
438 
439 /* Process the NBD_OPT_LIST command, with a potential series of replies.
440  * Return -errno on error, 0 on success. */
441 static int nbd_negotiate_handle_list(NBDClient *client, Error **errp)
442 {
443     NBDExport *exp;
444     assert(client->opt == NBD_OPT_LIST);
445 
446     /* For each export, send a NBD_REP_SERVER reply. */
447     QTAILQ_FOREACH(exp, &exports, next) {
448         if (nbd_negotiate_send_rep_list(client, exp, errp)) {
449             return -EINVAL;
450         }
451     }
452     /* Finish with a NBD_REP_ACK. */
453     return nbd_negotiate_send_rep(client, NBD_REP_ACK, errp);
454 }
455 
456 static void nbd_check_meta_export(NBDClient *client)
457 {
458     if (client->exp != client->export_meta.exp) {
459         client->export_meta.count = 0;
460     }
461 }
462 
463 /* Send a reply to NBD_OPT_EXPORT_NAME.
464  * Return -errno on error, 0 on success. */
465 static int nbd_negotiate_handle_export_name(NBDClient *client, bool no_zeroes,
466                                             Error **errp)
467 {
468     ERRP_GUARD();
469     g_autofree char *name = NULL;
470     char buf[NBD_REPLY_EXPORT_NAME_SIZE] = "";
471     size_t len;
472     int ret;
473     uint16_t myflags;
474 
475     /* Client sends:
476         [20 ..  xx]   export name (length bytes)
477        Server replies:
478         [ 0 ..   7]   size
479         [ 8 ..   9]   export flags
480         [10 .. 133]   reserved     (0) [unless no_zeroes]
481      */
482     trace_nbd_negotiate_handle_export_name();
483     if (client->optlen > NBD_MAX_STRING_SIZE) {
484         error_setg(errp, "Bad length received");
485         return -EINVAL;
486     }
487     name = g_malloc(client->optlen + 1);
488     if (nbd_read(client->ioc, name, client->optlen, "export name", errp) < 0) {
489         return -EIO;
490     }
491     name[client->optlen] = '\0';
492     client->optlen = 0;
493 
494     trace_nbd_negotiate_handle_export_name_request(name);
495 
496     client->exp = nbd_export_find(name);
497     if (!client->exp) {
498         error_setg(errp, "export not found");
499         return -EINVAL;
500     }
501 
502     myflags = client->exp->nbdflags;
503     if (client->structured_reply) {
504         myflags |= NBD_FLAG_SEND_DF;
505     }
506     trace_nbd_negotiate_new_style_size_flags(client->exp->size, myflags);
507     stq_be_p(buf, client->exp->size);
508     stw_be_p(buf + 8, myflags);
509     len = no_zeroes ? 10 : sizeof(buf);
510     ret = nbd_write(client->ioc, buf, len, errp);
511     if (ret < 0) {
512         error_prepend(errp, "write failed: ");
513         return ret;
514     }
515 
516     QTAILQ_INSERT_TAIL(&client->exp->clients, client, next);
517     blk_exp_ref(&client->exp->common);
518     nbd_check_meta_export(client);
519 
520     return 0;
521 }
522 
523 /* Send a single NBD_REP_INFO, with a buffer @buf of @length bytes.
524  * The buffer does NOT include the info type prefix.
525  * Return -errno on error, 0 if ready to send more. */
526 static int nbd_negotiate_send_info(NBDClient *client,
527                                    uint16_t info, uint32_t length, void *buf,
528                                    Error **errp)
529 {
530     int rc;
531 
532     trace_nbd_negotiate_send_info(info, nbd_info_lookup(info), length);
533     rc = nbd_negotiate_send_rep_len(client, NBD_REP_INFO,
534                                     sizeof(info) + length, errp);
535     if (rc < 0) {
536         return rc;
537     }
538     info = cpu_to_be16(info);
539     if (nbd_write(client->ioc, &info, sizeof(info), errp) < 0) {
540         return -EIO;
541     }
542     if (nbd_write(client->ioc, buf, length, errp) < 0) {
543         return -EIO;
544     }
545     return 0;
546 }
547 
548 /* nbd_reject_length: Handle any unexpected payload.
549  * @fatal requests that we quit talking to the client, even if we are able
550  * to successfully send an error reply.
551  * Return:
552  * -errno  transmission error occurred or @fatal was requested, errp is set
553  * 0       error message successfully sent to client, errp is not set
554  */
555 static int nbd_reject_length(NBDClient *client, bool fatal, Error **errp)
556 {
557     int ret;
558 
559     assert(client->optlen);
560     ret = nbd_opt_invalid(client, errp, "option '%s' has unexpected length",
561                           nbd_opt_lookup(client->opt));
562     if (fatal && !ret) {
563         error_setg(errp, "option '%s' has unexpected length",
564                    nbd_opt_lookup(client->opt));
565         return -EINVAL;
566     }
567     return ret;
568 }
569 
570 /* Handle NBD_OPT_INFO and NBD_OPT_GO.
571  * Return -errno on error, 0 if ready for next option, and 1 to move
572  * into transmission phase.  */
573 static int nbd_negotiate_handle_info(NBDClient *client, Error **errp)
574 {
575     int rc;
576     g_autofree char *name = NULL;
577     NBDExport *exp;
578     uint16_t requests;
579     uint16_t request;
580     uint32_t namelen = 0;
581     bool sendname = false;
582     bool blocksize = false;
583     uint32_t sizes[3];
584     char buf[sizeof(uint64_t) + sizeof(uint16_t)];
585     uint32_t check_align = 0;
586     uint16_t myflags;
587 
588     /* Client sends:
589         4 bytes: L, name length (can be 0)
590         L bytes: export name
591         2 bytes: N, number of requests (can be 0)
592         N * 2 bytes: N requests
593     */
594     rc = nbd_opt_read_name(client, &name, &namelen, errp);
595     if (rc <= 0) {
596         return rc;
597     }
598     trace_nbd_negotiate_handle_export_name_request(name);
599 
600     rc = nbd_opt_read(client, &requests, sizeof(requests), false, errp);
601     if (rc <= 0) {
602         return rc;
603     }
604     requests = be16_to_cpu(requests);
605     trace_nbd_negotiate_handle_info_requests(requests);
606     while (requests--) {
607         rc = nbd_opt_read(client, &request, sizeof(request), false, errp);
608         if (rc <= 0) {
609             return rc;
610         }
611         request = be16_to_cpu(request);
612         trace_nbd_negotiate_handle_info_request(request,
613                                                 nbd_info_lookup(request));
614         /* We care about NBD_INFO_NAME and NBD_INFO_BLOCK_SIZE;
615          * everything else is either a request we don't know or
616          * something we send regardless of request */
617         switch (request) {
618         case NBD_INFO_NAME:
619             sendname = true;
620             break;
621         case NBD_INFO_BLOCK_SIZE:
622             blocksize = true;
623             break;
624         }
625     }
626     if (client->optlen) {
627         return nbd_reject_length(client, false, errp);
628     }
629 
630     exp = nbd_export_find(name);
631     if (!exp) {
632         g_autofree char *sane_name = nbd_sanitize_name(name);
633 
634         return nbd_negotiate_send_rep_err(client, NBD_REP_ERR_UNKNOWN,
635                                           errp, "export '%s' not present",
636                                           sane_name);
637     }
638 
639     /* Don't bother sending NBD_INFO_NAME unless client requested it */
640     if (sendname) {
641         rc = nbd_negotiate_send_info(client, NBD_INFO_NAME, namelen, name,
642                                      errp);
643         if (rc < 0) {
644             return rc;
645         }
646     }
647 
648     /* Send NBD_INFO_DESCRIPTION only if available, regardless of
649      * client request */
650     if (exp->description) {
651         size_t len = strlen(exp->description);
652 
653         assert(len <= NBD_MAX_STRING_SIZE);
654         rc = nbd_negotiate_send_info(client, NBD_INFO_DESCRIPTION,
655                                      len, exp->description, errp);
656         if (rc < 0) {
657             return rc;
658         }
659     }
660 
661     /* Send NBD_INFO_BLOCK_SIZE always, but tweak the minimum size
662      * according to whether the client requested it, and according to
663      * whether this is OPT_INFO or OPT_GO. */
664     /* minimum - 1 for back-compat, or actual if client will obey it. */
665     if (client->opt == NBD_OPT_INFO || blocksize) {
666         check_align = sizes[0] = blk_get_request_alignment(exp->common.blk);
667     } else {
668         sizes[0] = 1;
669     }
670     assert(sizes[0] <= NBD_MAX_BUFFER_SIZE);
671     /* preferred - Hard-code to 4096 for now.
672      * TODO: is blk_bs(blk)->bl.opt_transfer appropriate? */
673     sizes[1] = MAX(4096, sizes[0]);
674     /* maximum - At most 32M, but smaller as appropriate. */
675     sizes[2] = MIN(blk_get_max_transfer(exp->common.blk), NBD_MAX_BUFFER_SIZE);
676     trace_nbd_negotiate_handle_info_block_size(sizes[0], sizes[1], sizes[2]);
677     sizes[0] = cpu_to_be32(sizes[0]);
678     sizes[1] = cpu_to_be32(sizes[1]);
679     sizes[2] = cpu_to_be32(sizes[2]);
680     rc = nbd_negotiate_send_info(client, NBD_INFO_BLOCK_SIZE,
681                                  sizeof(sizes), sizes, errp);
682     if (rc < 0) {
683         return rc;
684     }
685 
686     /* Send NBD_INFO_EXPORT always */
687     myflags = exp->nbdflags;
688     if (client->structured_reply) {
689         myflags |= NBD_FLAG_SEND_DF;
690     }
691     trace_nbd_negotiate_new_style_size_flags(exp->size, myflags);
692     stq_be_p(buf, exp->size);
693     stw_be_p(buf + 8, myflags);
694     rc = nbd_negotiate_send_info(client, NBD_INFO_EXPORT,
695                                  sizeof(buf), buf, errp);
696     if (rc < 0) {
697         return rc;
698     }
699 
700     /*
701      * If the client is just asking for NBD_OPT_INFO, but forgot to
702      * request block sizes in a situation that would impact
703      * performance, then return an error. But for NBD_OPT_GO, we
704      * tolerate all clients, regardless of alignments.
705      */
706     if (client->opt == NBD_OPT_INFO && !blocksize &&
707         blk_get_request_alignment(exp->common.blk) > 1) {
708         return nbd_negotiate_send_rep_err(client,
709                                           NBD_REP_ERR_BLOCK_SIZE_REQD,
710                                           errp,
711                                           "request NBD_INFO_BLOCK_SIZE to "
712                                           "use this export");
713     }
714 
715     /* Final reply */
716     rc = nbd_negotiate_send_rep(client, NBD_REP_ACK, errp);
717     if (rc < 0) {
718         return rc;
719     }
720 
721     if (client->opt == NBD_OPT_GO) {
722         client->exp = exp;
723         client->check_align = check_align;
724         QTAILQ_INSERT_TAIL(&client->exp->clients, client, next);
725         blk_exp_ref(&client->exp->common);
726         nbd_check_meta_export(client);
727         rc = 1;
728     }
729     return rc;
730 }
731 
732 
733 /* Handle NBD_OPT_STARTTLS. Return NULL to drop connection, or else the
734  * new channel for all further (now-encrypted) communication. */
735 static QIOChannel *nbd_negotiate_handle_starttls(NBDClient *client,
736                                                  Error **errp)
737 {
738     QIOChannel *ioc;
739     QIOChannelTLS *tioc;
740     struct NBDTLSHandshakeData data = { 0 };
741 
742     assert(client->opt == NBD_OPT_STARTTLS);
743 
744     trace_nbd_negotiate_handle_starttls();
745     ioc = client->ioc;
746 
747     if (nbd_negotiate_send_rep(client, NBD_REP_ACK, errp) < 0) {
748         return NULL;
749     }
750 
751     tioc = qio_channel_tls_new_server(ioc,
752                                       client->tlscreds,
753                                       client->tlsauthz,
754                                       errp);
755     if (!tioc) {
756         return NULL;
757     }
758 
759     qio_channel_set_name(QIO_CHANNEL(tioc), "nbd-server-tls");
760     trace_nbd_negotiate_handle_starttls_handshake();
761     data.loop = g_main_loop_new(g_main_context_default(), FALSE);
762     qio_channel_tls_handshake(tioc,
763                               nbd_tls_handshake,
764                               &data,
765                               NULL,
766                               NULL);
767 
768     if (!data.complete) {
769         g_main_loop_run(data.loop);
770     }
771     g_main_loop_unref(data.loop);
772     if (data.error) {
773         object_unref(OBJECT(tioc));
774         error_propagate(errp, data.error);
775         return NULL;
776     }
777 
778     return QIO_CHANNEL(tioc);
779 }
780 
781 /* nbd_negotiate_send_meta_context
782  *
783  * Send one chunk of reply to NBD_OPT_{LIST,SET}_META_CONTEXT
784  *
785  * For NBD_OPT_LIST_META_CONTEXT @context_id is ignored, 0 is used instead.
786  */
787 static int nbd_negotiate_send_meta_context(NBDClient *client,
788                                            const char *context,
789                                            uint32_t context_id,
790                                            Error **errp)
791 {
792     NBDOptionReplyMetaContext opt;
793     struct iovec iov[] = {
794         {.iov_base = &opt, .iov_len = sizeof(opt)},
795         {.iov_base = (void *)context, .iov_len = strlen(context)}
796     };
797 
798     assert(iov[1].iov_len <= NBD_MAX_STRING_SIZE);
799     if (client->opt == NBD_OPT_LIST_META_CONTEXT) {
800         context_id = 0;
801     }
802 
803     trace_nbd_negotiate_meta_query_reply(context, context_id);
804     set_be_option_rep(&opt.h, client->opt, NBD_REP_META_CONTEXT,
805                       sizeof(opt) - sizeof(opt.h) + iov[1].iov_len);
806     stl_be_p(&opt.context_id, context_id);
807 
808     return qio_channel_writev_all(client->ioc, iov, 2, errp) < 0 ? -EIO : 0;
809 }
810 
811 /*
812  * Return true if @query matches @pattern, or if @query is empty when
813  * the @client is performing _LIST_.
814  */
815 static bool nbd_meta_empty_or_pattern(NBDClient *client, const char *pattern,
816                                       const char *query)
817 {
818     if (!*query) {
819         trace_nbd_negotiate_meta_query_parse("empty");
820         return client->opt == NBD_OPT_LIST_META_CONTEXT;
821     }
822     if (strcmp(query, pattern) == 0) {
823         trace_nbd_negotiate_meta_query_parse(pattern);
824         return true;
825     }
826     trace_nbd_negotiate_meta_query_skip("pattern not matched");
827     return false;
828 }
829 
830 /*
831  * Return true and adjust @str in place if it begins with @prefix.
832  */
833 static bool nbd_strshift(const char **str, const char *prefix)
834 {
835     size_t len = strlen(prefix);
836 
837     if (strncmp(*str, prefix, len) == 0) {
838         *str += len;
839         return true;
840     }
841     return false;
842 }
843 
844 /* nbd_meta_base_query
845  *
846  * Handle queries to 'base' namespace. For now, only the base:allocation
847  * context is available.  Return true if @query has been handled.
848  */
849 static bool nbd_meta_base_query(NBDClient *client, NBDExportMetaContexts *meta,
850                                 const char *query)
851 {
852     if (!nbd_strshift(&query, "base:")) {
853         return false;
854     }
855     trace_nbd_negotiate_meta_query_parse("base:");
856 
857     if (nbd_meta_empty_or_pattern(client, "allocation", query)) {
858         meta->base_allocation = true;
859     }
860     return true;
861 }
862 
863 /* nbd_meta_qemu_query
864  *
865  * Handle queries to 'qemu' namespace. For now, only the qemu:dirty-bitmap:
866  * and qemu:allocation-depth contexts are available.  Return true if @query
867  * has been handled.
868  */
869 static bool nbd_meta_qemu_query(NBDClient *client, NBDExportMetaContexts *meta,
870                                 const char *query)
871 {
872     size_t i;
873 
874     if (!nbd_strshift(&query, "qemu:")) {
875         return false;
876     }
877     trace_nbd_negotiate_meta_query_parse("qemu:");
878 
879     if (!*query) {
880         if (client->opt == NBD_OPT_LIST_META_CONTEXT) {
881             meta->allocation_depth = meta->exp->allocation_depth;
882             if (meta->exp->nr_export_bitmaps) {
883                 memset(meta->bitmaps, 1, meta->exp->nr_export_bitmaps);
884             }
885         }
886         trace_nbd_negotiate_meta_query_parse("empty");
887         return true;
888     }
889 
890     if (strcmp(query, "allocation-depth") == 0) {
891         trace_nbd_negotiate_meta_query_parse("allocation-depth");
892         meta->allocation_depth = meta->exp->allocation_depth;
893         return true;
894     }
895 
896     if (nbd_strshift(&query, "dirty-bitmap:")) {
897         trace_nbd_negotiate_meta_query_parse("dirty-bitmap:");
898         if (!*query) {
899             if (client->opt == NBD_OPT_LIST_META_CONTEXT &&
900                 meta->exp->nr_export_bitmaps) {
901                 memset(meta->bitmaps, 1, meta->exp->nr_export_bitmaps);
902             }
903             trace_nbd_negotiate_meta_query_parse("empty");
904             return true;
905         }
906 
907         for (i = 0; i < meta->exp->nr_export_bitmaps; i++) {
908             const char *bm_name;
909 
910             bm_name = bdrv_dirty_bitmap_name(meta->exp->export_bitmaps[i]);
911             if (strcmp(bm_name, query) == 0) {
912                 meta->bitmaps[i] = true;
913                 trace_nbd_negotiate_meta_query_parse(query);
914                 return true;
915             }
916         }
917         trace_nbd_negotiate_meta_query_skip("no dirty-bitmap match");
918         return true;
919     }
920 
921     trace_nbd_negotiate_meta_query_skip("unknown qemu context");
922     return true;
923 }
924 
925 /* nbd_negotiate_meta_query
926  *
927  * Parse namespace name and call corresponding function to parse body of the
928  * query.
929  *
930  * The only supported namespaces are 'base' and 'qemu'.
931  *
932  * Return -errno on I/O error, 0 if option was completely handled by
933  * sending a reply about inconsistent lengths, or 1 on success. */
934 static int nbd_negotiate_meta_query(NBDClient *client,
935                                     NBDExportMetaContexts *meta, Error **errp)
936 {
937     int ret;
938     g_autofree char *query = NULL;
939     uint32_t len;
940 
941     ret = nbd_opt_read(client, &len, sizeof(len), false, errp);
942     if (ret <= 0) {
943         return ret;
944     }
945     len = cpu_to_be32(len);
946 
947     if (len > NBD_MAX_STRING_SIZE) {
948         trace_nbd_negotiate_meta_query_skip("length too long");
949         return nbd_opt_skip(client, len, errp);
950     }
951 
952     query = g_malloc(len + 1);
953     ret = nbd_opt_read(client, query, len, true, errp);
954     if (ret <= 0) {
955         return ret;
956     }
957     query[len] = '\0';
958 
959     if (nbd_meta_base_query(client, meta, query)) {
960         return 1;
961     }
962     if (nbd_meta_qemu_query(client, meta, query)) {
963         return 1;
964     }
965 
966     trace_nbd_negotiate_meta_query_skip("unknown namespace");
967     return 1;
968 }
969 
970 /* nbd_negotiate_meta_queries
971  * Handle NBD_OPT_LIST_META_CONTEXT and NBD_OPT_SET_META_CONTEXT
972  *
973  * Return -errno on I/O error, or 0 if option was completely handled. */
974 static int nbd_negotiate_meta_queries(NBDClient *client,
975                                       NBDExportMetaContexts *meta, Error **errp)
976 {
977     int ret;
978     g_autofree char *export_name = NULL;
979     /* Mark unused to work around https://bugs.llvm.org/show_bug.cgi?id=3888 */
980     g_autofree G_GNUC_UNUSED bool *bitmaps = NULL;
981     NBDExportMetaContexts local_meta = {0};
982     uint32_t nb_queries;
983     size_t i;
984     size_t count = 0;
985 
986     if (client->opt == NBD_OPT_SET_META_CONTEXT && !client->structured_reply) {
987         return nbd_opt_invalid(client, errp,
988                                "request option '%s' when structured reply "
989                                "is not negotiated",
990                                nbd_opt_lookup(client->opt));
991     }
992 
993     if (client->opt == NBD_OPT_LIST_META_CONTEXT) {
994         /* Only change the caller's meta on SET. */
995         meta = &local_meta;
996     }
997 
998     g_free(meta->bitmaps);
999     memset(meta, 0, sizeof(*meta));
1000 
1001     ret = nbd_opt_read_name(client, &export_name, NULL, errp);
1002     if (ret <= 0) {
1003         return ret;
1004     }
1005 
1006     meta->exp = nbd_export_find(export_name);
1007     if (meta->exp == NULL) {
1008         g_autofree char *sane_name = nbd_sanitize_name(export_name);
1009 
1010         return nbd_opt_drop(client, NBD_REP_ERR_UNKNOWN, errp,
1011                             "export '%s' not present", sane_name);
1012     }
1013     meta->bitmaps = g_new0(bool, meta->exp->nr_export_bitmaps);
1014     if (client->opt == NBD_OPT_LIST_META_CONTEXT) {
1015         bitmaps = meta->bitmaps;
1016     }
1017 
1018     ret = nbd_opt_read(client, &nb_queries, sizeof(nb_queries), false, errp);
1019     if (ret <= 0) {
1020         return ret;
1021     }
1022     nb_queries = cpu_to_be32(nb_queries);
1023     trace_nbd_negotiate_meta_context(nbd_opt_lookup(client->opt),
1024                                      export_name, nb_queries);
1025 
1026     if (client->opt == NBD_OPT_LIST_META_CONTEXT && !nb_queries) {
1027         /* enable all known contexts */
1028         meta->base_allocation = true;
1029         meta->allocation_depth = meta->exp->allocation_depth;
1030         if (meta->exp->nr_export_bitmaps) {
1031             memset(meta->bitmaps, 1, meta->exp->nr_export_bitmaps);
1032         }
1033     } else {
1034         for (i = 0; i < nb_queries; ++i) {
1035             ret = nbd_negotiate_meta_query(client, meta, errp);
1036             if (ret <= 0) {
1037                 return ret;
1038             }
1039         }
1040     }
1041 
1042     if (meta->base_allocation) {
1043         ret = nbd_negotiate_send_meta_context(client, "base:allocation",
1044                                               NBD_META_ID_BASE_ALLOCATION,
1045                                               errp);
1046         if (ret < 0) {
1047             return ret;
1048         }
1049         count++;
1050     }
1051 
1052     if (meta->allocation_depth) {
1053         ret = nbd_negotiate_send_meta_context(client, "qemu:allocation-depth",
1054                                               NBD_META_ID_ALLOCATION_DEPTH,
1055                                               errp);
1056         if (ret < 0) {
1057             return ret;
1058         }
1059         count++;
1060     }
1061 
1062     for (i = 0; i < meta->exp->nr_export_bitmaps; i++) {
1063         const char *bm_name;
1064         g_autofree char *context = NULL;
1065 
1066         if (!meta->bitmaps[i]) {
1067             continue;
1068         }
1069 
1070         bm_name = bdrv_dirty_bitmap_name(meta->exp->export_bitmaps[i]);
1071         context = g_strdup_printf("qemu:dirty-bitmap:%s", bm_name);
1072 
1073         ret = nbd_negotiate_send_meta_context(client, context,
1074                                               NBD_META_ID_DIRTY_BITMAP + i,
1075                                               errp);
1076         if (ret < 0) {
1077             return ret;
1078         }
1079         count++;
1080     }
1081 
1082     ret = nbd_negotiate_send_rep(client, NBD_REP_ACK, errp);
1083     if (ret == 0) {
1084         meta->count = count;
1085     }
1086 
1087     return ret;
1088 }
1089 
1090 /* nbd_negotiate_options
1091  * Process all NBD_OPT_* client option commands, during fixed newstyle
1092  * negotiation.
1093  * Return:
1094  * -errno  on error, errp is set
1095  * 0       on successful negotiation, errp is not set
1096  * 1       if client sent NBD_OPT_ABORT, i.e. on valid disconnect,
1097  *         errp is not set
1098  */
1099 static int nbd_negotiate_options(NBDClient *client, Error **errp)
1100 {
1101     uint32_t flags;
1102     bool fixedNewstyle = false;
1103     bool no_zeroes = false;
1104 
1105     /* Client sends:
1106         [ 0 ..   3]   client flags
1107 
1108        Then we loop until NBD_OPT_EXPORT_NAME or NBD_OPT_GO:
1109         [ 0 ..   7]   NBD_OPTS_MAGIC
1110         [ 8 ..  11]   NBD option
1111         [12 ..  15]   Data length
1112         ...           Rest of request
1113 
1114         [ 0 ..   7]   NBD_OPTS_MAGIC
1115         [ 8 ..  11]   Second NBD option
1116         [12 ..  15]   Data length
1117         ...           Rest of request
1118     */
1119 
1120     if (nbd_read32(client->ioc, &flags, "flags", errp) < 0) {
1121         return -EIO;
1122     }
1123     trace_nbd_negotiate_options_flags(flags);
1124     if (flags & NBD_FLAG_C_FIXED_NEWSTYLE) {
1125         fixedNewstyle = true;
1126         flags &= ~NBD_FLAG_C_FIXED_NEWSTYLE;
1127     }
1128     if (flags & NBD_FLAG_C_NO_ZEROES) {
1129         no_zeroes = true;
1130         flags &= ~NBD_FLAG_C_NO_ZEROES;
1131     }
1132     if (flags != 0) {
1133         error_setg(errp, "Unknown client flags 0x%" PRIx32 " received", flags);
1134         return -EINVAL;
1135     }
1136 
1137     while (1) {
1138         int ret;
1139         uint32_t option, length;
1140         uint64_t magic;
1141 
1142         if (nbd_read64(client->ioc, &magic, "opts magic", errp) < 0) {
1143             return -EINVAL;
1144         }
1145         trace_nbd_negotiate_options_check_magic(magic);
1146         if (magic != NBD_OPTS_MAGIC) {
1147             error_setg(errp, "Bad magic received");
1148             return -EINVAL;
1149         }
1150 
1151         if (nbd_read32(client->ioc, &option, "option", errp) < 0) {
1152             return -EINVAL;
1153         }
1154         client->opt = option;
1155 
1156         if (nbd_read32(client->ioc, &length, "option length", errp) < 0) {
1157             return -EINVAL;
1158         }
1159         assert(!client->optlen);
1160         client->optlen = length;
1161 
1162         if (length > NBD_MAX_BUFFER_SIZE) {
1163             error_setg(errp, "len (%" PRIu32" ) is larger than max len (%u)",
1164                        length, NBD_MAX_BUFFER_SIZE);
1165             return -EINVAL;
1166         }
1167 
1168         trace_nbd_negotiate_options_check_option(option,
1169                                                  nbd_opt_lookup(option));
1170         if (client->tlscreds &&
1171             client->ioc == (QIOChannel *)client->sioc) {
1172             QIOChannel *tioc;
1173             if (!fixedNewstyle) {
1174                 error_setg(errp, "Unsupported option 0x%" PRIx32, option);
1175                 return -EINVAL;
1176             }
1177             switch (option) {
1178             case NBD_OPT_STARTTLS:
1179                 if (length) {
1180                     /* Unconditionally drop the connection if the client
1181                      * can't start a TLS negotiation correctly */
1182                     return nbd_reject_length(client, true, errp);
1183                 }
1184                 tioc = nbd_negotiate_handle_starttls(client, errp);
1185                 if (!tioc) {
1186                     return -EIO;
1187                 }
1188                 ret = 0;
1189                 object_unref(OBJECT(client->ioc));
1190                 client->ioc = QIO_CHANNEL(tioc);
1191                 break;
1192 
1193             case NBD_OPT_EXPORT_NAME:
1194                 /* No way to return an error to client, so drop connection */
1195                 error_setg(errp, "Option 0x%x not permitted before TLS",
1196                            option);
1197                 return -EINVAL;
1198 
1199             default:
1200                 /* Let the client keep trying, unless they asked to
1201                  * quit. Always try to give an error back to the
1202                  * client; but when replying to OPT_ABORT, be aware
1203                  * that the client may hang up before receiving the
1204                  * error, in which case we are fine ignoring the
1205                  * resulting EPIPE. */
1206                 ret = nbd_opt_drop(client, NBD_REP_ERR_TLS_REQD,
1207                                    option == NBD_OPT_ABORT ? NULL : errp,
1208                                    "Option 0x%" PRIx32
1209                                    " not permitted before TLS", option);
1210                 if (option == NBD_OPT_ABORT) {
1211                     return 1;
1212                 }
1213                 break;
1214             }
1215         } else if (fixedNewstyle) {
1216             switch (option) {
1217             case NBD_OPT_LIST:
1218                 if (length) {
1219                     ret = nbd_reject_length(client, false, errp);
1220                 } else {
1221                     ret = nbd_negotiate_handle_list(client, errp);
1222                 }
1223                 break;
1224 
1225             case NBD_OPT_ABORT:
1226                 /* NBD spec says we must try to reply before
1227                  * disconnecting, but that we must also tolerate
1228                  * guests that don't wait for our reply. */
1229                 nbd_negotiate_send_rep(client, NBD_REP_ACK, NULL);
1230                 return 1;
1231 
1232             case NBD_OPT_EXPORT_NAME:
1233                 return nbd_negotiate_handle_export_name(client, no_zeroes,
1234                                                         errp);
1235 
1236             case NBD_OPT_INFO:
1237             case NBD_OPT_GO:
1238                 ret = nbd_negotiate_handle_info(client, errp);
1239                 if (ret == 1) {
1240                     assert(option == NBD_OPT_GO);
1241                     return 0;
1242                 }
1243                 break;
1244 
1245             case NBD_OPT_STARTTLS:
1246                 if (length) {
1247                     ret = nbd_reject_length(client, false, errp);
1248                 } else if (client->tlscreds) {
1249                     ret = nbd_negotiate_send_rep_err(client,
1250                                                      NBD_REP_ERR_INVALID, errp,
1251                                                      "TLS already enabled");
1252                 } else {
1253                     ret = nbd_negotiate_send_rep_err(client,
1254                                                      NBD_REP_ERR_POLICY, errp,
1255                                                      "TLS not configured");
1256                 }
1257                 break;
1258 
1259             case NBD_OPT_STRUCTURED_REPLY:
1260                 if (length) {
1261                     ret = nbd_reject_length(client, false, errp);
1262                 } else if (client->structured_reply) {
1263                     ret = nbd_negotiate_send_rep_err(
1264                         client, NBD_REP_ERR_INVALID, errp,
1265                         "structured reply already negotiated");
1266                 } else {
1267                     ret = nbd_negotiate_send_rep(client, NBD_REP_ACK, errp);
1268                     client->structured_reply = true;
1269                 }
1270                 break;
1271 
1272             case NBD_OPT_LIST_META_CONTEXT:
1273             case NBD_OPT_SET_META_CONTEXT:
1274                 ret = nbd_negotiate_meta_queries(client, &client->export_meta,
1275                                                  errp);
1276                 break;
1277 
1278             default:
1279                 ret = nbd_opt_drop(client, NBD_REP_ERR_UNSUP, errp,
1280                                    "Unsupported option %" PRIu32 " (%s)",
1281                                    option, nbd_opt_lookup(option));
1282                 break;
1283             }
1284         } else {
1285             /*
1286              * If broken new-style we should drop the connection
1287              * for anything except NBD_OPT_EXPORT_NAME
1288              */
1289             switch (option) {
1290             case NBD_OPT_EXPORT_NAME:
1291                 return nbd_negotiate_handle_export_name(client, no_zeroes,
1292                                                         errp);
1293 
1294             default:
1295                 error_setg(errp, "Unsupported option %" PRIu32 " (%s)",
1296                            option, nbd_opt_lookup(option));
1297                 return -EINVAL;
1298             }
1299         }
1300         if (ret < 0) {
1301             return ret;
1302         }
1303     }
1304 }
1305 
1306 /* nbd_negotiate
1307  * Return:
1308  * -errno  on error, errp is set
1309  * 0       on successful negotiation, errp is not set
1310  * 1       if client sent NBD_OPT_ABORT, i.e. on valid disconnect,
1311  *         errp is not set
1312  */
1313 static coroutine_fn int nbd_negotiate(NBDClient *client, Error **errp)
1314 {
1315     ERRP_GUARD();
1316     char buf[NBD_OLDSTYLE_NEGOTIATE_SIZE] = "";
1317     int ret;
1318 
1319     /* Old style negotiation header, no room for options
1320         [ 0 ..   7]   passwd       ("NBDMAGIC")
1321         [ 8 ..  15]   magic        (NBD_CLIENT_MAGIC)
1322         [16 ..  23]   size
1323         [24 ..  27]   export flags (zero-extended)
1324         [28 .. 151]   reserved     (0)
1325 
1326        New style negotiation header, client can send options
1327         [ 0 ..   7]   passwd       ("NBDMAGIC")
1328         [ 8 ..  15]   magic        (NBD_OPTS_MAGIC)
1329         [16 ..  17]   server flags (0)
1330         ....options sent, ending in NBD_OPT_EXPORT_NAME or NBD_OPT_GO....
1331      */
1332 
1333     qio_channel_set_blocking(client->ioc, false, NULL);
1334 
1335     trace_nbd_negotiate_begin();
1336     memcpy(buf, "NBDMAGIC", 8);
1337 
1338     stq_be_p(buf + 8, NBD_OPTS_MAGIC);
1339     stw_be_p(buf + 16, NBD_FLAG_FIXED_NEWSTYLE | NBD_FLAG_NO_ZEROES);
1340 
1341     if (nbd_write(client->ioc, buf, 18, errp) < 0) {
1342         error_prepend(errp, "write failed: ");
1343         return -EINVAL;
1344     }
1345     ret = nbd_negotiate_options(client, errp);
1346     if (ret != 0) {
1347         if (ret < 0) {
1348             error_prepend(errp, "option negotiation failed: ");
1349         }
1350         return ret;
1351     }
1352 
1353     /* Attach the channel to the same AioContext as the export */
1354     if (client->exp && client->exp->common.ctx) {
1355         qio_channel_attach_aio_context(client->ioc, client->exp->common.ctx);
1356     }
1357 
1358     assert(!client->optlen);
1359     trace_nbd_negotiate_success();
1360 
1361     return 0;
1362 }
1363 
1364 /* nbd_read_eof
1365  * Tries to read @size bytes from @ioc. This is a local implementation of
1366  * qio_channel_readv_all_eof. We have it here because we need it to be
1367  * interruptible and to know when the coroutine is yielding.
1368  * Returns 1 on success
1369  *         0 on eof, when no data was read (errp is not set)
1370  *         negative errno on failure (errp is set)
1371  */
1372 static inline int coroutine_fn
1373 nbd_read_eof(NBDClient *client, void *buffer, size_t size, Error **errp)
1374 {
1375     bool partial = false;
1376 
1377     assert(size);
1378     while (size > 0) {
1379         struct iovec iov = { .iov_base = buffer, .iov_len = size };
1380         ssize_t len;
1381 
1382         len = qio_channel_readv(client->ioc, &iov, 1, errp);
1383         if (len == QIO_CHANNEL_ERR_BLOCK) {
1384             client->read_yielding = true;
1385             qio_channel_yield(client->ioc, G_IO_IN);
1386             client->read_yielding = false;
1387             if (client->quiescing) {
1388                 return -EAGAIN;
1389             }
1390             continue;
1391         } else if (len < 0) {
1392             return -EIO;
1393         } else if (len == 0) {
1394             if (partial) {
1395                 error_setg(errp,
1396                            "Unexpected end-of-file before all bytes were read");
1397                 return -EIO;
1398             } else {
1399                 return 0;
1400             }
1401         }
1402 
1403         partial = true;
1404         size -= len;
1405         buffer = (uint8_t *) buffer + len;
1406     }
1407     return 1;
1408 }
1409 
1410 static int nbd_receive_request(NBDClient *client, NBDRequest *request,
1411                                Error **errp)
1412 {
1413     uint8_t buf[NBD_REQUEST_SIZE];
1414     uint32_t magic;
1415     int ret;
1416 
1417     ret = nbd_read_eof(client, buf, sizeof(buf), errp);
1418     if (ret < 0) {
1419         return ret;
1420     }
1421 
1422     /* Request
1423        [ 0 ..  3]   magic   (NBD_REQUEST_MAGIC)
1424        [ 4 ..  5]   flags   (NBD_CMD_FLAG_FUA, ...)
1425        [ 6 ..  7]   type    (NBD_CMD_READ, ...)
1426        [ 8 .. 15]   handle
1427        [16 .. 23]   from
1428        [24 .. 27]   len
1429      */
1430 
1431     magic = ldl_be_p(buf);
1432     request->flags  = lduw_be_p(buf + 4);
1433     request->type   = lduw_be_p(buf + 6);
1434     request->handle = ldq_be_p(buf + 8);
1435     request->from   = ldq_be_p(buf + 16);
1436     request->len    = ldl_be_p(buf + 24);
1437 
1438     trace_nbd_receive_request(magic, request->flags, request->type,
1439                               request->from, request->len);
1440 
1441     if (magic != NBD_REQUEST_MAGIC) {
1442         error_setg(errp, "invalid magic (got 0x%" PRIx32 ")", magic);
1443         return -EINVAL;
1444     }
1445     return 0;
1446 }
1447 
1448 #define MAX_NBD_REQUESTS 16
1449 
1450 void nbd_client_get(NBDClient *client)
1451 {
1452     client->refcount++;
1453 }
1454 
1455 void nbd_client_put(NBDClient *client)
1456 {
1457     if (--client->refcount == 0) {
1458         /* The last reference should be dropped by client->close,
1459          * which is called by client_close.
1460          */
1461         assert(client->closing);
1462 
1463         qio_channel_detach_aio_context(client->ioc);
1464         object_unref(OBJECT(client->sioc));
1465         object_unref(OBJECT(client->ioc));
1466         if (client->tlscreds) {
1467             object_unref(OBJECT(client->tlscreds));
1468         }
1469         g_free(client->tlsauthz);
1470         if (client->exp) {
1471             QTAILQ_REMOVE(&client->exp->clients, client, next);
1472             blk_exp_unref(&client->exp->common);
1473         }
1474         g_free(client->export_meta.bitmaps);
1475         g_free(client);
1476     }
1477 }
1478 
1479 static void client_close(NBDClient *client, bool negotiated)
1480 {
1481     if (client->closing) {
1482         return;
1483     }
1484 
1485     client->closing = true;
1486 
1487     /* Force requests to finish.  They will drop their own references,
1488      * then we'll close the socket and free the NBDClient.
1489      */
1490     qio_channel_shutdown(client->ioc, QIO_CHANNEL_SHUTDOWN_BOTH,
1491                          NULL);
1492 
1493     /* Also tell the client, so that they release their reference.  */
1494     if (client->close_fn) {
1495         client->close_fn(client, negotiated);
1496     }
1497 }
1498 
1499 static NBDRequestData *nbd_request_get(NBDClient *client)
1500 {
1501     NBDRequestData *req;
1502 
1503     assert(client->nb_requests <= MAX_NBD_REQUESTS - 1);
1504     client->nb_requests++;
1505 
1506     req = g_new0(NBDRequestData, 1);
1507     nbd_client_get(client);
1508     req->client = client;
1509     return req;
1510 }
1511 
1512 static void nbd_request_put(NBDRequestData *req)
1513 {
1514     NBDClient *client = req->client;
1515 
1516     if (req->data) {
1517         qemu_vfree(req->data);
1518     }
1519     g_free(req);
1520 
1521     client->nb_requests--;
1522 
1523     if (client->quiescing && client->nb_requests == 0) {
1524         aio_wait_kick();
1525     }
1526 
1527     nbd_client_receive_next_request(client);
1528 
1529     nbd_client_put(client);
1530 }
1531 
1532 static void blk_aio_attached(AioContext *ctx, void *opaque)
1533 {
1534     NBDExport *exp = opaque;
1535     NBDClient *client;
1536 
1537     trace_nbd_blk_aio_attached(exp->name, ctx);
1538 
1539     exp->common.ctx = ctx;
1540 
1541     QTAILQ_FOREACH(client, &exp->clients, next) {
1542         qio_channel_attach_aio_context(client->ioc, ctx);
1543 
1544         assert(client->nb_requests == 0);
1545         assert(client->recv_coroutine == NULL);
1546         assert(client->send_coroutine == NULL);
1547     }
1548 }
1549 
1550 static void blk_aio_detach(void *opaque)
1551 {
1552     NBDExport *exp = opaque;
1553     NBDClient *client;
1554 
1555     trace_nbd_blk_aio_detach(exp->name, exp->common.ctx);
1556 
1557     QTAILQ_FOREACH(client, &exp->clients, next) {
1558         qio_channel_detach_aio_context(client->ioc);
1559     }
1560 
1561     exp->common.ctx = NULL;
1562 }
1563 
1564 static void nbd_drained_begin(void *opaque)
1565 {
1566     NBDExport *exp = opaque;
1567     NBDClient *client;
1568 
1569     QTAILQ_FOREACH(client, &exp->clients, next) {
1570         client->quiescing = true;
1571     }
1572 }
1573 
1574 static void nbd_drained_end(void *opaque)
1575 {
1576     NBDExport *exp = opaque;
1577     NBDClient *client;
1578 
1579     QTAILQ_FOREACH(client, &exp->clients, next) {
1580         client->quiescing = false;
1581         nbd_client_receive_next_request(client);
1582     }
1583 }
1584 
1585 static bool nbd_drained_poll(void *opaque)
1586 {
1587     NBDExport *exp = opaque;
1588     NBDClient *client;
1589 
1590     QTAILQ_FOREACH(client, &exp->clients, next) {
1591         if (client->nb_requests != 0) {
1592             /*
1593              * If there's a coroutine waiting for a request on nbd_read_eof()
1594              * enter it here so we don't depend on the client to wake it up.
1595              */
1596             if (client->recv_coroutine != NULL && client->read_yielding) {
1597                 qemu_aio_coroutine_enter(exp->common.ctx,
1598                                          client->recv_coroutine);
1599             }
1600 
1601             return true;
1602         }
1603     }
1604 
1605     return false;
1606 }
1607 
1608 static void nbd_eject_notifier(Notifier *n, void *data)
1609 {
1610     NBDExport *exp = container_of(n, NBDExport, eject_notifier);
1611 
1612     blk_exp_request_shutdown(&exp->common);
1613 }
1614 
1615 void nbd_export_set_on_eject_blk(BlockExport *exp, BlockBackend *blk)
1616 {
1617     NBDExport *nbd_exp = container_of(exp, NBDExport, common);
1618     assert(exp->drv == &blk_exp_nbd);
1619     assert(nbd_exp->eject_notifier_blk == NULL);
1620 
1621     blk_ref(blk);
1622     nbd_exp->eject_notifier_blk = blk;
1623     nbd_exp->eject_notifier.notify = nbd_eject_notifier;
1624     blk_add_remove_bs_notifier(blk, &nbd_exp->eject_notifier);
1625 }
1626 
1627 static const BlockDevOps nbd_block_ops = {
1628     .drained_begin = nbd_drained_begin,
1629     .drained_end = nbd_drained_end,
1630     .drained_poll = nbd_drained_poll,
1631 };
1632 
1633 static int nbd_export_create(BlockExport *blk_exp, BlockExportOptions *exp_args,
1634                              Error **errp)
1635 {
1636     NBDExport *exp = container_of(blk_exp, NBDExport, common);
1637     BlockExportOptionsNbd *arg = &exp_args->u.nbd;
1638     BlockBackend *blk = blk_exp->blk;
1639     int64_t size;
1640     uint64_t perm, shared_perm;
1641     bool readonly = !exp_args->writable;
1642     bool shared = !exp_args->writable;
1643     strList *bitmaps;
1644     size_t i;
1645     int ret;
1646 
1647     assert(exp_args->type == BLOCK_EXPORT_TYPE_NBD);
1648 
1649     if (!nbd_server_is_running()) {
1650         error_setg(errp, "NBD server not running");
1651         return -EINVAL;
1652     }
1653 
1654     if (!arg->has_name) {
1655         arg->name = exp_args->node_name;
1656     }
1657 
1658     if (strlen(arg->name) > NBD_MAX_STRING_SIZE) {
1659         error_setg(errp, "export name '%s' too long", arg->name);
1660         return -EINVAL;
1661     }
1662 
1663     if (arg->description && strlen(arg->description) > NBD_MAX_STRING_SIZE) {
1664         error_setg(errp, "description '%s' too long", arg->description);
1665         return -EINVAL;
1666     }
1667 
1668     if (nbd_export_find(arg->name)) {
1669         error_setg(errp, "NBD server already has export named '%s'", arg->name);
1670         return -EEXIST;
1671     }
1672 
1673     size = blk_getlength(blk);
1674     if (size < 0) {
1675         error_setg_errno(errp, -size,
1676                          "Failed to determine the NBD export's length");
1677         return size;
1678     }
1679 
1680     /* Don't allow resize while the NBD server is running, otherwise we don't
1681      * care what happens with the node. */
1682     blk_get_perm(blk, &perm, &shared_perm);
1683     ret = blk_set_perm(blk, perm, shared_perm & ~BLK_PERM_RESIZE, errp);
1684     if (ret < 0) {
1685         return ret;
1686     }
1687 
1688     QTAILQ_INIT(&exp->clients);
1689     exp->name = g_strdup(arg->name);
1690     exp->description = g_strdup(arg->description);
1691     exp->nbdflags = (NBD_FLAG_HAS_FLAGS | NBD_FLAG_SEND_FLUSH |
1692                      NBD_FLAG_SEND_FUA | NBD_FLAG_SEND_CACHE);
1693     if (readonly) {
1694         exp->nbdflags |= NBD_FLAG_READ_ONLY;
1695         if (shared) {
1696             exp->nbdflags |= NBD_FLAG_CAN_MULTI_CONN;
1697         }
1698     } else {
1699         exp->nbdflags |= (NBD_FLAG_SEND_TRIM | NBD_FLAG_SEND_WRITE_ZEROES |
1700                           NBD_FLAG_SEND_FAST_ZERO);
1701     }
1702     exp->size = QEMU_ALIGN_DOWN(size, BDRV_SECTOR_SIZE);
1703 
1704     for (bitmaps = arg->bitmaps; bitmaps; bitmaps = bitmaps->next) {
1705         exp->nr_export_bitmaps++;
1706     }
1707     exp->export_bitmaps = g_new0(BdrvDirtyBitmap *, exp->nr_export_bitmaps);
1708     for (i = 0, bitmaps = arg->bitmaps; bitmaps;
1709          i++, bitmaps = bitmaps->next) {
1710         const char *bitmap = bitmaps->value;
1711         BlockDriverState *bs = blk_bs(blk);
1712         BdrvDirtyBitmap *bm = NULL;
1713 
1714         while (bs) {
1715             bm = bdrv_find_dirty_bitmap(bs, bitmap);
1716             if (bm != NULL) {
1717                 break;
1718             }
1719 
1720             bs = bdrv_filter_or_cow_bs(bs);
1721         }
1722 
1723         if (bm == NULL) {
1724             ret = -ENOENT;
1725             error_setg(errp, "Bitmap '%s' is not found", bitmap);
1726             goto fail;
1727         }
1728 
1729         if (bdrv_dirty_bitmap_check(bm, BDRV_BITMAP_ALLOW_RO, errp)) {
1730             ret = -EINVAL;
1731             goto fail;
1732         }
1733 
1734         if (readonly && bdrv_is_writable(bs) &&
1735             bdrv_dirty_bitmap_enabled(bm)) {
1736             ret = -EINVAL;
1737             error_setg(errp,
1738                        "Enabled bitmap '%s' incompatible with readonly export",
1739                        bitmap);
1740             goto fail;
1741         }
1742 
1743         exp->export_bitmaps[i] = bm;
1744         assert(strlen(bitmap) <= BDRV_BITMAP_MAX_NAME_SIZE);
1745     }
1746 
1747     /* Mark bitmaps busy in a separate loop, to simplify roll-back concerns. */
1748     for (i = 0; i < exp->nr_export_bitmaps; i++) {
1749         bdrv_dirty_bitmap_set_busy(exp->export_bitmaps[i], true);
1750     }
1751 
1752     exp->allocation_depth = arg->allocation_depth;
1753 
1754     /*
1755      * We need to inhibit request queuing in the block layer to ensure we can
1756      * be properly quiesced when entering a drained section, as our coroutines
1757      * servicing pending requests might enter blk_pread().
1758      */
1759     blk_set_disable_request_queuing(blk, true);
1760 
1761     blk_add_aio_context_notifier(blk, blk_aio_attached, blk_aio_detach, exp);
1762 
1763     blk_set_dev_ops(blk, &nbd_block_ops, exp);
1764 
1765     QTAILQ_INSERT_TAIL(&exports, exp, next);
1766 
1767     return 0;
1768 
1769 fail:
1770     g_free(exp->export_bitmaps);
1771     g_free(exp->name);
1772     g_free(exp->description);
1773     return ret;
1774 }
1775 
1776 NBDExport *nbd_export_find(const char *name)
1777 {
1778     NBDExport *exp;
1779     QTAILQ_FOREACH(exp, &exports, next) {
1780         if (strcmp(name, exp->name) == 0) {
1781             return exp;
1782         }
1783     }
1784 
1785     return NULL;
1786 }
1787 
1788 AioContext *
1789 nbd_export_aio_context(NBDExport *exp)
1790 {
1791     return exp->common.ctx;
1792 }
1793 
1794 static void nbd_export_request_shutdown(BlockExport *blk_exp)
1795 {
1796     NBDExport *exp = container_of(blk_exp, NBDExport, common);
1797     NBDClient *client, *next;
1798 
1799     blk_exp_ref(&exp->common);
1800     /*
1801      * TODO: Should we expand QMP NbdServerRemoveNode enum to allow a
1802      * close mode that stops advertising the export to new clients but
1803      * still permits existing clients to run to completion? Because of
1804      * that possibility, nbd_export_close() can be called more than
1805      * once on an export.
1806      */
1807     QTAILQ_FOREACH_SAFE(client, &exp->clients, next, next) {
1808         client_close(client, true);
1809     }
1810     if (exp->name) {
1811         g_free(exp->name);
1812         exp->name = NULL;
1813         QTAILQ_REMOVE(&exports, exp, next);
1814     }
1815     blk_exp_unref(&exp->common);
1816 }
1817 
1818 static void nbd_export_delete(BlockExport *blk_exp)
1819 {
1820     size_t i;
1821     NBDExport *exp = container_of(blk_exp, NBDExport, common);
1822 
1823     assert(exp->name == NULL);
1824     assert(QTAILQ_EMPTY(&exp->clients));
1825 
1826     g_free(exp->description);
1827     exp->description = NULL;
1828 
1829     if (exp->common.blk) {
1830         if (exp->eject_notifier_blk) {
1831             notifier_remove(&exp->eject_notifier);
1832             blk_unref(exp->eject_notifier_blk);
1833         }
1834         blk_remove_aio_context_notifier(exp->common.blk, blk_aio_attached,
1835                                         blk_aio_detach, exp);
1836         blk_set_disable_request_queuing(exp->common.blk, false);
1837     }
1838 
1839     for (i = 0; i < exp->nr_export_bitmaps; i++) {
1840         bdrv_dirty_bitmap_set_busy(exp->export_bitmaps[i], false);
1841     }
1842 }
1843 
1844 const BlockExportDriver blk_exp_nbd = {
1845     .type               = BLOCK_EXPORT_TYPE_NBD,
1846     .instance_size      = sizeof(NBDExport),
1847     .create             = nbd_export_create,
1848     .delete             = nbd_export_delete,
1849     .request_shutdown   = nbd_export_request_shutdown,
1850 };
1851 
1852 static int coroutine_fn nbd_co_send_iov(NBDClient *client, struct iovec *iov,
1853                                         unsigned niov, Error **errp)
1854 {
1855     int ret;
1856 
1857     g_assert(qemu_in_coroutine());
1858     qemu_co_mutex_lock(&client->send_lock);
1859     client->send_coroutine = qemu_coroutine_self();
1860 
1861     ret = qio_channel_writev_all(client->ioc, iov, niov, errp) < 0 ? -EIO : 0;
1862 
1863     client->send_coroutine = NULL;
1864     qemu_co_mutex_unlock(&client->send_lock);
1865 
1866     return ret;
1867 }
1868 
1869 static inline void set_be_simple_reply(NBDSimpleReply *reply, uint64_t error,
1870                                        uint64_t handle)
1871 {
1872     stl_be_p(&reply->magic, NBD_SIMPLE_REPLY_MAGIC);
1873     stl_be_p(&reply->error, error);
1874     stq_be_p(&reply->handle, handle);
1875 }
1876 
1877 static int nbd_co_send_simple_reply(NBDClient *client,
1878                                     uint64_t handle,
1879                                     uint32_t error,
1880                                     void *data,
1881                                     size_t len,
1882                                     Error **errp)
1883 {
1884     NBDSimpleReply reply;
1885     int nbd_err = system_errno_to_nbd_errno(error);
1886     struct iovec iov[] = {
1887         {.iov_base = &reply, .iov_len = sizeof(reply)},
1888         {.iov_base = data, .iov_len = len}
1889     };
1890 
1891     trace_nbd_co_send_simple_reply(handle, nbd_err, nbd_err_lookup(nbd_err),
1892                                    len);
1893     set_be_simple_reply(&reply, nbd_err, handle);
1894 
1895     return nbd_co_send_iov(client, iov, len ? 2 : 1, errp);
1896 }
1897 
1898 static inline void set_be_chunk(NBDStructuredReplyChunk *chunk, uint16_t flags,
1899                                 uint16_t type, uint64_t handle, uint32_t length)
1900 {
1901     stl_be_p(&chunk->magic, NBD_STRUCTURED_REPLY_MAGIC);
1902     stw_be_p(&chunk->flags, flags);
1903     stw_be_p(&chunk->type, type);
1904     stq_be_p(&chunk->handle, handle);
1905     stl_be_p(&chunk->length, length);
1906 }
1907 
1908 static int coroutine_fn nbd_co_send_structured_done(NBDClient *client,
1909                                                     uint64_t handle,
1910                                                     Error **errp)
1911 {
1912     NBDStructuredReplyChunk chunk;
1913     struct iovec iov[] = {
1914         {.iov_base = &chunk, .iov_len = sizeof(chunk)},
1915     };
1916 
1917     trace_nbd_co_send_structured_done(handle);
1918     set_be_chunk(&chunk, NBD_REPLY_FLAG_DONE, NBD_REPLY_TYPE_NONE, handle, 0);
1919 
1920     return nbd_co_send_iov(client, iov, 1, errp);
1921 }
1922 
1923 static int coroutine_fn nbd_co_send_structured_read(NBDClient *client,
1924                                                     uint64_t handle,
1925                                                     uint64_t offset,
1926                                                     void *data,
1927                                                     size_t size,
1928                                                     bool final,
1929                                                     Error **errp)
1930 {
1931     NBDStructuredReadData chunk;
1932     struct iovec iov[] = {
1933         {.iov_base = &chunk, .iov_len = sizeof(chunk)},
1934         {.iov_base = data, .iov_len = size}
1935     };
1936 
1937     assert(size);
1938     trace_nbd_co_send_structured_read(handle, offset, data, size);
1939     set_be_chunk(&chunk.h, final ? NBD_REPLY_FLAG_DONE : 0,
1940                  NBD_REPLY_TYPE_OFFSET_DATA, handle,
1941                  sizeof(chunk) - sizeof(chunk.h) + size);
1942     stq_be_p(&chunk.offset, offset);
1943 
1944     return nbd_co_send_iov(client, iov, 2, errp);
1945 }
1946 
1947 static int coroutine_fn nbd_co_send_structured_error(NBDClient *client,
1948                                                      uint64_t handle,
1949                                                      uint32_t error,
1950                                                      const char *msg,
1951                                                      Error **errp)
1952 {
1953     NBDStructuredError chunk;
1954     int nbd_err = system_errno_to_nbd_errno(error);
1955     struct iovec iov[] = {
1956         {.iov_base = &chunk, .iov_len = sizeof(chunk)},
1957         {.iov_base = (char *)msg, .iov_len = msg ? strlen(msg) : 0},
1958     };
1959 
1960     assert(nbd_err);
1961     trace_nbd_co_send_structured_error(handle, nbd_err,
1962                                        nbd_err_lookup(nbd_err), msg ? msg : "");
1963     set_be_chunk(&chunk.h, NBD_REPLY_FLAG_DONE, NBD_REPLY_TYPE_ERROR, handle,
1964                  sizeof(chunk) - sizeof(chunk.h) + iov[1].iov_len);
1965     stl_be_p(&chunk.error, nbd_err);
1966     stw_be_p(&chunk.message_length, iov[1].iov_len);
1967 
1968     return nbd_co_send_iov(client, iov, 1 + !!iov[1].iov_len, errp);
1969 }
1970 
1971 /* Do a sparse read and send the structured reply to the client.
1972  * Returns -errno if sending fails. bdrv_block_status_above() failure is
1973  * reported to the client, at which point this function succeeds.
1974  */
1975 static int coroutine_fn nbd_co_send_sparse_read(NBDClient *client,
1976                                                 uint64_t handle,
1977                                                 uint64_t offset,
1978                                                 uint8_t *data,
1979                                                 size_t size,
1980                                                 Error **errp)
1981 {
1982     int ret = 0;
1983     NBDExport *exp = client->exp;
1984     size_t progress = 0;
1985 
1986     while (progress < size) {
1987         int64_t pnum;
1988         int status = bdrv_block_status_above(blk_bs(exp->common.blk), NULL,
1989                                              offset + progress,
1990                                              size - progress, &pnum, NULL,
1991                                              NULL);
1992         bool final;
1993 
1994         if (status < 0) {
1995             char *msg = g_strdup_printf("unable to check for holes: %s",
1996                                         strerror(-status));
1997 
1998             ret = nbd_co_send_structured_error(client, handle, -status, msg,
1999                                                errp);
2000             g_free(msg);
2001             return ret;
2002         }
2003         assert(pnum && pnum <= size - progress);
2004         final = progress + pnum == size;
2005         if (status & BDRV_BLOCK_ZERO) {
2006             NBDStructuredReadHole chunk;
2007             struct iovec iov[] = {
2008                 {.iov_base = &chunk, .iov_len = sizeof(chunk)},
2009             };
2010 
2011             trace_nbd_co_send_structured_read_hole(handle, offset + progress,
2012                                                    pnum);
2013             set_be_chunk(&chunk.h, final ? NBD_REPLY_FLAG_DONE : 0,
2014                          NBD_REPLY_TYPE_OFFSET_HOLE,
2015                          handle, sizeof(chunk) - sizeof(chunk.h));
2016             stq_be_p(&chunk.offset, offset + progress);
2017             stl_be_p(&chunk.length, pnum);
2018             ret = nbd_co_send_iov(client, iov, 1, errp);
2019         } else {
2020             ret = blk_pread(exp->common.blk, offset + progress,
2021                             data + progress, pnum);
2022             if (ret < 0) {
2023                 error_setg_errno(errp, -ret, "reading from file failed");
2024                 break;
2025             }
2026             ret = nbd_co_send_structured_read(client, handle, offset + progress,
2027                                               data + progress, pnum, final,
2028                                               errp);
2029         }
2030 
2031         if (ret < 0) {
2032             break;
2033         }
2034         progress += pnum;
2035     }
2036     return ret;
2037 }
2038 
2039 typedef struct NBDExtentArray {
2040     NBDExtent *extents;
2041     unsigned int nb_alloc;
2042     unsigned int count;
2043     uint64_t total_length;
2044     bool can_add;
2045     bool converted_to_be;
2046 } NBDExtentArray;
2047 
2048 static NBDExtentArray *nbd_extent_array_new(unsigned int nb_alloc)
2049 {
2050     NBDExtentArray *ea = g_new0(NBDExtentArray, 1);
2051 
2052     ea->nb_alloc = nb_alloc;
2053     ea->extents = g_new(NBDExtent, nb_alloc);
2054     ea->can_add = true;
2055 
2056     return ea;
2057 }
2058 
2059 static void nbd_extent_array_free(NBDExtentArray *ea)
2060 {
2061     g_free(ea->extents);
2062     g_free(ea);
2063 }
2064 G_DEFINE_AUTOPTR_CLEANUP_FUNC(NBDExtentArray, nbd_extent_array_free);
2065 
2066 /* Further modifications of the array after conversion are abandoned */
2067 static void nbd_extent_array_convert_to_be(NBDExtentArray *ea)
2068 {
2069     int i;
2070 
2071     assert(!ea->converted_to_be);
2072     ea->can_add = false;
2073     ea->converted_to_be = true;
2074 
2075     for (i = 0; i < ea->count; i++) {
2076         ea->extents[i].flags = cpu_to_be32(ea->extents[i].flags);
2077         ea->extents[i].length = cpu_to_be32(ea->extents[i].length);
2078     }
2079 }
2080 
2081 /*
2082  * Add extent to NBDExtentArray. If extent can't be added (no available space),
2083  * return -1.
2084  * For safety, when returning -1 for the first time, .can_add is set to false,
2085  * further call to nbd_extent_array_add() will crash.
2086  * (to avoid the situation, when after failing to add an extent (returned -1),
2087  * user miss this failure and add another extent, which is successfully added
2088  * (array is full, but new extent may be squashed into the last one), then we
2089  * have invalid array with skipped extent)
2090  */
2091 static int nbd_extent_array_add(NBDExtentArray *ea,
2092                                 uint32_t length, uint32_t flags)
2093 {
2094     assert(ea->can_add);
2095 
2096     if (!length) {
2097         return 0;
2098     }
2099 
2100     /* Extend previous extent if flags are the same */
2101     if (ea->count > 0 && flags == ea->extents[ea->count - 1].flags) {
2102         uint64_t sum = (uint64_t)length + ea->extents[ea->count - 1].length;
2103 
2104         if (sum <= UINT32_MAX) {
2105             ea->extents[ea->count - 1].length = sum;
2106             ea->total_length += length;
2107             return 0;
2108         }
2109     }
2110 
2111     if (ea->count >= ea->nb_alloc) {
2112         ea->can_add = false;
2113         return -1;
2114     }
2115 
2116     ea->total_length += length;
2117     ea->extents[ea->count] = (NBDExtent) {.length = length, .flags = flags};
2118     ea->count++;
2119 
2120     return 0;
2121 }
2122 
2123 static int blockstatus_to_extents(BlockDriverState *bs, uint64_t offset,
2124                                   uint64_t bytes, NBDExtentArray *ea)
2125 {
2126     while (bytes) {
2127         uint32_t flags;
2128         int64_t num;
2129         int ret = bdrv_block_status_above(bs, NULL, offset, bytes, &num,
2130                                           NULL, NULL);
2131 
2132         if (ret < 0) {
2133             return ret;
2134         }
2135 
2136         flags = (ret & BDRV_BLOCK_DATA ? 0 : NBD_STATE_HOLE) |
2137                 (ret & BDRV_BLOCK_ZERO ? NBD_STATE_ZERO : 0);
2138 
2139         if (nbd_extent_array_add(ea, num, flags) < 0) {
2140             return 0;
2141         }
2142 
2143         offset += num;
2144         bytes -= num;
2145     }
2146 
2147     return 0;
2148 }
2149 
2150 static int blockalloc_to_extents(BlockDriverState *bs, uint64_t offset,
2151                                  uint64_t bytes, NBDExtentArray *ea)
2152 {
2153     while (bytes) {
2154         int64_t num;
2155         int ret = bdrv_is_allocated_above(bs, NULL, false, offset, bytes,
2156                                           &num);
2157 
2158         if (ret < 0) {
2159             return ret;
2160         }
2161 
2162         if (nbd_extent_array_add(ea, num, ret) < 0) {
2163             return 0;
2164         }
2165 
2166         offset += num;
2167         bytes -= num;
2168     }
2169 
2170     return 0;
2171 }
2172 
2173 /*
2174  * nbd_co_send_extents
2175  *
2176  * @ea is converted to BE by the function
2177  * @last controls whether NBD_REPLY_FLAG_DONE is sent.
2178  */
2179 static int nbd_co_send_extents(NBDClient *client, uint64_t handle,
2180                                NBDExtentArray *ea,
2181                                bool last, uint32_t context_id, Error **errp)
2182 {
2183     NBDStructuredMeta chunk;
2184     struct iovec iov[] = {
2185         {.iov_base = &chunk, .iov_len = sizeof(chunk)},
2186         {.iov_base = ea->extents, .iov_len = ea->count * sizeof(ea->extents[0])}
2187     };
2188 
2189     nbd_extent_array_convert_to_be(ea);
2190 
2191     trace_nbd_co_send_extents(handle, ea->count, context_id, ea->total_length,
2192                               last);
2193     set_be_chunk(&chunk.h, last ? NBD_REPLY_FLAG_DONE : 0,
2194                  NBD_REPLY_TYPE_BLOCK_STATUS,
2195                  handle, sizeof(chunk) - sizeof(chunk.h) + iov[1].iov_len);
2196     stl_be_p(&chunk.context_id, context_id);
2197 
2198     return nbd_co_send_iov(client, iov, 2, errp);
2199 }
2200 
2201 /* Get block status from the exported device and send it to the client */
2202 static int nbd_co_send_block_status(NBDClient *client, uint64_t handle,
2203                                     BlockDriverState *bs, uint64_t offset,
2204                                     uint32_t length, bool dont_fragment,
2205                                     bool last, uint32_t context_id,
2206                                     Error **errp)
2207 {
2208     int ret;
2209     unsigned int nb_extents = dont_fragment ? 1 : NBD_MAX_BLOCK_STATUS_EXTENTS;
2210     g_autoptr(NBDExtentArray) ea = nbd_extent_array_new(nb_extents);
2211 
2212     if (context_id == NBD_META_ID_BASE_ALLOCATION) {
2213         ret = blockstatus_to_extents(bs, offset, length, ea);
2214     } else {
2215         ret = blockalloc_to_extents(bs, offset, length, ea);
2216     }
2217     if (ret < 0) {
2218         return nbd_co_send_structured_error(
2219                 client, handle, -ret, "can't get block status", errp);
2220     }
2221 
2222     return nbd_co_send_extents(client, handle, ea, last, context_id, errp);
2223 }
2224 
2225 /* Populate @ea from a dirty bitmap. */
2226 static void bitmap_to_extents(BdrvDirtyBitmap *bitmap,
2227                               uint64_t offset, uint64_t length,
2228                               NBDExtentArray *es)
2229 {
2230     int64_t start, dirty_start, dirty_count;
2231     int64_t end = offset + length;
2232     bool full = false;
2233 
2234     bdrv_dirty_bitmap_lock(bitmap);
2235 
2236     for (start = offset;
2237          bdrv_dirty_bitmap_next_dirty_area(bitmap, start, end, INT32_MAX,
2238                                            &dirty_start, &dirty_count);
2239          start = dirty_start + dirty_count)
2240     {
2241         if ((nbd_extent_array_add(es, dirty_start - start, 0) < 0) ||
2242             (nbd_extent_array_add(es, dirty_count, NBD_STATE_DIRTY) < 0))
2243         {
2244             full = true;
2245             break;
2246         }
2247     }
2248 
2249     if (!full) {
2250         /* last non dirty extent, nothing to do if array is now full */
2251         (void) nbd_extent_array_add(es, end - start, 0);
2252     }
2253 
2254     bdrv_dirty_bitmap_unlock(bitmap);
2255 }
2256 
2257 static int nbd_co_send_bitmap(NBDClient *client, uint64_t handle,
2258                               BdrvDirtyBitmap *bitmap, uint64_t offset,
2259                               uint32_t length, bool dont_fragment, bool last,
2260                               uint32_t context_id, Error **errp)
2261 {
2262     unsigned int nb_extents = dont_fragment ? 1 : NBD_MAX_BLOCK_STATUS_EXTENTS;
2263     g_autoptr(NBDExtentArray) ea = nbd_extent_array_new(nb_extents);
2264 
2265     bitmap_to_extents(bitmap, offset, length, ea);
2266 
2267     return nbd_co_send_extents(client, handle, ea, last, context_id, errp);
2268 }
2269 
2270 /* nbd_co_receive_request
2271  * Collect a client request. Return 0 if request looks valid, -EIO to drop
2272  * connection right away, -EAGAIN to indicate we were interrupted and the
2273  * channel should be quiesced, and any other negative value to report an error
2274  * to the client (although the caller may still need to disconnect after
2275  * reporting the error).
2276  */
2277 static int nbd_co_receive_request(NBDRequestData *req, NBDRequest *request,
2278                                   Error **errp)
2279 {
2280     NBDClient *client = req->client;
2281     int valid_flags;
2282     int ret;
2283 
2284     g_assert(qemu_in_coroutine());
2285     assert(client->recv_coroutine == qemu_coroutine_self());
2286     ret = nbd_receive_request(client, request, errp);
2287     if (ret < 0) {
2288         return  ret;
2289     }
2290 
2291     trace_nbd_co_receive_request_decode_type(request->handle, request->type,
2292                                              nbd_cmd_lookup(request->type));
2293 
2294     if (request->type != NBD_CMD_WRITE) {
2295         /* No payload, we are ready to read the next request.  */
2296         req->complete = true;
2297     }
2298 
2299     if (request->type == NBD_CMD_DISC) {
2300         /* Special case: we're going to disconnect without a reply,
2301          * whether or not flags, from, or len are bogus */
2302         return -EIO;
2303     }
2304 
2305     if (request->type == NBD_CMD_READ || request->type == NBD_CMD_WRITE ||
2306         request->type == NBD_CMD_CACHE)
2307     {
2308         if (request->len > NBD_MAX_BUFFER_SIZE) {
2309             error_setg(errp, "len (%" PRIu32" ) is larger than max len (%u)",
2310                        request->len, NBD_MAX_BUFFER_SIZE);
2311             return -EINVAL;
2312         }
2313 
2314         if (request->type != NBD_CMD_CACHE) {
2315             req->data = blk_try_blockalign(client->exp->common.blk,
2316                                            request->len);
2317             if (req->data == NULL) {
2318                 error_setg(errp, "No memory");
2319                 return -ENOMEM;
2320             }
2321         }
2322     }
2323 
2324     if (request->type == NBD_CMD_WRITE) {
2325         if (nbd_read(client->ioc, req->data, request->len, "CMD_WRITE data",
2326                      errp) < 0)
2327         {
2328             return -EIO;
2329         }
2330         req->complete = true;
2331 
2332         trace_nbd_co_receive_request_payload_received(request->handle,
2333                                                       request->len);
2334     }
2335 
2336     /* Sanity checks. */
2337     if (client->exp->nbdflags & NBD_FLAG_READ_ONLY &&
2338         (request->type == NBD_CMD_WRITE ||
2339          request->type == NBD_CMD_WRITE_ZEROES ||
2340          request->type == NBD_CMD_TRIM)) {
2341         error_setg(errp, "Export is read-only");
2342         return -EROFS;
2343     }
2344     if (request->from > client->exp->size ||
2345         request->len > client->exp->size - request->from) {
2346         error_setg(errp, "operation past EOF; From: %" PRIu64 ", Len: %" PRIu32
2347                    ", Size: %" PRIu64, request->from, request->len,
2348                    client->exp->size);
2349         return (request->type == NBD_CMD_WRITE ||
2350                 request->type == NBD_CMD_WRITE_ZEROES) ? -ENOSPC : -EINVAL;
2351     }
2352     if (client->check_align && !QEMU_IS_ALIGNED(request->from | request->len,
2353                                                 client->check_align)) {
2354         /*
2355          * The block layer gracefully handles unaligned requests, but
2356          * it's still worth tracing client non-compliance
2357          */
2358         trace_nbd_co_receive_align_compliance(nbd_cmd_lookup(request->type),
2359                                               request->from,
2360                                               request->len,
2361                                               client->check_align);
2362     }
2363     valid_flags = NBD_CMD_FLAG_FUA;
2364     if (request->type == NBD_CMD_READ && client->structured_reply) {
2365         valid_flags |= NBD_CMD_FLAG_DF;
2366     } else if (request->type == NBD_CMD_WRITE_ZEROES) {
2367         valid_flags |= NBD_CMD_FLAG_NO_HOLE | NBD_CMD_FLAG_FAST_ZERO;
2368     } else if (request->type == NBD_CMD_BLOCK_STATUS) {
2369         valid_flags |= NBD_CMD_FLAG_REQ_ONE;
2370     }
2371     if (request->flags & ~valid_flags) {
2372         error_setg(errp, "unsupported flags for command %s (got 0x%x)",
2373                    nbd_cmd_lookup(request->type), request->flags);
2374         return -EINVAL;
2375     }
2376 
2377     return 0;
2378 }
2379 
2380 /* Send simple reply without a payload, or a structured error
2381  * @error_msg is ignored if @ret >= 0
2382  * Returns 0 if connection is still live, -errno on failure to talk to client
2383  */
2384 static coroutine_fn int nbd_send_generic_reply(NBDClient *client,
2385                                                uint64_t handle,
2386                                                int ret,
2387                                                const char *error_msg,
2388                                                Error **errp)
2389 {
2390     if (client->structured_reply && ret < 0) {
2391         return nbd_co_send_structured_error(client, handle, -ret, error_msg,
2392                                             errp);
2393     } else {
2394         return nbd_co_send_simple_reply(client, handle, ret < 0 ? -ret : 0,
2395                                         NULL, 0, errp);
2396     }
2397 }
2398 
2399 /* Handle NBD_CMD_READ request.
2400  * Return -errno if sending fails. Other errors are reported directly to the
2401  * client as an error reply. */
2402 static coroutine_fn int nbd_do_cmd_read(NBDClient *client, NBDRequest *request,
2403                                         uint8_t *data, Error **errp)
2404 {
2405     int ret;
2406     NBDExport *exp = client->exp;
2407 
2408     assert(request->type == NBD_CMD_READ);
2409 
2410     /* XXX: NBD Protocol only documents use of FUA with WRITE */
2411     if (request->flags & NBD_CMD_FLAG_FUA) {
2412         ret = blk_co_flush(exp->common.blk);
2413         if (ret < 0) {
2414             return nbd_send_generic_reply(client, request->handle, ret,
2415                                           "flush failed", errp);
2416         }
2417     }
2418 
2419     if (client->structured_reply && !(request->flags & NBD_CMD_FLAG_DF) &&
2420         request->len)
2421     {
2422         return nbd_co_send_sparse_read(client, request->handle, request->from,
2423                                        data, request->len, errp);
2424     }
2425 
2426     ret = blk_pread(exp->common.blk, request->from, data, request->len);
2427     if (ret < 0) {
2428         return nbd_send_generic_reply(client, request->handle, ret,
2429                                       "reading from file failed", errp);
2430     }
2431 
2432     if (client->structured_reply) {
2433         if (request->len) {
2434             return nbd_co_send_structured_read(client, request->handle,
2435                                                request->from, data,
2436                                                request->len, true, errp);
2437         } else {
2438             return nbd_co_send_structured_done(client, request->handle, errp);
2439         }
2440     } else {
2441         return nbd_co_send_simple_reply(client, request->handle, 0,
2442                                         data, request->len, errp);
2443     }
2444 }
2445 
2446 /*
2447  * nbd_do_cmd_cache
2448  *
2449  * Handle NBD_CMD_CACHE request.
2450  * Return -errno if sending fails. Other errors are reported directly to the
2451  * client as an error reply.
2452  */
2453 static coroutine_fn int nbd_do_cmd_cache(NBDClient *client, NBDRequest *request,
2454                                          Error **errp)
2455 {
2456     int ret;
2457     NBDExport *exp = client->exp;
2458 
2459     assert(request->type == NBD_CMD_CACHE);
2460 
2461     ret = blk_co_preadv(exp->common.blk, request->from, request->len,
2462                         NULL, BDRV_REQ_COPY_ON_READ | BDRV_REQ_PREFETCH);
2463 
2464     return nbd_send_generic_reply(client, request->handle, ret,
2465                                   "caching data failed", errp);
2466 }
2467 
2468 /* Handle NBD request.
2469  * Return -errno if sending fails. Other errors are reported directly to the
2470  * client as an error reply. */
2471 static coroutine_fn int nbd_handle_request(NBDClient *client,
2472                                            NBDRequest *request,
2473                                            uint8_t *data, Error **errp)
2474 {
2475     int ret;
2476     int flags;
2477     NBDExport *exp = client->exp;
2478     char *msg;
2479     size_t i;
2480 
2481     switch (request->type) {
2482     case NBD_CMD_CACHE:
2483         return nbd_do_cmd_cache(client, request, errp);
2484 
2485     case NBD_CMD_READ:
2486         return nbd_do_cmd_read(client, request, data, errp);
2487 
2488     case NBD_CMD_WRITE:
2489         flags = 0;
2490         if (request->flags & NBD_CMD_FLAG_FUA) {
2491             flags |= BDRV_REQ_FUA;
2492         }
2493         ret = blk_pwrite(exp->common.blk, request->from, data, request->len,
2494                          flags);
2495         return nbd_send_generic_reply(client, request->handle, ret,
2496                                       "writing to file failed", errp);
2497 
2498     case NBD_CMD_WRITE_ZEROES:
2499         flags = 0;
2500         if (request->flags & NBD_CMD_FLAG_FUA) {
2501             flags |= BDRV_REQ_FUA;
2502         }
2503         if (!(request->flags & NBD_CMD_FLAG_NO_HOLE)) {
2504             flags |= BDRV_REQ_MAY_UNMAP;
2505         }
2506         if (request->flags & NBD_CMD_FLAG_FAST_ZERO) {
2507             flags |= BDRV_REQ_NO_FALLBACK;
2508         }
2509         ret = 0;
2510         /* FIXME simplify this when blk_pwrite_zeroes switches to 64-bit */
2511         while (ret >= 0 && request->len) {
2512             int align = client->check_align ?: 1;
2513             int len = MIN(request->len, QEMU_ALIGN_DOWN(BDRV_REQUEST_MAX_BYTES,
2514                                                         align));
2515             ret = blk_pwrite_zeroes(exp->common.blk, request->from, len, flags);
2516             request->len -= len;
2517             request->from += len;
2518         }
2519         return nbd_send_generic_reply(client, request->handle, ret,
2520                                       "writing to file failed", errp);
2521 
2522     case NBD_CMD_DISC:
2523         /* unreachable, thanks to special case in nbd_co_receive_request() */
2524         abort();
2525 
2526     case NBD_CMD_FLUSH:
2527         ret = blk_co_flush(exp->common.blk);
2528         return nbd_send_generic_reply(client, request->handle, ret,
2529                                       "flush failed", errp);
2530 
2531     case NBD_CMD_TRIM:
2532         ret = 0;
2533         /* FIXME simplify this when blk_co_pdiscard switches to 64-bit */
2534         while (ret >= 0 && request->len) {
2535             int align = client->check_align ?: 1;
2536             int len = MIN(request->len, QEMU_ALIGN_DOWN(BDRV_REQUEST_MAX_BYTES,
2537                                                         align));
2538             ret = blk_co_pdiscard(exp->common.blk, request->from, len);
2539             request->len -= len;
2540             request->from += len;
2541         }
2542         if (ret >= 0 && request->flags & NBD_CMD_FLAG_FUA) {
2543             ret = blk_co_flush(exp->common.blk);
2544         }
2545         return nbd_send_generic_reply(client, request->handle, ret,
2546                                       "discard failed", errp);
2547 
2548     case NBD_CMD_BLOCK_STATUS:
2549         if (!request->len) {
2550             return nbd_send_generic_reply(client, request->handle, -EINVAL,
2551                                           "need non-zero length", errp);
2552         }
2553         if (client->export_meta.count) {
2554             bool dont_fragment = request->flags & NBD_CMD_FLAG_REQ_ONE;
2555             int contexts_remaining = client->export_meta.count;
2556 
2557             if (client->export_meta.base_allocation) {
2558                 ret = nbd_co_send_block_status(client, request->handle,
2559                                                blk_bs(exp->common.blk),
2560                                                request->from,
2561                                                request->len, dont_fragment,
2562                                                !--contexts_remaining,
2563                                                NBD_META_ID_BASE_ALLOCATION,
2564                                                errp);
2565                 if (ret < 0) {
2566                     return ret;
2567                 }
2568             }
2569 
2570             if (client->export_meta.allocation_depth) {
2571                 ret = nbd_co_send_block_status(client, request->handle,
2572                                                blk_bs(exp->common.blk),
2573                                                request->from, request->len,
2574                                                dont_fragment,
2575                                                !--contexts_remaining,
2576                                                NBD_META_ID_ALLOCATION_DEPTH,
2577                                                errp);
2578                 if (ret < 0) {
2579                     return ret;
2580                 }
2581             }
2582 
2583             for (i = 0; i < client->exp->nr_export_bitmaps; i++) {
2584                 if (!client->export_meta.bitmaps[i]) {
2585                     continue;
2586                 }
2587                 ret = nbd_co_send_bitmap(client, request->handle,
2588                                          client->exp->export_bitmaps[i],
2589                                          request->from, request->len,
2590                                          dont_fragment, !--contexts_remaining,
2591                                          NBD_META_ID_DIRTY_BITMAP + i, errp);
2592                 if (ret < 0) {
2593                     return ret;
2594                 }
2595             }
2596 
2597             assert(!contexts_remaining);
2598 
2599             return 0;
2600         } else {
2601             return nbd_send_generic_reply(client, request->handle, -EINVAL,
2602                                           "CMD_BLOCK_STATUS not negotiated",
2603                                           errp);
2604         }
2605 
2606     default:
2607         msg = g_strdup_printf("invalid request type (%" PRIu32 ") received",
2608                               request->type);
2609         ret = nbd_send_generic_reply(client, request->handle, -EINVAL, msg,
2610                                      errp);
2611         g_free(msg);
2612         return ret;
2613     }
2614 }
2615 
2616 /* Owns a reference to the NBDClient passed as opaque.  */
2617 static coroutine_fn void nbd_trip(void *opaque)
2618 {
2619     NBDClient *client = opaque;
2620     NBDRequestData *req;
2621     NBDRequest request = { 0 };    /* GCC thinks it can be used uninitialized */
2622     int ret;
2623     Error *local_err = NULL;
2624 
2625     trace_nbd_trip();
2626     if (client->closing) {
2627         nbd_client_put(client);
2628         return;
2629     }
2630 
2631     if (client->quiescing) {
2632         /*
2633          * We're switching between AIO contexts. Don't attempt to receive a new
2634          * request and kick the main context which may be waiting for us.
2635          */
2636         nbd_client_put(client);
2637         client->recv_coroutine = NULL;
2638         aio_wait_kick();
2639         return;
2640     }
2641 
2642     req = nbd_request_get(client);
2643     ret = nbd_co_receive_request(req, &request, &local_err);
2644     client->recv_coroutine = NULL;
2645 
2646     if (client->closing) {
2647         /*
2648          * The client may be closed when we are blocked in
2649          * nbd_co_receive_request()
2650          */
2651         goto done;
2652     }
2653 
2654     if (ret == -EAGAIN) {
2655         assert(client->quiescing);
2656         goto done;
2657     }
2658 
2659     nbd_client_receive_next_request(client);
2660     if (ret == -EIO) {
2661         goto disconnect;
2662     }
2663 
2664     if (ret < 0) {
2665         /* It wans't -EIO, so, according to nbd_co_receive_request()
2666          * semantics, we should return the error to the client. */
2667         Error *export_err = local_err;
2668 
2669         local_err = NULL;
2670         ret = nbd_send_generic_reply(client, request.handle, -EINVAL,
2671                                      error_get_pretty(export_err), &local_err);
2672         error_free(export_err);
2673     } else {
2674         ret = nbd_handle_request(client, &request, req->data, &local_err);
2675     }
2676     if (ret < 0) {
2677         error_prepend(&local_err, "Failed to send reply: ");
2678         goto disconnect;
2679     }
2680 
2681     /* We must disconnect after NBD_CMD_WRITE if we did not
2682      * read the payload.
2683      */
2684     if (!req->complete) {
2685         error_setg(&local_err, "Request handling failed in intermediate state");
2686         goto disconnect;
2687     }
2688 
2689 done:
2690     nbd_request_put(req);
2691     nbd_client_put(client);
2692     return;
2693 
2694 disconnect:
2695     if (local_err) {
2696         error_reportf_err(local_err, "Disconnect client, due to: ");
2697     }
2698     nbd_request_put(req);
2699     client_close(client, true);
2700     nbd_client_put(client);
2701 }
2702 
2703 static void nbd_client_receive_next_request(NBDClient *client)
2704 {
2705     if (!client->recv_coroutine && client->nb_requests < MAX_NBD_REQUESTS &&
2706         !client->quiescing) {
2707         nbd_client_get(client);
2708         client->recv_coroutine = qemu_coroutine_create(nbd_trip, client);
2709         aio_co_schedule(client->exp->common.ctx, client->recv_coroutine);
2710     }
2711 }
2712 
2713 static coroutine_fn void nbd_co_client_start(void *opaque)
2714 {
2715     NBDClient *client = opaque;
2716     Error *local_err = NULL;
2717 
2718     qemu_co_mutex_init(&client->send_lock);
2719 
2720     if (nbd_negotiate(client, &local_err)) {
2721         if (local_err) {
2722             error_report_err(local_err);
2723         }
2724         client_close(client, false);
2725         return;
2726     }
2727 
2728     nbd_client_receive_next_request(client);
2729 }
2730 
2731 /*
2732  * Create a new client listener using the given channel @sioc.
2733  * Begin servicing it in a coroutine.  When the connection closes, call
2734  * @close_fn with an indication of whether the client completed negotiation.
2735  */
2736 void nbd_client_new(QIOChannelSocket *sioc,
2737                     QCryptoTLSCreds *tlscreds,
2738                     const char *tlsauthz,
2739                     void (*close_fn)(NBDClient *, bool))
2740 {
2741     NBDClient *client;
2742     Coroutine *co;
2743 
2744     client = g_new0(NBDClient, 1);
2745     client->refcount = 1;
2746     client->tlscreds = tlscreds;
2747     if (tlscreds) {
2748         object_ref(OBJECT(client->tlscreds));
2749     }
2750     client->tlsauthz = g_strdup(tlsauthz);
2751     client->sioc = sioc;
2752     object_ref(OBJECT(client->sioc));
2753     client->ioc = QIO_CHANNEL(sioc);
2754     object_ref(OBJECT(client->ioc));
2755     client->close_fn = close_fn;
2756 
2757     co = qemu_coroutine_create(nbd_co_client_start, client);
2758     qemu_coroutine_enter(co);
2759 }
2760