xref: /openbmc/qemu/include/block/nbd.h (revision f14eced5)
1 /*
2  *  Copyright Red Hat
3  *  Copyright (C) 2005  Anthony Liguori <anthony@codemonkey.ws>
4  *
5  *  Network Block Device
6  *
7  *  This program is free software; you can redistribute it and/or modify
8  *  it under the terms of the GNU General Public License as published by
9  *  the Free Software Foundation; under version 2 of the License.
10  *
11  *  This program is distributed in the hope that it will be useful,
12  *  but WITHOUT ANY WARRANTY; without even the implied warranty of
13  *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  *  GNU General Public License for more details.
15  *
16  *  You should have received a copy of the GNU General Public License
17  *  along with this program; if not, see <http://www.gnu.org/licenses/>.
18  */
19 
20 #ifndef NBD_H
21 #define NBD_H
22 
23 #include "block/export.h"
24 #include "io/channel-socket.h"
25 #include "crypto/tlscreds.h"
26 #include "qapi/error.h"
27 #include "qemu/bswap.h"
28 
29 typedef struct NBDExport NBDExport;
30 typedef struct NBDClient NBDClient;
31 typedef struct NBDClientConnection NBDClientConnection;
32 
33 extern const BlockExportDriver blk_exp_nbd;
34 
35 /* Handshake phase structs - this struct is passed on the wire */
36 
37 typedef struct NBDOption {
38     uint64_t magic; /* NBD_OPTS_MAGIC */
39     uint32_t option; /* NBD_OPT_* */
40     uint32_t length;
41 } QEMU_PACKED NBDOption;
42 
43 typedef struct NBDOptionReply {
44     uint64_t magic; /* NBD_REP_MAGIC */
45     uint32_t option; /* NBD_OPT_* */
46     uint32_t type; /* NBD_REP_* */
47     uint32_t length;
48 } QEMU_PACKED NBDOptionReply;
49 
50 typedef struct NBDOptionReplyMetaContext {
51     NBDOptionReply h; /* h.type = NBD_REP_META_CONTEXT, h.length > 4 */
52     uint32_t context_id;
53     /* metadata context name follows */
54 } QEMU_PACKED NBDOptionReplyMetaContext;
55 
56 /* Track results of negotiation */
57 typedef enum NBDMode {
58     /* Keep this list in a continuum of increasing features. */
59     NBD_MODE_OLDSTYLE,     /* server lacks newstyle negotiation */
60     NBD_MODE_EXPORT_NAME,  /* newstyle but only OPT_EXPORT_NAME safe */
61     NBD_MODE_SIMPLE,       /* newstyle but only simple replies */
62     NBD_MODE_STRUCTURED,   /* newstyle, structured replies enabled */
63     NBD_MODE_EXTENDED,     /* newstyle, extended headers enabled */
64 } NBDMode;
65 
66 /* Transmission phase structs */
67 
68 /*
69  * Note: NBDRequest is _NOT_ the same as the network representation of an NBD
70  * request!
71  */
72 typedef struct NBDRequest {
73     uint64_t cookie;
74     uint64_t from;  /* Offset touched by the command */
75     uint64_t len;   /* Effect length; 32 bit limit without extended headers */
76     uint16_t flags; /* NBD_CMD_FLAG_* */
77     uint16_t type;  /* NBD_CMD_* */
78     NBDMode mode;   /* Determines which network representation to use */
79 } NBDRequest;
80 
81 typedef struct NBDSimpleReply {
82     uint32_t magic;  /* NBD_SIMPLE_REPLY_MAGIC */
83     uint32_t error;
84     uint64_t cookie;
85 } QEMU_PACKED NBDSimpleReply;
86 
87 /* Header of all structured replies */
88 typedef struct NBDStructuredReplyChunk {
89     uint32_t magic;  /* NBD_STRUCTURED_REPLY_MAGIC */
90     uint16_t flags;  /* combination of NBD_REPLY_FLAG_* */
91     uint16_t type;   /* NBD_REPLY_TYPE_* */
92     uint64_t cookie; /* request handle */
93     uint32_t length; /* length of payload */
94 } QEMU_PACKED NBDStructuredReplyChunk;
95 
96 typedef struct NBDExtendedReplyChunk {
97     uint32_t magic;  /* NBD_EXTENDED_REPLY_MAGIC */
98     uint16_t flags;  /* combination of NBD_REPLY_FLAG_* */
99     uint16_t type;   /* NBD_REPLY_TYPE_* */
100     uint64_t cookie; /* request handle */
101     uint64_t offset; /* request offset */
102     uint64_t length; /* length of payload */
103 } QEMU_PACKED NBDExtendedReplyChunk;
104 
105 typedef union NBDReply {
106     NBDSimpleReply simple;
107     NBDStructuredReplyChunk structured;
108     NBDExtendedReplyChunk extended;
109     struct {
110         /*
111          * @magic and @cookie fields have the same offset and size in all
112          * forms of replies, so let them be accessible without ".simple.",
113          * ".structured.", or ".extended." specifications.
114          */
115         uint32_t magic;
116         uint32_t _skip;
117         uint64_t cookie;
118     };
119 } NBDReply;
120 QEMU_BUILD_BUG_ON(offsetof(NBDReply, simple.cookie) !=
121                   offsetof(NBDReply, cookie));
122 QEMU_BUILD_BUG_ON(offsetof(NBDReply, structured.cookie) !=
123                   offsetof(NBDReply, cookie));
124 QEMU_BUILD_BUG_ON(offsetof(NBDReply, extended.cookie) !=
125                   offsetof(NBDReply, cookie));
126 
127 /* Header of chunk for NBD_REPLY_TYPE_OFFSET_DATA */
128 typedef struct NBDStructuredReadData {
129     /* header's .length >= 9 */
130     uint64_t offset;
131     /* At least one byte of data payload follows, calculated from h.length */
132 } QEMU_PACKED NBDStructuredReadData;
133 
134 /* Complete chunk for NBD_REPLY_TYPE_OFFSET_HOLE */
135 typedef struct NBDStructuredReadHole {
136     /* header's length == 12 */
137     uint64_t offset;
138     uint32_t length;
139 } QEMU_PACKED NBDStructuredReadHole;
140 
141 /* Header of all NBD_REPLY_TYPE_ERROR* errors */
142 typedef struct NBDStructuredError {
143     /* header's length >= 6 */
144     uint32_t error;
145     uint16_t message_length;
146 } QEMU_PACKED NBDStructuredError;
147 
148 /* Header of NBD_REPLY_TYPE_BLOCK_STATUS */
149 typedef struct NBDStructuredMeta {
150     /* header's length >= 12 (at least one extent) */
151     uint32_t context_id;
152     /* NBDExtent32 extents[] follows, array length implied by header */
153 } QEMU_PACKED NBDStructuredMeta;
154 
155 /* Extent array element for NBD_REPLY_TYPE_BLOCK_STATUS */
156 typedef struct NBDExtent32 {
157     uint32_t length;
158     uint32_t flags; /* NBD_STATE_* */
159 } QEMU_PACKED NBDExtent32;
160 
161 /* Header of NBD_REPLY_TYPE_BLOCK_STATUS_EXT */
162 typedef struct NBDExtendedMeta {
163     /* header's length >= 24 (at least one extent) */
164     uint32_t context_id;
165     uint32_t count; /* header length must be count * 16 + 8 */
166     /* NBDExtent64 extents[count] follows */
167 } QEMU_PACKED NBDExtendedMeta;
168 
169 /* Extent array element for NBD_REPLY_TYPE_BLOCK_STATUS_EXT */
170 typedef struct NBDExtent64 {
171     uint64_t length;
172     uint64_t flags; /* NBD_STATE_* */
173 } QEMU_PACKED NBDExtent64;
174 
175 /* Client payload for limiting NBD_CMD_BLOCK_STATUS reply */
176 typedef struct NBDBlockStatusPayload {
177     uint64_t effect_length;
178     /* uint32_t ids[] follows, array length implied by header */
179 } QEMU_PACKED NBDBlockStatusPayload;
180 
181 /* Transmission (export) flags: sent from server to client during handshake,
182    but describe what will happen during transmission */
183 enum {
184     NBD_FLAG_HAS_FLAGS_BIT          =  0, /* Flags are there */
185     NBD_FLAG_READ_ONLY_BIT          =  1, /* Device is read-only */
186     NBD_FLAG_SEND_FLUSH_BIT         =  2, /* Send FLUSH */
187     NBD_FLAG_SEND_FUA_BIT           =  3, /* Send FUA (Force Unit Access) */
188     NBD_FLAG_ROTATIONAL_BIT         =  4, /* Use elevator algorithm -
189                                              rotational media */
190     NBD_FLAG_SEND_TRIM_BIT          =  5, /* Send TRIM (discard) */
191     NBD_FLAG_SEND_WRITE_ZEROES_BIT  =  6, /* Send WRITE_ZEROES */
192     NBD_FLAG_SEND_DF_BIT            =  7, /* Send DF (Do not Fragment) */
193     NBD_FLAG_CAN_MULTI_CONN_BIT     =  8, /* Multi-client cache consistent */
194     NBD_FLAG_SEND_RESIZE_BIT        =  9, /* Send resize */
195     NBD_FLAG_SEND_CACHE_BIT         = 10, /* Send CACHE (prefetch) */
196     NBD_FLAG_SEND_FAST_ZERO_BIT     = 11, /* FAST_ZERO flag for WRITE_ZEROES */
197     NBD_FLAG_BLOCK_STAT_PAYLOAD_BIT = 12, /* PAYLOAD flag for BLOCK_STATUS */
198 };
199 
200 #define NBD_FLAG_HAS_FLAGS          (1 << NBD_FLAG_HAS_FLAGS_BIT)
201 #define NBD_FLAG_READ_ONLY          (1 << NBD_FLAG_READ_ONLY_BIT)
202 #define NBD_FLAG_SEND_FLUSH         (1 << NBD_FLAG_SEND_FLUSH_BIT)
203 #define NBD_FLAG_SEND_FUA           (1 << NBD_FLAG_SEND_FUA_BIT)
204 #define NBD_FLAG_ROTATIONAL         (1 << NBD_FLAG_ROTATIONAL_BIT)
205 #define NBD_FLAG_SEND_TRIM          (1 << NBD_FLAG_SEND_TRIM_BIT)
206 #define NBD_FLAG_SEND_WRITE_ZEROES  (1 << NBD_FLAG_SEND_WRITE_ZEROES_BIT)
207 #define NBD_FLAG_SEND_DF            (1 << NBD_FLAG_SEND_DF_BIT)
208 #define NBD_FLAG_CAN_MULTI_CONN     (1 << NBD_FLAG_CAN_MULTI_CONN_BIT)
209 #define NBD_FLAG_SEND_RESIZE        (1 << NBD_FLAG_SEND_RESIZE_BIT)
210 #define NBD_FLAG_SEND_CACHE         (1 << NBD_FLAG_SEND_CACHE_BIT)
211 #define NBD_FLAG_SEND_FAST_ZERO     (1 << NBD_FLAG_SEND_FAST_ZERO_BIT)
212 #define NBD_FLAG_BLOCK_STAT_PAYLOAD (1 << NBD_FLAG_BLOCK_STAT_PAYLOAD_BIT)
213 
214 /* New-style handshake (global) flags, sent from server to client, and
215    control what will happen during handshake phase. */
216 #define NBD_FLAG_FIXED_NEWSTYLE   (1 << 0) /* Fixed newstyle protocol. */
217 #define NBD_FLAG_NO_ZEROES        (1 << 1) /* End handshake without zeroes. */
218 
219 /* New-style client flags, sent from client to server to control what happens
220    during handshake phase. */
221 #define NBD_FLAG_C_FIXED_NEWSTYLE (1 << 0) /* Fixed newstyle protocol. */
222 #define NBD_FLAG_C_NO_ZEROES      (1 << 1) /* End handshake without zeroes. */
223 
224 /* Option requests. */
225 #define NBD_OPT_EXPORT_NAME       (1)
226 #define NBD_OPT_ABORT             (2)
227 #define NBD_OPT_LIST              (3)
228 /* #define NBD_OPT_PEEK_EXPORT    (4) not in use */
229 #define NBD_OPT_STARTTLS          (5)
230 #define NBD_OPT_INFO              (6)
231 #define NBD_OPT_GO                (7)
232 #define NBD_OPT_STRUCTURED_REPLY  (8)
233 #define NBD_OPT_LIST_META_CONTEXT (9)
234 #define NBD_OPT_SET_META_CONTEXT  (10)
235 #define NBD_OPT_EXTENDED_HEADERS  (11)
236 
237 /* Option reply types. */
238 #define NBD_REP_ERR(value) ((UINT32_C(1) << 31) | (value))
239 
240 #define NBD_REP_ACK             (1)    /* Data sending finished. */
241 #define NBD_REP_SERVER          (2)    /* Export description. */
242 #define NBD_REP_INFO            (3)    /* NBD_OPT_INFO/GO. */
243 #define NBD_REP_META_CONTEXT    (4)    /* NBD_OPT_{LIST,SET}_META_CONTEXT */
244 
245 #define NBD_REP_ERR_UNSUP           NBD_REP_ERR(1)  /* Unknown option */
246 #define NBD_REP_ERR_POLICY          NBD_REP_ERR(2)  /* Server denied */
247 #define NBD_REP_ERR_INVALID         NBD_REP_ERR(3)  /* Invalid length */
248 #define NBD_REP_ERR_PLATFORM        NBD_REP_ERR(4)  /* Not compiled in */
249 #define NBD_REP_ERR_TLS_REQD        NBD_REP_ERR(5)  /* TLS required */
250 #define NBD_REP_ERR_UNKNOWN         NBD_REP_ERR(6)  /* Export unknown */
251 #define NBD_REP_ERR_SHUTDOWN        NBD_REP_ERR(7)  /* Server shutting down */
252 #define NBD_REP_ERR_BLOCK_SIZE_REQD NBD_REP_ERR(8)  /* Need INFO_BLOCK_SIZE */
253 #define NBD_REP_ERR_TOO_BIG         NBD_REP_ERR(9)  /* Payload size overflow */
254 #define NBD_REP_ERR_EXT_HEADER_REQD NBD_REP_ERR(10) /* Need extended headers */
255 
256 /* Info types, used during NBD_REP_INFO */
257 #define NBD_INFO_EXPORT         0
258 #define NBD_INFO_NAME           1
259 #define NBD_INFO_DESCRIPTION    2
260 #define NBD_INFO_BLOCK_SIZE     3
261 
262 /* Request flags, sent from client to server during transmission phase */
263 #define NBD_CMD_FLAG_FUA         (1 << 0) /* 'force unit access' during write */
264 #define NBD_CMD_FLAG_NO_HOLE     (1 << 1) /* don't punch hole on zero run */
265 #define NBD_CMD_FLAG_DF          (1 << 2) /* don't fragment structured read */
266 #define NBD_CMD_FLAG_REQ_ONE     (1 << 3) \
267     /* only one extent in BLOCK_STATUS reply chunk */
268 #define NBD_CMD_FLAG_FAST_ZERO   (1 << 4) /* fail if WRITE_ZEROES is not fast */
269 #define NBD_CMD_FLAG_PAYLOAD_LEN (1 << 5) \
270     /* length describes payload, not effect; only with ext header */
271 
272 /* Supported request types */
273 enum {
274     NBD_CMD_READ = 0,
275     NBD_CMD_WRITE = 1,
276     NBD_CMD_DISC = 2,
277     NBD_CMD_FLUSH = 3,
278     NBD_CMD_TRIM = 4,
279     NBD_CMD_CACHE = 5,
280     NBD_CMD_WRITE_ZEROES = 6,
281     NBD_CMD_BLOCK_STATUS = 7,
282 };
283 
284 #define NBD_DEFAULT_PORT	10809
285 
286 /* Maximum size of a single READ/WRITE data buffer */
287 #define NBD_MAX_BUFFER_SIZE (32 * 1024 * 1024)
288 
289 /*
290  * Maximum size of a protocol string (export name, metadata context name,
291  * etc.).  Use malloc rather than stack allocation for storage of a
292  * string.
293  */
294 #define NBD_MAX_STRING_SIZE 4096
295 
296 /* Two types of request structures, a given client will only use 1 */
297 #define NBD_REQUEST_MAGIC           0x25609513
298 #define NBD_EXTENDED_REQUEST_MAGIC  0x21e41c71
299 
300 /*
301  * Three types of reply structures, but what a client expects depends
302  * on NBD_OPT_STRUCTURED_REPLY and NBD_OPT_EXTENDED_HEADERS.
303  */
304 #define NBD_SIMPLE_REPLY_MAGIC      0x67446698
305 #define NBD_STRUCTURED_REPLY_MAGIC  0x668e33ef
306 #define NBD_EXTENDED_REPLY_MAGIC    0x6e8a278c
307 
308 /* Chunk reply flags (for structured and extended replies) */
309 #define NBD_REPLY_FLAG_DONE          (1 << 0) /* This reply-chunk is last */
310 
311 /* Chunk reply types */
312 #define NBD_REPLY_ERR(value)         ((1 << 15) | (value))
313 
314 #define NBD_REPLY_TYPE_NONE              0
315 #define NBD_REPLY_TYPE_OFFSET_DATA       1
316 #define NBD_REPLY_TYPE_OFFSET_HOLE       2
317 #define NBD_REPLY_TYPE_BLOCK_STATUS      5
318 #define NBD_REPLY_TYPE_BLOCK_STATUS_EXT  6
319 #define NBD_REPLY_TYPE_ERROR             NBD_REPLY_ERR(1)
320 #define NBD_REPLY_TYPE_ERROR_OFFSET      NBD_REPLY_ERR(2)
321 
322 /* Extent flags for base:allocation in NBD_REPLY_TYPE_BLOCK_STATUS */
323 #define NBD_STATE_HOLE (1 << 0)
324 #define NBD_STATE_ZERO (1 << 1)
325 
326 /* Extent flags for qemu:dirty-bitmap in NBD_REPLY_TYPE_BLOCK_STATUS */
327 #define NBD_STATE_DIRTY (1 << 0)
328 
329 /* No flags needed for qemu:allocation-depth in NBD_REPLY_TYPE_BLOCK_STATUS */
330 
331 static inline bool nbd_reply_type_is_error(int type)
332 {
333     return type & (1 << 15);
334 }
335 
336 /* NBD errors are based on errno numbers, so there is a 1:1 mapping,
337  * but only a limited set of errno values is specified in the protocol.
338  * Everything else is squashed to EINVAL.
339  */
340 #define NBD_SUCCESS    0
341 #define NBD_EPERM      1
342 #define NBD_EIO        5
343 #define NBD_ENOMEM     12
344 #define NBD_EINVAL     22
345 #define NBD_ENOSPC     28
346 #define NBD_EOVERFLOW  75
347 #define NBD_ENOTSUP    95
348 #define NBD_ESHUTDOWN  108
349 
350 /* Details collected by NBD_OPT_EXPORT_NAME and NBD_OPT_GO */
351 typedef struct NBDExportInfo {
352     /* Set by client before nbd_receive_negotiate() */
353     bool request_sizes;
354     char *x_dirty_bitmap;
355 
356     /* Set by client before nbd_receive_negotiate(), or by server results
357      * during nbd_receive_export_list() */
358     char *name; /* must be non-NULL */
359 
360     /* In-out fields, set by client before nbd_receive_negotiate() and
361      * updated by server results during nbd_receive_negotiate() */
362     NBDMode mode; /* input maximum mode tolerated; output actual mode chosen */
363     bool base_allocation; /* base:allocation context for NBD_CMD_BLOCK_STATUS */
364 
365     /* Set by server results during nbd_receive_negotiate() and
366      * nbd_receive_export_list() */
367     uint64_t size;
368     uint16_t flags;
369     uint32_t min_block;
370     uint32_t opt_block;
371     uint32_t max_block;
372 
373     uint32_t context_id;
374 
375     /* Set by server results during nbd_receive_export_list() */
376     char *description;
377     int n_contexts;
378     char **contexts;
379 } NBDExportInfo;
380 
381 int nbd_receive_negotiate(QIOChannel *ioc, QCryptoTLSCreds *tlscreds,
382                           const char *hostname, QIOChannel **outioc,
383                           NBDExportInfo *info, Error **errp);
384 void nbd_free_export_list(NBDExportInfo *info, int count);
385 int nbd_receive_export_list(QIOChannel *ioc, QCryptoTLSCreds *tlscreds,
386                             const char *hostname, NBDExportInfo **info,
387                             Error **errp);
388 int nbd_init(int fd, QIOChannelSocket *sioc, NBDExportInfo *info,
389              Error **errp);
390 int nbd_send_request(QIOChannel *ioc, NBDRequest *request);
391 int coroutine_fn nbd_receive_reply(BlockDriverState *bs, QIOChannel *ioc,
392                                    NBDReply *reply, Error **errp);
393 int nbd_client(int fd);
394 int nbd_disconnect(int fd);
395 int nbd_errno_to_system_errno(int err);
396 
397 void nbd_export_set_on_eject_blk(BlockExport *exp, BlockBackend *blk);
398 
399 AioContext *nbd_export_aio_context(NBDExport *exp);
400 NBDExport *nbd_export_find(const char *name);
401 
402 void nbd_client_new(QIOChannelSocket *sioc,
403                     QCryptoTLSCreds *tlscreds,
404                     const char *tlsauthz,
405                     void (*close_fn)(NBDClient *, bool));
406 void nbd_client_get(NBDClient *client);
407 void nbd_client_put(NBDClient *client);
408 
409 void nbd_server_is_qemu_nbd(int max_connections);
410 bool nbd_server_is_running(void);
411 int nbd_server_max_connections(void);
412 void nbd_server_start(SocketAddress *addr, const char *tls_creds,
413                       const char *tls_authz, uint32_t max_connections,
414                       Error **errp);
415 void nbd_server_start_options(NbdServerOptions *arg, Error **errp);
416 
417 /* nbd_read
418  * Reads @size bytes from @ioc. Returns 0 on success.
419  */
420 static inline int nbd_read(QIOChannel *ioc, void *buffer, size_t size,
421                            const char *desc, Error **errp)
422 {
423     ERRP_GUARD();
424     int ret = qio_channel_read_all(ioc, buffer, size, errp) < 0 ? -EIO : 0;
425 
426     if (ret < 0) {
427         if (desc) {
428             error_prepend(errp, "Failed to read %s: ", desc);
429         }
430         return ret;
431     }
432 
433     return 0;
434 }
435 
436 #define DEF_NBD_READ_N(bits)                                            \
437 static inline int nbd_read##bits(QIOChannel *ioc,                       \
438                                  uint##bits##_t *val,                   \
439                                  const char *desc, Error **errp)        \
440 {                                                                       \
441     int ret = nbd_read(ioc, val, sizeof(*val), desc, errp);             \
442     if (ret < 0) {                                                      \
443         return ret;                                                     \
444     }                                                                   \
445     *val = be##bits##_to_cpu(*val);                                     \
446     return 0;                                                           \
447 }
448 
449 DEF_NBD_READ_N(16) /* Defines nbd_read16(). */
450 DEF_NBD_READ_N(32) /* Defines nbd_read32(). */
451 DEF_NBD_READ_N(64) /* Defines nbd_read64(). */
452 
453 #undef DEF_NBD_READ_N
454 
455 static inline bool nbd_reply_is_simple(NBDReply *reply)
456 {
457     return reply->magic == NBD_SIMPLE_REPLY_MAGIC;
458 }
459 
460 static inline bool nbd_reply_is_structured(NBDReply *reply)
461 {
462     return reply->magic == NBD_STRUCTURED_REPLY_MAGIC;
463 }
464 
465 const char *nbd_reply_type_lookup(uint16_t type);
466 const char *nbd_opt_lookup(uint32_t opt);
467 const char *nbd_rep_lookup(uint32_t rep);
468 const char *nbd_info_lookup(uint16_t info);
469 const char *nbd_cmd_lookup(uint16_t info);
470 const char *nbd_err_lookup(int err);
471 const char *nbd_mode_lookup(NBDMode mode);
472 
473 /* nbd/client-connection.c */
474 void nbd_client_connection_enable_retry(NBDClientConnection *conn);
475 
476 NBDClientConnection *nbd_client_connection_new(const SocketAddress *saddr,
477                                                bool do_negotiation,
478                                                const char *export_name,
479                                                const char *x_dirty_bitmap,
480                                                QCryptoTLSCreds *tlscreds,
481                                                const char *tlshostname);
482 void nbd_client_connection_release(NBDClientConnection *conn);
483 
484 QIOChannel *coroutine_fn
485 nbd_co_establish_connection(NBDClientConnection *conn, NBDExportInfo *info,
486                             bool blocking, Error **errp);
487 
488 void nbd_co_establish_connection_cancel(NBDClientConnection *conn);
489 
490 #endif
491