1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Ceph msgr2 protocol implementation 4 * 5 * Copyright (C) 2020 Ilya Dryomov <idryomov@gmail.com> 6 */ 7 8 #include <linux/ceph/ceph_debug.h> 9 10 #include <crypto/aead.h> 11 #include <crypto/algapi.h> /* for crypto_memneq() */ 12 #include <crypto/hash.h> 13 #include <crypto/sha2.h> 14 #include <linux/bvec.h> 15 #include <linux/crc32c.h> 16 #include <linux/net.h> 17 #include <linux/scatterlist.h> 18 #include <linux/socket.h> 19 #include <linux/sched/mm.h> 20 #include <net/sock.h> 21 #include <net/tcp.h> 22 23 #include <linux/ceph/ceph_features.h> 24 #include <linux/ceph/decode.h> 25 #include <linux/ceph/libceph.h> 26 #include <linux/ceph/messenger.h> 27 28 #include "crypto.h" /* for CEPH_KEY_LEN and CEPH_MAX_CON_SECRET_LEN */ 29 30 #define FRAME_TAG_HELLO 1 31 #define FRAME_TAG_AUTH_REQUEST 2 32 #define FRAME_TAG_AUTH_BAD_METHOD 3 33 #define FRAME_TAG_AUTH_REPLY_MORE 4 34 #define FRAME_TAG_AUTH_REQUEST_MORE 5 35 #define FRAME_TAG_AUTH_DONE 6 36 #define FRAME_TAG_AUTH_SIGNATURE 7 37 #define FRAME_TAG_CLIENT_IDENT 8 38 #define FRAME_TAG_SERVER_IDENT 9 39 #define FRAME_TAG_IDENT_MISSING_FEATURES 10 40 #define FRAME_TAG_SESSION_RECONNECT 11 41 #define FRAME_TAG_SESSION_RESET 12 42 #define FRAME_TAG_SESSION_RETRY 13 43 #define FRAME_TAG_SESSION_RETRY_GLOBAL 14 44 #define FRAME_TAG_SESSION_RECONNECT_OK 15 45 #define FRAME_TAG_WAIT 16 46 #define FRAME_TAG_MESSAGE 17 47 #define FRAME_TAG_KEEPALIVE2 18 48 #define FRAME_TAG_KEEPALIVE2_ACK 19 49 #define FRAME_TAG_ACK 20 50 51 #define FRAME_LATE_STATUS_ABORTED 0x1 52 #define FRAME_LATE_STATUS_COMPLETE 0xe 53 #define FRAME_LATE_STATUS_ABORTED_MASK 0xf 54 55 #define IN_S_HANDLE_PREAMBLE 1 56 #define IN_S_HANDLE_CONTROL 2 57 #define IN_S_HANDLE_CONTROL_REMAINDER 3 58 #define IN_S_PREPARE_READ_DATA 4 59 #define IN_S_PREPARE_READ_DATA_CONT 5 60 #define IN_S_PREPARE_READ_ENC_PAGE 6 61 #define IN_S_HANDLE_EPILOGUE 7 62 #define IN_S_FINISH_SKIP 8 63 64 #define OUT_S_QUEUE_DATA 1 65 #define OUT_S_QUEUE_DATA_CONT 2 66 #define OUT_S_QUEUE_ENC_PAGE 3 67 #define OUT_S_QUEUE_ZEROS 4 68 #define OUT_S_FINISH_MESSAGE 5 69 #define OUT_S_GET_NEXT 6 70 71 #define CTRL_BODY(p) ((void *)(p) + CEPH_PREAMBLE_LEN) 72 #define FRONT_PAD(p) ((void *)(p) + CEPH_EPILOGUE_SECURE_LEN) 73 #define MIDDLE_PAD(p) (FRONT_PAD(p) + CEPH_GCM_BLOCK_LEN) 74 #define DATA_PAD(p) (MIDDLE_PAD(p) + CEPH_GCM_BLOCK_LEN) 75 76 #define CEPH_MSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL) 77 78 static int do_recvmsg(struct socket *sock, struct iov_iter *it) 79 { 80 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS }; 81 int ret; 82 83 msg.msg_iter = *it; 84 while (iov_iter_count(it)) { 85 ret = sock_recvmsg(sock, &msg, msg.msg_flags); 86 if (ret <= 0) { 87 if (ret == -EAGAIN) 88 ret = 0; 89 return ret; 90 } 91 92 iov_iter_advance(it, ret); 93 } 94 95 WARN_ON(msg_data_left(&msg)); 96 return 1; 97 } 98 99 /* 100 * Read as much as possible. 101 * 102 * Return: 103 * 1 - done, nothing (else) to read 104 * 0 - socket is empty, need to wait 105 * <0 - error 106 */ 107 static int ceph_tcp_recv(struct ceph_connection *con) 108 { 109 int ret; 110 111 dout("%s con %p %s %zu\n", __func__, con, 112 iov_iter_is_discard(&con->v2.in_iter) ? "discard" : "need", 113 iov_iter_count(&con->v2.in_iter)); 114 ret = do_recvmsg(con->sock, &con->v2.in_iter); 115 dout("%s con %p ret %d left %zu\n", __func__, con, ret, 116 iov_iter_count(&con->v2.in_iter)); 117 return ret; 118 } 119 120 static int do_sendmsg(struct socket *sock, struct iov_iter *it) 121 { 122 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS }; 123 int ret; 124 125 msg.msg_iter = *it; 126 while (iov_iter_count(it)) { 127 ret = sock_sendmsg(sock, &msg); 128 if (ret <= 0) { 129 if (ret == -EAGAIN) 130 ret = 0; 131 return ret; 132 } 133 134 iov_iter_advance(it, ret); 135 } 136 137 WARN_ON(msg_data_left(&msg)); 138 return 1; 139 } 140 141 static int do_try_sendpage(struct socket *sock, struct iov_iter *it) 142 { 143 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS }; 144 struct bio_vec bv; 145 int ret; 146 147 if (WARN_ON(!iov_iter_is_bvec(it))) 148 return -EINVAL; 149 150 while (iov_iter_count(it)) { 151 /* iov_iter_iovec() for ITER_BVEC */ 152 bv.bv_page = it->bvec->bv_page; 153 bv.bv_offset = it->bvec->bv_offset + it->iov_offset; 154 bv.bv_len = min(iov_iter_count(it), 155 it->bvec->bv_len - it->iov_offset); 156 157 /* 158 * sendpage cannot properly handle pages with 159 * page_count == 0, we need to fall back to sendmsg if 160 * that's the case. 161 * 162 * Same goes for slab pages: skb_can_coalesce() allows 163 * coalescing neighboring slab objects into a single frag 164 * which triggers one of hardened usercopy checks. 165 */ 166 if (sendpage_ok(bv.bv_page)) { 167 ret = sock->ops->sendpage(sock, bv.bv_page, 168 bv.bv_offset, bv.bv_len, 169 CEPH_MSG_FLAGS); 170 } else { 171 iov_iter_bvec(&msg.msg_iter, WRITE, &bv, 1, bv.bv_len); 172 ret = sock_sendmsg(sock, &msg); 173 } 174 if (ret <= 0) { 175 if (ret == -EAGAIN) 176 ret = 0; 177 return ret; 178 } 179 180 iov_iter_advance(it, ret); 181 } 182 183 return 1; 184 } 185 186 /* 187 * Write as much as possible. The socket is expected to be corked, 188 * so we don't bother with MSG_MORE/MSG_SENDPAGE_NOTLAST here. 189 * 190 * Return: 191 * 1 - done, nothing (else) to write 192 * 0 - socket is full, need to wait 193 * <0 - error 194 */ 195 static int ceph_tcp_send(struct ceph_connection *con) 196 { 197 int ret; 198 199 dout("%s con %p have %zu try_sendpage %d\n", __func__, con, 200 iov_iter_count(&con->v2.out_iter), con->v2.out_iter_sendpage); 201 if (con->v2.out_iter_sendpage) 202 ret = do_try_sendpage(con->sock, &con->v2.out_iter); 203 else 204 ret = do_sendmsg(con->sock, &con->v2.out_iter); 205 dout("%s con %p ret %d left %zu\n", __func__, con, ret, 206 iov_iter_count(&con->v2.out_iter)); 207 return ret; 208 } 209 210 static void add_in_kvec(struct ceph_connection *con, void *buf, int len) 211 { 212 BUG_ON(con->v2.in_kvec_cnt >= ARRAY_SIZE(con->v2.in_kvecs)); 213 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter)); 214 215 con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_base = buf; 216 con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_len = len; 217 con->v2.in_kvec_cnt++; 218 219 con->v2.in_iter.nr_segs++; 220 con->v2.in_iter.count += len; 221 } 222 223 static void reset_in_kvecs(struct ceph_connection *con) 224 { 225 WARN_ON(iov_iter_count(&con->v2.in_iter)); 226 227 con->v2.in_kvec_cnt = 0; 228 iov_iter_kvec(&con->v2.in_iter, READ, con->v2.in_kvecs, 0, 0); 229 } 230 231 static void set_in_bvec(struct ceph_connection *con, const struct bio_vec *bv) 232 { 233 WARN_ON(iov_iter_count(&con->v2.in_iter)); 234 235 con->v2.in_bvec = *bv; 236 iov_iter_bvec(&con->v2.in_iter, READ, &con->v2.in_bvec, 1, bv->bv_len); 237 } 238 239 static void set_in_skip(struct ceph_connection *con, int len) 240 { 241 WARN_ON(iov_iter_count(&con->v2.in_iter)); 242 243 dout("%s con %p len %d\n", __func__, con, len); 244 iov_iter_discard(&con->v2.in_iter, READ, len); 245 } 246 247 static void add_out_kvec(struct ceph_connection *con, void *buf, int len) 248 { 249 BUG_ON(con->v2.out_kvec_cnt >= ARRAY_SIZE(con->v2.out_kvecs)); 250 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter)); 251 WARN_ON(con->v2.out_zero); 252 253 con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_base = buf; 254 con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_len = len; 255 con->v2.out_kvec_cnt++; 256 257 con->v2.out_iter.nr_segs++; 258 con->v2.out_iter.count += len; 259 } 260 261 static void reset_out_kvecs(struct ceph_connection *con) 262 { 263 WARN_ON(iov_iter_count(&con->v2.out_iter)); 264 WARN_ON(con->v2.out_zero); 265 266 con->v2.out_kvec_cnt = 0; 267 268 iov_iter_kvec(&con->v2.out_iter, WRITE, con->v2.out_kvecs, 0, 0); 269 con->v2.out_iter_sendpage = false; 270 } 271 272 static void set_out_bvec(struct ceph_connection *con, const struct bio_vec *bv, 273 bool zerocopy) 274 { 275 WARN_ON(iov_iter_count(&con->v2.out_iter)); 276 WARN_ON(con->v2.out_zero); 277 278 con->v2.out_bvec = *bv; 279 con->v2.out_iter_sendpage = zerocopy; 280 iov_iter_bvec(&con->v2.out_iter, WRITE, &con->v2.out_bvec, 1, 281 con->v2.out_bvec.bv_len); 282 } 283 284 static void set_out_bvec_zero(struct ceph_connection *con) 285 { 286 WARN_ON(iov_iter_count(&con->v2.out_iter)); 287 WARN_ON(!con->v2.out_zero); 288 289 con->v2.out_bvec.bv_page = ceph_zero_page; 290 con->v2.out_bvec.bv_offset = 0; 291 con->v2.out_bvec.bv_len = min(con->v2.out_zero, (int)PAGE_SIZE); 292 con->v2.out_iter_sendpage = true; 293 iov_iter_bvec(&con->v2.out_iter, WRITE, &con->v2.out_bvec, 1, 294 con->v2.out_bvec.bv_len); 295 } 296 297 static void out_zero_add(struct ceph_connection *con, int len) 298 { 299 dout("%s con %p len %d\n", __func__, con, len); 300 con->v2.out_zero += len; 301 } 302 303 static void *alloc_conn_buf(struct ceph_connection *con, int len) 304 { 305 void *buf; 306 307 dout("%s con %p len %d\n", __func__, con, len); 308 309 if (WARN_ON(con->v2.conn_buf_cnt >= ARRAY_SIZE(con->v2.conn_bufs))) 310 return NULL; 311 312 buf = kvmalloc(len, GFP_NOIO); 313 if (!buf) 314 return NULL; 315 316 con->v2.conn_bufs[con->v2.conn_buf_cnt++] = buf; 317 return buf; 318 } 319 320 static void free_conn_bufs(struct ceph_connection *con) 321 { 322 while (con->v2.conn_buf_cnt) 323 kvfree(con->v2.conn_bufs[--con->v2.conn_buf_cnt]); 324 } 325 326 static void add_in_sign_kvec(struct ceph_connection *con, void *buf, int len) 327 { 328 BUG_ON(con->v2.in_sign_kvec_cnt >= ARRAY_SIZE(con->v2.in_sign_kvecs)); 329 330 con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_base = buf; 331 con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_len = len; 332 con->v2.in_sign_kvec_cnt++; 333 } 334 335 static void clear_in_sign_kvecs(struct ceph_connection *con) 336 { 337 con->v2.in_sign_kvec_cnt = 0; 338 } 339 340 static void add_out_sign_kvec(struct ceph_connection *con, void *buf, int len) 341 { 342 BUG_ON(con->v2.out_sign_kvec_cnt >= ARRAY_SIZE(con->v2.out_sign_kvecs)); 343 344 con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_base = buf; 345 con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_len = len; 346 con->v2.out_sign_kvec_cnt++; 347 } 348 349 static void clear_out_sign_kvecs(struct ceph_connection *con) 350 { 351 con->v2.out_sign_kvec_cnt = 0; 352 } 353 354 static bool con_secure(struct ceph_connection *con) 355 { 356 return con->v2.con_mode == CEPH_CON_MODE_SECURE; 357 } 358 359 static int front_len(const struct ceph_msg *msg) 360 { 361 return le32_to_cpu(msg->hdr.front_len); 362 } 363 364 static int middle_len(const struct ceph_msg *msg) 365 { 366 return le32_to_cpu(msg->hdr.middle_len); 367 } 368 369 static int data_len(const struct ceph_msg *msg) 370 { 371 return le32_to_cpu(msg->hdr.data_len); 372 } 373 374 static bool need_padding(int len) 375 { 376 return !IS_ALIGNED(len, CEPH_GCM_BLOCK_LEN); 377 } 378 379 static int padded_len(int len) 380 { 381 return ALIGN(len, CEPH_GCM_BLOCK_LEN); 382 } 383 384 static int padding_len(int len) 385 { 386 return padded_len(len) - len; 387 } 388 389 /* preamble + control segment */ 390 static int head_onwire_len(int ctrl_len, bool secure) 391 { 392 int head_len; 393 int rem_len; 394 395 if (secure) { 396 head_len = CEPH_PREAMBLE_SECURE_LEN; 397 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) { 398 rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN; 399 head_len += padded_len(rem_len) + CEPH_GCM_TAG_LEN; 400 } 401 } else { 402 head_len = CEPH_PREAMBLE_PLAIN_LEN; 403 if (ctrl_len) 404 head_len += ctrl_len + CEPH_CRC_LEN; 405 } 406 return head_len; 407 } 408 409 /* front, middle and data segments + epilogue */ 410 static int __tail_onwire_len(int front_len, int middle_len, int data_len, 411 bool secure) 412 { 413 if (!front_len && !middle_len && !data_len) 414 return 0; 415 416 if (!secure) 417 return front_len + middle_len + data_len + 418 CEPH_EPILOGUE_PLAIN_LEN; 419 420 return padded_len(front_len) + padded_len(middle_len) + 421 padded_len(data_len) + CEPH_EPILOGUE_SECURE_LEN; 422 } 423 424 static int tail_onwire_len(const struct ceph_msg *msg, bool secure) 425 { 426 return __tail_onwire_len(front_len(msg), middle_len(msg), 427 data_len(msg), secure); 428 } 429 430 /* head_onwire_len(sizeof(struct ceph_msg_header2), false) */ 431 #define MESSAGE_HEAD_PLAIN_LEN (CEPH_PREAMBLE_PLAIN_LEN + \ 432 sizeof(struct ceph_msg_header2) + \ 433 CEPH_CRC_LEN) 434 435 static const int frame_aligns[] = { 436 sizeof(void *), 437 sizeof(void *), 438 sizeof(void *), 439 PAGE_SIZE 440 }; 441 442 /* 443 * Discards trailing empty segments, unless there is just one segment. 444 * A frame always has at least one (possibly empty) segment. 445 */ 446 static int calc_segment_count(const int *lens, int len_cnt) 447 { 448 int i; 449 450 for (i = len_cnt - 1; i >= 0; i--) { 451 if (lens[i]) 452 return i + 1; 453 } 454 455 return 1; 456 } 457 458 static void init_frame_desc(struct ceph_frame_desc *desc, int tag, 459 const int *lens, int len_cnt) 460 { 461 int i; 462 463 memset(desc, 0, sizeof(*desc)); 464 465 desc->fd_tag = tag; 466 desc->fd_seg_cnt = calc_segment_count(lens, len_cnt); 467 BUG_ON(desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT); 468 for (i = 0; i < desc->fd_seg_cnt; i++) { 469 desc->fd_lens[i] = lens[i]; 470 desc->fd_aligns[i] = frame_aligns[i]; 471 } 472 } 473 474 /* 475 * Preamble crc covers everything up to itself (28 bytes) and 476 * is calculated and verified irrespective of the connection mode 477 * (i.e. even if the frame is encrypted). 478 */ 479 static void encode_preamble(const struct ceph_frame_desc *desc, void *p) 480 { 481 void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN; 482 void *start = p; 483 int i; 484 485 memset(p, 0, CEPH_PREAMBLE_LEN); 486 487 ceph_encode_8(&p, desc->fd_tag); 488 ceph_encode_8(&p, desc->fd_seg_cnt); 489 for (i = 0; i < desc->fd_seg_cnt; i++) { 490 ceph_encode_32(&p, desc->fd_lens[i]); 491 ceph_encode_16(&p, desc->fd_aligns[i]); 492 } 493 494 put_unaligned_le32(crc32c(0, start, crcp - start), crcp); 495 } 496 497 static int decode_preamble(void *p, struct ceph_frame_desc *desc) 498 { 499 void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN; 500 u32 crc, expected_crc; 501 int i; 502 503 crc = crc32c(0, p, crcp - p); 504 expected_crc = get_unaligned_le32(crcp); 505 if (crc != expected_crc) { 506 pr_err("bad preamble crc, calculated %u, expected %u\n", 507 crc, expected_crc); 508 return -EBADMSG; 509 } 510 511 memset(desc, 0, sizeof(*desc)); 512 513 desc->fd_tag = ceph_decode_8(&p); 514 desc->fd_seg_cnt = ceph_decode_8(&p); 515 if (desc->fd_seg_cnt < 1 || 516 desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT) { 517 pr_err("bad segment count %d\n", desc->fd_seg_cnt); 518 return -EINVAL; 519 } 520 for (i = 0; i < desc->fd_seg_cnt; i++) { 521 desc->fd_lens[i] = ceph_decode_32(&p); 522 desc->fd_aligns[i] = ceph_decode_16(&p); 523 } 524 525 /* 526 * This would fire for FRAME_TAG_WAIT (it has one empty 527 * segment), but we should never get it as client. 528 */ 529 if (!desc->fd_lens[desc->fd_seg_cnt - 1]) { 530 pr_err("last segment empty\n"); 531 return -EINVAL; 532 } 533 534 if (desc->fd_lens[0] > CEPH_MSG_MAX_CONTROL_LEN) { 535 pr_err("control segment too big %d\n", desc->fd_lens[0]); 536 return -EINVAL; 537 } 538 if (desc->fd_lens[1] > CEPH_MSG_MAX_FRONT_LEN) { 539 pr_err("front segment too big %d\n", desc->fd_lens[1]); 540 return -EINVAL; 541 } 542 if (desc->fd_lens[2] > CEPH_MSG_MAX_MIDDLE_LEN) { 543 pr_err("middle segment too big %d\n", desc->fd_lens[2]); 544 return -EINVAL; 545 } 546 if (desc->fd_lens[3] > CEPH_MSG_MAX_DATA_LEN) { 547 pr_err("data segment too big %d\n", desc->fd_lens[3]); 548 return -EINVAL; 549 } 550 551 return 0; 552 } 553 554 static void encode_epilogue_plain(struct ceph_connection *con, bool aborted) 555 { 556 con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED : 557 FRAME_LATE_STATUS_COMPLETE; 558 cpu_to_le32s(&con->v2.out_epil.front_crc); 559 cpu_to_le32s(&con->v2.out_epil.middle_crc); 560 cpu_to_le32s(&con->v2.out_epil.data_crc); 561 } 562 563 static void encode_epilogue_secure(struct ceph_connection *con, bool aborted) 564 { 565 memset(&con->v2.out_epil, 0, sizeof(con->v2.out_epil)); 566 con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED : 567 FRAME_LATE_STATUS_COMPLETE; 568 } 569 570 static int decode_epilogue(void *p, u32 *front_crc, u32 *middle_crc, 571 u32 *data_crc) 572 { 573 u8 late_status; 574 575 late_status = ceph_decode_8(&p); 576 if ((late_status & FRAME_LATE_STATUS_ABORTED_MASK) != 577 FRAME_LATE_STATUS_COMPLETE) { 578 /* we should never get an aborted message as client */ 579 pr_err("bad late_status 0x%x\n", late_status); 580 return -EINVAL; 581 } 582 583 if (front_crc && middle_crc && data_crc) { 584 *front_crc = ceph_decode_32(&p); 585 *middle_crc = ceph_decode_32(&p); 586 *data_crc = ceph_decode_32(&p); 587 } 588 589 return 0; 590 } 591 592 static void fill_header(struct ceph_msg_header *hdr, 593 const struct ceph_msg_header2 *hdr2, 594 int front_len, int middle_len, int data_len, 595 const struct ceph_entity_name *peer_name) 596 { 597 hdr->seq = hdr2->seq; 598 hdr->tid = hdr2->tid; 599 hdr->type = hdr2->type; 600 hdr->priority = hdr2->priority; 601 hdr->version = hdr2->version; 602 hdr->front_len = cpu_to_le32(front_len); 603 hdr->middle_len = cpu_to_le32(middle_len); 604 hdr->data_len = cpu_to_le32(data_len); 605 hdr->data_off = hdr2->data_off; 606 hdr->src = *peer_name; 607 hdr->compat_version = hdr2->compat_version; 608 hdr->reserved = 0; 609 hdr->crc = 0; 610 } 611 612 static void fill_header2(struct ceph_msg_header2 *hdr2, 613 const struct ceph_msg_header *hdr, u64 ack_seq) 614 { 615 hdr2->seq = hdr->seq; 616 hdr2->tid = hdr->tid; 617 hdr2->type = hdr->type; 618 hdr2->priority = hdr->priority; 619 hdr2->version = hdr->version; 620 hdr2->data_pre_padding_len = 0; 621 hdr2->data_off = hdr->data_off; 622 hdr2->ack_seq = cpu_to_le64(ack_seq); 623 hdr2->flags = 0; 624 hdr2->compat_version = hdr->compat_version; 625 hdr2->reserved = 0; 626 } 627 628 static int verify_control_crc(struct ceph_connection *con) 629 { 630 int ctrl_len = con->v2.in_desc.fd_lens[0]; 631 u32 crc, expected_crc; 632 633 WARN_ON(con->v2.in_kvecs[0].iov_len != ctrl_len); 634 WARN_ON(con->v2.in_kvecs[1].iov_len != CEPH_CRC_LEN); 635 636 crc = crc32c(-1, con->v2.in_kvecs[0].iov_base, ctrl_len); 637 expected_crc = get_unaligned_le32(con->v2.in_kvecs[1].iov_base); 638 if (crc != expected_crc) { 639 pr_err("bad control crc, calculated %u, expected %u\n", 640 crc, expected_crc); 641 return -EBADMSG; 642 } 643 644 return 0; 645 } 646 647 static int verify_epilogue_crcs(struct ceph_connection *con, u32 front_crc, 648 u32 middle_crc, u32 data_crc) 649 { 650 if (front_len(con->in_msg)) { 651 con->in_front_crc = crc32c(-1, con->in_msg->front.iov_base, 652 front_len(con->in_msg)); 653 } else { 654 WARN_ON(!middle_len(con->in_msg) && !data_len(con->in_msg)); 655 con->in_front_crc = -1; 656 } 657 658 if (middle_len(con->in_msg)) 659 con->in_middle_crc = crc32c(-1, 660 con->in_msg->middle->vec.iov_base, 661 middle_len(con->in_msg)); 662 else if (data_len(con->in_msg)) 663 con->in_middle_crc = -1; 664 else 665 con->in_middle_crc = 0; 666 667 if (!data_len(con->in_msg)) 668 con->in_data_crc = 0; 669 670 dout("%s con %p msg %p crcs %u %u %u\n", __func__, con, con->in_msg, 671 con->in_front_crc, con->in_middle_crc, con->in_data_crc); 672 673 if (con->in_front_crc != front_crc) { 674 pr_err("bad front crc, calculated %u, expected %u\n", 675 con->in_front_crc, front_crc); 676 return -EBADMSG; 677 } 678 if (con->in_middle_crc != middle_crc) { 679 pr_err("bad middle crc, calculated %u, expected %u\n", 680 con->in_middle_crc, middle_crc); 681 return -EBADMSG; 682 } 683 if (con->in_data_crc != data_crc) { 684 pr_err("bad data crc, calculated %u, expected %u\n", 685 con->in_data_crc, data_crc); 686 return -EBADMSG; 687 } 688 689 return 0; 690 } 691 692 static int setup_crypto(struct ceph_connection *con, 693 const u8 *session_key, int session_key_len, 694 const u8 *con_secret, int con_secret_len) 695 { 696 unsigned int noio_flag; 697 int ret; 698 699 dout("%s con %p con_mode %d session_key_len %d con_secret_len %d\n", 700 __func__, con, con->v2.con_mode, session_key_len, con_secret_len); 701 WARN_ON(con->v2.hmac_tfm || con->v2.gcm_tfm || con->v2.gcm_req); 702 703 if (con->v2.con_mode != CEPH_CON_MODE_CRC && 704 con->v2.con_mode != CEPH_CON_MODE_SECURE) { 705 pr_err("bad con_mode %d\n", con->v2.con_mode); 706 return -EINVAL; 707 } 708 709 if (!session_key_len) { 710 WARN_ON(con->v2.con_mode != CEPH_CON_MODE_CRC); 711 WARN_ON(con_secret_len); 712 return 0; /* auth_none */ 713 } 714 715 noio_flag = memalloc_noio_save(); 716 con->v2.hmac_tfm = crypto_alloc_shash("hmac(sha256)", 0, 0); 717 memalloc_noio_restore(noio_flag); 718 if (IS_ERR(con->v2.hmac_tfm)) { 719 ret = PTR_ERR(con->v2.hmac_tfm); 720 con->v2.hmac_tfm = NULL; 721 pr_err("failed to allocate hmac tfm context: %d\n", ret); 722 return ret; 723 } 724 725 WARN_ON((unsigned long)session_key & 726 crypto_shash_alignmask(con->v2.hmac_tfm)); 727 ret = crypto_shash_setkey(con->v2.hmac_tfm, session_key, 728 session_key_len); 729 if (ret) { 730 pr_err("failed to set hmac key: %d\n", ret); 731 return ret; 732 } 733 734 if (con->v2.con_mode == CEPH_CON_MODE_CRC) { 735 WARN_ON(con_secret_len); 736 return 0; /* auth_x, plain mode */ 737 } 738 739 if (con_secret_len < CEPH_GCM_KEY_LEN + 2 * CEPH_GCM_IV_LEN) { 740 pr_err("con_secret too small %d\n", con_secret_len); 741 return -EINVAL; 742 } 743 744 noio_flag = memalloc_noio_save(); 745 con->v2.gcm_tfm = crypto_alloc_aead("gcm(aes)", 0, 0); 746 memalloc_noio_restore(noio_flag); 747 if (IS_ERR(con->v2.gcm_tfm)) { 748 ret = PTR_ERR(con->v2.gcm_tfm); 749 con->v2.gcm_tfm = NULL; 750 pr_err("failed to allocate gcm tfm context: %d\n", ret); 751 return ret; 752 } 753 754 WARN_ON((unsigned long)con_secret & 755 crypto_aead_alignmask(con->v2.gcm_tfm)); 756 ret = crypto_aead_setkey(con->v2.gcm_tfm, con_secret, CEPH_GCM_KEY_LEN); 757 if (ret) { 758 pr_err("failed to set gcm key: %d\n", ret); 759 return ret; 760 } 761 762 WARN_ON(crypto_aead_ivsize(con->v2.gcm_tfm) != CEPH_GCM_IV_LEN); 763 ret = crypto_aead_setauthsize(con->v2.gcm_tfm, CEPH_GCM_TAG_LEN); 764 if (ret) { 765 pr_err("failed to set gcm tag size: %d\n", ret); 766 return ret; 767 } 768 769 con->v2.gcm_req = aead_request_alloc(con->v2.gcm_tfm, GFP_NOIO); 770 if (!con->v2.gcm_req) { 771 pr_err("failed to allocate gcm request\n"); 772 return -ENOMEM; 773 } 774 775 crypto_init_wait(&con->v2.gcm_wait); 776 aead_request_set_callback(con->v2.gcm_req, CRYPTO_TFM_REQ_MAY_BACKLOG, 777 crypto_req_done, &con->v2.gcm_wait); 778 779 memcpy(&con->v2.in_gcm_nonce, con_secret + CEPH_GCM_KEY_LEN, 780 CEPH_GCM_IV_LEN); 781 memcpy(&con->v2.out_gcm_nonce, 782 con_secret + CEPH_GCM_KEY_LEN + CEPH_GCM_IV_LEN, 783 CEPH_GCM_IV_LEN); 784 return 0; /* auth_x, secure mode */ 785 } 786 787 static int hmac_sha256(struct ceph_connection *con, const struct kvec *kvecs, 788 int kvec_cnt, u8 *hmac) 789 { 790 SHASH_DESC_ON_STACK(desc, con->v2.hmac_tfm); /* tfm arg is ignored */ 791 int ret; 792 int i; 793 794 dout("%s con %p hmac_tfm %p kvec_cnt %d\n", __func__, con, 795 con->v2.hmac_tfm, kvec_cnt); 796 797 if (!con->v2.hmac_tfm) { 798 memset(hmac, 0, SHA256_DIGEST_SIZE); 799 return 0; /* auth_none */ 800 } 801 802 desc->tfm = con->v2.hmac_tfm; 803 ret = crypto_shash_init(desc); 804 if (ret) 805 goto out; 806 807 for (i = 0; i < kvec_cnt; i++) { 808 WARN_ON((unsigned long)kvecs[i].iov_base & 809 crypto_shash_alignmask(con->v2.hmac_tfm)); 810 ret = crypto_shash_update(desc, kvecs[i].iov_base, 811 kvecs[i].iov_len); 812 if (ret) 813 goto out; 814 } 815 816 ret = crypto_shash_final(desc, hmac); 817 818 out: 819 shash_desc_zero(desc); 820 return ret; /* auth_x, both plain and secure modes */ 821 } 822 823 static void gcm_inc_nonce(struct ceph_gcm_nonce *nonce) 824 { 825 u64 counter; 826 827 counter = le64_to_cpu(nonce->counter); 828 nonce->counter = cpu_to_le64(counter + 1); 829 } 830 831 static int gcm_crypt(struct ceph_connection *con, bool encrypt, 832 struct scatterlist *src, struct scatterlist *dst, 833 int src_len) 834 { 835 struct ceph_gcm_nonce *nonce; 836 int ret; 837 838 nonce = encrypt ? &con->v2.out_gcm_nonce : &con->v2.in_gcm_nonce; 839 840 aead_request_set_ad(con->v2.gcm_req, 0); /* no AAD */ 841 aead_request_set_crypt(con->v2.gcm_req, src, dst, src_len, (u8 *)nonce); 842 ret = crypto_wait_req(encrypt ? crypto_aead_encrypt(con->v2.gcm_req) : 843 crypto_aead_decrypt(con->v2.gcm_req), 844 &con->v2.gcm_wait); 845 if (ret) 846 return ret; 847 848 gcm_inc_nonce(nonce); 849 return 0; 850 } 851 852 static void get_bvec_at(struct ceph_msg_data_cursor *cursor, 853 struct bio_vec *bv) 854 { 855 struct page *page; 856 size_t off, len; 857 858 WARN_ON(!cursor->total_resid); 859 860 /* skip zero-length data items */ 861 while (!cursor->resid) 862 ceph_msg_data_advance(cursor, 0); 863 864 /* get a piece of data, cursor isn't advanced */ 865 page = ceph_msg_data_next(cursor, &off, &len, NULL); 866 867 bv->bv_page = page; 868 bv->bv_offset = off; 869 bv->bv_len = len; 870 } 871 872 static int calc_sg_cnt(void *buf, int buf_len) 873 { 874 int sg_cnt; 875 876 if (!buf_len) 877 return 0; 878 879 sg_cnt = need_padding(buf_len) ? 1 : 0; 880 if (is_vmalloc_addr(buf)) { 881 WARN_ON(offset_in_page(buf)); 882 sg_cnt += PAGE_ALIGN(buf_len) >> PAGE_SHIFT; 883 } else { 884 sg_cnt++; 885 } 886 887 return sg_cnt; 888 } 889 890 static int calc_sg_cnt_cursor(struct ceph_msg_data_cursor *cursor) 891 { 892 int data_len = cursor->total_resid; 893 struct bio_vec bv; 894 int sg_cnt; 895 896 if (!data_len) 897 return 0; 898 899 sg_cnt = need_padding(data_len) ? 1 : 0; 900 do { 901 get_bvec_at(cursor, &bv); 902 sg_cnt++; 903 904 ceph_msg_data_advance(cursor, bv.bv_len); 905 } while (cursor->total_resid); 906 907 return sg_cnt; 908 } 909 910 static void init_sgs(struct scatterlist **sg, void *buf, int buf_len, u8 *pad) 911 { 912 void *end = buf + buf_len; 913 struct page *page; 914 int len; 915 void *p; 916 917 if (!buf_len) 918 return; 919 920 if (is_vmalloc_addr(buf)) { 921 p = buf; 922 do { 923 page = vmalloc_to_page(p); 924 len = min_t(int, end - p, PAGE_SIZE); 925 WARN_ON(!page || !len || offset_in_page(p)); 926 sg_set_page(*sg, page, len, 0); 927 *sg = sg_next(*sg); 928 p += len; 929 } while (p != end); 930 } else { 931 sg_set_buf(*sg, buf, buf_len); 932 *sg = sg_next(*sg); 933 } 934 935 if (need_padding(buf_len)) { 936 sg_set_buf(*sg, pad, padding_len(buf_len)); 937 *sg = sg_next(*sg); 938 } 939 } 940 941 static void init_sgs_cursor(struct scatterlist **sg, 942 struct ceph_msg_data_cursor *cursor, u8 *pad) 943 { 944 int data_len = cursor->total_resid; 945 struct bio_vec bv; 946 947 if (!data_len) 948 return; 949 950 do { 951 get_bvec_at(cursor, &bv); 952 sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset); 953 *sg = sg_next(*sg); 954 955 ceph_msg_data_advance(cursor, bv.bv_len); 956 } while (cursor->total_resid); 957 958 if (need_padding(data_len)) { 959 sg_set_buf(*sg, pad, padding_len(data_len)); 960 *sg = sg_next(*sg); 961 } 962 } 963 964 static int setup_message_sgs(struct sg_table *sgt, struct ceph_msg *msg, 965 u8 *front_pad, u8 *middle_pad, u8 *data_pad, 966 void *epilogue, bool add_tag) 967 { 968 struct ceph_msg_data_cursor cursor; 969 struct scatterlist *cur_sg; 970 int sg_cnt; 971 int ret; 972 973 if (!front_len(msg) && !middle_len(msg) && !data_len(msg)) 974 return 0; 975 976 sg_cnt = 1; /* epilogue + [auth tag] */ 977 if (front_len(msg)) 978 sg_cnt += calc_sg_cnt(msg->front.iov_base, 979 front_len(msg)); 980 if (middle_len(msg)) 981 sg_cnt += calc_sg_cnt(msg->middle->vec.iov_base, 982 middle_len(msg)); 983 if (data_len(msg)) { 984 ceph_msg_data_cursor_init(&cursor, msg, data_len(msg)); 985 sg_cnt += calc_sg_cnt_cursor(&cursor); 986 } 987 988 ret = sg_alloc_table(sgt, sg_cnt, GFP_NOIO); 989 if (ret) 990 return ret; 991 992 cur_sg = sgt->sgl; 993 if (front_len(msg)) 994 init_sgs(&cur_sg, msg->front.iov_base, front_len(msg), 995 front_pad); 996 if (middle_len(msg)) 997 init_sgs(&cur_sg, msg->middle->vec.iov_base, middle_len(msg), 998 middle_pad); 999 if (data_len(msg)) { 1000 ceph_msg_data_cursor_init(&cursor, msg, data_len(msg)); 1001 init_sgs_cursor(&cur_sg, &cursor, data_pad); 1002 } 1003 1004 WARN_ON(!sg_is_last(cur_sg)); 1005 sg_set_buf(cur_sg, epilogue, 1006 CEPH_GCM_BLOCK_LEN + (add_tag ? CEPH_GCM_TAG_LEN : 0)); 1007 return 0; 1008 } 1009 1010 static int decrypt_preamble(struct ceph_connection *con) 1011 { 1012 struct scatterlist sg; 1013 1014 sg_init_one(&sg, con->v2.in_buf, CEPH_PREAMBLE_SECURE_LEN); 1015 return gcm_crypt(con, false, &sg, &sg, CEPH_PREAMBLE_SECURE_LEN); 1016 } 1017 1018 static int decrypt_control_remainder(struct ceph_connection *con) 1019 { 1020 int ctrl_len = con->v2.in_desc.fd_lens[0]; 1021 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN; 1022 int pt_len = padding_len(rem_len) + CEPH_GCM_TAG_LEN; 1023 struct scatterlist sgs[2]; 1024 1025 WARN_ON(con->v2.in_kvecs[0].iov_len != rem_len); 1026 WARN_ON(con->v2.in_kvecs[1].iov_len != pt_len); 1027 1028 sg_init_table(sgs, 2); 1029 sg_set_buf(&sgs[0], con->v2.in_kvecs[0].iov_base, rem_len); 1030 sg_set_buf(&sgs[1], con->v2.in_buf, pt_len); 1031 1032 return gcm_crypt(con, false, sgs, sgs, 1033 padded_len(rem_len) + CEPH_GCM_TAG_LEN); 1034 } 1035 1036 static int decrypt_tail(struct ceph_connection *con) 1037 { 1038 struct sg_table enc_sgt = {}; 1039 struct sg_table sgt = {}; 1040 int tail_len; 1041 int ret; 1042 1043 tail_len = tail_onwire_len(con->in_msg, true); 1044 ret = sg_alloc_table_from_pages(&enc_sgt, con->v2.in_enc_pages, 1045 con->v2.in_enc_page_cnt, 0, tail_len, 1046 GFP_NOIO); 1047 if (ret) 1048 goto out; 1049 1050 ret = setup_message_sgs(&sgt, con->in_msg, FRONT_PAD(con->v2.in_buf), 1051 MIDDLE_PAD(con->v2.in_buf), DATA_PAD(con->v2.in_buf), 1052 con->v2.in_buf, true); 1053 if (ret) 1054 goto out; 1055 1056 dout("%s con %p msg %p enc_page_cnt %d sg_cnt %d\n", __func__, con, 1057 con->in_msg, con->v2.in_enc_page_cnt, sgt.orig_nents); 1058 ret = gcm_crypt(con, false, enc_sgt.sgl, sgt.sgl, tail_len); 1059 if (ret) 1060 goto out; 1061 1062 WARN_ON(!con->v2.in_enc_page_cnt); 1063 ceph_release_page_vector(con->v2.in_enc_pages, 1064 con->v2.in_enc_page_cnt); 1065 con->v2.in_enc_pages = NULL; 1066 con->v2.in_enc_page_cnt = 0; 1067 1068 out: 1069 sg_free_table(&sgt); 1070 sg_free_table(&enc_sgt); 1071 return ret; 1072 } 1073 1074 static int prepare_banner(struct ceph_connection *con) 1075 { 1076 int buf_len = CEPH_BANNER_V2_LEN + 2 + 8 + 8; 1077 void *buf, *p; 1078 1079 buf = alloc_conn_buf(con, buf_len); 1080 if (!buf) 1081 return -ENOMEM; 1082 1083 p = buf; 1084 ceph_encode_copy(&p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN); 1085 ceph_encode_16(&p, sizeof(u64) + sizeof(u64)); 1086 ceph_encode_64(&p, CEPH_MSGR2_SUPPORTED_FEATURES); 1087 ceph_encode_64(&p, CEPH_MSGR2_REQUIRED_FEATURES); 1088 WARN_ON(p != buf + buf_len); 1089 1090 add_out_kvec(con, buf, buf_len); 1091 add_out_sign_kvec(con, buf, buf_len); 1092 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING); 1093 return 0; 1094 } 1095 1096 /* 1097 * base: 1098 * preamble 1099 * control body (ctrl_len bytes) 1100 * space for control crc 1101 * 1102 * extdata (optional): 1103 * control body (extdata_len bytes) 1104 * 1105 * Compute control crc and gather base and extdata into: 1106 * 1107 * preamble 1108 * control body (ctrl_len + extdata_len bytes) 1109 * control crc 1110 * 1111 * Preamble should already be encoded at the start of base. 1112 */ 1113 static void prepare_head_plain(struct ceph_connection *con, void *base, 1114 int ctrl_len, void *extdata, int extdata_len, 1115 bool to_be_signed) 1116 { 1117 int base_len = CEPH_PREAMBLE_LEN + ctrl_len + CEPH_CRC_LEN; 1118 void *crcp = base + base_len - CEPH_CRC_LEN; 1119 u32 crc; 1120 1121 crc = crc32c(-1, CTRL_BODY(base), ctrl_len); 1122 if (extdata_len) 1123 crc = crc32c(crc, extdata, extdata_len); 1124 put_unaligned_le32(crc, crcp); 1125 1126 if (!extdata_len) { 1127 add_out_kvec(con, base, base_len); 1128 if (to_be_signed) 1129 add_out_sign_kvec(con, base, base_len); 1130 return; 1131 } 1132 1133 add_out_kvec(con, base, crcp - base); 1134 add_out_kvec(con, extdata, extdata_len); 1135 add_out_kvec(con, crcp, CEPH_CRC_LEN); 1136 if (to_be_signed) { 1137 add_out_sign_kvec(con, base, crcp - base); 1138 add_out_sign_kvec(con, extdata, extdata_len); 1139 add_out_sign_kvec(con, crcp, CEPH_CRC_LEN); 1140 } 1141 } 1142 1143 static int prepare_head_secure_small(struct ceph_connection *con, 1144 void *base, int ctrl_len) 1145 { 1146 struct scatterlist sg; 1147 int ret; 1148 1149 /* inline buffer padding? */ 1150 if (ctrl_len < CEPH_PREAMBLE_INLINE_LEN) 1151 memset(CTRL_BODY(base) + ctrl_len, 0, 1152 CEPH_PREAMBLE_INLINE_LEN - ctrl_len); 1153 1154 sg_init_one(&sg, base, CEPH_PREAMBLE_SECURE_LEN); 1155 ret = gcm_crypt(con, true, &sg, &sg, 1156 CEPH_PREAMBLE_SECURE_LEN - CEPH_GCM_TAG_LEN); 1157 if (ret) 1158 return ret; 1159 1160 add_out_kvec(con, base, CEPH_PREAMBLE_SECURE_LEN); 1161 return 0; 1162 } 1163 1164 /* 1165 * base: 1166 * preamble 1167 * control body (ctrl_len bytes) 1168 * space for padding, if needed 1169 * space for control remainder auth tag 1170 * space for preamble auth tag 1171 * 1172 * Encrypt preamble and the inline portion, then encrypt the remainder 1173 * and gather into: 1174 * 1175 * preamble 1176 * control body (48 bytes) 1177 * preamble auth tag 1178 * control body (ctrl_len - 48 bytes) 1179 * zero padding, if needed 1180 * control remainder auth tag 1181 * 1182 * Preamble should already be encoded at the start of base. 1183 */ 1184 static int prepare_head_secure_big(struct ceph_connection *con, 1185 void *base, int ctrl_len) 1186 { 1187 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN; 1188 void *rem = CTRL_BODY(base) + CEPH_PREAMBLE_INLINE_LEN; 1189 void *rem_tag = rem + padded_len(rem_len); 1190 void *pmbl_tag = rem_tag + CEPH_GCM_TAG_LEN; 1191 struct scatterlist sgs[2]; 1192 int ret; 1193 1194 sg_init_table(sgs, 2); 1195 sg_set_buf(&sgs[0], base, rem - base); 1196 sg_set_buf(&sgs[1], pmbl_tag, CEPH_GCM_TAG_LEN); 1197 ret = gcm_crypt(con, true, sgs, sgs, rem - base); 1198 if (ret) 1199 return ret; 1200 1201 /* control remainder padding? */ 1202 if (need_padding(rem_len)) 1203 memset(rem + rem_len, 0, padding_len(rem_len)); 1204 1205 sg_init_one(&sgs[0], rem, pmbl_tag - rem); 1206 ret = gcm_crypt(con, true, sgs, sgs, rem_tag - rem); 1207 if (ret) 1208 return ret; 1209 1210 add_out_kvec(con, base, rem - base); 1211 add_out_kvec(con, pmbl_tag, CEPH_GCM_TAG_LEN); 1212 add_out_kvec(con, rem, pmbl_tag - rem); 1213 return 0; 1214 } 1215 1216 static int __prepare_control(struct ceph_connection *con, int tag, 1217 void *base, int ctrl_len, void *extdata, 1218 int extdata_len, bool to_be_signed) 1219 { 1220 int total_len = ctrl_len + extdata_len; 1221 struct ceph_frame_desc desc; 1222 int ret; 1223 1224 dout("%s con %p tag %d len %d (%d+%d)\n", __func__, con, tag, 1225 total_len, ctrl_len, extdata_len); 1226 1227 /* extdata may be vmalloc'ed but not base */ 1228 if (WARN_ON(is_vmalloc_addr(base) || !ctrl_len)) 1229 return -EINVAL; 1230 1231 init_frame_desc(&desc, tag, &total_len, 1); 1232 encode_preamble(&desc, base); 1233 1234 if (con_secure(con)) { 1235 if (WARN_ON(extdata_len || to_be_signed)) 1236 return -EINVAL; 1237 1238 if (ctrl_len <= CEPH_PREAMBLE_INLINE_LEN) 1239 /* fully inlined, inline buffer may need padding */ 1240 ret = prepare_head_secure_small(con, base, ctrl_len); 1241 else 1242 /* partially inlined, inline buffer is full */ 1243 ret = prepare_head_secure_big(con, base, ctrl_len); 1244 if (ret) 1245 return ret; 1246 } else { 1247 prepare_head_plain(con, base, ctrl_len, extdata, extdata_len, 1248 to_be_signed); 1249 } 1250 1251 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING); 1252 return 0; 1253 } 1254 1255 static int prepare_control(struct ceph_connection *con, int tag, 1256 void *base, int ctrl_len) 1257 { 1258 return __prepare_control(con, tag, base, ctrl_len, NULL, 0, false); 1259 } 1260 1261 static int prepare_hello(struct ceph_connection *con) 1262 { 1263 void *buf, *p; 1264 int ctrl_len; 1265 1266 ctrl_len = 1 + ceph_entity_addr_encoding_len(&con->peer_addr); 1267 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false)); 1268 if (!buf) 1269 return -ENOMEM; 1270 1271 p = CTRL_BODY(buf); 1272 ceph_encode_8(&p, CEPH_ENTITY_TYPE_CLIENT); 1273 ceph_encode_entity_addr(&p, &con->peer_addr); 1274 WARN_ON(p != CTRL_BODY(buf) + ctrl_len); 1275 1276 return __prepare_control(con, FRAME_TAG_HELLO, buf, ctrl_len, 1277 NULL, 0, true); 1278 } 1279 1280 /* so that head_onwire_len(AUTH_BUF_LEN, false) is 512 */ 1281 #define AUTH_BUF_LEN (512 - CEPH_CRC_LEN - CEPH_PREAMBLE_PLAIN_LEN) 1282 1283 static int prepare_auth_request(struct ceph_connection *con) 1284 { 1285 void *authorizer, *authorizer_copy; 1286 int ctrl_len, authorizer_len; 1287 void *buf; 1288 int ret; 1289 1290 ctrl_len = AUTH_BUF_LEN; 1291 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false)); 1292 if (!buf) 1293 return -ENOMEM; 1294 1295 mutex_unlock(&con->mutex); 1296 ret = con->ops->get_auth_request(con, CTRL_BODY(buf), &ctrl_len, 1297 &authorizer, &authorizer_len); 1298 mutex_lock(&con->mutex); 1299 if (con->state != CEPH_CON_S_V2_HELLO) { 1300 dout("%s con %p state changed to %d\n", __func__, con, 1301 con->state); 1302 return -EAGAIN; 1303 } 1304 1305 dout("%s con %p get_auth_request ret %d\n", __func__, con, ret); 1306 if (ret) 1307 return ret; 1308 1309 authorizer_copy = alloc_conn_buf(con, authorizer_len); 1310 if (!authorizer_copy) 1311 return -ENOMEM; 1312 1313 memcpy(authorizer_copy, authorizer, authorizer_len); 1314 1315 return __prepare_control(con, FRAME_TAG_AUTH_REQUEST, buf, ctrl_len, 1316 authorizer_copy, authorizer_len, true); 1317 } 1318 1319 static int prepare_auth_request_more(struct ceph_connection *con, 1320 void *reply, int reply_len) 1321 { 1322 int ctrl_len, authorizer_len; 1323 void *authorizer; 1324 void *buf; 1325 int ret; 1326 1327 ctrl_len = AUTH_BUF_LEN; 1328 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false)); 1329 if (!buf) 1330 return -ENOMEM; 1331 1332 mutex_unlock(&con->mutex); 1333 ret = con->ops->handle_auth_reply_more(con, reply, reply_len, 1334 CTRL_BODY(buf), &ctrl_len, 1335 &authorizer, &authorizer_len); 1336 mutex_lock(&con->mutex); 1337 if (con->state != CEPH_CON_S_V2_AUTH) { 1338 dout("%s con %p state changed to %d\n", __func__, con, 1339 con->state); 1340 return -EAGAIN; 1341 } 1342 1343 dout("%s con %p handle_auth_reply_more ret %d\n", __func__, con, ret); 1344 if (ret) 1345 return ret; 1346 1347 return __prepare_control(con, FRAME_TAG_AUTH_REQUEST_MORE, buf, 1348 ctrl_len, authorizer, authorizer_len, true); 1349 } 1350 1351 static int prepare_auth_signature(struct ceph_connection *con) 1352 { 1353 void *buf; 1354 int ret; 1355 1356 buf = alloc_conn_buf(con, head_onwire_len(SHA256_DIGEST_SIZE, 1357 con_secure(con))); 1358 if (!buf) 1359 return -ENOMEM; 1360 1361 ret = hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt, 1362 CTRL_BODY(buf)); 1363 if (ret) 1364 return ret; 1365 1366 return prepare_control(con, FRAME_TAG_AUTH_SIGNATURE, buf, 1367 SHA256_DIGEST_SIZE); 1368 } 1369 1370 static int prepare_client_ident(struct ceph_connection *con) 1371 { 1372 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr; 1373 struct ceph_client *client = from_msgr(con->msgr); 1374 u64 global_id = ceph_client_gid(client); 1375 void *buf, *p; 1376 int ctrl_len; 1377 1378 WARN_ON(con->v2.server_cookie); 1379 WARN_ON(con->v2.connect_seq); 1380 WARN_ON(con->v2.peer_global_seq); 1381 1382 if (!con->v2.client_cookie) { 1383 do { 1384 get_random_bytes(&con->v2.client_cookie, 1385 sizeof(con->v2.client_cookie)); 1386 } while (!con->v2.client_cookie); 1387 dout("%s con %p generated cookie 0x%llx\n", __func__, con, 1388 con->v2.client_cookie); 1389 } else { 1390 dout("%s con %p cookie already set 0x%llx\n", __func__, con, 1391 con->v2.client_cookie); 1392 } 1393 1394 dout("%s con %p my_addr %s/%u peer_addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx cookie 0x%llx\n", 1395 __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce), 1396 ceph_pr_addr(&con->peer_addr), le32_to_cpu(con->peer_addr.nonce), 1397 global_id, con->v2.global_seq, client->supported_features, 1398 client->required_features, con->v2.client_cookie); 1399 1400 ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) + 1401 ceph_entity_addr_encoding_len(&con->peer_addr) + 6 * 8; 1402 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con))); 1403 if (!buf) 1404 return -ENOMEM; 1405 1406 p = CTRL_BODY(buf); 1407 ceph_encode_8(&p, 2); /* addrvec marker */ 1408 ceph_encode_32(&p, 1); /* addr_cnt */ 1409 ceph_encode_entity_addr(&p, my_addr); 1410 ceph_encode_entity_addr(&p, &con->peer_addr); 1411 ceph_encode_64(&p, global_id); 1412 ceph_encode_64(&p, con->v2.global_seq); 1413 ceph_encode_64(&p, client->supported_features); 1414 ceph_encode_64(&p, client->required_features); 1415 ceph_encode_64(&p, 0); /* flags */ 1416 ceph_encode_64(&p, con->v2.client_cookie); 1417 WARN_ON(p != CTRL_BODY(buf) + ctrl_len); 1418 1419 return prepare_control(con, FRAME_TAG_CLIENT_IDENT, buf, ctrl_len); 1420 } 1421 1422 static int prepare_session_reconnect(struct ceph_connection *con) 1423 { 1424 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr; 1425 void *buf, *p; 1426 int ctrl_len; 1427 1428 WARN_ON(!con->v2.client_cookie); 1429 WARN_ON(!con->v2.server_cookie); 1430 WARN_ON(!con->v2.connect_seq); 1431 WARN_ON(!con->v2.peer_global_seq); 1432 1433 dout("%s con %p my_addr %s/%u client_cookie 0x%llx server_cookie 0x%llx global_seq %llu connect_seq %llu in_seq %llu\n", 1434 __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce), 1435 con->v2.client_cookie, con->v2.server_cookie, con->v2.global_seq, 1436 con->v2.connect_seq, con->in_seq); 1437 1438 ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) + 5 * 8; 1439 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con))); 1440 if (!buf) 1441 return -ENOMEM; 1442 1443 p = CTRL_BODY(buf); 1444 ceph_encode_8(&p, 2); /* entity_addrvec_t marker */ 1445 ceph_encode_32(&p, 1); /* my_addrs len */ 1446 ceph_encode_entity_addr(&p, my_addr); 1447 ceph_encode_64(&p, con->v2.client_cookie); 1448 ceph_encode_64(&p, con->v2.server_cookie); 1449 ceph_encode_64(&p, con->v2.global_seq); 1450 ceph_encode_64(&p, con->v2.connect_seq); 1451 ceph_encode_64(&p, con->in_seq); 1452 WARN_ON(p != CTRL_BODY(buf) + ctrl_len); 1453 1454 return prepare_control(con, FRAME_TAG_SESSION_RECONNECT, buf, ctrl_len); 1455 } 1456 1457 static int prepare_keepalive2(struct ceph_connection *con) 1458 { 1459 struct ceph_timespec *ts = CTRL_BODY(con->v2.out_buf); 1460 struct timespec64 now; 1461 1462 ktime_get_real_ts64(&now); 1463 dout("%s con %p timestamp %lld.%09ld\n", __func__, con, now.tv_sec, 1464 now.tv_nsec); 1465 1466 ceph_encode_timespec64(ts, &now); 1467 1468 reset_out_kvecs(con); 1469 return prepare_control(con, FRAME_TAG_KEEPALIVE2, con->v2.out_buf, 1470 sizeof(struct ceph_timespec)); 1471 } 1472 1473 static int prepare_ack(struct ceph_connection *con) 1474 { 1475 void *p; 1476 1477 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con, 1478 con->in_seq_acked, con->in_seq); 1479 con->in_seq_acked = con->in_seq; 1480 1481 p = CTRL_BODY(con->v2.out_buf); 1482 ceph_encode_64(&p, con->in_seq_acked); 1483 1484 reset_out_kvecs(con); 1485 return prepare_control(con, FRAME_TAG_ACK, con->v2.out_buf, 8); 1486 } 1487 1488 static void prepare_epilogue_plain(struct ceph_connection *con, bool aborted) 1489 { 1490 dout("%s con %p msg %p aborted %d crcs %u %u %u\n", __func__, con, 1491 con->out_msg, aborted, con->v2.out_epil.front_crc, 1492 con->v2.out_epil.middle_crc, con->v2.out_epil.data_crc); 1493 1494 encode_epilogue_plain(con, aborted); 1495 add_out_kvec(con, &con->v2.out_epil, CEPH_EPILOGUE_PLAIN_LEN); 1496 } 1497 1498 /* 1499 * For "used" empty segments, crc is -1. For unused (trailing) 1500 * segments, crc is 0. 1501 */ 1502 static void prepare_message_plain(struct ceph_connection *con) 1503 { 1504 struct ceph_msg *msg = con->out_msg; 1505 1506 prepare_head_plain(con, con->v2.out_buf, 1507 sizeof(struct ceph_msg_header2), NULL, 0, false); 1508 1509 if (!front_len(msg) && !middle_len(msg)) { 1510 if (!data_len(msg)) { 1511 /* 1512 * Empty message: once the head is written, 1513 * we are done -- there is no epilogue. 1514 */ 1515 con->v2.out_state = OUT_S_FINISH_MESSAGE; 1516 return; 1517 } 1518 1519 con->v2.out_epil.front_crc = -1; 1520 con->v2.out_epil.middle_crc = -1; 1521 con->v2.out_state = OUT_S_QUEUE_DATA; 1522 return; 1523 } 1524 1525 if (front_len(msg)) { 1526 con->v2.out_epil.front_crc = crc32c(-1, msg->front.iov_base, 1527 front_len(msg)); 1528 add_out_kvec(con, msg->front.iov_base, front_len(msg)); 1529 } else { 1530 /* middle (at least) is there, checked above */ 1531 con->v2.out_epil.front_crc = -1; 1532 } 1533 1534 if (middle_len(msg)) { 1535 con->v2.out_epil.middle_crc = 1536 crc32c(-1, msg->middle->vec.iov_base, middle_len(msg)); 1537 add_out_kvec(con, msg->middle->vec.iov_base, middle_len(msg)); 1538 } else { 1539 con->v2.out_epil.middle_crc = data_len(msg) ? -1 : 0; 1540 } 1541 1542 if (data_len(msg)) { 1543 con->v2.out_state = OUT_S_QUEUE_DATA; 1544 } else { 1545 con->v2.out_epil.data_crc = 0; 1546 prepare_epilogue_plain(con, false); 1547 con->v2.out_state = OUT_S_FINISH_MESSAGE; 1548 } 1549 } 1550 1551 /* 1552 * Unfortunately the kernel crypto API doesn't support streaming 1553 * (piecewise) operation for AEAD algorithms, so we can't get away 1554 * with a fixed size buffer and a couple sgs. Instead, we have to 1555 * allocate pages for the entire tail of the message (currently up 1556 * to ~32M) and two sgs arrays (up to ~256K each)... 1557 */ 1558 static int prepare_message_secure(struct ceph_connection *con) 1559 { 1560 void *zerop = page_address(ceph_zero_page); 1561 struct sg_table enc_sgt = {}; 1562 struct sg_table sgt = {}; 1563 struct page **enc_pages; 1564 int enc_page_cnt; 1565 int tail_len; 1566 int ret; 1567 1568 ret = prepare_head_secure_small(con, con->v2.out_buf, 1569 sizeof(struct ceph_msg_header2)); 1570 if (ret) 1571 return ret; 1572 1573 tail_len = tail_onwire_len(con->out_msg, true); 1574 if (!tail_len) { 1575 /* 1576 * Empty message: once the head is written, 1577 * we are done -- there is no epilogue. 1578 */ 1579 con->v2.out_state = OUT_S_FINISH_MESSAGE; 1580 return 0; 1581 } 1582 1583 encode_epilogue_secure(con, false); 1584 ret = setup_message_sgs(&sgt, con->out_msg, zerop, zerop, zerop, 1585 &con->v2.out_epil, false); 1586 if (ret) 1587 goto out; 1588 1589 enc_page_cnt = calc_pages_for(0, tail_len); 1590 enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO); 1591 if (IS_ERR(enc_pages)) { 1592 ret = PTR_ERR(enc_pages); 1593 goto out; 1594 } 1595 1596 WARN_ON(con->v2.out_enc_pages || con->v2.out_enc_page_cnt); 1597 con->v2.out_enc_pages = enc_pages; 1598 con->v2.out_enc_page_cnt = enc_page_cnt; 1599 con->v2.out_enc_resid = tail_len; 1600 con->v2.out_enc_i = 0; 1601 1602 ret = sg_alloc_table_from_pages(&enc_sgt, enc_pages, enc_page_cnt, 1603 0, tail_len, GFP_NOIO); 1604 if (ret) 1605 goto out; 1606 1607 ret = gcm_crypt(con, true, sgt.sgl, enc_sgt.sgl, 1608 tail_len - CEPH_GCM_TAG_LEN); 1609 if (ret) 1610 goto out; 1611 1612 dout("%s con %p msg %p sg_cnt %d enc_page_cnt %d\n", __func__, con, 1613 con->out_msg, sgt.orig_nents, enc_page_cnt); 1614 con->v2.out_state = OUT_S_QUEUE_ENC_PAGE; 1615 1616 out: 1617 sg_free_table(&sgt); 1618 sg_free_table(&enc_sgt); 1619 return ret; 1620 } 1621 1622 static int prepare_message(struct ceph_connection *con) 1623 { 1624 int lens[] = { 1625 sizeof(struct ceph_msg_header2), 1626 front_len(con->out_msg), 1627 middle_len(con->out_msg), 1628 data_len(con->out_msg) 1629 }; 1630 struct ceph_frame_desc desc; 1631 int ret; 1632 1633 dout("%s con %p msg %p logical %d+%d+%d+%d\n", __func__, con, 1634 con->out_msg, lens[0], lens[1], lens[2], lens[3]); 1635 1636 if (con->in_seq > con->in_seq_acked) { 1637 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con, 1638 con->in_seq_acked, con->in_seq); 1639 con->in_seq_acked = con->in_seq; 1640 } 1641 1642 reset_out_kvecs(con); 1643 init_frame_desc(&desc, FRAME_TAG_MESSAGE, lens, 4); 1644 encode_preamble(&desc, con->v2.out_buf); 1645 fill_header2(CTRL_BODY(con->v2.out_buf), &con->out_msg->hdr, 1646 con->in_seq_acked); 1647 1648 if (con_secure(con)) { 1649 ret = prepare_message_secure(con); 1650 if (ret) 1651 return ret; 1652 } else { 1653 prepare_message_plain(con); 1654 } 1655 1656 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING); 1657 return 0; 1658 } 1659 1660 static int prepare_read_banner_prefix(struct ceph_connection *con) 1661 { 1662 void *buf; 1663 1664 buf = alloc_conn_buf(con, CEPH_BANNER_V2_PREFIX_LEN); 1665 if (!buf) 1666 return -ENOMEM; 1667 1668 reset_in_kvecs(con); 1669 add_in_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN); 1670 add_in_sign_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN); 1671 con->state = CEPH_CON_S_V2_BANNER_PREFIX; 1672 return 0; 1673 } 1674 1675 static int prepare_read_banner_payload(struct ceph_connection *con, 1676 int payload_len) 1677 { 1678 void *buf; 1679 1680 buf = alloc_conn_buf(con, payload_len); 1681 if (!buf) 1682 return -ENOMEM; 1683 1684 reset_in_kvecs(con); 1685 add_in_kvec(con, buf, payload_len); 1686 add_in_sign_kvec(con, buf, payload_len); 1687 con->state = CEPH_CON_S_V2_BANNER_PAYLOAD; 1688 return 0; 1689 } 1690 1691 static void prepare_read_preamble(struct ceph_connection *con) 1692 { 1693 reset_in_kvecs(con); 1694 add_in_kvec(con, con->v2.in_buf, 1695 con_secure(con) ? CEPH_PREAMBLE_SECURE_LEN : 1696 CEPH_PREAMBLE_PLAIN_LEN); 1697 con->v2.in_state = IN_S_HANDLE_PREAMBLE; 1698 } 1699 1700 static int prepare_read_control(struct ceph_connection *con) 1701 { 1702 int ctrl_len = con->v2.in_desc.fd_lens[0]; 1703 int head_len; 1704 void *buf; 1705 1706 reset_in_kvecs(con); 1707 if (con->state == CEPH_CON_S_V2_HELLO || 1708 con->state == CEPH_CON_S_V2_AUTH) { 1709 head_len = head_onwire_len(ctrl_len, false); 1710 buf = alloc_conn_buf(con, head_len); 1711 if (!buf) 1712 return -ENOMEM; 1713 1714 /* preserve preamble */ 1715 memcpy(buf, con->v2.in_buf, CEPH_PREAMBLE_LEN); 1716 1717 add_in_kvec(con, CTRL_BODY(buf), ctrl_len); 1718 add_in_kvec(con, CTRL_BODY(buf) + ctrl_len, CEPH_CRC_LEN); 1719 add_in_sign_kvec(con, buf, head_len); 1720 } else { 1721 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) { 1722 buf = alloc_conn_buf(con, ctrl_len); 1723 if (!buf) 1724 return -ENOMEM; 1725 1726 add_in_kvec(con, buf, ctrl_len); 1727 } else { 1728 add_in_kvec(con, CTRL_BODY(con->v2.in_buf), ctrl_len); 1729 } 1730 add_in_kvec(con, con->v2.in_buf, CEPH_CRC_LEN); 1731 } 1732 con->v2.in_state = IN_S_HANDLE_CONTROL; 1733 return 0; 1734 } 1735 1736 static int prepare_read_control_remainder(struct ceph_connection *con) 1737 { 1738 int ctrl_len = con->v2.in_desc.fd_lens[0]; 1739 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN; 1740 void *buf; 1741 1742 buf = alloc_conn_buf(con, ctrl_len); 1743 if (!buf) 1744 return -ENOMEM; 1745 1746 memcpy(buf, CTRL_BODY(con->v2.in_buf), CEPH_PREAMBLE_INLINE_LEN); 1747 1748 reset_in_kvecs(con); 1749 add_in_kvec(con, buf + CEPH_PREAMBLE_INLINE_LEN, rem_len); 1750 add_in_kvec(con, con->v2.in_buf, 1751 padding_len(rem_len) + CEPH_GCM_TAG_LEN); 1752 con->v2.in_state = IN_S_HANDLE_CONTROL_REMAINDER; 1753 return 0; 1754 } 1755 1756 static void prepare_read_data(struct ceph_connection *con) 1757 { 1758 struct bio_vec bv; 1759 1760 con->in_data_crc = -1; 1761 ceph_msg_data_cursor_init(&con->v2.in_cursor, con->in_msg, 1762 data_len(con->in_msg)); 1763 1764 get_bvec_at(&con->v2.in_cursor, &bv); 1765 set_in_bvec(con, &bv); 1766 con->v2.in_state = IN_S_PREPARE_READ_DATA_CONT; 1767 } 1768 1769 static void prepare_read_data_cont(struct ceph_connection *con) 1770 { 1771 struct bio_vec bv; 1772 1773 con->in_data_crc = ceph_crc32c_page(con->in_data_crc, 1774 con->v2.in_bvec.bv_page, 1775 con->v2.in_bvec.bv_offset, 1776 con->v2.in_bvec.bv_len); 1777 1778 ceph_msg_data_advance(&con->v2.in_cursor, con->v2.in_bvec.bv_len); 1779 if (con->v2.in_cursor.total_resid) { 1780 get_bvec_at(&con->v2.in_cursor, &bv); 1781 set_in_bvec(con, &bv); 1782 WARN_ON(con->v2.in_state != IN_S_PREPARE_READ_DATA_CONT); 1783 return; 1784 } 1785 1786 /* 1787 * We've read all data. Prepare to read epilogue. 1788 */ 1789 reset_in_kvecs(con); 1790 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN); 1791 con->v2.in_state = IN_S_HANDLE_EPILOGUE; 1792 } 1793 1794 static void prepare_read_tail_plain(struct ceph_connection *con) 1795 { 1796 struct ceph_msg *msg = con->in_msg; 1797 1798 if (!front_len(msg) && !middle_len(msg)) { 1799 WARN_ON(!data_len(msg)); 1800 prepare_read_data(con); 1801 return; 1802 } 1803 1804 reset_in_kvecs(con); 1805 if (front_len(msg)) { 1806 add_in_kvec(con, msg->front.iov_base, front_len(msg)); 1807 WARN_ON(msg->front.iov_len != front_len(msg)); 1808 } 1809 if (middle_len(msg)) { 1810 add_in_kvec(con, msg->middle->vec.iov_base, middle_len(msg)); 1811 WARN_ON(msg->middle->vec.iov_len != middle_len(msg)); 1812 } 1813 1814 if (data_len(msg)) { 1815 con->v2.in_state = IN_S_PREPARE_READ_DATA; 1816 } else { 1817 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN); 1818 con->v2.in_state = IN_S_HANDLE_EPILOGUE; 1819 } 1820 } 1821 1822 static void prepare_read_enc_page(struct ceph_connection *con) 1823 { 1824 struct bio_vec bv; 1825 1826 dout("%s con %p i %d resid %d\n", __func__, con, con->v2.in_enc_i, 1827 con->v2.in_enc_resid); 1828 WARN_ON(!con->v2.in_enc_resid); 1829 1830 bv.bv_page = con->v2.in_enc_pages[con->v2.in_enc_i]; 1831 bv.bv_offset = 0; 1832 bv.bv_len = min(con->v2.in_enc_resid, (int)PAGE_SIZE); 1833 1834 set_in_bvec(con, &bv); 1835 con->v2.in_enc_i++; 1836 con->v2.in_enc_resid -= bv.bv_len; 1837 1838 if (con->v2.in_enc_resid) { 1839 con->v2.in_state = IN_S_PREPARE_READ_ENC_PAGE; 1840 return; 1841 } 1842 1843 /* 1844 * We are set to read the last piece of ciphertext (ending 1845 * with epilogue) + auth tag. 1846 */ 1847 WARN_ON(con->v2.in_enc_i != con->v2.in_enc_page_cnt); 1848 con->v2.in_state = IN_S_HANDLE_EPILOGUE; 1849 } 1850 1851 static int prepare_read_tail_secure(struct ceph_connection *con) 1852 { 1853 struct page **enc_pages; 1854 int enc_page_cnt; 1855 int tail_len; 1856 1857 tail_len = tail_onwire_len(con->in_msg, true); 1858 WARN_ON(!tail_len); 1859 1860 enc_page_cnt = calc_pages_for(0, tail_len); 1861 enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO); 1862 if (IS_ERR(enc_pages)) 1863 return PTR_ERR(enc_pages); 1864 1865 WARN_ON(con->v2.in_enc_pages || con->v2.in_enc_page_cnt); 1866 con->v2.in_enc_pages = enc_pages; 1867 con->v2.in_enc_page_cnt = enc_page_cnt; 1868 con->v2.in_enc_resid = tail_len; 1869 con->v2.in_enc_i = 0; 1870 1871 prepare_read_enc_page(con); 1872 return 0; 1873 } 1874 1875 static void __finish_skip(struct ceph_connection *con) 1876 { 1877 con->in_seq++; 1878 prepare_read_preamble(con); 1879 } 1880 1881 static void prepare_skip_message(struct ceph_connection *con) 1882 { 1883 struct ceph_frame_desc *desc = &con->v2.in_desc; 1884 int tail_len; 1885 1886 dout("%s con %p %d+%d+%d\n", __func__, con, desc->fd_lens[1], 1887 desc->fd_lens[2], desc->fd_lens[3]); 1888 1889 tail_len = __tail_onwire_len(desc->fd_lens[1], desc->fd_lens[2], 1890 desc->fd_lens[3], con_secure(con)); 1891 if (!tail_len) { 1892 __finish_skip(con); 1893 } else { 1894 set_in_skip(con, tail_len); 1895 con->v2.in_state = IN_S_FINISH_SKIP; 1896 } 1897 } 1898 1899 static int process_banner_prefix(struct ceph_connection *con) 1900 { 1901 int payload_len; 1902 void *p; 1903 1904 WARN_ON(con->v2.in_kvecs[0].iov_len != CEPH_BANNER_V2_PREFIX_LEN); 1905 1906 p = con->v2.in_kvecs[0].iov_base; 1907 if (memcmp(p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN)) { 1908 if (!memcmp(p, CEPH_BANNER, CEPH_BANNER_LEN)) 1909 con->error_msg = "server is speaking msgr1 protocol"; 1910 else 1911 con->error_msg = "protocol error, bad banner"; 1912 return -EINVAL; 1913 } 1914 1915 p += CEPH_BANNER_V2_LEN; 1916 payload_len = ceph_decode_16(&p); 1917 dout("%s con %p payload_len %d\n", __func__, con, payload_len); 1918 1919 return prepare_read_banner_payload(con, payload_len); 1920 } 1921 1922 static int process_banner_payload(struct ceph_connection *con) 1923 { 1924 void *end = con->v2.in_kvecs[0].iov_base + con->v2.in_kvecs[0].iov_len; 1925 u64 feat = CEPH_MSGR2_SUPPORTED_FEATURES; 1926 u64 req_feat = CEPH_MSGR2_REQUIRED_FEATURES; 1927 u64 server_feat, server_req_feat; 1928 void *p; 1929 int ret; 1930 1931 p = con->v2.in_kvecs[0].iov_base; 1932 ceph_decode_64_safe(&p, end, server_feat, bad); 1933 ceph_decode_64_safe(&p, end, server_req_feat, bad); 1934 1935 dout("%s con %p server_feat 0x%llx server_req_feat 0x%llx\n", 1936 __func__, con, server_feat, server_req_feat); 1937 1938 if (req_feat & ~server_feat) { 1939 pr_err("msgr2 feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n", 1940 server_feat, req_feat & ~server_feat); 1941 con->error_msg = "missing required protocol features"; 1942 return -EINVAL; 1943 } 1944 if (server_req_feat & ~feat) { 1945 pr_err("msgr2 feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n", 1946 feat, server_req_feat & ~feat); 1947 con->error_msg = "missing required protocol features"; 1948 return -EINVAL; 1949 } 1950 1951 /* no reset_out_kvecs() as our banner may still be pending */ 1952 ret = prepare_hello(con); 1953 if (ret) { 1954 pr_err("prepare_hello failed: %d\n", ret); 1955 return ret; 1956 } 1957 1958 con->state = CEPH_CON_S_V2_HELLO; 1959 prepare_read_preamble(con); 1960 return 0; 1961 1962 bad: 1963 pr_err("failed to decode banner payload\n"); 1964 return -EINVAL; 1965 } 1966 1967 static int process_hello(struct ceph_connection *con, void *p, void *end) 1968 { 1969 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr; 1970 struct ceph_entity_addr addr_for_me; 1971 u8 entity_type; 1972 int ret; 1973 1974 if (con->state != CEPH_CON_S_V2_HELLO) { 1975 con->error_msg = "protocol error, unexpected hello"; 1976 return -EINVAL; 1977 } 1978 1979 ceph_decode_8_safe(&p, end, entity_type, bad); 1980 ret = ceph_decode_entity_addr(&p, end, &addr_for_me); 1981 if (ret) { 1982 pr_err("failed to decode addr_for_me: %d\n", ret); 1983 return ret; 1984 } 1985 1986 dout("%s con %p entity_type %d addr_for_me %s\n", __func__, con, 1987 entity_type, ceph_pr_addr(&addr_for_me)); 1988 1989 if (entity_type != con->peer_name.type) { 1990 pr_err("bad peer type, want %d, got %d\n", 1991 con->peer_name.type, entity_type); 1992 con->error_msg = "wrong peer at address"; 1993 return -EINVAL; 1994 } 1995 1996 /* 1997 * Set our address to the address our first peer (i.e. monitor) 1998 * sees that we are connecting from. If we are behind some sort 1999 * of NAT and want to be identified by some private (not NATed) 2000 * address, ip option should be used. 2001 */ 2002 if (ceph_addr_is_blank(my_addr)) { 2003 memcpy(&my_addr->in_addr, &addr_for_me.in_addr, 2004 sizeof(my_addr->in_addr)); 2005 ceph_addr_set_port(my_addr, 0); 2006 dout("%s con %p set my addr %s, as seen by peer %s\n", 2007 __func__, con, ceph_pr_addr(my_addr), 2008 ceph_pr_addr(&con->peer_addr)); 2009 } else { 2010 dout("%s con %p my addr already set %s\n", 2011 __func__, con, ceph_pr_addr(my_addr)); 2012 } 2013 2014 WARN_ON(ceph_addr_is_blank(my_addr) || ceph_addr_port(my_addr)); 2015 WARN_ON(my_addr->type != CEPH_ENTITY_ADDR_TYPE_ANY); 2016 WARN_ON(!my_addr->nonce); 2017 2018 /* no reset_out_kvecs() as our hello may still be pending */ 2019 ret = prepare_auth_request(con); 2020 if (ret) { 2021 if (ret != -EAGAIN) 2022 pr_err("prepare_auth_request failed: %d\n", ret); 2023 return ret; 2024 } 2025 2026 con->state = CEPH_CON_S_V2_AUTH; 2027 return 0; 2028 2029 bad: 2030 pr_err("failed to decode hello\n"); 2031 return -EINVAL; 2032 } 2033 2034 static int process_auth_bad_method(struct ceph_connection *con, 2035 void *p, void *end) 2036 { 2037 int allowed_protos[8], allowed_modes[8]; 2038 int allowed_proto_cnt, allowed_mode_cnt; 2039 int used_proto, result; 2040 int ret; 2041 int i; 2042 2043 if (con->state != CEPH_CON_S_V2_AUTH) { 2044 con->error_msg = "protocol error, unexpected auth_bad_method"; 2045 return -EINVAL; 2046 } 2047 2048 ceph_decode_32_safe(&p, end, used_proto, bad); 2049 ceph_decode_32_safe(&p, end, result, bad); 2050 dout("%s con %p used_proto %d result %d\n", __func__, con, used_proto, 2051 result); 2052 2053 ceph_decode_32_safe(&p, end, allowed_proto_cnt, bad); 2054 if (allowed_proto_cnt > ARRAY_SIZE(allowed_protos)) { 2055 pr_err("allowed_protos too big %d\n", allowed_proto_cnt); 2056 return -EINVAL; 2057 } 2058 for (i = 0; i < allowed_proto_cnt; i++) { 2059 ceph_decode_32_safe(&p, end, allowed_protos[i], bad); 2060 dout("%s con %p allowed_protos[%d] %d\n", __func__, con, 2061 i, allowed_protos[i]); 2062 } 2063 2064 ceph_decode_32_safe(&p, end, allowed_mode_cnt, bad); 2065 if (allowed_mode_cnt > ARRAY_SIZE(allowed_modes)) { 2066 pr_err("allowed_modes too big %d\n", allowed_mode_cnt); 2067 return -EINVAL; 2068 } 2069 for (i = 0; i < allowed_mode_cnt; i++) { 2070 ceph_decode_32_safe(&p, end, allowed_modes[i], bad); 2071 dout("%s con %p allowed_modes[%d] %d\n", __func__, con, 2072 i, allowed_modes[i]); 2073 } 2074 2075 mutex_unlock(&con->mutex); 2076 ret = con->ops->handle_auth_bad_method(con, used_proto, result, 2077 allowed_protos, 2078 allowed_proto_cnt, 2079 allowed_modes, 2080 allowed_mode_cnt); 2081 mutex_lock(&con->mutex); 2082 if (con->state != CEPH_CON_S_V2_AUTH) { 2083 dout("%s con %p state changed to %d\n", __func__, con, 2084 con->state); 2085 return -EAGAIN; 2086 } 2087 2088 dout("%s con %p handle_auth_bad_method ret %d\n", __func__, con, ret); 2089 return ret; 2090 2091 bad: 2092 pr_err("failed to decode auth_bad_method\n"); 2093 return -EINVAL; 2094 } 2095 2096 static int process_auth_reply_more(struct ceph_connection *con, 2097 void *p, void *end) 2098 { 2099 int payload_len; 2100 int ret; 2101 2102 if (con->state != CEPH_CON_S_V2_AUTH) { 2103 con->error_msg = "protocol error, unexpected auth_reply_more"; 2104 return -EINVAL; 2105 } 2106 2107 ceph_decode_32_safe(&p, end, payload_len, bad); 2108 ceph_decode_need(&p, end, payload_len, bad); 2109 2110 dout("%s con %p payload_len %d\n", __func__, con, payload_len); 2111 2112 reset_out_kvecs(con); 2113 ret = prepare_auth_request_more(con, p, payload_len); 2114 if (ret) { 2115 if (ret != -EAGAIN) 2116 pr_err("prepare_auth_request_more failed: %d\n", ret); 2117 return ret; 2118 } 2119 2120 return 0; 2121 2122 bad: 2123 pr_err("failed to decode auth_reply_more\n"); 2124 return -EINVAL; 2125 } 2126 2127 /* 2128 * Align session_key and con_secret to avoid GFP_ATOMIC allocation 2129 * inside crypto_shash_setkey() and crypto_aead_setkey() called from 2130 * setup_crypto(). __aligned(16) isn't guaranteed to work for stack 2131 * objects, so do it by hand. 2132 */ 2133 static int process_auth_done(struct ceph_connection *con, void *p, void *end) 2134 { 2135 u8 session_key_buf[CEPH_KEY_LEN + 16]; 2136 u8 con_secret_buf[CEPH_MAX_CON_SECRET_LEN + 16]; 2137 u8 *session_key = PTR_ALIGN(&session_key_buf[0], 16); 2138 u8 *con_secret = PTR_ALIGN(&con_secret_buf[0], 16); 2139 int session_key_len, con_secret_len; 2140 int payload_len; 2141 u64 global_id; 2142 int ret; 2143 2144 if (con->state != CEPH_CON_S_V2_AUTH) { 2145 con->error_msg = "protocol error, unexpected auth_done"; 2146 return -EINVAL; 2147 } 2148 2149 ceph_decode_64_safe(&p, end, global_id, bad); 2150 ceph_decode_32_safe(&p, end, con->v2.con_mode, bad); 2151 ceph_decode_32_safe(&p, end, payload_len, bad); 2152 2153 dout("%s con %p global_id %llu con_mode %d payload_len %d\n", 2154 __func__, con, global_id, con->v2.con_mode, payload_len); 2155 2156 mutex_unlock(&con->mutex); 2157 session_key_len = 0; 2158 con_secret_len = 0; 2159 ret = con->ops->handle_auth_done(con, global_id, p, payload_len, 2160 session_key, &session_key_len, 2161 con_secret, &con_secret_len); 2162 mutex_lock(&con->mutex); 2163 if (con->state != CEPH_CON_S_V2_AUTH) { 2164 dout("%s con %p state changed to %d\n", __func__, con, 2165 con->state); 2166 ret = -EAGAIN; 2167 goto out; 2168 } 2169 2170 dout("%s con %p handle_auth_done ret %d\n", __func__, con, ret); 2171 if (ret) 2172 goto out; 2173 2174 ret = setup_crypto(con, session_key, session_key_len, con_secret, 2175 con_secret_len); 2176 if (ret) 2177 goto out; 2178 2179 reset_out_kvecs(con); 2180 ret = prepare_auth_signature(con); 2181 if (ret) { 2182 pr_err("prepare_auth_signature failed: %d\n", ret); 2183 goto out; 2184 } 2185 2186 con->state = CEPH_CON_S_V2_AUTH_SIGNATURE; 2187 2188 out: 2189 memzero_explicit(session_key_buf, sizeof(session_key_buf)); 2190 memzero_explicit(con_secret_buf, sizeof(con_secret_buf)); 2191 return ret; 2192 2193 bad: 2194 pr_err("failed to decode auth_done\n"); 2195 return -EINVAL; 2196 } 2197 2198 static int process_auth_signature(struct ceph_connection *con, 2199 void *p, void *end) 2200 { 2201 u8 hmac[SHA256_DIGEST_SIZE]; 2202 int ret; 2203 2204 if (con->state != CEPH_CON_S_V2_AUTH_SIGNATURE) { 2205 con->error_msg = "protocol error, unexpected auth_signature"; 2206 return -EINVAL; 2207 } 2208 2209 ret = hmac_sha256(con, con->v2.out_sign_kvecs, 2210 con->v2.out_sign_kvec_cnt, hmac); 2211 if (ret) 2212 return ret; 2213 2214 ceph_decode_need(&p, end, SHA256_DIGEST_SIZE, bad); 2215 if (crypto_memneq(p, hmac, SHA256_DIGEST_SIZE)) { 2216 con->error_msg = "integrity error, bad auth signature"; 2217 return -EBADMSG; 2218 } 2219 2220 dout("%s con %p auth signature ok\n", __func__, con); 2221 2222 /* no reset_out_kvecs() as our auth_signature may still be pending */ 2223 if (!con->v2.server_cookie) { 2224 ret = prepare_client_ident(con); 2225 if (ret) { 2226 pr_err("prepare_client_ident failed: %d\n", ret); 2227 return ret; 2228 } 2229 2230 con->state = CEPH_CON_S_V2_SESSION_CONNECT; 2231 } else { 2232 ret = prepare_session_reconnect(con); 2233 if (ret) { 2234 pr_err("prepare_session_reconnect failed: %d\n", ret); 2235 return ret; 2236 } 2237 2238 con->state = CEPH_CON_S_V2_SESSION_RECONNECT; 2239 } 2240 2241 return 0; 2242 2243 bad: 2244 pr_err("failed to decode auth_signature\n"); 2245 return -EINVAL; 2246 } 2247 2248 static int process_server_ident(struct ceph_connection *con, 2249 void *p, void *end) 2250 { 2251 struct ceph_client *client = from_msgr(con->msgr); 2252 u64 features, required_features; 2253 struct ceph_entity_addr addr; 2254 u64 global_seq; 2255 u64 global_id; 2256 u64 cookie; 2257 u64 flags; 2258 int ret; 2259 2260 if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) { 2261 con->error_msg = "protocol error, unexpected server_ident"; 2262 return -EINVAL; 2263 } 2264 2265 ret = ceph_decode_entity_addrvec(&p, end, true, &addr); 2266 if (ret) { 2267 pr_err("failed to decode server addrs: %d\n", ret); 2268 return ret; 2269 } 2270 2271 ceph_decode_64_safe(&p, end, global_id, bad); 2272 ceph_decode_64_safe(&p, end, global_seq, bad); 2273 ceph_decode_64_safe(&p, end, features, bad); 2274 ceph_decode_64_safe(&p, end, required_features, bad); 2275 ceph_decode_64_safe(&p, end, flags, bad); 2276 ceph_decode_64_safe(&p, end, cookie, bad); 2277 2278 dout("%s con %p addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx flags 0x%llx cookie 0x%llx\n", 2279 __func__, con, ceph_pr_addr(&addr), le32_to_cpu(addr.nonce), 2280 global_id, global_seq, features, required_features, flags, cookie); 2281 2282 /* is this who we intended to talk to? */ 2283 if (memcmp(&addr, &con->peer_addr, sizeof(con->peer_addr))) { 2284 pr_err("bad peer addr/nonce, want %s/%u, got %s/%u\n", 2285 ceph_pr_addr(&con->peer_addr), 2286 le32_to_cpu(con->peer_addr.nonce), 2287 ceph_pr_addr(&addr), le32_to_cpu(addr.nonce)); 2288 con->error_msg = "wrong peer at address"; 2289 return -EINVAL; 2290 } 2291 2292 if (client->required_features & ~features) { 2293 pr_err("RADOS feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n", 2294 features, client->required_features & ~features); 2295 con->error_msg = "missing required protocol features"; 2296 return -EINVAL; 2297 } 2298 2299 /* 2300 * Both name->type and name->num are set in ceph_con_open() but 2301 * name->num may be bogus in the initial monmap. name->type is 2302 * verified in handle_hello(). 2303 */ 2304 WARN_ON(!con->peer_name.type); 2305 con->peer_name.num = cpu_to_le64(global_id); 2306 con->v2.peer_global_seq = global_seq; 2307 con->peer_features = features; 2308 WARN_ON(required_features & ~client->supported_features); 2309 con->v2.server_cookie = cookie; 2310 2311 if (flags & CEPH_MSG_CONNECT_LOSSY) { 2312 ceph_con_flag_set(con, CEPH_CON_F_LOSSYTX); 2313 WARN_ON(con->v2.server_cookie); 2314 } else { 2315 WARN_ON(!con->v2.server_cookie); 2316 } 2317 2318 clear_in_sign_kvecs(con); 2319 clear_out_sign_kvecs(con); 2320 free_conn_bufs(con); 2321 con->delay = 0; /* reset backoff memory */ 2322 2323 con->state = CEPH_CON_S_OPEN; 2324 con->v2.out_state = OUT_S_GET_NEXT; 2325 return 0; 2326 2327 bad: 2328 pr_err("failed to decode server_ident\n"); 2329 return -EINVAL; 2330 } 2331 2332 static int process_ident_missing_features(struct ceph_connection *con, 2333 void *p, void *end) 2334 { 2335 struct ceph_client *client = from_msgr(con->msgr); 2336 u64 missing_features; 2337 2338 if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) { 2339 con->error_msg = "protocol error, unexpected ident_missing_features"; 2340 return -EINVAL; 2341 } 2342 2343 ceph_decode_64_safe(&p, end, missing_features, bad); 2344 pr_err("RADOS feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n", 2345 client->supported_features, missing_features); 2346 con->error_msg = "missing required protocol features"; 2347 return -EINVAL; 2348 2349 bad: 2350 pr_err("failed to decode ident_missing_features\n"); 2351 return -EINVAL; 2352 } 2353 2354 static int process_session_reconnect_ok(struct ceph_connection *con, 2355 void *p, void *end) 2356 { 2357 u64 seq; 2358 2359 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { 2360 con->error_msg = "protocol error, unexpected session_reconnect_ok"; 2361 return -EINVAL; 2362 } 2363 2364 ceph_decode_64_safe(&p, end, seq, bad); 2365 2366 dout("%s con %p seq %llu\n", __func__, con, seq); 2367 ceph_con_discard_requeued(con, seq); 2368 2369 clear_in_sign_kvecs(con); 2370 clear_out_sign_kvecs(con); 2371 free_conn_bufs(con); 2372 con->delay = 0; /* reset backoff memory */ 2373 2374 con->state = CEPH_CON_S_OPEN; 2375 con->v2.out_state = OUT_S_GET_NEXT; 2376 return 0; 2377 2378 bad: 2379 pr_err("failed to decode session_reconnect_ok\n"); 2380 return -EINVAL; 2381 } 2382 2383 static int process_session_retry(struct ceph_connection *con, 2384 void *p, void *end) 2385 { 2386 u64 connect_seq; 2387 int ret; 2388 2389 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { 2390 con->error_msg = "protocol error, unexpected session_retry"; 2391 return -EINVAL; 2392 } 2393 2394 ceph_decode_64_safe(&p, end, connect_seq, bad); 2395 2396 dout("%s con %p connect_seq %llu\n", __func__, con, connect_seq); 2397 WARN_ON(connect_seq <= con->v2.connect_seq); 2398 con->v2.connect_seq = connect_seq + 1; 2399 2400 free_conn_bufs(con); 2401 2402 reset_out_kvecs(con); 2403 ret = prepare_session_reconnect(con); 2404 if (ret) { 2405 pr_err("prepare_session_reconnect (cseq) failed: %d\n", ret); 2406 return ret; 2407 } 2408 2409 return 0; 2410 2411 bad: 2412 pr_err("failed to decode session_retry\n"); 2413 return -EINVAL; 2414 } 2415 2416 static int process_session_retry_global(struct ceph_connection *con, 2417 void *p, void *end) 2418 { 2419 u64 global_seq; 2420 int ret; 2421 2422 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { 2423 con->error_msg = "protocol error, unexpected session_retry_global"; 2424 return -EINVAL; 2425 } 2426 2427 ceph_decode_64_safe(&p, end, global_seq, bad); 2428 2429 dout("%s con %p global_seq %llu\n", __func__, con, global_seq); 2430 WARN_ON(global_seq <= con->v2.global_seq); 2431 con->v2.global_seq = ceph_get_global_seq(con->msgr, global_seq); 2432 2433 free_conn_bufs(con); 2434 2435 reset_out_kvecs(con); 2436 ret = prepare_session_reconnect(con); 2437 if (ret) { 2438 pr_err("prepare_session_reconnect (gseq) failed: %d\n", ret); 2439 return ret; 2440 } 2441 2442 return 0; 2443 2444 bad: 2445 pr_err("failed to decode session_retry_global\n"); 2446 return -EINVAL; 2447 } 2448 2449 static int process_session_reset(struct ceph_connection *con, 2450 void *p, void *end) 2451 { 2452 bool full; 2453 int ret; 2454 2455 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { 2456 con->error_msg = "protocol error, unexpected session_reset"; 2457 return -EINVAL; 2458 } 2459 2460 ceph_decode_8_safe(&p, end, full, bad); 2461 if (!full) { 2462 con->error_msg = "protocol error, bad session_reset"; 2463 return -EINVAL; 2464 } 2465 2466 pr_info("%s%lld %s session reset\n", ENTITY_NAME(con->peer_name), 2467 ceph_pr_addr(&con->peer_addr)); 2468 ceph_con_reset_session(con); 2469 2470 mutex_unlock(&con->mutex); 2471 if (con->ops->peer_reset) 2472 con->ops->peer_reset(con); 2473 mutex_lock(&con->mutex); 2474 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { 2475 dout("%s con %p state changed to %d\n", __func__, con, 2476 con->state); 2477 return -EAGAIN; 2478 } 2479 2480 free_conn_bufs(con); 2481 2482 reset_out_kvecs(con); 2483 ret = prepare_client_ident(con); 2484 if (ret) { 2485 pr_err("prepare_client_ident (rst) failed: %d\n", ret); 2486 return ret; 2487 } 2488 2489 con->state = CEPH_CON_S_V2_SESSION_CONNECT; 2490 return 0; 2491 2492 bad: 2493 pr_err("failed to decode session_reset\n"); 2494 return -EINVAL; 2495 } 2496 2497 static int process_keepalive2_ack(struct ceph_connection *con, 2498 void *p, void *end) 2499 { 2500 if (con->state != CEPH_CON_S_OPEN) { 2501 con->error_msg = "protocol error, unexpected keepalive2_ack"; 2502 return -EINVAL; 2503 } 2504 2505 ceph_decode_need(&p, end, sizeof(struct ceph_timespec), bad); 2506 ceph_decode_timespec64(&con->last_keepalive_ack, p); 2507 2508 dout("%s con %p timestamp %lld.%09ld\n", __func__, con, 2509 con->last_keepalive_ack.tv_sec, con->last_keepalive_ack.tv_nsec); 2510 2511 return 0; 2512 2513 bad: 2514 pr_err("failed to decode keepalive2_ack\n"); 2515 return -EINVAL; 2516 } 2517 2518 static int process_ack(struct ceph_connection *con, void *p, void *end) 2519 { 2520 u64 seq; 2521 2522 if (con->state != CEPH_CON_S_OPEN) { 2523 con->error_msg = "protocol error, unexpected ack"; 2524 return -EINVAL; 2525 } 2526 2527 ceph_decode_64_safe(&p, end, seq, bad); 2528 2529 dout("%s con %p seq %llu\n", __func__, con, seq); 2530 ceph_con_discard_sent(con, seq); 2531 return 0; 2532 2533 bad: 2534 pr_err("failed to decode ack\n"); 2535 return -EINVAL; 2536 } 2537 2538 static int process_control(struct ceph_connection *con, void *p, void *end) 2539 { 2540 int tag = con->v2.in_desc.fd_tag; 2541 int ret; 2542 2543 dout("%s con %p tag %d len %d\n", __func__, con, tag, (int)(end - p)); 2544 2545 switch (tag) { 2546 case FRAME_TAG_HELLO: 2547 ret = process_hello(con, p, end); 2548 break; 2549 case FRAME_TAG_AUTH_BAD_METHOD: 2550 ret = process_auth_bad_method(con, p, end); 2551 break; 2552 case FRAME_TAG_AUTH_REPLY_MORE: 2553 ret = process_auth_reply_more(con, p, end); 2554 break; 2555 case FRAME_TAG_AUTH_DONE: 2556 ret = process_auth_done(con, p, end); 2557 break; 2558 case FRAME_TAG_AUTH_SIGNATURE: 2559 ret = process_auth_signature(con, p, end); 2560 break; 2561 case FRAME_TAG_SERVER_IDENT: 2562 ret = process_server_ident(con, p, end); 2563 break; 2564 case FRAME_TAG_IDENT_MISSING_FEATURES: 2565 ret = process_ident_missing_features(con, p, end); 2566 break; 2567 case FRAME_TAG_SESSION_RECONNECT_OK: 2568 ret = process_session_reconnect_ok(con, p, end); 2569 break; 2570 case FRAME_TAG_SESSION_RETRY: 2571 ret = process_session_retry(con, p, end); 2572 break; 2573 case FRAME_TAG_SESSION_RETRY_GLOBAL: 2574 ret = process_session_retry_global(con, p, end); 2575 break; 2576 case FRAME_TAG_SESSION_RESET: 2577 ret = process_session_reset(con, p, end); 2578 break; 2579 case FRAME_TAG_KEEPALIVE2_ACK: 2580 ret = process_keepalive2_ack(con, p, end); 2581 break; 2582 case FRAME_TAG_ACK: 2583 ret = process_ack(con, p, end); 2584 break; 2585 default: 2586 pr_err("bad tag %d\n", tag); 2587 con->error_msg = "protocol error, bad tag"; 2588 return -EINVAL; 2589 } 2590 if (ret) { 2591 dout("%s con %p error %d\n", __func__, con, ret); 2592 return ret; 2593 } 2594 2595 prepare_read_preamble(con); 2596 return 0; 2597 } 2598 2599 /* 2600 * Return: 2601 * 1 - con->in_msg set, read message 2602 * 0 - skip message 2603 * <0 - error 2604 */ 2605 static int process_message_header(struct ceph_connection *con, 2606 void *p, void *end) 2607 { 2608 struct ceph_frame_desc *desc = &con->v2.in_desc; 2609 struct ceph_msg_header2 *hdr2 = p; 2610 struct ceph_msg_header hdr; 2611 int skip; 2612 int ret; 2613 u64 seq; 2614 2615 /* verify seq# */ 2616 seq = le64_to_cpu(hdr2->seq); 2617 if ((s64)seq - (s64)con->in_seq < 1) { 2618 pr_info("%s%lld %s skipping old message: seq %llu, expected %llu\n", 2619 ENTITY_NAME(con->peer_name), 2620 ceph_pr_addr(&con->peer_addr), 2621 seq, con->in_seq + 1); 2622 return 0; 2623 } 2624 if ((s64)seq - (s64)con->in_seq > 1) { 2625 pr_err("bad seq %llu, expected %llu\n", seq, con->in_seq + 1); 2626 con->error_msg = "bad message sequence # for incoming message"; 2627 return -EBADE; 2628 } 2629 2630 ceph_con_discard_sent(con, le64_to_cpu(hdr2->ack_seq)); 2631 2632 fill_header(&hdr, hdr2, desc->fd_lens[1], desc->fd_lens[2], 2633 desc->fd_lens[3], &con->peer_name); 2634 ret = ceph_con_in_msg_alloc(con, &hdr, &skip); 2635 if (ret) 2636 return ret; 2637 2638 WARN_ON(!con->in_msg ^ skip); 2639 if (skip) 2640 return 0; 2641 2642 WARN_ON(!con->in_msg); 2643 WARN_ON(con->in_msg->con != con); 2644 return 1; 2645 } 2646 2647 static int process_message(struct ceph_connection *con) 2648 { 2649 ceph_con_process_message(con); 2650 2651 /* 2652 * We could have been closed by ceph_con_close() because 2653 * ceph_con_process_message() temporarily drops con->mutex. 2654 */ 2655 if (con->state != CEPH_CON_S_OPEN) { 2656 dout("%s con %p state changed to %d\n", __func__, con, 2657 con->state); 2658 return -EAGAIN; 2659 } 2660 2661 prepare_read_preamble(con); 2662 return 0; 2663 } 2664 2665 static int __handle_control(struct ceph_connection *con, void *p) 2666 { 2667 void *end = p + con->v2.in_desc.fd_lens[0]; 2668 struct ceph_msg *msg; 2669 int ret; 2670 2671 if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE) 2672 return process_control(con, p, end); 2673 2674 ret = process_message_header(con, p, end); 2675 if (ret < 0) 2676 return ret; 2677 if (ret == 0) { 2678 prepare_skip_message(con); 2679 return 0; 2680 } 2681 2682 msg = con->in_msg; /* set in process_message_header() */ 2683 if (front_len(msg)) { 2684 WARN_ON(front_len(msg) > msg->front_alloc_len); 2685 msg->front.iov_len = front_len(msg); 2686 } else { 2687 msg->front.iov_len = 0; 2688 } 2689 if (middle_len(msg)) { 2690 WARN_ON(middle_len(msg) > msg->middle->alloc_len); 2691 msg->middle->vec.iov_len = middle_len(msg); 2692 } else if (msg->middle) { 2693 msg->middle->vec.iov_len = 0; 2694 } 2695 2696 if (!front_len(msg) && !middle_len(msg) && !data_len(msg)) 2697 return process_message(con); 2698 2699 if (con_secure(con)) 2700 return prepare_read_tail_secure(con); 2701 2702 prepare_read_tail_plain(con); 2703 return 0; 2704 } 2705 2706 static int handle_preamble(struct ceph_connection *con) 2707 { 2708 struct ceph_frame_desc *desc = &con->v2.in_desc; 2709 int ret; 2710 2711 if (con_secure(con)) { 2712 ret = decrypt_preamble(con); 2713 if (ret) { 2714 if (ret == -EBADMSG) 2715 con->error_msg = "integrity error, bad preamble auth tag"; 2716 return ret; 2717 } 2718 } 2719 2720 ret = decode_preamble(con->v2.in_buf, desc); 2721 if (ret) { 2722 if (ret == -EBADMSG) 2723 con->error_msg = "integrity error, bad crc"; 2724 else 2725 con->error_msg = "protocol error, bad preamble"; 2726 return ret; 2727 } 2728 2729 dout("%s con %p tag %d seg_cnt %d %d+%d+%d+%d\n", __func__, 2730 con, desc->fd_tag, desc->fd_seg_cnt, desc->fd_lens[0], 2731 desc->fd_lens[1], desc->fd_lens[2], desc->fd_lens[3]); 2732 2733 if (!con_secure(con)) 2734 return prepare_read_control(con); 2735 2736 if (desc->fd_lens[0] > CEPH_PREAMBLE_INLINE_LEN) 2737 return prepare_read_control_remainder(con); 2738 2739 return __handle_control(con, CTRL_BODY(con->v2.in_buf)); 2740 } 2741 2742 static int handle_control(struct ceph_connection *con) 2743 { 2744 int ctrl_len = con->v2.in_desc.fd_lens[0]; 2745 void *buf; 2746 int ret; 2747 2748 WARN_ON(con_secure(con)); 2749 2750 ret = verify_control_crc(con); 2751 if (ret) { 2752 con->error_msg = "integrity error, bad crc"; 2753 return ret; 2754 } 2755 2756 if (con->state == CEPH_CON_S_V2_AUTH) { 2757 buf = alloc_conn_buf(con, ctrl_len); 2758 if (!buf) 2759 return -ENOMEM; 2760 2761 memcpy(buf, con->v2.in_kvecs[0].iov_base, ctrl_len); 2762 return __handle_control(con, buf); 2763 } 2764 2765 return __handle_control(con, con->v2.in_kvecs[0].iov_base); 2766 } 2767 2768 static int handle_control_remainder(struct ceph_connection *con) 2769 { 2770 int ret; 2771 2772 WARN_ON(!con_secure(con)); 2773 2774 ret = decrypt_control_remainder(con); 2775 if (ret) { 2776 if (ret == -EBADMSG) 2777 con->error_msg = "integrity error, bad control remainder auth tag"; 2778 return ret; 2779 } 2780 2781 return __handle_control(con, con->v2.in_kvecs[0].iov_base - 2782 CEPH_PREAMBLE_INLINE_LEN); 2783 } 2784 2785 static int handle_epilogue(struct ceph_connection *con) 2786 { 2787 u32 front_crc, middle_crc, data_crc; 2788 int ret; 2789 2790 if (con_secure(con)) { 2791 ret = decrypt_tail(con); 2792 if (ret) { 2793 if (ret == -EBADMSG) 2794 con->error_msg = "integrity error, bad epilogue auth tag"; 2795 return ret; 2796 } 2797 2798 /* just late_status */ 2799 ret = decode_epilogue(con->v2.in_buf, NULL, NULL, NULL); 2800 if (ret) { 2801 con->error_msg = "protocol error, bad epilogue"; 2802 return ret; 2803 } 2804 } else { 2805 ret = decode_epilogue(con->v2.in_buf, &front_crc, 2806 &middle_crc, &data_crc); 2807 if (ret) { 2808 con->error_msg = "protocol error, bad epilogue"; 2809 return ret; 2810 } 2811 2812 ret = verify_epilogue_crcs(con, front_crc, middle_crc, 2813 data_crc); 2814 if (ret) { 2815 con->error_msg = "integrity error, bad crc"; 2816 return ret; 2817 } 2818 } 2819 2820 return process_message(con); 2821 } 2822 2823 static void finish_skip(struct ceph_connection *con) 2824 { 2825 dout("%s con %p\n", __func__, con); 2826 2827 if (con_secure(con)) 2828 gcm_inc_nonce(&con->v2.in_gcm_nonce); 2829 2830 __finish_skip(con); 2831 } 2832 2833 static int populate_in_iter(struct ceph_connection *con) 2834 { 2835 int ret; 2836 2837 dout("%s con %p state %d in_state %d\n", __func__, con, con->state, 2838 con->v2.in_state); 2839 WARN_ON(iov_iter_count(&con->v2.in_iter)); 2840 2841 if (con->state == CEPH_CON_S_V2_BANNER_PREFIX) { 2842 ret = process_banner_prefix(con); 2843 } else if (con->state == CEPH_CON_S_V2_BANNER_PAYLOAD) { 2844 ret = process_banner_payload(con); 2845 } else if ((con->state >= CEPH_CON_S_V2_HELLO && 2846 con->state <= CEPH_CON_S_V2_SESSION_RECONNECT) || 2847 con->state == CEPH_CON_S_OPEN) { 2848 switch (con->v2.in_state) { 2849 case IN_S_HANDLE_PREAMBLE: 2850 ret = handle_preamble(con); 2851 break; 2852 case IN_S_HANDLE_CONTROL: 2853 ret = handle_control(con); 2854 break; 2855 case IN_S_HANDLE_CONTROL_REMAINDER: 2856 ret = handle_control_remainder(con); 2857 break; 2858 case IN_S_PREPARE_READ_DATA: 2859 prepare_read_data(con); 2860 ret = 0; 2861 break; 2862 case IN_S_PREPARE_READ_DATA_CONT: 2863 prepare_read_data_cont(con); 2864 ret = 0; 2865 break; 2866 case IN_S_PREPARE_READ_ENC_PAGE: 2867 prepare_read_enc_page(con); 2868 ret = 0; 2869 break; 2870 case IN_S_HANDLE_EPILOGUE: 2871 ret = handle_epilogue(con); 2872 break; 2873 case IN_S_FINISH_SKIP: 2874 finish_skip(con); 2875 ret = 0; 2876 break; 2877 default: 2878 WARN(1, "bad in_state %d", con->v2.in_state); 2879 return -EINVAL; 2880 } 2881 } else { 2882 WARN(1, "bad state %d", con->state); 2883 return -EINVAL; 2884 } 2885 if (ret) { 2886 dout("%s con %p error %d\n", __func__, con, ret); 2887 return ret; 2888 } 2889 2890 if (WARN_ON(!iov_iter_count(&con->v2.in_iter))) 2891 return -ENODATA; 2892 dout("%s con %p populated %zu\n", __func__, con, 2893 iov_iter_count(&con->v2.in_iter)); 2894 return 1; 2895 } 2896 2897 int ceph_con_v2_try_read(struct ceph_connection *con) 2898 { 2899 int ret; 2900 2901 dout("%s con %p state %d need %zu\n", __func__, con, con->state, 2902 iov_iter_count(&con->v2.in_iter)); 2903 2904 if (con->state == CEPH_CON_S_PREOPEN) 2905 return 0; 2906 2907 /* 2908 * We should always have something pending here. If not, 2909 * avoid calling populate_in_iter() as if we read something 2910 * (ceph_tcp_recv() would immediately return 1). 2911 */ 2912 if (WARN_ON(!iov_iter_count(&con->v2.in_iter))) 2913 return -ENODATA; 2914 2915 for (;;) { 2916 ret = ceph_tcp_recv(con); 2917 if (ret <= 0) 2918 return ret; 2919 2920 ret = populate_in_iter(con); 2921 if (ret <= 0) { 2922 if (ret && ret != -EAGAIN && !con->error_msg) 2923 con->error_msg = "read processing error"; 2924 return ret; 2925 } 2926 } 2927 } 2928 2929 static void queue_data(struct ceph_connection *con) 2930 { 2931 struct bio_vec bv; 2932 2933 con->v2.out_epil.data_crc = -1; 2934 ceph_msg_data_cursor_init(&con->v2.out_cursor, con->out_msg, 2935 data_len(con->out_msg)); 2936 2937 get_bvec_at(&con->v2.out_cursor, &bv); 2938 set_out_bvec(con, &bv, true); 2939 con->v2.out_state = OUT_S_QUEUE_DATA_CONT; 2940 } 2941 2942 static void queue_data_cont(struct ceph_connection *con) 2943 { 2944 struct bio_vec bv; 2945 2946 con->v2.out_epil.data_crc = ceph_crc32c_page( 2947 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page, 2948 con->v2.out_bvec.bv_offset, con->v2.out_bvec.bv_len); 2949 2950 ceph_msg_data_advance(&con->v2.out_cursor, con->v2.out_bvec.bv_len); 2951 if (con->v2.out_cursor.total_resid) { 2952 get_bvec_at(&con->v2.out_cursor, &bv); 2953 set_out_bvec(con, &bv, true); 2954 WARN_ON(con->v2.out_state != OUT_S_QUEUE_DATA_CONT); 2955 return; 2956 } 2957 2958 /* 2959 * We've written all data. Queue epilogue. Once it's written, 2960 * we are done. 2961 */ 2962 reset_out_kvecs(con); 2963 prepare_epilogue_plain(con, false); 2964 con->v2.out_state = OUT_S_FINISH_MESSAGE; 2965 } 2966 2967 static void queue_enc_page(struct ceph_connection *con) 2968 { 2969 struct bio_vec bv; 2970 2971 dout("%s con %p i %d resid %d\n", __func__, con, con->v2.out_enc_i, 2972 con->v2.out_enc_resid); 2973 WARN_ON(!con->v2.out_enc_resid); 2974 2975 bv.bv_page = con->v2.out_enc_pages[con->v2.out_enc_i]; 2976 bv.bv_offset = 0; 2977 bv.bv_len = min(con->v2.out_enc_resid, (int)PAGE_SIZE); 2978 2979 set_out_bvec(con, &bv, false); 2980 con->v2.out_enc_i++; 2981 con->v2.out_enc_resid -= bv.bv_len; 2982 2983 if (con->v2.out_enc_resid) { 2984 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE); 2985 return; 2986 } 2987 2988 /* 2989 * We've queued the last piece of ciphertext (ending with 2990 * epilogue) + auth tag. Once it's written, we are done. 2991 */ 2992 WARN_ON(con->v2.out_enc_i != con->v2.out_enc_page_cnt); 2993 con->v2.out_state = OUT_S_FINISH_MESSAGE; 2994 } 2995 2996 static void queue_zeros(struct ceph_connection *con) 2997 { 2998 dout("%s con %p out_zero %d\n", __func__, con, con->v2.out_zero); 2999 3000 if (con->v2.out_zero) { 3001 set_out_bvec_zero(con); 3002 con->v2.out_zero -= con->v2.out_bvec.bv_len; 3003 con->v2.out_state = OUT_S_QUEUE_ZEROS; 3004 return; 3005 } 3006 3007 /* 3008 * We've zero-filled everything up to epilogue. Queue epilogue 3009 * with late_status set to ABORTED and crcs adjusted for zeros. 3010 * Once it's written, we are done patching up for the revoke. 3011 */ 3012 reset_out_kvecs(con); 3013 prepare_epilogue_plain(con, true); 3014 con->v2.out_state = OUT_S_FINISH_MESSAGE; 3015 } 3016 3017 static void finish_message(struct ceph_connection *con) 3018 { 3019 dout("%s con %p msg %p\n", __func__, con, con->out_msg); 3020 3021 /* we end up here both plain and secure modes */ 3022 if (con->v2.out_enc_pages) { 3023 WARN_ON(!con->v2.out_enc_page_cnt); 3024 ceph_release_page_vector(con->v2.out_enc_pages, 3025 con->v2.out_enc_page_cnt); 3026 con->v2.out_enc_pages = NULL; 3027 con->v2.out_enc_page_cnt = 0; 3028 } 3029 /* message may have been revoked */ 3030 if (con->out_msg) { 3031 ceph_msg_put(con->out_msg); 3032 con->out_msg = NULL; 3033 } 3034 3035 con->v2.out_state = OUT_S_GET_NEXT; 3036 } 3037 3038 static int populate_out_iter(struct ceph_connection *con) 3039 { 3040 int ret; 3041 3042 dout("%s con %p state %d out_state %d\n", __func__, con, con->state, 3043 con->v2.out_state); 3044 WARN_ON(iov_iter_count(&con->v2.out_iter)); 3045 3046 if (con->state != CEPH_CON_S_OPEN) { 3047 WARN_ON(con->state < CEPH_CON_S_V2_BANNER_PREFIX || 3048 con->state > CEPH_CON_S_V2_SESSION_RECONNECT); 3049 goto nothing_pending; 3050 } 3051 3052 switch (con->v2.out_state) { 3053 case OUT_S_QUEUE_DATA: 3054 WARN_ON(!con->out_msg); 3055 queue_data(con); 3056 goto populated; 3057 case OUT_S_QUEUE_DATA_CONT: 3058 WARN_ON(!con->out_msg); 3059 queue_data_cont(con); 3060 goto populated; 3061 case OUT_S_QUEUE_ENC_PAGE: 3062 queue_enc_page(con); 3063 goto populated; 3064 case OUT_S_QUEUE_ZEROS: 3065 WARN_ON(con->out_msg); /* revoked */ 3066 queue_zeros(con); 3067 goto populated; 3068 case OUT_S_FINISH_MESSAGE: 3069 finish_message(con); 3070 break; 3071 case OUT_S_GET_NEXT: 3072 break; 3073 default: 3074 WARN(1, "bad out_state %d", con->v2.out_state); 3075 return -EINVAL; 3076 } 3077 3078 WARN_ON(con->v2.out_state != OUT_S_GET_NEXT); 3079 if (ceph_con_flag_test_and_clear(con, CEPH_CON_F_KEEPALIVE_PENDING)) { 3080 ret = prepare_keepalive2(con); 3081 if (ret) { 3082 pr_err("prepare_keepalive2 failed: %d\n", ret); 3083 return ret; 3084 } 3085 } else if (!list_empty(&con->out_queue)) { 3086 ceph_con_get_out_msg(con); 3087 ret = prepare_message(con); 3088 if (ret) { 3089 pr_err("prepare_message failed: %d\n", ret); 3090 return ret; 3091 } 3092 } else if (con->in_seq > con->in_seq_acked) { 3093 ret = prepare_ack(con); 3094 if (ret) { 3095 pr_err("prepare_ack failed: %d\n", ret); 3096 return ret; 3097 } 3098 } else { 3099 goto nothing_pending; 3100 } 3101 3102 populated: 3103 if (WARN_ON(!iov_iter_count(&con->v2.out_iter))) 3104 return -ENODATA; 3105 dout("%s con %p populated %zu\n", __func__, con, 3106 iov_iter_count(&con->v2.out_iter)); 3107 return 1; 3108 3109 nothing_pending: 3110 WARN_ON(iov_iter_count(&con->v2.out_iter)); 3111 dout("%s con %p nothing pending\n", __func__, con); 3112 ceph_con_flag_clear(con, CEPH_CON_F_WRITE_PENDING); 3113 return 0; 3114 } 3115 3116 int ceph_con_v2_try_write(struct ceph_connection *con) 3117 { 3118 int ret; 3119 3120 dout("%s con %p state %d have %zu\n", __func__, con, con->state, 3121 iov_iter_count(&con->v2.out_iter)); 3122 3123 /* open the socket first? */ 3124 if (con->state == CEPH_CON_S_PREOPEN) { 3125 WARN_ON(con->peer_addr.type != CEPH_ENTITY_ADDR_TYPE_MSGR2); 3126 3127 /* 3128 * Always bump global_seq. Bump connect_seq only if 3129 * there is a session (i.e. we are reconnecting and will 3130 * send session_reconnect instead of client_ident). 3131 */ 3132 con->v2.global_seq = ceph_get_global_seq(con->msgr, 0); 3133 if (con->v2.server_cookie) 3134 con->v2.connect_seq++; 3135 3136 ret = prepare_read_banner_prefix(con); 3137 if (ret) { 3138 pr_err("prepare_read_banner_prefix failed: %d\n", ret); 3139 con->error_msg = "connect error"; 3140 return ret; 3141 } 3142 3143 reset_out_kvecs(con); 3144 ret = prepare_banner(con); 3145 if (ret) { 3146 pr_err("prepare_banner failed: %d\n", ret); 3147 con->error_msg = "connect error"; 3148 return ret; 3149 } 3150 3151 ret = ceph_tcp_connect(con); 3152 if (ret) { 3153 pr_err("ceph_tcp_connect failed: %d\n", ret); 3154 con->error_msg = "connect error"; 3155 return ret; 3156 } 3157 } 3158 3159 if (!iov_iter_count(&con->v2.out_iter)) { 3160 ret = populate_out_iter(con); 3161 if (ret <= 0) { 3162 if (ret && ret != -EAGAIN && !con->error_msg) 3163 con->error_msg = "write processing error"; 3164 return ret; 3165 } 3166 } 3167 3168 tcp_sock_set_cork(con->sock->sk, true); 3169 for (;;) { 3170 ret = ceph_tcp_send(con); 3171 if (ret <= 0) 3172 break; 3173 3174 ret = populate_out_iter(con); 3175 if (ret <= 0) { 3176 if (ret && ret != -EAGAIN && !con->error_msg) 3177 con->error_msg = "write processing error"; 3178 break; 3179 } 3180 } 3181 3182 tcp_sock_set_cork(con->sock->sk, false); 3183 return ret; 3184 } 3185 3186 static u32 crc32c_zeros(u32 crc, int zero_len) 3187 { 3188 int len; 3189 3190 while (zero_len) { 3191 len = min(zero_len, (int)PAGE_SIZE); 3192 crc = crc32c(crc, page_address(ceph_zero_page), len); 3193 zero_len -= len; 3194 } 3195 3196 return crc; 3197 } 3198 3199 static void prepare_zero_front(struct ceph_connection *con, int resid) 3200 { 3201 int sent; 3202 3203 WARN_ON(!resid || resid > front_len(con->out_msg)); 3204 sent = front_len(con->out_msg) - resid; 3205 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid); 3206 3207 if (sent) { 3208 con->v2.out_epil.front_crc = 3209 crc32c(-1, con->out_msg->front.iov_base, sent); 3210 con->v2.out_epil.front_crc = 3211 crc32c_zeros(con->v2.out_epil.front_crc, resid); 3212 } else { 3213 con->v2.out_epil.front_crc = crc32c_zeros(-1, resid); 3214 } 3215 3216 con->v2.out_iter.count -= resid; 3217 out_zero_add(con, resid); 3218 } 3219 3220 static void prepare_zero_middle(struct ceph_connection *con, int resid) 3221 { 3222 int sent; 3223 3224 WARN_ON(!resid || resid > middle_len(con->out_msg)); 3225 sent = middle_len(con->out_msg) - resid; 3226 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid); 3227 3228 if (sent) { 3229 con->v2.out_epil.middle_crc = 3230 crc32c(-1, con->out_msg->middle->vec.iov_base, sent); 3231 con->v2.out_epil.middle_crc = 3232 crc32c_zeros(con->v2.out_epil.middle_crc, resid); 3233 } else { 3234 con->v2.out_epil.middle_crc = crc32c_zeros(-1, resid); 3235 } 3236 3237 con->v2.out_iter.count -= resid; 3238 out_zero_add(con, resid); 3239 } 3240 3241 static void prepare_zero_data(struct ceph_connection *con) 3242 { 3243 dout("%s con %p\n", __func__, con); 3244 con->v2.out_epil.data_crc = crc32c_zeros(-1, data_len(con->out_msg)); 3245 out_zero_add(con, data_len(con->out_msg)); 3246 } 3247 3248 static void revoke_at_queue_data(struct ceph_connection *con) 3249 { 3250 int boundary; 3251 int resid; 3252 3253 WARN_ON(!data_len(con->out_msg)); 3254 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter)); 3255 resid = iov_iter_count(&con->v2.out_iter); 3256 3257 boundary = front_len(con->out_msg) + middle_len(con->out_msg); 3258 if (resid > boundary) { 3259 resid -= boundary; 3260 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN); 3261 dout("%s con %p was sending head\n", __func__, con); 3262 if (front_len(con->out_msg)) 3263 prepare_zero_front(con, front_len(con->out_msg)); 3264 if (middle_len(con->out_msg)) 3265 prepare_zero_middle(con, middle_len(con->out_msg)); 3266 prepare_zero_data(con); 3267 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid); 3268 con->v2.out_state = OUT_S_QUEUE_ZEROS; 3269 return; 3270 } 3271 3272 boundary = middle_len(con->out_msg); 3273 if (resid > boundary) { 3274 resid -= boundary; 3275 dout("%s con %p was sending front\n", __func__, con); 3276 prepare_zero_front(con, resid); 3277 if (middle_len(con->out_msg)) 3278 prepare_zero_middle(con, middle_len(con->out_msg)); 3279 prepare_zero_data(con); 3280 queue_zeros(con); 3281 return; 3282 } 3283 3284 WARN_ON(!resid); 3285 dout("%s con %p was sending middle\n", __func__, con); 3286 prepare_zero_middle(con, resid); 3287 prepare_zero_data(con); 3288 queue_zeros(con); 3289 } 3290 3291 static void revoke_at_queue_data_cont(struct ceph_connection *con) 3292 { 3293 int sent, resid; /* current piece of data */ 3294 3295 WARN_ON(!data_len(con->out_msg)); 3296 WARN_ON(!iov_iter_is_bvec(&con->v2.out_iter)); 3297 resid = iov_iter_count(&con->v2.out_iter); 3298 WARN_ON(!resid || resid > con->v2.out_bvec.bv_len); 3299 sent = con->v2.out_bvec.bv_len - resid; 3300 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid); 3301 3302 if (sent) { 3303 con->v2.out_epil.data_crc = ceph_crc32c_page( 3304 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page, 3305 con->v2.out_bvec.bv_offset, sent); 3306 ceph_msg_data_advance(&con->v2.out_cursor, sent); 3307 } 3308 WARN_ON(resid > con->v2.out_cursor.total_resid); 3309 con->v2.out_epil.data_crc = crc32c_zeros(con->v2.out_epil.data_crc, 3310 con->v2.out_cursor.total_resid); 3311 3312 con->v2.out_iter.count -= resid; 3313 out_zero_add(con, con->v2.out_cursor.total_resid); 3314 queue_zeros(con); 3315 } 3316 3317 static void revoke_at_finish_message(struct ceph_connection *con) 3318 { 3319 int boundary; 3320 int resid; 3321 3322 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter)); 3323 resid = iov_iter_count(&con->v2.out_iter); 3324 3325 if (!front_len(con->out_msg) && !middle_len(con->out_msg) && 3326 !data_len(con->out_msg)) { 3327 WARN_ON(!resid || resid > MESSAGE_HEAD_PLAIN_LEN); 3328 dout("%s con %p was sending head (empty message) - noop\n", 3329 __func__, con); 3330 return; 3331 } 3332 3333 boundary = front_len(con->out_msg) + middle_len(con->out_msg) + 3334 CEPH_EPILOGUE_PLAIN_LEN; 3335 if (resid > boundary) { 3336 resid -= boundary; 3337 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN); 3338 dout("%s con %p was sending head\n", __func__, con); 3339 if (front_len(con->out_msg)) 3340 prepare_zero_front(con, front_len(con->out_msg)); 3341 if (middle_len(con->out_msg)) 3342 prepare_zero_middle(con, middle_len(con->out_msg)); 3343 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN; 3344 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid); 3345 con->v2.out_state = OUT_S_QUEUE_ZEROS; 3346 return; 3347 } 3348 3349 boundary = middle_len(con->out_msg) + CEPH_EPILOGUE_PLAIN_LEN; 3350 if (resid > boundary) { 3351 resid -= boundary; 3352 dout("%s con %p was sending front\n", __func__, con); 3353 prepare_zero_front(con, resid); 3354 if (middle_len(con->out_msg)) 3355 prepare_zero_middle(con, middle_len(con->out_msg)); 3356 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN; 3357 queue_zeros(con); 3358 return; 3359 } 3360 3361 boundary = CEPH_EPILOGUE_PLAIN_LEN; 3362 if (resid > boundary) { 3363 resid -= boundary; 3364 dout("%s con %p was sending middle\n", __func__, con); 3365 prepare_zero_middle(con, resid); 3366 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN; 3367 queue_zeros(con); 3368 return; 3369 } 3370 3371 WARN_ON(!resid); 3372 dout("%s con %p was sending epilogue - noop\n", __func__, con); 3373 } 3374 3375 void ceph_con_v2_revoke(struct ceph_connection *con) 3376 { 3377 WARN_ON(con->v2.out_zero); 3378 3379 if (con_secure(con)) { 3380 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE && 3381 con->v2.out_state != OUT_S_FINISH_MESSAGE); 3382 dout("%s con %p secure - noop\n", __func__, con); 3383 return; 3384 } 3385 3386 switch (con->v2.out_state) { 3387 case OUT_S_QUEUE_DATA: 3388 revoke_at_queue_data(con); 3389 break; 3390 case OUT_S_QUEUE_DATA_CONT: 3391 revoke_at_queue_data_cont(con); 3392 break; 3393 case OUT_S_FINISH_MESSAGE: 3394 revoke_at_finish_message(con); 3395 break; 3396 default: 3397 WARN(1, "bad out_state %d", con->v2.out_state); 3398 break; 3399 } 3400 } 3401 3402 static void revoke_at_prepare_read_data(struct ceph_connection *con) 3403 { 3404 int remaining; 3405 int resid; 3406 3407 WARN_ON(con_secure(con)); 3408 WARN_ON(!data_len(con->in_msg)); 3409 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter)); 3410 resid = iov_iter_count(&con->v2.in_iter); 3411 WARN_ON(!resid); 3412 3413 remaining = data_len(con->in_msg) + CEPH_EPILOGUE_PLAIN_LEN; 3414 dout("%s con %p resid %d remaining %d\n", __func__, con, resid, 3415 remaining); 3416 con->v2.in_iter.count -= resid; 3417 set_in_skip(con, resid + remaining); 3418 con->v2.in_state = IN_S_FINISH_SKIP; 3419 } 3420 3421 static void revoke_at_prepare_read_data_cont(struct ceph_connection *con) 3422 { 3423 int recved, resid; /* current piece of data */ 3424 int remaining; 3425 3426 WARN_ON(con_secure(con)); 3427 WARN_ON(!data_len(con->in_msg)); 3428 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter)); 3429 resid = iov_iter_count(&con->v2.in_iter); 3430 WARN_ON(!resid || resid > con->v2.in_bvec.bv_len); 3431 recved = con->v2.in_bvec.bv_len - resid; 3432 dout("%s con %p recved %d resid %d\n", __func__, con, recved, resid); 3433 3434 if (recved) 3435 ceph_msg_data_advance(&con->v2.in_cursor, recved); 3436 WARN_ON(resid > con->v2.in_cursor.total_resid); 3437 3438 remaining = CEPH_EPILOGUE_PLAIN_LEN; 3439 dout("%s con %p total_resid %zu remaining %d\n", __func__, con, 3440 con->v2.in_cursor.total_resid, remaining); 3441 con->v2.in_iter.count -= resid; 3442 set_in_skip(con, con->v2.in_cursor.total_resid + remaining); 3443 con->v2.in_state = IN_S_FINISH_SKIP; 3444 } 3445 3446 static void revoke_at_prepare_read_enc_page(struct ceph_connection *con) 3447 { 3448 int resid; /* current enc page (not necessarily data) */ 3449 3450 WARN_ON(!con_secure(con)); 3451 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter)); 3452 resid = iov_iter_count(&con->v2.in_iter); 3453 WARN_ON(!resid || resid > con->v2.in_bvec.bv_len); 3454 3455 dout("%s con %p resid %d enc_resid %d\n", __func__, con, resid, 3456 con->v2.in_enc_resid); 3457 con->v2.in_iter.count -= resid; 3458 set_in_skip(con, resid + con->v2.in_enc_resid); 3459 con->v2.in_state = IN_S_FINISH_SKIP; 3460 } 3461 3462 static void revoke_at_handle_epilogue(struct ceph_connection *con) 3463 { 3464 int resid; 3465 3466 resid = iov_iter_count(&con->v2.in_iter); 3467 WARN_ON(!resid); 3468 3469 dout("%s con %p resid %d\n", __func__, con, resid); 3470 con->v2.in_iter.count -= resid; 3471 set_in_skip(con, resid); 3472 con->v2.in_state = IN_S_FINISH_SKIP; 3473 } 3474 3475 void ceph_con_v2_revoke_incoming(struct ceph_connection *con) 3476 { 3477 switch (con->v2.in_state) { 3478 case IN_S_PREPARE_READ_DATA: 3479 revoke_at_prepare_read_data(con); 3480 break; 3481 case IN_S_PREPARE_READ_DATA_CONT: 3482 revoke_at_prepare_read_data_cont(con); 3483 break; 3484 case IN_S_PREPARE_READ_ENC_PAGE: 3485 revoke_at_prepare_read_enc_page(con); 3486 break; 3487 case IN_S_HANDLE_EPILOGUE: 3488 revoke_at_handle_epilogue(con); 3489 break; 3490 default: 3491 WARN(1, "bad in_state %d", con->v2.in_state); 3492 break; 3493 } 3494 } 3495 3496 bool ceph_con_v2_opened(struct ceph_connection *con) 3497 { 3498 return con->v2.peer_global_seq; 3499 } 3500 3501 void ceph_con_v2_reset_session(struct ceph_connection *con) 3502 { 3503 con->v2.client_cookie = 0; 3504 con->v2.server_cookie = 0; 3505 con->v2.global_seq = 0; 3506 con->v2.connect_seq = 0; 3507 con->v2.peer_global_seq = 0; 3508 } 3509 3510 void ceph_con_v2_reset_protocol(struct ceph_connection *con) 3511 { 3512 iov_iter_truncate(&con->v2.in_iter, 0); 3513 iov_iter_truncate(&con->v2.out_iter, 0); 3514 con->v2.out_zero = 0; 3515 3516 clear_in_sign_kvecs(con); 3517 clear_out_sign_kvecs(con); 3518 free_conn_bufs(con); 3519 3520 if (con->v2.in_enc_pages) { 3521 WARN_ON(!con->v2.in_enc_page_cnt); 3522 ceph_release_page_vector(con->v2.in_enc_pages, 3523 con->v2.in_enc_page_cnt); 3524 con->v2.in_enc_pages = NULL; 3525 con->v2.in_enc_page_cnt = 0; 3526 } 3527 if (con->v2.out_enc_pages) { 3528 WARN_ON(!con->v2.out_enc_page_cnt); 3529 ceph_release_page_vector(con->v2.out_enc_pages, 3530 con->v2.out_enc_page_cnt); 3531 con->v2.out_enc_pages = NULL; 3532 con->v2.out_enc_page_cnt = 0; 3533 } 3534 3535 con->v2.con_mode = CEPH_CON_MODE_UNKNOWN; 3536 memzero_explicit(&con->v2.in_gcm_nonce, CEPH_GCM_IV_LEN); 3537 memzero_explicit(&con->v2.out_gcm_nonce, CEPH_GCM_IV_LEN); 3538 3539 if (con->v2.hmac_tfm) { 3540 crypto_free_shash(con->v2.hmac_tfm); 3541 con->v2.hmac_tfm = NULL; 3542 } 3543 if (con->v2.gcm_req) { 3544 aead_request_free(con->v2.gcm_req); 3545 con->v2.gcm_req = NULL; 3546 } 3547 if (con->v2.gcm_tfm) { 3548 crypto_free_aead(con->v2.gcm_tfm); 3549 con->v2.gcm_tfm = NULL; 3550 } 3551 } 3552