1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Ceph msgr2 protocol implementation 4 * 5 * Copyright (C) 2020 Ilya Dryomov <idryomov@gmail.com> 6 */ 7 8 #include <linux/ceph/ceph_debug.h> 9 10 #include <crypto/aead.h> 11 #include <crypto/hash.h> 12 #include <crypto/sha2.h> 13 #include <crypto/utils.h> 14 #include <linux/bvec.h> 15 #include <linux/crc32c.h> 16 #include <linux/net.h> 17 #include <linux/scatterlist.h> 18 #include <linux/socket.h> 19 #include <linux/sched/mm.h> 20 #include <net/sock.h> 21 #include <net/tcp.h> 22 23 #include <linux/ceph/ceph_features.h> 24 #include <linux/ceph/decode.h> 25 #include <linux/ceph/libceph.h> 26 #include <linux/ceph/messenger.h> 27 28 #include "crypto.h" /* for CEPH_KEY_LEN and CEPH_MAX_CON_SECRET_LEN */ 29 30 #define FRAME_TAG_HELLO 1 31 #define FRAME_TAG_AUTH_REQUEST 2 32 #define FRAME_TAG_AUTH_BAD_METHOD 3 33 #define FRAME_TAG_AUTH_REPLY_MORE 4 34 #define FRAME_TAG_AUTH_REQUEST_MORE 5 35 #define FRAME_TAG_AUTH_DONE 6 36 #define FRAME_TAG_AUTH_SIGNATURE 7 37 #define FRAME_TAG_CLIENT_IDENT 8 38 #define FRAME_TAG_SERVER_IDENT 9 39 #define FRAME_TAG_IDENT_MISSING_FEATURES 10 40 #define FRAME_TAG_SESSION_RECONNECT 11 41 #define FRAME_TAG_SESSION_RESET 12 42 #define FRAME_TAG_SESSION_RETRY 13 43 #define FRAME_TAG_SESSION_RETRY_GLOBAL 14 44 #define FRAME_TAG_SESSION_RECONNECT_OK 15 45 #define FRAME_TAG_WAIT 16 46 #define FRAME_TAG_MESSAGE 17 47 #define FRAME_TAG_KEEPALIVE2 18 48 #define FRAME_TAG_KEEPALIVE2_ACK 19 49 #define FRAME_TAG_ACK 20 50 51 #define FRAME_LATE_STATUS_ABORTED 0x1 52 #define FRAME_LATE_STATUS_COMPLETE 0xe 53 #define FRAME_LATE_STATUS_ABORTED_MASK 0xf 54 55 #define IN_S_HANDLE_PREAMBLE 1 56 #define IN_S_HANDLE_CONTROL 2 57 #define IN_S_HANDLE_CONTROL_REMAINDER 3 58 #define IN_S_PREPARE_READ_DATA 4 59 #define IN_S_PREPARE_READ_DATA_CONT 5 60 #define IN_S_PREPARE_READ_ENC_PAGE 6 61 #define IN_S_PREPARE_SPARSE_DATA 7 62 #define IN_S_PREPARE_SPARSE_DATA_CONT 8 63 #define IN_S_HANDLE_EPILOGUE 9 64 #define IN_S_FINISH_SKIP 10 65 66 #define OUT_S_QUEUE_DATA 1 67 #define OUT_S_QUEUE_DATA_CONT 2 68 #define OUT_S_QUEUE_ENC_PAGE 3 69 #define OUT_S_QUEUE_ZEROS 4 70 #define OUT_S_FINISH_MESSAGE 5 71 #define OUT_S_GET_NEXT 6 72 73 #define CTRL_BODY(p) ((void *)(p) + CEPH_PREAMBLE_LEN) 74 #define FRONT_PAD(p) ((void *)(p) + CEPH_EPILOGUE_SECURE_LEN) 75 #define MIDDLE_PAD(p) (FRONT_PAD(p) + CEPH_GCM_BLOCK_LEN) 76 #define DATA_PAD(p) (MIDDLE_PAD(p) + CEPH_GCM_BLOCK_LEN) 77 78 #define CEPH_MSG_FLAGS (MSG_DONTWAIT | MSG_NOSIGNAL) 79 80 static int do_recvmsg(struct socket *sock, struct iov_iter *it) 81 { 82 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS }; 83 int ret; 84 85 msg.msg_iter = *it; 86 while (iov_iter_count(it)) { 87 ret = sock_recvmsg(sock, &msg, msg.msg_flags); 88 if (ret <= 0) { 89 if (ret == -EAGAIN) 90 ret = 0; 91 return ret; 92 } 93 94 iov_iter_advance(it, ret); 95 } 96 97 WARN_ON(msg_data_left(&msg)); 98 return 1; 99 } 100 101 /* 102 * Read as much as possible. 103 * 104 * Return: 105 * 1 - done, nothing (else) to read 106 * 0 - socket is empty, need to wait 107 * <0 - error 108 */ 109 static int ceph_tcp_recv(struct ceph_connection *con) 110 { 111 int ret; 112 113 dout("%s con %p %s %zu\n", __func__, con, 114 iov_iter_is_discard(&con->v2.in_iter) ? "discard" : "need", 115 iov_iter_count(&con->v2.in_iter)); 116 ret = do_recvmsg(con->sock, &con->v2.in_iter); 117 dout("%s con %p ret %d left %zu\n", __func__, con, ret, 118 iov_iter_count(&con->v2.in_iter)); 119 return ret; 120 } 121 122 static int do_sendmsg(struct socket *sock, struct iov_iter *it) 123 { 124 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS }; 125 int ret; 126 127 msg.msg_iter = *it; 128 while (iov_iter_count(it)) { 129 ret = sock_sendmsg(sock, &msg); 130 if (ret <= 0) { 131 if (ret == -EAGAIN) 132 ret = 0; 133 return ret; 134 } 135 136 iov_iter_advance(it, ret); 137 } 138 139 WARN_ON(msg_data_left(&msg)); 140 return 1; 141 } 142 143 static int do_try_sendpage(struct socket *sock, struct iov_iter *it) 144 { 145 struct msghdr msg = { .msg_flags = CEPH_MSG_FLAGS }; 146 struct bio_vec bv; 147 int ret; 148 149 if (WARN_ON(!iov_iter_is_bvec(it))) 150 return -EINVAL; 151 152 while (iov_iter_count(it)) { 153 /* iov_iter_iovec() for ITER_BVEC */ 154 bvec_set_page(&bv, it->bvec->bv_page, 155 min(iov_iter_count(it), 156 it->bvec->bv_len - it->iov_offset), 157 it->bvec->bv_offset + it->iov_offset); 158 159 /* 160 * MSG_SPLICE_PAGES cannot properly handle pages with 161 * page_count == 0, we need to fall back to sendmsg if 162 * that's the case. 163 * 164 * Same goes for slab pages: skb_can_coalesce() allows 165 * coalescing neighboring slab objects into a single frag 166 * which triggers one of hardened usercopy checks. 167 */ 168 if (sendpage_ok(bv.bv_page)) 169 msg.msg_flags |= MSG_SPLICE_PAGES; 170 else 171 msg.msg_flags &= ~MSG_SPLICE_PAGES; 172 173 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bv, 1, bv.bv_len); 174 ret = sock_sendmsg(sock, &msg); 175 if (ret <= 0) { 176 if (ret == -EAGAIN) 177 ret = 0; 178 return ret; 179 } 180 181 iov_iter_advance(it, ret); 182 } 183 184 return 1; 185 } 186 187 /* 188 * Write as much as possible. The socket is expected to be corked, 189 * so we don't bother with MSG_MORE here. 190 * 191 * Return: 192 * 1 - done, nothing (else) to write 193 * 0 - socket is full, need to wait 194 * <0 - error 195 */ 196 static int ceph_tcp_send(struct ceph_connection *con) 197 { 198 int ret; 199 200 dout("%s con %p have %zu try_sendpage %d\n", __func__, con, 201 iov_iter_count(&con->v2.out_iter), con->v2.out_iter_sendpage); 202 if (con->v2.out_iter_sendpage) 203 ret = do_try_sendpage(con->sock, &con->v2.out_iter); 204 else 205 ret = do_sendmsg(con->sock, &con->v2.out_iter); 206 dout("%s con %p ret %d left %zu\n", __func__, con, ret, 207 iov_iter_count(&con->v2.out_iter)); 208 return ret; 209 } 210 211 static void add_in_kvec(struct ceph_connection *con, void *buf, int len) 212 { 213 BUG_ON(con->v2.in_kvec_cnt >= ARRAY_SIZE(con->v2.in_kvecs)); 214 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter)); 215 216 con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_base = buf; 217 con->v2.in_kvecs[con->v2.in_kvec_cnt].iov_len = len; 218 con->v2.in_kvec_cnt++; 219 220 con->v2.in_iter.nr_segs++; 221 con->v2.in_iter.count += len; 222 } 223 224 static void reset_in_kvecs(struct ceph_connection *con) 225 { 226 WARN_ON(iov_iter_count(&con->v2.in_iter)); 227 228 con->v2.in_kvec_cnt = 0; 229 iov_iter_kvec(&con->v2.in_iter, ITER_DEST, con->v2.in_kvecs, 0, 0); 230 } 231 232 static void set_in_bvec(struct ceph_connection *con, const struct bio_vec *bv) 233 { 234 WARN_ON(iov_iter_count(&con->v2.in_iter)); 235 236 con->v2.in_bvec = *bv; 237 iov_iter_bvec(&con->v2.in_iter, ITER_DEST, &con->v2.in_bvec, 1, bv->bv_len); 238 } 239 240 static void set_in_skip(struct ceph_connection *con, int len) 241 { 242 WARN_ON(iov_iter_count(&con->v2.in_iter)); 243 244 dout("%s con %p len %d\n", __func__, con, len); 245 iov_iter_discard(&con->v2.in_iter, ITER_DEST, len); 246 } 247 248 static void add_out_kvec(struct ceph_connection *con, void *buf, int len) 249 { 250 BUG_ON(con->v2.out_kvec_cnt >= ARRAY_SIZE(con->v2.out_kvecs)); 251 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter)); 252 WARN_ON(con->v2.out_zero); 253 254 con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_base = buf; 255 con->v2.out_kvecs[con->v2.out_kvec_cnt].iov_len = len; 256 con->v2.out_kvec_cnt++; 257 258 con->v2.out_iter.nr_segs++; 259 con->v2.out_iter.count += len; 260 } 261 262 static void reset_out_kvecs(struct ceph_connection *con) 263 { 264 WARN_ON(iov_iter_count(&con->v2.out_iter)); 265 WARN_ON(con->v2.out_zero); 266 267 con->v2.out_kvec_cnt = 0; 268 269 iov_iter_kvec(&con->v2.out_iter, ITER_SOURCE, con->v2.out_kvecs, 0, 0); 270 con->v2.out_iter_sendpage = false; 271 } 272 273 static void set_out_bvec(struct ceph_connection *con, const struct bio_vec *bv, 274 bool zerocopy) 275 { 276 WARN_ON(iov_iter_count(&con->v2.out_iter)); 277 WARN_ON(con->v2.out_zero); 278 279 con->v2.out_bvec = *bv; 280 con->v2.out_iter_sendpage = zerocopy; 281 iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1, 282 con->v2.out_bvec.bv_len); 283 } 284 285 static void set_out_bvec_zero(struct ceph_connection *con) 286 { 287 WARN_ON(iov_iter_count(&con->v2.out_iter)); 288 WARN_ON(!con->v2.out_zero); 289 290 bvec_set_page(&con->v2.out_bvec, ceph_zero_page, 291 min(con->v2.out_zero, (int)PAGE_SIZE), 0); 292 con->v2.out_iter_sendpage = true; 293 iov_iter_bvec(&con->v2.out_iter, ITER_SOURCE, &con->v2.out_bvec, 1, 294 con->v2.out_bvec.bv_len); 295 } 296 297 static void out_zero_add(struct ceph_connection *con, int len) 298 { 299 dout("%s con %p len %d\n", __func__, con, len); 300 con->v2.out_zero += len; 301 } 302 303 static void *alloc_conn_buf(struct ceph_connection *con, int len) 304 { 305 void *buf; 306 307 dout("%s con %p len %d\n", __func__, con, len); 308 309 if (WARN_ON(con->v2.conn_buf_cnt >= ARRAY_SIZE(con->v2.conn_bufs))) 310 return NULL; 311 312 buf = kvmalloc(len, GFP_NOIO); 313 if (!buf) 314 return NULL; 315 316 con->v2.conn_bufs[con->v2.conn_buf_cnt++] = buf; 317 return buf; 318 } 319 320 static void free_conn_bufs(struct ceph_connection *con) 321 { 322 while (con->v2.conn_buf_cnt) 323 kvfree(con->v2.conn_bufs[--con->v2.conn_buf_cnt]); 324 } 325 326 static void add_in_sign_kvec(struct ceph_connection *con, void *buf, int len) 327 { 328 BUG_ON(con->v2.in_sign_kvec_cnt >= ARRAY_SIZE(con->v2.in_sign_kvecs)); 329 330 con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_base = buf; 331 con->v2.in_sign_kvecs[con->v2.in_sign_kvec_cnt].iov_len = len; 332 con->v2.in_sign_kvec_cnt++; 333 } 334 335 static void clear_in_sign_kvecs(struct ceph_connection *con) 336 { 337 con->v2.in_sign_kvec_cnt = 0; 338 } 339 340 static void add_out_sign_kvec(struct ceph_connection *con, void *buf, int len) 341 { 342 BUG_ON(con->v2.out_sign_kvec_cnt >= ARRAY_SIZE(con->v2.out_sign_kvecs)); 343 344 con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_base = buf; 345 con->v2.out_sign_kvecs[con->v2.out_sign_kvec_cnt].iov_len = len; 346 con->v2.out_sign_kvec_cnt++; 347 } 348 349 static void clear_out_sign_kvecs(struct ceph_connection *con) 350 { 351 con->v2.out_sign_kvec_cnt = 0; 352 } 353 354 static bool con_secure(struct ceph_connection *con) 355 { 356 return con->v2.con_mode == CEPH_CON_MODE_SECURE; 357 } 358 359 static int front_len(const struct ceph_msg *msg) 360 { 361 return le32_to_cpu(msg->hdr.front_len); 362 } 363 364 static int middle_len(const struct ceph_msg *msg) 365 { 366 return le32_to_cpu(msg->hdr.middle_len); 367 } 368 369 static int data_len(const struct ceph_msg *msg) 370 { 371 return le32_to_cpu(msg->hdr.data_len); 372 } 373 374 static bool need_padding(int len) 375 { 376 return !IS_ALIGNED(len, CEPH_GCM_BLOCK_LEN); 377 } 378 379 static int padded_len(int len) 380 { 381 return ALIGN(len, CEPH_GCM_BLOCK_LEN); 382 } 383 384 static int padding_len(int len) 385 { 386 return padded_len(len) - len; 387 } 388 389 /* preamble + control segment */ 390 static int head_onwire_len(int ctrl_len, bool secure) 391 { 392 int head_len; 393 int rem_len; 394 395 BUG_ON(ctrl_len < 0 || ctrl_len > CEPH_MSG_MAX_CONTROL_LEN); 396 397 if (secure) { 398 head_len = CEPH_PREAMBLE_SECURE_LEN; 399 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) { 400 rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN; 401 head_len += padded_len(rem_len) + CEPH_GCM_TAG_LEN; 402 } 403 } else { 404 head_len = CEPH_PREAMBLE_PLAIN_LEN; 405 if (ctrl_len) 406 head_len += ctrl_len + CEPH_CRC_LEN; 407 } 408 return head_len; 409 } 410 411 /* front, middle and data segments + epilogue */ 412 static int __tail_onwire_len(int front_len, int middle_len, int data_len, 413 bool secure) 414 { 415 BUG_ON(front_len < 0 || front_len > CEPH_MSG_MAX_FRONT_LEN || 416 middle_len < 0 || middle_len > CEPH_MSG_MAX_MIDDLE_LEN || 417 data_len < 0 || data_len > CEPH_MSG_MAX_DATA_LEN); 418 419 if (!front_len && !middle_len && !data_len) 420 return 0; 421 422 if (!secure) 423 return front_len + middle_len + data_len + 424 CEPH_EPILOGUE_PLAIN_LEN; 425 426 return padded_len(front_len) + padded_len(middle_len) + 427 padded_len(data_len) + CEPH_EPILOGUE_SECURE_LEN; 428 } 429 430 static int tail_onwire_len(const struct ceph_msg *msg, bool secure) 431 { 432 return __tail_onwire_len(front_len(msg), middle_len(msg), 433 data_len(msg), secure); 434 } 435 436 /* head_onwire_len(sizeof(struct ceph_msg_header2), false) */ 437 #define MESSAGE_HEAD_PLAIN_LEN (CEPH_PREAMBLE_PLAIN_LEN + \ 438 sizeof(struct ceph_msg_header2) + \ 439 CEPH_CRC_LEN) 440 441 static const int frame_aligns[] = { 442 sizeof(void *), 443 sizeof(void *), 444 sizeof(void *), 445 PAGE_SIZE 446 }; 447 448 /* 449 * Discards trailing empty segments, unless there is just one segment. 450 * A frame always has at least one (possibly empty) segment. 451 */ 452 static int calc_segment_count(const int *lens, int len_cnt) 453 { 454 int i; 455 456 for (i = len_cnt - 1; i >= 0; i--) { 457 if (lens[i]) 458 return i + 1; 459 } 460 461 return 1; 462 } 463 464 static void init_frame_desc(struct ceph_frame_desc *desc, int tag, 465 const int *lens, int len_cnt) 466 { 467 int i; 468 469 memset(desc, 0, sizeof(*desc)); 470 471 desc->fd_tag = tag; 472 desc->fd_seg_cnt = calc_segment_count(lens, len_cnt); 473 BUG_ON(desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT); 474 for (i = 0; i < desc->fd_seg_cnt; i++) { 475 desc->fd_lens[i] = lens[i]; 476 desc->fd_aligns[i] = frame_aligns[i]; 477 } 478 } 479 480 /* 481 * Preamble crc covers everything up to itself (28 bytes) and 482 * is calculated and verified irrespective of the connection mode 483 * (i.e. even if the frame is encrypted). 484 */ 485 static void encode_preamble(const struct ceph_frame_desc *desc, void *p) 486 { 487 void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN; 488 void *start = p; 489 int i; 490 491 memset(p, 0, CEPH_PREAMBLE_LEN); 492 493 ceph_encode_8(&p, desc->fd_tag); 494 ceph_encode_8(&p, desc->fd_seg_cnt); 495 for (i = 0; i < desc->fd_seg_cnt; i++) { 496 ceph_encode_32(&p, desc->fd_lens[i]); 497 ceph_encode_16(&p, desc->fd_aligns[i]); 498 } 499 500 put_unaligned_le32(crc32c(0, start, crcp - start), crcp); 501 } 502 503 static int decode_preamble(void *p, struct ceph_frame_desc *desc) 504 { 505 void *crcp = p + CEPH_PREAMBLE_LEN - CEPH_CRC_LEN; 506 u32 crc, expected_crc; 507 int i; 508 509 crc = crc32c(0, p, crcp - p); 510 expected_crc = get_unaligned_le32(crcp); 511 if (crc != expected_crc) { 512 pr_err("bad preamble crc, calculated %u, expected %u\n", 513 crc, expected_crc); 514 return -EBADMSG; 515 } 516 517 memset(desc, 0, sizeof(*desc)); 518 519 desc->fd_tag = ceph_decode_8(&p); 520 desc->fd_seg_cnt = ceph_decode_8(&p); 521 if (desc->fd_seg_cnt < 1 || 522 desc->fd_seg_cnt > CEPH_FRAME_MAX_SEGMENT_COUNT) { 523 pr_err("bad segment count %d\n", desc->fd_seg_cnt); 524 return -EINVAL; 525 } 526 for (i = 0; i < desc->fd_seg_cnt; i++) { 527 desc->fd_lens[i] = ceph_decode_32(&p); 528 desc->fd_aligns[i] = ceph_decode_16(&p); 529 } 530 531 if (desc->fd_lens[0] < 0 || 532 desc->fd_lens[0] > CEPH_MSG_MAX_CONTROL_LEN) { 533 pr_err("bad control segment length %d\n", desc->fd_lens[0]); 534 return -EINVAL; 535 } 536 if (desc->fd_lens[1] < 0 || 537 desc->fd_lens[1] > CEPH_MSG_MAX_FRONT_LEN) { 538 pr_err("bad front segment length %d\n", desc->fd_lens[1]); 539 return -EINVAL; 540 } 541 if (desc->fd_lens[2] < 0 || 542 desc->fd_lens[2] > CEPH_MSG_MAX_MIDDLE_LEN) { 543 pr_err("bad middle segment length %d\n", desc->fd_lens[2]); 544 return -EINVAL; 545 } 546 if (desc->fd_lens[3] < 0 || 547 desc->fd_lens[3] > CEPH_MSG_MAX_DATA_LEN) { 548 pr_err("bad data segment length %d\n", desc->fd_lens[3]); 549 return -EINVAL; 550 } 551 552 /* 553 * This would fire for FRAME_TAG_WAIT (it has one empty 554 * segment), but we should never get it as client. 555 */ 556 if (!desc->fd_lens[desc->fd_seg_cnt - 1]) { 557 pr_err("last segment empty, segment count %d\n", 558 desc->fd_seg_cnt); 559 return -EINVAL; 560 } 561 562 return 0; 563 } 564 565 static void encode_epilogue_plain(struct ceph_connection *con, bool aborted) 566 { 567 con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED : 568 FRAME_LATE_STATUS_COMPLETE; 569 cpu_to_le32s(&con->v2.out_epil.front_crc); 570 cpu_to_le32s(&con->v2.out_epil.middle_crc); 571 cpu_to_le32s(&con->v2.out_epil.data_crc); 572 } 573 574 static void encode_epilogue_secure(struct ceph_connection *con, bool aborted) 575 { 576 memset(&con->v2.out_epil, 0, sizeof(con->v2.out_epil)); 577 con->v2.out_epil.late_status = aborted ? FRAME_LATE_STATUS_ABORTED : 578 FRAME_LATE_STATUS_COMPLETE; 579 } 580 581 static int decode_epilogue(void *p, u32 *front_crc, u32 *middle_crc, 582 u32 *data_crc) 583 { 584 u8 late_status; 585 586 late_status = ceph_decode_8(&p); 587 if ((late_status & FRAME_LATE_STATUS_ABORTED_MASK) != 588 FRAME_LATE_STATUS_COMPLETE) { 589 /* we should never get an aborted message as client */ 590 pr_err("bad late_status 0x%x\n", late_status); 591 return -EINVAL; 592 } 593 594 if (front_crc && middle_crc && data_crc) { 595 *front_crc = ceph_decode_32(&p); 596 *middle_crc = ceph_decode_32(&p); 597 *data_crc = ceph_decode_32(&p); 598 } 599 600 return 0; 601 } 602 603 static void fill_header(struct ceph_msg_header *hdr, 604 const struct ceph_msg_header2 *hdr2, 605 int front_len, int middle_len, int data_len, 606 const struct ceph_entity_name *peer_name) 607 { 608 hdr->seq = hdr2->seq; 609 hdr->tid = hdr2->tid; 610 hdr->type = hdr2->type; 611 hdr->priority = hdr2->priority; 612 hdr->version = hdr2->version; 613 hdr->front_len = cpu_to_le32(front_len); 614 hdr->middle_len = cpu_to_le32(middle_len); 615 hdr->data_len = cpu_to_le32(data_len); 616 hdr->data_off = hdr2->data_off; 617 hdr->src = *peer_name; 618 hdr->compat_version = hdr2->compat_version; 619 hdr->reserved = 0; 620 hdr->crc = 0; 621 } 622 623 static void fill_header2(struct ceph_msg_header2 *hdr2, 624 const struct ceph_msg_header *hdr, u64 ack_seq) 625 { 626 hdr2->seq = hdr->seq; 627 hdr2->tid = hdr->tid; 628 hdr2->type = hdr->type; 629 hdr2->priority = hdr->priority; 630 hdr2->version = hdr->version; 631 hdr2->data_pre_padding_len = 0; 632 hdr2->data_off = hdr->data_off; 633 hdr2->ack_seq = cpu_to_le64(ack_seq); 634 hdr2->flags = 0; 635 hdr2->compat_version = hdr->compat_version; 636 hdr2->reserved = 0; 637 } 638 639 static int verify_control_crc(struct ceph_connection *con) 640 { 641 int ctrl_len = con->v2.in_desc.fd_lens[0]; 642 u32 crc, expected_crc; 643 644 WARN_ON(con->v2.in_kvecs[0].iov_len != ctrl_len); 645 WARN_ON(con->v2.in_kvecs[1].iov_len != CEPH_CRC_LEN); 646 647 crc = crc32c(-1, con->v2.in_kvecs[0].iov_base, ctrl_len); 648 expected_crc = get_unaligned_le32(con->v2.in_kvecs[1].iov_base); 649 if (crc != expected_crc) { 650 pr_err("bad control crc, calculated %u, expected %u\n", 651 crc, expected_crc); 652 return -EBADMSG; 653 } 654 655 return 0; 656 } 657 658 static int verify_epilogue_crcs(struct ceph_connection *con, u32 front_crc, 659 u32 middle_crc, u32 data_crc) 660 { 661 if (front_len(con->in_msg)) { 662 con->in_front_crc = crc32c(-1, con->in_msg->front.iov_base, 663 front_len(con->in_msg)); 664 } else { 665 WARN_ON(!middle_len(con->in_msg) && !data_len(con->in_msg)); 666 con->in_front_crc = -1; 667 } 668 669 if (middle_len(con->in_msg)) 670 con->in_middle_crc = crc32c(-1, 671 con->in_msg->middle->vec.iov_base, 672 middle_len(con->in_msg)); 673 else if (data_len(con->in_msg)) 674 con->in_middle_crc = -1; 675 else 676 con->in_middle_crc = 0; 677 678 if (!data_len(con->in_msg)) 679 con->in_data_crc = 0; 680 681 dout("%s con %p msg %p crcs %u %u %u\n", __func__, con, con->in_msg, 682 con->in_front_crc, con->in_middle_crc, con->in_data_crc); 683 684 if (con->in_front_crc != front_crc) { 685 pr_err("bad front crc, calculated %u, expected %u\n", 686 con->in_front_crc, front_crc); 687 return -EBADMSG; 688 } 689 if (con->in_middle_crc != middle_crc) { 690 pr_err("bad middle crc, calculated %u, expected %u\n", 691 con->in_middle_crc, middle_crc); 692 return -EBADMSG; 693 } 694 if (con->in_data_crc != data_crc) { 695 pr_err("bad data crc, calculated %u, expected %u\n", 696 con->in_data_crc, data_crc); 697 return -EBADMSG; 698 } 699 700 return 0; 701 } 702 703 static int setup_crypto(struct ceph_connection *con, 704 const u8 *session_key, int session_key_len, 705 const u8 *con_secret, int con_secret_len) 706 { 707 unsigned int noio_flag; 708 int ret; 709 710 dout("%s con %p con_mode %d session_key_len %d con_secret_len %d\n", 711 __func__, con, con->v2.con_mode, session_key_len, con_secret_len); 712 WARN_ON(con->v2.hmac_tfm || con->v2.gcm_tfm || con->v2.gcm_req); 713 714 if (con->v2.con_mode != CEPH_CON_MODE_CRC && 715 con->v2.con_mode != CEPH_CON_MODE_SECURE) { 716 pr_err("bad con_mode %d\n", con->v2.con_mode); 717 return -EINVAL; 718 } 719 720 if (!session_key_len) { 721 WARN_ON(con->v2.con_mode != CEPH_CON_MODE_CRC); 722 WARN_ON(con_secret_len); 723 return 0; /* auth_none */ 724 } 725 726 noio_flag = memalloc_noio_save(); 727 con->v2.hmac_tfm = crypto_alloc_shash("hmac(sha256)", 0, 0); 728 memalloc_noio_restore(noio_flag); 729 if (IS_ERR(con->v2.hmac_tfm)) { 730 ret = PTR_ERR(con->v2.hmac_tfm); 731 con->v2.hmac_tfm = NULL; 732 pr_err("failed to allocate hmac tfm context: %d\n", ret); 733 return ret; 734 } 735 736 WARN_ON((unsigned long)session_key & 737 crypto_shash_alignmask(con->v2.hmac_tfm)); 738 ret = crypto_shash_setkey(con->v2.hmac_tfm, session_key, 739 session_key_len); 740 if (ret) { 741 pr_err("failed to set hmac key: %d\n", ret); 742 return ret; 743 } 744 745 if (con->v2.con_mode == CEPH_CON_MODE_CRC) { 746 WARN_ON(con_secret_len); 747 return 0; /* auth_x, plain mode */ 748 } 749 750 if (con_secret_len < CEPH_GCM_KEY_LEN + 2 * CEPH_GCM_IV_LEN) { 751 pr_err("con_secret too small %d\n", con_secret_len); 752 return -EINVAL; 753 } 754 755 noio_flag = memalloc_noio_save(); 756 con->v2.gcm_tfm = crypto_alloc_aead("gcm(aes)", 0, 0); 757 memalloc_noio_restore(noio_flag); 758 if (IS_ERR(con->v2.gcm_tfm)) { 759 ret = PTR_ERR(con->v2.gcm_tfm); 760 con->v2.gcm_tfm = NULL; 761 pr_err("failed to allocate gcm tfm context: %d\n", ret); 762 return ret; 763 } 764 765 WARN_ON((unsigned long)con_secret & 766 crypto_aead_alignmask(con->v2.gcm_tfm)); 767 ret = crypto_aead_setkey(con->v2.gcm_tfm, con_secret, CEPH_GCM_KEY_LEN); 768 if (ret) { 769 pr_err("failed to set gcm key: %d\n", ret); 770 return ret; 771 } 772 773 WARN_ON(crypto_aead_ivsize(con->v2.gcm_tfm) != CEPH_GCM_IV_LEN); 774 ret = crypto_aead_setauthsize(con->v2.gcm_tfm, CEPH_GCM_TAG_LEN); 775 if (ret) { 776 pr_err("failed to set gcm tag size: %d\n", ret); 777 return ret; 778 } 779 780 con->v2.gcm_req = aead_request_alloc(con->v2.gcm_tfm, GFP_NOIO); 781 if (!con->v2.gcm_req) { 782 pr_err("failed to allocate gcm request\n"); 783 return -ENOMEM; 784 } 785 786 crypto_init_wait(&con->v2.gcm_wait); 787 aead_request_set_callback(con->v2.gcm_req, CRYPTO_TFM_REQ_MAY_BACKLOG, 788 crypto_req_done, &con->v2.gcm_wait); 789 790 memcpy(&con->v2.in_gcm_nonce, con_secret + CEPH_GCM_KEY_LEN, 791 CEPH_GCM_IV_LEN); 792 memcpy(&con->v2.out_gcm_nonce, 793 con_secret + CEPH_GCM_KEY_LEN + CEPH_GCM_IV_LEN, 794 CEPH_GCM_IV_LEN); 795 return 0; /* auth_x, secure mode */ 796 } 797 798 static int hmac_sha256(struct ceph_connection *con, const struct kvec *kvecs, 799 int kvec_cnt, u8 *hmac) 800 { 801 SHASH_DESC_ON_STACK(desc, con->v2.hmac_tfm); /* tfm arg is ignored */ 802 int ret; 803 int i; 804 805 dout("%s con %p hmac_tfm %p kvec_cnt %d\n", __func__, con, 806 con->v2.hmac_tfm, kvec_cnt); 807 808 if (!con->v2.hmac_tfm) { 809 memset(hmac, 0, SHA256_DIGEST_SIZE); 810 return 0; /* auth_none */ 811 } 812 813 desc->tfm = con->v2.hmac_tfm; 814 ret = crypto_shash_init(desc); 815 if (ret) 816 goto out; 817 818 for (i = 0; i < kvec_cnt; i++) { 819 WARN_ON((unsigned long)kvecs[i].iov_base & 820 crypto_shash_alignmask(con->v2.hmac_tfm)); 821 ret = crypto_shash_update(desc, kvecs[i].iov_base, 822 kvecs[i].iov_len); 823 if (ret) 824 goto out; 825 } 826 827 ret = crypto_shash_final(desc, hmac); 828 829 out: 830 shash_desc_zero(desc); 831 return ret; /* auth_x, both plain and secure modes */ 832 } 833 834 static void gcm_inc_nonce(struct ceph_gcm_nonce *nonce) 835 { 836 u64 counter; 837 838 counter = le64_to_cpu(nonce->counter); 839 nonce->counter = cpu_to_le64(counter + 1); 840 } 841 842 static int gcm_crypt(struct ceph_connection *con, bool encrypt, 843 struct scatterlist *src, struct scatterlist *dst, 844 int src_len) 845 { 846 struct ceph_gcm_nonce *nonce; 847 int ret; 848 849 nonce = encrypt ? &con->v2.out_gcm_nonce : &con->v2.in_gcm_nonce; 850 851 aead_request_set_ad(con->v2.gcm_req, 0); /* no AAD */ 852 aead_request_set_crypt(con->v2.gcm_req, src, dst, src_len, (u8 *)nonce); 853 ret = crypto_wait_req(encrypt ? crypto_aead_encrypt(con->v2.gcm_req) : 854 crypto_aead_decrypt(con->v2.gcm_req), 855 &con->v2.gcm_wait); 856 if (ret) 857 return ret; 858 859 gcm_inc_nonce(nonce); 860 return 0; 861 } 862 863 static void get_bvec_at(struct ceph_msg_data_cursor *cursor, 864 struct bio_vec *bv) 865 { 866 struct page *page; 867 size_t off, len; 868 869 WARN_ON(!cursor->total_resid); 870 871 /* skip zero-length data items */ 872 while (!cursor->resid) 873 ceph_msg_data_advance(cursor, 0); 874 875 /* get a piece of data, cursor isn't advanced */ 876 page = ceph_msg_data_next(cursor, &off, &len); 877 bvec_set_page(bv, page, len, off); 878 } 879 880 static int calc_sg_cnt(void *buf, int buf_len) 881 { 882 int sg_cnt; 883 884 if (!buf_len) 885 return 0; 886 887 sg_cnt = need_padding(buf_len) ? 1 : 0; 888 if (is_vmalloc_addr(buf)) { 889 WARN_ON(offset_in_page(buf)); 890 sg_cnt += PAGE_ALIGN(buf_len) >> PAGE_SHIFT; 891 } else { 892 sg_cnt++; 893 } 894 895 return sg_cnt; 896 } 897 898 static int calc_sg_cnt_cursor(struct ceph_msg_data_cursor *cursor) 899 { 900 int data_len = cursor->total_resid; 901 struct bio_vec bv; 902 int sg_cnt; 903 904 if (!data_len) 905 return 0; 906 907 sg_cnt = need_padding(data_len) ? 1 : 0; 908 do { 909 get_bvec_at(cursor, &bv); 910 sg_cnt++; 911 912 ceph_msg_data_advance(cursor, bv.bv_len); 913 } while (cursor->total_resid); 914 915 return sg_cnt; 916 } 917 918 static void init_sgs(struct scatterlist **sg, void *buf, int buf_len, u8 *pad) 919 { 920 void *end = buf + buf_len; 921 struct page *page; 922 int len; 923 void *p; 924 925 if (!buf_len) 926 return; 927 928 if (is_vmalloc_addr(buf)) { 929 p = buf; 930 do { 931 page = vmalloc_to_page(p); 932 len = min_t(int, end - p, PAGE_SIZE); 933 WARN_ON(!page || !len || offset_in_page(p)); 934 sg_set_page(*sg, page, len, 0); 935 *sg = sg_next(*sg); 936 p += len; 937 } while (p != end); 938 } else { 939 sg_set_buf(*sg, buf, buf_len); 940 *sg = sg_next(*sg); 941 } 942 943 if (need_padding(buf_len)) { 944 sg_set_buf(*sg, pad, padding_len(buf_len)); 945 *sg = sg_next(*sg); 946 } 947 } 948 949 static void init_sgs_cursor(struct scatterlist **sg, 950 struct ceph_msg_data_cursor *cursor, u8 *pad) 951 { 952 int data_len = cursor->total_resid; 953 struct bio_vec bv; 954 955 if (!data_len) 956 return; 957 958 do { 959 get_bvec_at(cursor, &bv); 960 sg_set_page(*sg, bv.bv_page, bv.bv_len, bv.bv_offset); 961 *sg = sg_next(*sg); 962 963 ceph_msg_data_advance(cursor, bv.bv_len); 964 } while (cursor->total_resid); 965 966 if (need_padding(data_len)) { 967 sg_set_buf(*sg, pad, padding_len(data_len)); 968 *sg = sg_next(*sg); 969 } 970 } 971 972 /** 973 * init_sgs_pages: set up scatterlist on an array of page pointers 974 * @sg: scatterlist to populate 975 * @pages: pointer to page array 976 * @dpos: position in the array to start (bytes) 977 * @dlen: len to add to sg (bytes) 978 * @pad: pointer to pad destination (if any) 979 * 980 * Populate the scatterlist from the page array, starting at an arbitrary 981 * byte in the array and running for a specified length. 982 */ 983 static void init_sgs_pages(struct scatterlist **sg, struct page **pages, 984 int dpos, int dlen, u8 *pad) 985 { 986 int idx = dpos >> PAGE_SHIFT; 987 int off = offset_in_page(dpos); 988 int resid = dlen; 989 990 do { 991 int len = min(resid, (int)PAGE_SIZE - off); 992 993 sg_set_page(*sg, pages[idx], len, off); 994 *sg = sg_next(*sg); 995 off = 0; 996 ++idx; 997 resid -= len; 998 } while (resid); 999 1000 if (need_padding(dlen)) { 1001 sg_set_buf(*sg, pad, padding_len(dlen)); 1002 *sg = sg_next(*sg); 1003 } 1004 } 1005 1006 static int setup_message_sgs(struct sg_table *sgt, struct ceph_msg *msg, 1007 u8 *front_pad, u8 *middle_pad, u8 *data_pad, 1008 void *epilogue, struct page **pages, int dpos, 1009 bool add_tag) 1010 { 1011 struct ceph_msg_data_cursor cursor; 1012 struct scatterlist *cur_sg; 1013 int dlen = data_len(msg); 1014 int sg_cnt; 1015 int ret; 1016 1017 if (!front_len(msg) && !middle_len(msg) && !data_len(msg)) 1018 return 0; 1019 1020 sg_cnt = 1; /* epilogue + [auth tag] */ 1021 if (front_len(msg)) 1022 sg_cnt += calc_sg_cnt(msg->front.iov_base, 1023 front_len(msg)); 1024 if (middle_len(msg)) 1025 sg_cnt += calc_sg_cnt(msg->middle->vec.iov_base, 1026 middle_len(msg)); 1027 if (dlen) { 1028 if (pages) { 1029 sg_cnt += calc_pages_for(dpos, dlen); 1030 if (need_padding(dlen)) 1031 sg_cnt++; 1032 } else { 1033 ceph_msg_data_cursor_init(&cursor, msg, dlen); 1034 sg_cnt += calc_sg_cnt_cursor(&cursor); 1035 } 1036 } 1037 1038 ret = sg_alloc_table(sgt, sg_cnt, GFP_NOIO); 1039 if (ret) 1040 return ret; 1041 1042 cur_sg = sgt->sgl; 1043 if (front_len(msg)) 1044 init_sgs(&cur_sg, msg->front.iov_base, front_len(msg), 1045 front_pad); 1046 if (middle_len(msg)) 1047 init_sgs(&cur_sg, msg->middle->vec.iov_base, middle_len(msg), 1048 middle_pad); 1049 if (dlen) { 1050 if (pages) { 1051 init_sgs_pages(&cur_sg, pages, dpos, dlen, data_pad); 1052 } else { 1053 ceph_msg_data_cursor_init(&cursor, msg, dlen); 1054 init_sgs_cursor(&cur_sg, &cursor, data_pad); 1055 } 1056 } 1057 1058 WARN_ON(!sg_is_last(cur_sg)); 1059 sg_set_buf(cur_sg, epilogue, 1060 CEPH_GCM_BLOCK_LEN + (add_tag ? CEPH_GCM_TAG_LEN : 0)); 1061 return 0; 1062 } 1063 1064 static int decrypt_preamble(struct ceph_connection *con) 1065 { 1066 struct scatterlist sg; 1067 1068 sg_init_one(&sg, con->v2.in_buf, CEPH_PREAMBLE_SECURE_LEN); 1069 return gcm_crypt(con, false, &sg, &sg, CEPH_PREAMBLE_SECURE_LEN); 1070 } 1071 1072 static int decrypt_control_remainder(struct ceph_connection *con) 1073 { 1074 int ctrl_len = con->v2.in_desc.fd_lens[0]; 1075 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN; 1076 int pt_len = padding_len(rem_len) + CEPH_GCM_TAG_LEN; 1077 struct scatterlist sgs[2]; 1078 1079 WARN_ON(con->v2.in_kvecs[0].iov_len != rem_len); 1080 WARN_ON(con->v2.in_kvecs[1].iov_len != pt_len); 1081 1082 sg_init_table(sgs, 2); 1083 sg_set_buf(&sgs[0], con->v2.in_kvecs[0].iov_base, rem_len); 1084 sg_set_buf(&sgs[1], con->v2.in_buf, pt_len); 1085 1086 return gcm_crypt(con, false, sgs, sgs, 1087 padded_len(rem_len) + CEPH_GCM_TAG_LEN); 1088 } 1089 1090 /* Process sparse read data that lives in a buffer */ 1091 static int process_v2_sparse_read(struct ceph_connection *con, 1092 struct page **pages, int spos) 1093 { 1094 struct ceph_msg_data_cursor *cursor = &con->v2.in_cursor; 1095 int ret; 1096 1097 for (;;) { 1098 char *buf = NULL; 1099 1100 ret = con->ops->sparse_read(con, cursor, &buf); 1101 if (ret <= 0) 1102 return ret; 1103 1104 dout("%s: sparse_read return %x buf %p\n", __func__, ret, buf); 1105 1106 do { 1107 int idx = spos >> PAGE_SHIFT; 1108 int soff = offset_in_page(spos); 1109 struct page *spage = con->v2.in_enc_pages[idx]; 1110 int len = min_t(int, ret, PAGE_SIZE - soff); 1111 1112 if (buf) { 1113 memcpy_from_page(buf, spage, soff, len); 1114 buf += len; 1115 } else { 1116 struct bio_vec bv; 1117 1118 get_bvec_at(cursor, &bv); 1119 len = min_t(int, len, bv.bv_len); 1120 memcpy_page(bv.bv_page, bv.bv_offset, 1121 spage, soff, len); 1122 ceph_msg_data_advance(cursor, len); 1123 } 1124 spos += len; 1125 ret -= len; 1126 } while (ret); 1127 } 1128 } 1129 1130 static int decrypt_tail(struct ceph_connection *con) 1131 { 1132 struct sg_table enc_sgt = {}; 1133 struct sg_table sgt = {}; 1134 struct page **pages = NULL; 1135 bool sparse = !!con->in_msg->sparse_read_total; 1136 int dpos = 0; 1137 int tail_len; 1138 int ret; 1139 1140 tail_len = tail_onwire_len(con->in_msg, true); 1141 ret = sg_alloc_table_from_pages(&enc_sgt, con->v2.in_enc_pages, 1142 con->v2.in_enc_page_cnt, 0, tail_len, 1143 GFP_NOIO); 1144 if (ret) 1145 goto out; 1146 1147 if (sparse) { 1148 dpos = padded_len(front_len(con->in_msg) + padded_len(middle_len(con->in_msg))); 1149 pages = con->v2.in_enc_pages; 1150 } 1151 1152 ret = setup_message_sgs(&sgt, con->in_msg, FRONT_PAD(con->v2.in_buf), 1153 MIDDLE_PAD(con->v2.in_buf), DATA_PAD(con->v2.in_buf), 1154 con->v2.in_buf, pages, dpos, true); 1155 if (ret) 1156 goto out; 1157 1158 dout("%s con %p msg %p enc_page_cnt %d sg_cnt %d\n", __func__, con, 1159 con->in_msg, con->v2.in_enc_page_cnt, sgt.orig_nents); 1160 ret = gcm_crypt(con, false, enc_sgt.sgl, sgt.sgl, tail_len); 1161 if (ret) 1162 goto out; 1163 1164 if (sparse && data_len(con->in_msg)) { 1165 ret = process_v2_sparse_read(con, con->v2.in_enc_pages, dpos); 1166 if (ret) 1167 goto out; 1168 } 1169 1170 WARN_ON(!con->v2.in_enc_page_cnt); 1171 ceph_release_page_vector(con->v2.in_enc_pages, 1172 con->v2.in_enc_page_cnt); 1173 con->v2.in_enc_pages = NULL; 1174 con->v2.in_enc_page_cnt = 0; 1175 1176 out: 1177 sg_free_table(&sgt); 1178 sg_free_table(&enc_sgt); 1179 return ret; 1180 } 1181 1182 static int prepare_banner(struct ceph_connection *con) 1183 { 1184 int buf_len = CEPH_BANNER_V2_LEN + 2 + 8 + 8; 1185 void *buf, *p; 1186 1187 buf = alloc_conn_buf(con, buf_len); 1188 if (!buf) 1189 return -ENOMEM; 1190 1191 p = buf; 1192 ceph_encode_copy(&p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN); 1193 ceph_encode_16(&p, sizeof(u64) + sizeof(u64)); 1194 ceph_encode_64(&p, CEPH_MSGR2_SUPPORTED_FEATURES); 1195 ceph_encode_64(&p, CEPH_MSGR2_REQUIRED_FEATURES); 1196 WARN_ON(p != buf + buf_len); 1197 1198 add_out_kvec(con, buf, buf_len); 1199 add_out_sign_kvec(con, buf, buf_len); 1200 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING); 1201 return 0; 1202 } 1203 1204 /* 1205 * base: 1206 * preamble 1207 * control body (ctrl_len bytes) 1208 * space for control crc 1209 * 1210 * extdata (optional): 1211 * control body (extdata_len bytes) 1212 * 1213 * Compute control crc and gather base and extdata into: 1214 * 1215 * preamble 1216 * control body (ctrl_len + extdata_len bytes) 1217 * control crc 1218 * 1219 * Preamble should already be encoded at the start of base. 1220 */ 1221 static void prepare_head_plain(struct ceph_connection *con, void *base, 1222 int ctrl_len, void *extdata, int extdata_len, 1223 bool to_be_signed) 1224 { 1225 int base_len = CEPH_PREAMBLE_LEN + ctrl_len + CEPH_CRC_LEN; 1226 void *crcp = base + base_len - CEPH_CRC_LEN; 1227 u32 crc; 1228 1229 crc = crc32c(-1, CTRL_BODY(base), ctrl_len); 1230 if (extdata_len) 1231 crc = crc32c(crc, extdata, extdata_len); 1232 put_unaligned_le32(crc, crcp); 1233 1234 if (!extdata_len) { 1235 add_out_kvec(con, base, base_len); 1236 if (to_be_signed) 1237 add_out_sign_kvec(con, base, base_len); 1238 return; 1239 } 1240 1241 add_out_kvec(con, base, crcp - base); 1242 add_out_kvec(con, extdata, extdata_len); 1243 add_out_kvec(con, crcp, CEPH_CRC_LEN); 1244 if (to_be_signed) { 1245 add_out_sign_kvec(con, base, crcp - base); 1246 add_out_sign_kvec(con, extdata, extdata_len); 1247 add_out_sign_kvec(con, crcp, CEPH_CRC_LEN); 1248 } 1249 } 1250 1251 static int prepare_head_secure_small(struct ceph_connection *con, 1252 void *base, int ctrl_len) 1253 { 1254 struct scatterlist sg; 1255 int ret; 1256 1257 /* inline buffer padding? */ 1258 if (ctrl_len < CEPH_PREAMBLE_INLINE_LEN) 1259 memset(CTRL_BODY(base) + ctrl_len, 0, 1260 CEPH_PREAMBLE_INLINE_LEN - ctrl_len); 1261 1262 sg_init_one(&sg, base, CEPH_PREAMBLE_SECURE_LEN); 1263 ret = gcm_crypt(con, true, &sg, &sg, 1264 CEPH_PREAMBLE_SECURE_LEN - CEPH_GCM_TAG_LEN); 1265 if (ret) 1266 return ret; 1267 1268 add_out_kvec(con, base, CEPH_PREAMBLE_SECURE_LEN); 1269 return 0; 1270 } 1271 1272 /* 1273 * base: 1274 * preamble 1275 * control body (ctrl_len bytes) 1276 * space for padding, if needed 1277 * space for control remainder auth tag 1278 * space for preamble auth tag 1279 * 1280 * Encrypt preamble and the inline portion, then encrypt the remainder 1281 * and gather into: 1282 * 1283 * preamble 1284 * control body (48 bytes) 1285 * preamble auth tag 1286 * control body (ctrl_len - 48 bytes) 1287 * zero padding, if needed 1288 * control remainder auth tag 1289 * 1290 * Preamble should already be encoded at the start of base. 1291 */ 1292 static int prepare_head_secure_big(struct ceph_connection *con, 1293 void *base, int ctrl_len) 1294 { 1295 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN; 1296 void *rem = CTRL_BODY(base) + CEPH_PREAMBLE_INLINE_LEN; 1297 void *rem_tag = rem + padded_len(rem_len); 1298 void *pmbl_tag = rem_tag + CEPH_GCM_TAG_LEN; 1299 struct scatterlist sgs[2]; 1300 int ret; 1301 1302 sg_init_table(sgs, 2); 1303 sg_set_buf(&sgs[0], base, rem - base); 1304 sg_set_buf(&sgs[1], pmbl_tag, CEPH_GCM_TAG_LEN); 1305 ret = gcm_crypt(con, true, sgs, sgs, rem - base); 1306 if (ret) 1307 return ret; 1308 1309 /* control remainder padding? */ 1310 if (need_padding(rem_len)) 1311 memset(rem + rem_len, 0, padding_len(rem_len)); 1312 1313 sg_init_one(&sgs[0], rem, pmbl_tag - rem); 1314 ret = gcm_crypt(con, true, sgs, sgs, rem_tag - rem); 1315 if (ret) 1316 return ret; 1317 1318 add_out_kvec(con, base, rem - base); 1319 add_out_kvec(con, pmbl_tag, CEPH_GCM_TAG_LEN); 1320 add_out_kvec(con, rem, pmbl_tag - rem); 1321 return 0; 1322 } 1323 1324 static int __prepare_control(struct ceph_connection *con, int tag, 1325 void *base, int ctrl_len, void *extdata, 1326 int extdata_len, bool to_be_signed) 1327 { 1328 int total_len = ctrl_len + extdata_len; 1329 struct ceph_frame_desc desc; 1330 int ret; 1331 1332 dout("%s con %p tag %d len %d (%d+%d)\n", __func__, con, tag, 1333 total_len, ctrl_len, extdata_len); 1334 1335 /* extdata may be vmalloc'ed but not base */ 1336 if (WARN_ON(is_vmalloc_addr(base) || !ctrl_len)) 1337 return -EINVAL; 1338 1339 init_frame_desc(&desc, tag, &total_len, 1); 1340 encode_preamble(&desc, base); 1341 1342 if (con_secure(con)) { 1343 if (WARN_ON(extdata_len || to_be_signed)) 1344 return -EINVAL; 1345 1346 if (ctrl_len <= CEPH_PREAMBLE_INLINE_LEN) 1347 /* fully inlined, inline buffer may need padding */ 1348 ret = prepare_head_secure_small(con, base, ctrl_len); 1349 else 1350 /* partially inlined, inline buffer is full */ 1351 ret = prepare_head_secure_big(con, base, ctrl_len); 1352 if (ret) 1353 return ret; 1354 } else { 1355 prepare_head_plain(con, base, ctrl_len, extdata, extdata_len, 1356 to_be_signed); 1357 } 1358 1359 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING); 1360 return 0; 1361 } 1362 1363 static int prepare_control(struct ceph_connection *con, int tag, 1364 void *base, int ctrl_len) 1365 { 1366 return __prepare_control(con, tag, base, ctrl_len, NULL, 0, false); 1367 } 1368 1369 static int prepare_hello(struct ceph_connection *con) 1370 { 1371 void *buf, *p; 1372 int ctrl_len; 1373 1374 ctrl_len = 1 + ceph_entity_addr_encoding_len(&con->peer_addr); 1375 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false)); 1376 if (!buf) 1377 return -ENOMEM; 1378 1379 p = CTRL_BODY(buf); 1380 ceph_encode_8(&p, CEPH_ENTITY_TYPE_CLIENT); 1381 ceph_encode_entity_addr(&p, &con->peer_addr); 1382 WARN_ON(p != CTRL_BODY(buf) + ctrl_len); 1383 1384 return __prepare_control(con, FRAME_TAG_HELLO, buf, ctrl_len, 1385 NULL, 0, true); 1386 } 1387 1388 /* so that head_onwire_len(AUTH_BUF_LEN, false) is 512 */ 1389 #define AUTH_BUF_LEN (512 - CEPH_CRC_LEN - CEPH_PREAMBLE_PLAIN_LEN) 1390 1391 static int prepare_auth_request(struct ceph_connection *con) 1392 { 1393 void *authorizer, *authorizer_copy; 1394 int ctrl_len, authorizer_len; 1395 void *buf; 1396 int ret; 1397 1398 ctrl_len = AUTH_BUF_LEN; 1399 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false)); 1400 if (!buf) 1401 return -ENOMEM; 1402 1403 mutex_unlock(&con->mutex); 1404 ret = con->ops->get_auth_request(con, CTRL_BODY(buf), &ctrl_len, 1405 &authorizer, &authorizer_len); 1406 mutex_lock(&con->mutex); 1407 if (con->state != CEPH_CON_S_V2_HELLO) { 1408 dout("%s con %p state changed to %d\n", __func__, con, 1409 con->state); 1410 return -EAGAIN; 1411 } 1412 1413 dout("%s con %p get_auth_request ret %d\n", __func__, con, ret); 1414 if (ret) 1415 return ret; 1416 1417 authorizer_copy = alloc_conn_buf(con, authorizer_len); 1418 if (!authorizer_copy) 1419 return -ENOMEM; 1420 1421 memcpy(authorizer_copy, authorizer, authorizer_len); 1422 1423 return __prepare_control(con, FRAME_TAG_AUTH_REQUEST, buf, ctrl_len, 1424 authorizer_copy, authorizer_len, true); 1425 } 1426 1427 static int prepare_auth_request_more(struct ceph_connection *con, 1428 void *reply, int reply_len) 1429 { 1430 int ctrl_len, authorizer_len; 1431 void *authorizer; 1432 void *buf; 1433 int ret; 1434 1435 ctrl_len = AUTH_BUF_LEN; 1436 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, false)); 1437 if (!buf) 1438 return -ENOMEM; 1439 1440 mutex_unlock(&con->mutex); 1441 ret = con->ops->handle_auth_reply_more(con, reply, reply_len, 1442 CTRL_BODY(buf), &ctrl_len, 1443 &authorizer, &authorizer_len); 1444 mutex_lock(&con->mutex); 1445 if (con->state != CEPH_CON_S_V2_AUTH) { 1446 dout("%s con %p state changed to %d\n", __func__, con, 1447 con->state); 1448 return -EAGAIN; 1449 } 1450 1451 dout("%s con %p handle_auth_reply_more ret %d\n", __func__, con, ret); 1452 if (ret) 1453 return ret; 1454 1455 return __prepare_control(con, FRAME_TAG_AUTH_REQUEST_MORE, buf, 1456 ctrl_len, authorizer, authorizer_len, true); 1457 } 1458 1459 static int prepare_auth_signature(struct ceph_connection *con) 1460 { 1461 void *buf; 1462 int ret; 1463 1464 buf = alloc_conn_buf(con, head_onwire_len(SHA256_DIGEST_SIZE, 1465 con_secure(con))); 1466 if (!buf) 1467 return -ENOMEM; 1468 1469 ret = hmac_sha256(con, con->v2.in_sign_kvecs, con->v2.in_sign_kvec_cnt, 1470 CTRL_BODY(buf)); 1471 if (ret) 1472 return ret; 1473 1474 return prepare_control(con, FRAME_TAG_AUTH_SIGNATURE, buf, 1475 SHA256_DIGEST_SIZE); 1476 } 1477 1478 static int prepare_client_ident(struct ceph_connection *con) 1479 { 1480 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr; 1481 struct ceph_client *client = from_msgr(con->msgr); 1482 u64 global_id = ceph_client_gid(client); 1483 void *buf, *p; 1484 int ctrl_len; 1485 1486 WARN_ON(con->v2.server_cookie); 1487 WARN_ON(con->v2.connect_seq); 1488 WARN_ON(con->v2.peer_global_seq); 1489 1490 if (!con->v2.client_cookie) { 1491 do { 1492 get_random_bytes(&con->v2.client_cookie, 1493 sizeof(con->v2.client_cookie)); 1494 } while (!con->v2.client_cookie); 1495 dout("%s con %p generated cookie 0x%llx\n", __func__, con, 1496 con->v2.client_cookie); 1497 } else { 1498 dout("%s con %p cookie already set 0x%llx\n", __func__, con, 1499 con->v2.client_cookie); 1500 } 1501 1502 dout("%s con %p my_addr %s/%u peer_addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx cookie 0x%llx\n", 1503 __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce), 1504 ceph_pr_addr(&con->peer_addr), le32_to_cpu(con->peer_addr.nonce), 1505 global_id, con->v2.global_seq, client->supported_features, 1506 client->required_features, con->v2.client_cookie); 1507 1508 ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) + 1509 ceph_entity_addr_encoding_len(&con->peer_addr) + 6 * 8; 1510 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con))); 1511 if (!buf) 1512 return -ENOMEM; 1513 1514 p = CTRL_BODY(buf); 1515 ceph_encode_8(&p, 2); /* addrvec marker */ 1516 ceph_encode_32(&p, 1); /* addr_cnt */ 1517 ceph_encode_entity_addr(&p, my_addr); 1518 ceph_encode_entity_addr(&p, &con->peer_addr); 1519 ceph_encode_64(&p, global_id); 1520 ceph_encode_64(&p, con->v2.global_seq); 1521 ceph_encode_64(&p, client->supported_features); 1522 ceph_encode_64(&p, client->required_features); 1523 ceph_encode_64(&p, 0); /* flags */ 1524 ceph_encode_64(&p, con->v2.client_cookie); 1525 WARN_ON(p != CTRL_BODY(buf) + ctrl_len); 1526 1527 return prepare_control(con, FRAME_TAG_CLIENT_IDENT, buf, ctrl_len); 1528 } 1529 1530 static int prepare_session_reconnect(struct ceph_connection *con) 1531 { 1532 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr; 1533 void *buf, *p; 1534 int ctrl_len; 1535 1536 WARN_ON(!con->v2.client_cookie); 1537 WARN_ON(!con->v2.server_cookie); 1538 WARN_ON(!con->v2.connect_seq); 1539 WARN_ON(!con->v2.peer_global_seq); 1540 1541 dout("%s con %p my_addr %s/%u client_cookie 0x%llx server_cookie 0x%llx global_seq %llu connect_seq %llu in_seq %llu\n", 1542 __func__, con, ceph_pr_addr(my_addr), le32_to_cpu(my_addr->nonce), 1543 con->v2.client_cookie, con->v2.server_cookie, con->v2.global_seq, 1544 con->v2.connect_seq, con->in_seq); 1545 1546 ctrl_len = 1 + 4 + ceph_entity_addr_encoding_len(my_addr) + 5 * 8; 1547 buf = alloc_conn_buf(con, head_onwire_len(ctrl_len, con_secure(con))); 1548 if (!buf) 1549 return -ENOMEM; 1550 1551 p = CTRL_BODY(buf); 1552 ceph_encode_8(&p, 2); /* entity_addrvec_t marker */ 1553 ceph_encode_32(&p, 1); /* my_addrs len */ 1554 ceph_encode_entity_addr(&p, my_addr); 1555 ceph_encode_64(&p, con->v2.client_cookie); 1556 ceph_encode_64(&p, con->v2.server_cookie); 1557 ceph_encode_64(&p, con->v2.global_seq); 1558 ceph_encode_64(&p, con->v2.connect_seq); 1559 ceph_encode_64(&p, con->in_seq); 1560 WARN_ON(p != CTRL_BODY(buf) + ctrl_len); 1561 1562 return prepare_control(con, FRAME_TAG_SESSION_RECONNECT, buf, ctrl_len); 1563 } 1564 1565 static int prepare_keepalive2(struct ceph_connection *con) 1566 { 1567 struct ceph_timespec *ts = CTRL_BODY(con->v2.out_buf); 1568 struct timespec64 now; 1569 1570 ktime_get_real_ts64(&now); 1571 dout("%s con %p timestamp %lld.%09ld\n", __func__, con, now.tv_sec, 1572 now.tv_nsec); 1573 1574 ceph_encode_timespec64(ts, &now); 1575 1576 reset_out_kvecs(con); 1577 return prepare_control(con, FRAME_TAG_KEEPALIVE2, con->v2.out_buf, 1578 sizeof(struct ceph_timespec)); 1579 } 1580 1581 static int prepare_ack(struct ceph_connection *con) 1582 { 1583 void *p; 1584 1585 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con, 1586 con->in_seq_acked, con->in_seq); 1587 con->in_seq_acked = con->in_seq; 1588 1589 p = CTRL_BODY(con->v2.out_buf); 1590 ceph_encode_64(&p, con->in_seq_acked); 1591 1592 reset_out_kvecs(con); 1593 return prepare_control(con, FRAME_TAG_ACK, con->v2.out_buf, 8); 1594 } 1595 1596 static void prepare_epilogue_plain(struct ceph_connection *con, bool aborted) 1597 { 1598 dout("%s con %p msg %p aborted %d crcs %u %u %u\n", __func__, con, 1599 con->out_msg, aborted, con->v2.out_epil.front_crc, 1600 con->v2.out_epil.middle_crc, con->v2.out_epil.data_crc); 1601 1602 encode_epilogue_plain(con, aborted); 1603 add_out_kvec(con, &con->v2.out_epil, CEPH_EPILOGUE_PLAIN_LEN); 1604 } 1605 1606 /* 1607 * For "used" empty segments, crc is -1. For unused (trailing) 1608 * segments, crc is 0. 1609 */ 1610 static void prepare_message_plain(struct ceph_connection *con) 1611 { 1612 struct ceph_msg *msg = con->out_msg; 1613 1614 prepare_head_plain(con, con->v2.out_buf, 1615 sizeof(struct ceph_msg_header2), NULL, 0, false); 1616 1617 if (!front_len(msg) && !middle_len(msg)) { 1618 if (!data_len(msg)) { 1619 /* 1620 * Empty message: once the head is written, 1621 * we are done -- there is no epilogue. 1622 */ 1623 con->v2.out_state = OUT_S_FINISH_MESSAGE; 1624 return; 1625 } 1626 1627 con->v2.out_epil.front_crc = -1; 1628 con->v2.out_epil.middle_crc = -1; 1629 con->v2.out_state = OUT_S_QUEUE_DATA; 1630 return; 1631 } 1632 1633 if (front_len(msg)) { 1634 con->v2.out_epil.front_crc = crc32c(-1, msg->front.iov_base, 1635 front_len(msg)); 1636 add_out_kvec(con, msg->front.iov_base, front_len(msg)); 1637 } else { 1638 /* middle (at least) is there, checked above */ 1639 con->v2.out_epil.front_crc = -1; 1640 } 1641 1642 if (middle_len(msg)) { 1643 con->v2.out_epil.middle_crc = 1644 crc32c(-1, msg->middle->vec.iov_base, middle_len(msg)); 1645 add_out_kvec(con, msg->middle->vec.iov_base, middle_len(msg)); 1646 } else { 1647 con->v2.out_epil.middle_crc = data_len(msg) ? -1 : 0; 1648 } 1649 1650 if (data_len(msg)) { 1651 con->v2.out_state = OUT_S_QUEUE_DATA; 1652 } else { 1653 con->v2.out_epil.data_crc = 0; 1654 prepare_epilogue_plain(con, false); 1655 con->v2.out_state = OUT_S_FINISH_MESSAGE; 1656 } 1657 } 1658 1659 /* 1660 * Unfortunately the kernel crypto API doesn't support streaming 1661 * (piecewise) operation for AEAD algorithms, so we can't get away 1662 * with a fixed size buffer and a couple sgs. Instead, we have to 1663 * allocate pages for the entire tail of the message (currently up 1664 * to ~32M) and two sgs arrays (up to ~256K each)... 1665 */ 1666 static int prepare_message_secure(struct ceph_connection *con) 1667 { 1668 void *zerop = page_address(ceph_zero_page); 1669 struct sg_table enc_sgt = {}; 1670 struct sg_table sgt = {}; 1671 struct page **enc_pages; 1672 int enc_page_cnt; 1673 int tail_len; 1674 int ret; 1675 1676 ret = prepare_head_secure_small(con, con->v2.out_buf, 1677 sizeof(struct ceph_msg_header2)); 1678 if (ret) 1679 return ret; 1680 1681 tail_len = tail_onwire_len(con->out_msg, true); 1682 if (!tail_len) { 1683 /* 1684 * Empty message: once the head is written, 1685 * we are done -- there is no epilogue. 1686 */ 1687 con->v2.out_state = OUT_S_FINISH_MESSAGE; 1688 return 0; 1689 } 1690 1691 encode_epilogue_secure(con, false); 1692 ret = setup_message_sgs(&sgt, con->out_msg, zerop, zerop, zerop, 1693 &con->v2.out_epil, NULL, 0, false); 1694 if (ret) 1695 goto out; 1696 1697 enc_page_cnt = calc_pages_for(0, tail_len); 1698 enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO); 1699 if (IS_ERR(enc_pages)) { 1700 ret = PTR_ERR(enc_pages); 1701 goto out; 1702 } 1703 1704 WARN_ON(con->v2.out_enc_pages || con->v2.out_enc_page_cnt); 1705 con->v2.out_enc_pages = enc_pages; 1706 con->v2.out_enc_page_cnt = enc_page_cnt; 1707 con->v2.out_enc_resid = tail_len; 1708 con->v2.out_enc_i = 0; 1709 1710 ret = sg_alloc_table_from_pages(&enc_sgt, enc_pages, enc_page_cnt, 1711 0, tail_len, GFP_NOIO); 1712 if (ret) 1713 goto out; 1714 1715 ret = gcm_crypt(con, true, sgt.sgl, enc_sgt.sgl, 1716 tail_len - CEPH_GCM_TAG_LEN); 1717 if (ret) 1718 goto out; 1719 1720 dout("%s con %p msg %p sg_cnt %d enc_page_cnt %d\n", __func__, con, 1721 con->out_msg, sgt.orig_nents, enc_page_cnt); 1722 con->v2.out_state = OUT_S_QUEUE_ENC_PAGE; 1723 1724 out: 1725 sg_free_table(&sgt); 1726 sg_free_table(&enc_sgt); 1727 return ret; 1728 } 1729 1730 static int prepare_message(struct ceph_connection *con) 1731 { 1732 int lens[] = { 1733 sizeof(struct ceph_msg_header2), 1734 front_len(con->out_msg), 1735 middle_len(con->out_msg), 1736 data_len(con->out_msg) 1737 }; 1738 struct ceph_frame_desc desc; 1739 int ret; 1740 1741 dout("%s con %p msg %p logical %d+%d+%d+%d\n", __func__, con, 1742 con->out_msg, lens[0], lens[1], lens[2], lens[3]); 1743 1744 if (con->in_seq > con->in_seq_acked) { 1745 dout("%s con %p in_seq_acked %llu -> %llu\n", __func__, con, 1746 con->in_seq_acked, con->in_seq); 1747 con->in_seq_acked = con->in_seq; 1748 } 1749 1750 reset_out_kvecs(con); 1751 init_frame_desc(&desc, FRAME_TAG_MESSAGE, lens, 4); 1752 encode_preamble(&desc, con->v2.out_buf); 1753 fill_header2(CTRL_BODY(con->v2.out_buf), &con->out_msg->hdr, 1754 con->in_seq_acked); 1755 1756 if (con_secure(con)) { 1757 ret = prepare_message_secure(con); 1758 if (ret) 1759 return ret; 1760 } else { 1761 prepare_message_plain(con); 1762 } 1763 1764 ceph_con_flag_set(con, CEPH_CON_F_WRITE_PENDING); 1765 return 0; 1766 } 1767 1768 static int prepare_read_banner_prefix(struct ceph_connection *con) 1769 { 1770 void *buf; 1771 1772 buf = alloc_conn_buf(con, CEPH_BANNER_V2_PREFIX_LEN); 1773 if (!buf) 1774 return -ENOMEM; 1775 1776 reset_in_kvecs(con); 1777 add_in_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN); 1778 add_in_sign_kvec(con, buf, CEPH_BANNER_V2_PREFIX_LEN); 1779 con->state = CEPH_CON_S_V2_BANNER_PREFIX; 1780 return 0; 1781 } 1782 1783 static int prepare_read_banner_payload(struct ceph_connection *con, 1784 int payload_len) 1785 { 1786 void *buf; 1787 1788 buf = alloc_conn_buf(con, payload_len); 1789 if (!buf) 1790 return -ENOMEM; 1791 1792 reset_in_kvecs(con); 1793 add_in_kvec(con, buf, payload_len); 1794 add_in_sign_kvec(con, buf, payload_len); 1795 con->state = CEPH_CON_S_V2_BANNER_PAYLOAD; 1796 return 0; 1797 } 1798 1799 static void prepare_read_preamble(struct ceph_connection *con) 1800 { 1801 reset_in_kvecs(con); 1802 add_in_kvec(con, con->v2.in_buf, 1803 con_secure(con) ? CEPH_PREAMBLE_SECURE_LEN : 1804 CEPH_PREAMBLE_PLAIN_LEN); 1805 con->v2.in_state = IN_S_HANDLE_PREAMBLE; 1806 } 1807 1808 static int prepare_read_control(struct ceph_connection *con) 1809 { 1810 int ctrl_len = con->v2.in_desc.fd_lens[0]; 1811 int head_len; 1812 void *buf; 1813 1814 reset_in_kvecs(con); 1815 if (con->state == CEPH_CON_S_V2_HELLO || 1816 con->state == CEPH_CON_S_V2_AUTH) { 1817 head_len = head_onwire_len(ctrl_len, false); 1818 buf = alloc_conn_buf(con, head_len); 1819 if (!buf) 1820 return -ENOMEM; 1821 1822 /* preserve preamble */ 1823 memcpy(buf, con->v2.in_buf, CEPH_PREAMBLE_LEN); 1824 1825 add_in_kvec(con, CTRL_BODY(buf), ctrl_len); 1826 add_in_kvec(con, CTRL_BODY(buf) + ctrl_len, CEPH_CRC_LEN); 1827 add_in_sign_kvec(con, buf, head_len); 1828 } else { 1829 if (ctrl_len > CEPH_PREAMBLE_INLINE_LEN) { 1830 buf = alloc_conn_buf(con, ctrl_len); 1831 if (!buf) 1832 return -ENOMEM; 1833 1834 add_in_kvec(con, buf, ctrl_len); 1835 } else { 1836 add_in_kvec(con, CTRL_BODY(con->v2.in_buf), ctrl_len); 1837 } 1838 add_in_kvec(con, con->v2.in_buf, CEPH_CRC_LEN); 1839 } 1840 con->v2.in_state = IN_S_HANDLE_CONTROL; 1841 return 0; 1842 } 1843 1844 static int prepare_read_control_remainder(struct ceph_connection *con) 1845 { 1846 int ctrl_len = con->v2.in_desc.fd_lens[0]; 1847 int rem_len = ctrl_len - CEPH_PREAMBLE_INLINE_LEN; 1848 void *buf; 1849 1850 buf = alloc_conn_buf(con, ctrl_len); 1851 if (!buf) 1852 return -ENOMEM; 1853 1854 memcpy(buf, CTRL_BODY(con->v2.in_buf), CEPH_PREAMBLE_INLINE_LEN); 1855 1856 reset_in_kvecs(con); 1857 add_in_kvec(con, buf + CEPH_PREAMBLE_INLINE_LEN, rem_len); 1858 add_in_kvec(con, con->v2.in_buf, 1859 padding_len(rem_len) + CEPH_GCM_TAG_LEN); 1860 con->v2.in_state = IN_S_HANDLE_CONTROL_REMAINDER; 1861 return 0; 1862 } 1863 1864 static int prepare_read_data(struct ceph_connection *con) 1865 { 1866 struct bio_vec bv; 1867 1868 con->in_data_crc = -1; 1869 ceph_msg_data_cursor_init(&con->v2.in_cursor, con->in_msg, 1870 data_len(con->in_msg)); 1871 1872 get_bvec_at(&con->v2.in_cursor, &bv); 1873 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) { 1874 if (unlikely(!con->bounce_page)) { 1875 con->bounce_page = alloc_page(GFP_NOIO); 1876 if (!con->bounce_page) { 1877 pr_err("failed to allocate bounce page\n"); 1878 return -ENOMEM; 1879 } 1880 } 1881 1882 bv.bv_page = con->bounce_page; 1883 bv.bv_offset = 0; 1884 } 1885 set_in_bvec(con, &bv); 1886 con->v2.in_state = IN_S_PREPARE_READ_DATA_CONT; 1887 return 0; 1888 } 1889 1890 static void prepare_read_data_cont(struct ceph_connection *con) 1891 { 1892 struct bio_vec bv; 1893 1894 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) { 1895 con->in_data_crc = crc32c(con->in_data_crc, 1896 page_address(con->bounce_page), 1897 con->v2.in_bvec.bv_len); 1898 1899 get_bvec_at(&con->v2.in_cursor, &bv); 1900 memcpy_to_page(bv.bv_page, bv.bv_offset, 1901 page_address(con->bounce_page), 1902 con->v2.in_bvec.bv_len); 1903 } else { 1904 con->in_data_crc = ceph_crc32c_page(con->in_data_crc, 1905 con->v2.in_bvec.bv_page, 1906 con->v2.in_bvec.bv_offset, 1907 con->v2.in_bvec.bv_len); 1908 } 1909 1910 ceph_msg_data_advance(&con->v2.in_cursor, con->v2.in_bvec.bv_len); 1911 if (con->v2.in_cursor.total_resid) { 1912 get_bvec_at(&con->v2.in_cursor, &bv); 1913 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) { 1914 bv.bv_page = con->bounce_page; 1915 bv.bv_offset = 0; 1916 } 1917 set_in_bvec(con, &bv); 1918 WARN_ON(con->v2.in_state != IN_S_PREPARE_READ_DATA_CONT); 1919 return; 1920 } 1921 1922 /* 1923 * We've read all data. Prepare to read epilogue. 1924 */ 1925 reset_in_kvecs(con); 1926 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN); 1927 con->v2.in_state = IN_S_HANDLE_EPILOGUE; 1928 } 1929 1930 static int prepare_sparse_read_cont(struct ceph_connection *con) 1931 { 1932 int ret; 1933 struct bio_vec bv; 1934 char *buf = NULL; 1935 struct ceph_msg_data_cursor *cursor = &con->v2.in_cursor; 1936 1937 WARN_ON(con->v2.in_state != IN_S_PREPARE_SPARSE_DATA_CONT); 1938 1939 if (iov_iter_is_bvec(&con->v2.in_iter)) { 1940 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) { 1941 con->in_data_crc = crc32c(con->in_data_crc, 1942 page_address(con->bounce_page), 1943 con->v2.in_bvec.bv_len); 1944 get_bvec_at(cursor, &bv); 1945 memcpy_to_page(bv.bv_page, bv.bv_offset, 1946 page_address(con->bounce_page), 1947 con->v2.in_bvec.bv_len); 1948 } else { 1949 con->in_data_crc = ceph_crc32c_page(con->in_data_crc, 1950 con->v2.in_bvec.bv_page, 1951 con->v2.in_bvec.bv_offset, 1952 con->v2.in_bvec.bv_len); 1953 } 1954 1955 ceph_msg_data_advance(cursor, con->v2.in_bvec.bv_len); 1956 cursor->sr_resid -= con->v2.in_bvec.bv_len; 1957 dout("%s: advance by 0x%x sr_resid 0x%x\n", __func__, 1958 con->v2.in_bvec.bv_len, cursor->sr_resid); 1959 WARN_ON_ONCE(cursor->sr_resid > cursor->total_resid); 1960 if (cursor->sr_resid) { 1961 get_bvec_at(cursor, &bv); 1962 if (bv.bv_len > cursor->sr_resid) 1963 bv.bv_len = cursor->sr_resid; 1964 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) { 1965 bv.bv_page = con->bounce_page; 1966 bv.bv_offset = 0; 1967 } 1968 set_in_bvec(con, &bv); 1969 con->v2.data_len_remain -= bv.bv_len; 1970 return 0; 1971 } 1972 } else if (iov_iter_is_kvec(&con->v2.in_iter)) { 1973 /* On first call, we have no kvec so don't compute crc */ 1974 if (con->v2.in_kvec_cnt) { 1975 WARN_ON_ONCE(con->v2.in_kvec_cnt > 1); 1976 con->in_data_crc = crc32c(con->in_data_crc, 1977 con->v2.in_kvecs[0].iov_base, 1978 con->v2.in_kvecs[0].iov_len); 1979 } 1980 } else { 1981 return -EIO; 1982 } 1983 1984 /* get next extent */ 1985 ret = con->ops->sparse_read(con, cursor, &buf); 1986 if (ret <= 0) { 1987 if (ret < 0) 1988 return ret; 1989 1990 reset_in_kvecs(con); 1991 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN); 1992 con->v2.in_state = IN_S_HANDLE_EPILOGUE; 1993 return 0; 1994 } 1995 1996 if (buf) { 1997 /* receive into buffer */ 1998 reset_in_kvecs(con); 1999 add_in_kvec(con, buf, ret); 2000 con->v2.data_len_remain -= ret; 2001 return 0; 2002 } 2003 2004 if (ret > cursor->total_resid) { 2005 pr_warn("%s: ret 0x%x total_resid 0x%zx resid 0x%zx\n", 2006 __func__, ret, cursor->total_resid, cursor->resid); 2007 return -EIO; 2008 } 2009 get_bvec_at(cursor, &bv); 2010 if (bv.bv_len > cursor->sr_resid) 2011 bv.bv_len = cursor->sr_resid; 2012 if (ceph_test_opt(from_msgr(con->msgr), RXBOUNCE)) { 2013 if (unlikely(!con->bounce_page)) { 2014 con->bounce_page = alloc_page(GFP_NOIO); 2015 if (!con->bounce_page) { 2016 pr_err("failed to allocate bounce page\n"); 2017 return -ENOMEM; 2018 } 2019 } 2020 2021 bv.bv_page = con->bounce_page; 2022 bv.bv_offset = 0; 2023 } 2024 set_in_bvec(con, &bv); 2025 con->v2.data_len_remain -= ret; 2026 return ret; 2027 } 2028 2029 static int prepare_sparse_read_data(struct ceph_connection *con) 2030 { 2031 struct ceph_msg *msg = con->in_msg; 2032 2033 dout("%s: starting sparse read\n", __func__); 2034 2035 if (WARN_ON_ONCE(!con->ops->sparse_read)) 2036 return -EOPNOTSUPP; 2037 2038 if (!con_secure(con)) 2039 con->in_data_crc = -1; 2040 2041 reset_in_kvecs(con); 2042 con->v2.in_state = IN_S_PREPARE_SPARSE_DATA_CONT; 2043 con->v2.data_len_remain = data_len(msg); 2044 return prepare_sparse_read_cont(con); 2045 } 2046 2047 static int prepare_read_tail_plain(struct ceph_connection *con) 2048 { 2049 struct ceph_msg *msg = con->in_msg; 2050 2051 if (!front_len(msg) && !middle_len(msg)) { 2052 WARN_ON(!data_len(msg)); 2053 return prepare_read_data(con); 2054 } 2055 2056 reset_in_kvecs(con); 2057 if (front_len(msg)) { 2058 add_in_kvec(con, msg->front.iov_base, front_len(msg)); 2059 WARN_ON(msg->front.iov_len != front_len(msg)); 2060 } 2061 if (middle_len(msg)) { 2062 add_in_kvec(con, msg->middle->vec.iov_base, middle_len(msg)); 2063 WARN_ON(msg->middle->vec.iov_len != middle_len(msg)); 2064 } 2065 2066 if (data_len(msg)) { 2067 if (msg->sparse_read_total) 2068 con->v2.in_state = IN_S_PREPARE_SPARSE_DATA; 2069 else 2070 con->v2.in_state = IN_S_PREPARE_READ_DATA; 2071 } else { 2072 add_in_kvec(con, con->v2.in_buf, CEPH_EPILOGUE_PLAIN_LEN); 2073 con->v2.in_state = IN_S_HANDLE_EPILOGUE; 2074 } 2075 return 0; 2076 } 2077 2078 static void prepare_read_enc_page(struct ceph_connection *con) 2079 { 2080 struct bio_vec bv; 2081 2082 dout("%s con %p i %d resid %d\n", __func__, con, con->v2.in_enc_i, 2083 con->v2.in_enc_resid); 2084 WARN_ON(!con->v2.in_enc_resid); 2085 2086 bvec_set_page(&bv, con->v2.in_enc_pages[con->v2.in_enc_i], 2087 min(con->v2.in_enc_resid, (int)PAGE_SIZE), 0); 2088 2089 set_in_bvec(con, &bv); 2090 con->v2.in_enc_i++; 2091 con->v2.in_enc_resid -= bv.bv_len; 2092 2093 if (con->v2.in_enc_resid) { 2094 con->v2.in_state = IN_S_PREPARE_READ_ENC_PAGE; 2095 return; 2096 } 2097 2098 /* 2099 * We are set to read the last piece of ciphertext (ending 2100 * with epilogue) + auth tag. 2101 */ 2102 WARN_ON(con->v2.in_enc_i != con->v2.in_enc_page_cnt); 2103 con->v2.in_state = IN_S_HANDLE_EPILOGUE; 2104 } 2105 2106 static int prepare_read_tail_secure(struct ceph_connection *con) 2107 { 2108 struct page **enc_pages; 2109 int enc_page_cnt; 2110 int tail_len; 2111 2112 tail_len = tail_onwire_len(con->in_msg, true); 2113 WARN_ON(!tail_len); 2114 2115 enc_page_cnt = calc_pages_for(0, tail_len); 2116 enc_pages = ceph_alloc_page_vector(enc_page_cnt, GFP_NOIO); 2117 if (IS_ERR(enc_pages)) 2118 return PTR_ERR(enc_pages); 2119 2120 WARN_ON(con->v2.in_enc_pages || con->v2.in_enc_page_cnt); 2121 con->v2.in_enc_pages = enc_pages; 2122 con->v2.in_enc_page_cnt = enc_page_cnt; 2123 con->v2.in_enc_resid = tail_len; 2124 con->v2.in_enc_i = 0; 2125 2126 prepare_read_enc_page(con); 2127 return 0; 2128 } 2129 2130 static void __finish_skip(struct ceph_connection *con) 2131 { 2132 con->in_seq++; 2133 prepare_read_preamble(con); 2134 } 2135 2136 static void prepare_skip_message(struct ceph_connection *con) 2137 { 2138 struct ceph_frame_desc *desc = &con->v2.in_desc; 2139 int tail_len; 2140 2141 dout("%s con %p %d+%d+%d\n", __func__, con, desc->fd_lens[1], 2142 desc->fd_lens[2], desc->fd_lens[3]); 2143 2144 tail_len = __tail_onwire_len(desc->fd_lens[1], desc->fd_lens[2], 2145 desc->fd_lens[3], con_secure(con)); 2146 if (!tail_len) { 2147 __finish_skip(con); 2148 } else { 2149 set_in_skip(con, tail_len); 2150 con->v2.in_state = IN_S_FINISH_SKIP; 2151 } 2152 } 2153 2154 static int process_banner_prefix(struct ceph_connection *con) 2155 { 2156 int payload_len; 2157 void *p; 2158 2159 WARN_ON(con->v2.in_kvecs[0].iov_len != CEPH_BANNER_V2_PREFIX_LEN); 2160 2161 p = con->v2.in_kvecs[0].iov_base; 2162 if (memcmp(p, CEPH_BANNER_V2, CEPH_BANNER_V2_LEN)) { 2163 if (!memcmp(p, CEPH_BANNER, CEPH_BANNER_LEN)) 2164 con->error_msg = "server is speaking msgr1 protocol"; 2165 else 2166 con->error_msg = "protocol error, bad banner"; 2167 return -EINVAL; 2168 } 2169 2170 p += CEPH_BANNER_V2_LEN; 2171 payload_len = ceph_decode_16(&p); 2172 dout("%s con %p payload_len %d\n", __func__, con, payload_len); 2173 2174 return prepare_read_banner_payload(con, payload_len); 2175 } 2176 2177 static int process_banner_payload(struct ceph_connection *con) 2178 { 2179 void *end = con->v2.in_kvecs[0].iov_base + con->v2.in_kvecs[0].iov_len; 2180 u64 feat = CEPH_MSGR2_SUPPORTED_FEATURES; 2181 u64 req_feat = CEPH_MSGR2_REQUIRED_FEATURES; 2182 u64 server_feat, server_req_feat; 2183 void *p; 2184 int ret; 2185 2186 p = con->v2.in_kvecs[0].iov_base; 2187 ceph_decode_64_safe(&p, end, server_feat, bad); 2188 ceph_decode_64_safe(&p, end, server_req_feat, bad); 2189 2190 dout("%s con %p server_feat 0x%llx server_req_feat 0x%llx\n", 2191 __func__, con, server_feat, server_req_feat); 2192 2193 if (req_feat & ~server_feat) { 2194 pr_err("msgr2 feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n", 2195 server_feat, req_feat & ~server_feat); 2196 con->error_msg = "missing required protocol features"; 2197 return -EINVAL; 2198 } 2199 if (server_req_feat & ~feat) { 2200 pr_err("msgr2 feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n", 2201 feat, server_req_feat & ~feat); 2202 con->error_msg = "missing required protocol features"; 2203 return -EINVAL; 2204 } 2205 2206 /* no reset_out_kvecs() as our banner may still be pending */ 2207 ret = prepare_hello(con); 2208 if (ret) { 2209 pr_err("prepare_hello failed: %d\n", ret); 2210 return ret; 2211 } 2212 2213 con->state = CEPH_CON_S_V2_HELLO; 2214 prepare_read_preamble(con); 2215 return 0; 2216 2217 bad: 2218 pr_err("failed to decode banner payload\n"); 2219 return -EINVAL; 2220 } 2221 2222 static int process_hello(struct ceph_connection *con, void *p, void *end) 2223 { 2224 struct ceph_entity_addr *my_addr = &con->msgr->inst.addr; 2225 struct ceph_entity_addr addr_for_me; 2226 u8 entity_type; 2227 int ret; 2228 2229 if (con->state != CEPH_CON_S_V2_HELLO) { 2230 con->error_msg = "protocol error, unexpected hello"; 2231 return -EINVAL; 2232 } 2233 2234 ceph_decode_8_safe(&p, end, entity_type, bad); 2235 ret = ceph_decode_entity_addr(&p, end, &addr_for_me); 2236 if (ret) { 2237 pr_err("failed to decode addr_for_me: %d\n", ret); 2238 return ret; 2239 } 2240 2241 dout("%s con %p entity_type %d addr_for_me %s\n", __func__, con, 2242 entity_type, ceph_pr_addr(&addr_for_me)); 2243 2244 if (entity_type != con->peer_name.type) { 2245 pr_err("bad peer type, want %d, got %d\n", 2246 con->peer_name.type, entity_type); 2247 con->error_msg = "wrong peer at address"; 2248 return -EINVAL; 2249 } 2250 2251 /* 2252 * Set our address to the address our first peer (i.e. monitor) 2253 * sees that we are connecting from. If we are behind some sort 2254 * of NAT and want to be identified by some private (not NATed) 2255 * address, ip option should be used. 2256 */ 2257 if (ceph_addr_is_blank(my_addr)) { 2258 memcpy(&my_addr->in_addr, &addr_for_me.in_addr, 2259 sizeof(my_addr->in_addr)); 2260 ceph_addr_set_port(my_addr, 0); 2261 dout("%s con %p set my addr %s, as seen by peer %s\n", 2262 __func__, con, ceph_pr_addr(my_addr), 2263 ceph_pr_addr(&con->peer_addr)); 2264 } else { 2265 dout("%s con %p my addr already set %s\n", 2266 __func__, con, ceph_pr_addr(my_addr)); 2267 } 2268 2269 WARN_ON(ceph_addr_is_blank(my_addr) || ceph_addr_port(my_addr)); 2270 WARN_ON(my_addr->type != CEPH_ENTITY_ADDR_TYPE_ANY); 2271 WARN_ON(!my_addr->nonce); 2272 2273 /* no reset_out_kvecs() as our hello may still be pending */ 2274 ret = prepare_auth_request(con); 2275 if (ret) { 2276 if (ret != -EAGAIN) 2277 pr_err("prepare_auth_request failed: %d\n", ret); 2278 return ret; 2279 } 2280 2281 con->state = CEPH_CON_S_V2_AUTH; 2282 return 0; 2283 2284 bad: 2285 pr_err("failed to decode hello\n"); 2286 return -EINVAL; 2287 } 2288 2289 static int process_auth_bad_method(struct ceph_connection *con, 2290 void *p, void *end) 2291 { 2292 int allowed_protos[8], allowed_modes[8]; 2293 int allowed_proto_cnt, allowed_mode_cnt; 2294 int used_proto, result; 2295 int ret; 2296 int i; 2297 2298 if (con->state != CEPH_CON_S_V2_AUTH) { 2299 con->error_msg = "protocol error, unexpected auth_bad_method"; 2300 return -EINVAL; 2301 } 2302 2303 ceph_decode_32_safe(&p, end, used_proto, bad); 2304 ceph_decode_32_safe(&p, end, result, bad); 2305 dout("%s con %p used_proto %d result %d\n", __func__, con, used_proto, 2306 result); 2307 2308 ceph_decode_32_safe(&p, end, allowed_proto_cnt, bad); 2309 if (allowed_proto_cnt > ARRAY_SIZE(allowed_protos)) { 2310 pr_err("allowed_protos too big %d\n", allowed_proto_cnt); 2311 return -EINVAL; 2312 } 2313 for (i = 0; i < allowed_proto_cnt; i++) { 2314 ceph_decode_32_safe(&p, end, allowed_protos[i], bad); 2315 dout("%s con %p allowed_protos[%d] %d\n", __func__, con, 2316 i, allowed_protos[i]); 2317 } 2318 2319 ceph_decode_32_safe(&p, end, allowed_mode_cnt, bad); 2320 if (allowed_mode_cnt > ARRAY_SIZE(allowed_modes)) { 2321 pr_err("allowed_modes too big %d\n", allowed_mode_cnt); 2322 return -EINVAL; 2323 } 2324 for (i = 0; i < allowed_mode_cnt; i++) { 2325 ceph_decode_32_safe(&p, end, allowed_modes[i], bad); 2326 dout("%s con %p allowed_modes[%d] %d\n", __func__, con, 2327 i, allowed_modes[i]); 2328 } 2329 2330 mutex_unlock(&con->mutex); 2331 ret = con->ops->handle_auth_bad_method(con, used_proto, result, 2332 allowed_protos, 2333 allowed_proto_cnt, 2334 allowed_modes, 2335 allowed_mode_cnt); 2336 mutex_lock(&con->mutex); 2337 if (con->state != CEPH_CON_S_V2_AUTH) { 2338 dout("%s con %p state changed to %d\n", __func__, con, 2339 con->state); 2340 return -EAGAIN; 2341 } 2342 2343 dout("%s con %p handle_auth_bad_method ret %d\n", __func__, con, ret); 2344 return ret; 2345 2346 bad: 2347 pr_err("failed to decode auth_bad_method\n"); 2348 return -EINVAL; 2349 } 2350 2351 static int process_auth_reply_more(struct ceph_connection *con, 2352 void *p, void *end) 2353 { 2354 int payload_len; 2355 int ret; 2356 2357 if (con->state != CEPH_CON_S_V2_AUTH) { 2358 con->error_msg = "protocol error, unexpected auth_reply_more"; 2359 return -EINVAL; 2360 } 2361 2362 ceph_decode_32_safe(&p, end, payload_len, bad); 2363 ceph_decode_need(&p, end, payload_len, bad); 2364 2365 dout("%s con %p payload_len %d\n", __func__, con, payload_len); 2366 2367 reset_out_kvecs(con); 2368 ret = prepare_auth_request_more(con, p, payload_len); 2369 if (ret) { 2370 if (ret != -EAGAIN) 2371 pr_err("prepare_auth_request_more failed: %d\n", ret); 2372 return ret; 2373 } 2374 2375 return 0; 2376 2377 bad: 2378 pr_err("failed to decode auth_reply_more\n"); 2379 return -EINVAL; 2380 } 2381 2382 /* 2383 * Align session_key and con_secret to avoid GFP_ATOMIC allocation 2384 * inside crypto_shash_setkey() and crypto_aead_setkey() called from 2385 * setup_crypto(). __aligned(16) isn't guaranteed to work for stack 2386 * objects, so do it by hand. 2387 */ 2388 static int process_auth_done(struct ceph_connection *con, void *p, void *end) 2389 { 2390 u8 session_key_buf[CEPH_KEY_LEN + 16]; 2391 u8 con_secret_buf[CEPH_MAX_CON_SECRET_LEN + 16]; 2392 u8 *session_key = PTR_ALIGN(&session_key_buf[0], 16); 2393 u8 *con_secret = PTR_ALIGN(&con_secret_buf[0], 16); 2394 int session_key_len, con_secret_len; 2395 int payload_len; 2396 u64 global_id; 2397 int ret; 2398 2399 if (con->state != CEPH_CON_S_V2_AUTH) { 2400 con->error_msg = "protocol error, unexpected auth_done"; 2401 return -EINVAL; 2402 } 2403 2404 ceph_decode_64_safe(&p, end, global_id, bad); 2405 ceph_decode_32_safe(&p, end, con->v2.con_mode, bad); 2406 ceph_decode_32_safe(&p, end, payload_len, bad); 2407 2408 dout("%s con %p global_id %llu con_mode %d payload_len %d\n", 2409 __func__, con, global_id, con->v2.con_mode, payload_len); 2410 2411 mutex_unlock(&con->mutex); 2412 session_key_len = 0; 2413 con_secret_len = 0; 2414 ret = con->ops->handle_auth_done(con, global_id, p, payload_len, 2415 session_key, &session_key_len, 2416 con_secret, &con_secret_len); 2417 mutex_lock(&con->mutex); 2418 if (con->state != CEPH_CON_S_V2_AUTH) { 2419 dout("%s con %p state changed to %d\n", __func__, con, 2420 con->state); 2421 ret = -EAGAIN; 2422 goto out; 2423 } 2424 2425 dout("%s con %p handle_auth_done ret %d\n", __func__, con, ret); 2426 if (ret) 2427 goto out; 2428 2429 ret = setup_crypto(con, session_key, session_key_len, con_secret, 2430 con_secret_len); 2431 if (ret) 2432 goto out; 2433 2434 reset_out_kvecs(con); 2435 ret = prepare_auth_signature(con); 2436 if (ret) { 2437 pr_err("prepare_auth_signature failed: %d\n", ret); 2438 goto out; 2439 } 2440 2441 con->state = CEPH_CON_S_V2_AUTH_SIGNATURE; 2442 2443 out: 2444 memzero_explicit(session_key_buf, sizeof(session_key_buf)); 2445 memzero_explicit(con_secret_buf, sizeof(con_secret_buf)); 2446 return ret; 2447 2448 bad: 2449 pr_err("failed to decode auth_done\n"); 2450 return -EINVAL; 2451 } 2452 2453 static int process_auth_signature(struct ceph_connection *con, 2454 void *p, void *end) 2455 { 2456 u8 hmac[SHA256_DIGEST_SIZE]; 2457 int ret; 2458 2459 if (con->state != CEPH_CON_S_V2_AUTH_SIGNATURE) { 2460 con->error_msg = "protocol error, unexpected auth_signature"; 2461 return -EINVAL; 2462 } 2463 2464 ret = hmac_sha256(con, con->v2.out_sign_kvecs, 2465 con->v2.out_sign_kvec_cnt, hmac); 2466 if (ret) 2467 return ret; 2468 2469 ceph_decode_need(&p, end, SHA256_DIGEST_SIZE, bad); 2470 if (crypto_memneq(p, hmac, SHA256_DIGEST_SIZE)) { 2471 con->error_msg = "integrity error, bad auth signature"; 2472 return -EBADMSG; 2473 } 2474 2475 dout("%s con %p auth signature ok\n", __func__, con); 2476 2477 /* no reset_out_kvecs() as our auth_signature may still be pending */ 2478 if (!con->v2.server_cookie) { 2479 ret = prepare_client_ident(con); 2480 if (ret) { 2481 pr_err("prepare_client_ident failed: %d\n", ret); 2482 return ret; 2483 } 2484 2485 con->state = CEPH_CON_S_V2_SESSION_CONNECT; 2486 } else { 2487 ret = prepare_session_reconnect(con); 2488 if (ret) { 2489 pr_err("prepare_session_reconnect failed: %d\n", ret); 2490 return ret; 2491 } 2492 2493 con->state = CEPH_CON_S_V2_SESSION_RECONNECT; 2494 } 2495 2496 return 0; 2497 2498 bad: 2499 pr_err("failed to decode auth_signature\n"); 2500 return -EINVAL; 2501 } 2502 2503 static int process_server_ident(struct ceph_connection *con, 2504 void *p, void *end) 2505 { 2506 struct ceph_client *client = from_msgr(con->msgr); 2507 u64 features, required_features; 2508 struct ceph_entity_addr addr; 2509 u64 global_seq; 2510 u64 global_id; 2511 u64 cookie; 2512 u64 flags; 2513 int ret; 2514 2515 if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) { 2516 con->error_msg = "protocol error, unexpected server_ident"; 2517 return -EINVAL; 2518 } 2519 2520 ret = ceph_decode_entity_addrvec(&p, end, true, &addr); 2521 if (ret) { 2522 pr_err("failed to decode server addrs: %d\n", ret); 2523 return ret; 2524 } 2525 2526 ceph_decode_64_safe(&p, end, global_id, bad); 2527 ceph_decode_64_safe(&p, end, global_seq, bad); 2528 ceph_decode_64_safe(&p, end, features, bad); 2529 ceph_decode_64_safe(&p, end, required_features, bad); 2530 ceph_decode_64_safe(&p, end, flags, bad); 2531 ceph_decode_64_safe(&p, end, cookie, bad); 2532 2533 dout("%s con %p addr %s/%u global_id %llu global_seq %llu features 0x%llx required_features 0x%llx flags 0x%llx cookie 0x%llx\n", 2534 __func__, con, ceph_pr_addr(&addr), le32_to_cpu(addr.nonce), 2535 global_id, global_seq, features, required_features, flags, cookie); 2536 2537 /* is this who we intended to talk to? */ 2538 if (memcmp(&addr, &con->peer_addr, sizeof(con->peer_addr))) { 2539 pr_err("bad peer addr/nonce, want %s/%u, got %s/%u\n", 2540 ceph_pr_addr(&con->peer_addr), 2541 le32_to_cpu(con->peer_addr.nonce), 2542 ceph_pr_addr(&addr), le32_to_cpu(addr.nonce)); 2543 con->error_msg = "wrong peer at address"; 2544 return -EINVAL; 2545 } 2546 2547 if (client->required_features & ~features) { 2548 pr_err("RADOS feature set mismatch: my required > server's supported 0x%llx, need 0x%llx\n", 2549 features, client->required_features & ~features); 2550 con->error_msg = "missing required protocol features"; 2551 return -EINVAL; 2552 } 2553 2554 /* 2555 * Both name->type and name->num are set in ceph_con_open() but 2556 * name->num may be bogus in the initial monmap. name->type is 2557 * verified in handle_hello(). 2558 */ 2559 WARN_ON(!con->peer_name.type); 2560 con->peer_name.num = cpu_to_le64(global_id); 2561 con->v2.peer_global_seq = global_seq; 2562 con->peer_features = features; 2563 WARN_ON(required_features & ~client->supported_features); 2564 con->v2.server_cookie = cookie; 2565 2566 if (flags & CEPH_MSG_CONNECT_LOSSY) { 2567 ceph_con_flag_set(con, CEPH_CON_F_LOSSYTX); 2568 WARN_ON(con->v2.server_cookie); 2569 } else { 2570 WARN_ON(!con->v2.server_cookie); 2571 } 2572 2573 clear_in_sign_kvecs(con); 2574 clear_out_sign_kvecs(con); 2575 free_conn_bufs(con); 2576 con->delay = 0; /* reset backoff memory */ 2577 2578 con->state = CEPH_CON_S_OPEN; 2579 con->v2.out_state = OUT_S_GET_NEXT; 2580 return 0; 2581 2582 bad: 2583 pr_err("failed to decode server_ident\n"); 2584 return -EINVAL; 2585 } 2586 2587 static int process_ident_missing_features(struct ceph_connection *con, 2588 void *p, void *end) 2589 { 2590 struct ceph_client *client = from_msgr(con->msgr); 2591 u64 missing_features; 2592 2593 if (con->state != CEPH_CON_S_V2_SESSION_CONNECT) { 2594 con->error_msg = "protocol error, unexpected ident_missing_features"; 2595 return -EINVAL; 2596 } 2597 2598 ceph_decode_64_safe(&p, end, missing_features, bad); 2599 pr_err("RADOS feature set mismatch: server's required > my supported 0x%llx, missing 0x%llx\n", 2600 client->supported_features, missing_features); 2601 con->error_msg = "missing required protocol features"; 2602 return -EINVAL; 2603 2604 bad: 2605 pr_err("failed to decode ident_missing_features\n"); 2606 return -EINVAL; 2607 } 2608 2609 static int process_session_reconnect_ok(struct ceph_connection *con, 2610 void *p, void *end) 2611 { 2612 u64 seq; 2613 2614 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { 2615 con->error_msg = "protocol error, unexpected session_reconnect_ok"; 2616 return -EINVAL; 2617 } 2618 2619 ceph_decode_64_safe(&p, end, seq, bad); 2620 2621 dout("%s con %p seq %llu\n", __func__, con, seq); 2622 ceph_con_discard_requeued(con, seq); 2623 2624 clear_in_sign_kvecs(con); 2625 clear_out_sign_kvecs(con); 2626 free_conn_bufs(con); 2627 con->delay = 0; /* reset backoff memory */ 2628 2629 con->state = CEPH_CON_S_OPEN; 2630 con->v2.out_state = OUT_S_GET_NEXT; 2631 return 0; 2632 2633 bad: 2634 pr_err("failed to decode session_reconnect_ok\n"); 2635 return -EINVAL; 2636 } 2637 2638 static int process_session_retry(struct ceph_connection *con, 2639 void *p, void *end) 2640 { 2641 u64 connect_seq; 2642 int ret; 2643 2644 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { 2645 con->error_msg = "protocol error, unexpected session_retry"; 2646 return -EINVAL; 2647 } 2648 2649 ceph_decode_64_safe(&p, end, connect_seq, bad); 2650 2651 dout("%s con %p connect_seq %llu\n", __func__, con, connect_seq); 2652 WARN_ON(connect_seq <= con->v2.connect_seq); 2653 con->v2.connect_seq = connect_seq + 1; 2654 2655 free_conn_bufs(con); 2656 2657 reset_out_kvecs(con); 2658 ret = prepare_session_reconnect(con); 2659 if (ret) { 2660 pr_err("prepare_session_reconnect (cseq) failed: %d\n", ret); 2661 return ret; 2662 } 2663 2664 return 0; 2665 2666 bad: 2667 pr_err("failed to decode session_retry\n"); 2668 return -EINVAL; 2669 } 2670 2671 static int process_session_retry_global(struct ceph_connection *con, 2672 void *p, void *end) 2673 { 2674 u64 global_seq; 2675 int ret; 2676 2677 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { 2678 con->error_msg = "protocol error, unexpected session_retry_global"; 2679 return -EINVAL; 2680 } 2681 2682 ceph_decode_64_safe(&p, end, global_seq, bad); 2683 2684 dout("%s con %p global_seq %llu\n", __func__, con, global_seq); 2685 WARN_ON(global_seq <= con->v2.global_seq); 2686 con->v2.global_seq = ceph_get_global_seq(con->msgr, global_seq); 2687 2688 free_conn_bufs(con); 2689 2690 reset_out_kvecs(con); 2691 ret = prepare_session_reconnect(con); 2692 if (ret) { 2693 pr_err("prepare_session_reconnect (gseq) failed: %d\n", ret); 2694 return ret; 2695 } 2696 2697 return 0; 2698 2699 bad: 2700 pr_err("failed to decode session_retry_global\n"); 2701 return -EINVAL; 2702 } 2703 2704 static int process_session_reset(struct ceph_connection *con, 2705 void *p, void *end) 2706 { 2707 bool full; 2708 int ret; 2709 2710 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { 2711 con->error_msg = "protocol error, unexpected session_reset"; 2712 return -EINVAL; 2713 } 2714 2715 ceph_decode_8_safe(&p, end, full, bad); 2716 if (!full) { 2717 con->error_msg = "protocol error, bad session_reset"; 2718 return -EINVAL; 2719 } 2720 2721 pr_info("%s%lld %s session reset\n", ENTITY_NAME(con->peer_name), 2722 ceph_pr_addr(&con->peer_addr)); 2723 ceph_con_reset_session(con); 2724 2725 mutex_unlock(&con->mutex); 2726 if (con->ops->peer_reset) 2727 con->ops->peer_reset(con); 2728 mutex_lock(&con->mutex); 2729 if (con->state != CEPH_CON_S_V2_SESSION_RECONNECT) { 2730 dout("%s con %p state changed to %d\n", __func__, con, 2731 con->state); 2732 return -EAGAIN; 2733 } 2734 2735 free_conn_bufs(con); 2736 2737 reset_out_kvecs(con); 2738 ret = prepare_client_ident(con); 2739 if (ret) { 2740 pr_err("prepare_client_ident (rst) failed: %d\n", ret); 2741 return ret; 2742 } 2743 2744 con->state = CEPH_CON_S_V2_SESSION_CONNECT; 2745 return 0; 2746 2747 bad: 2748 pr_err("failed to decode session_reset\n"); 2749 return -EINVAL; 2750 } 2751 2752 static int process_keepalive2_ack(struct ceph_connection *con, 2753 void *p, void *end) 2754 { 2755 if (con->state != CEPH_CON_S_OPEN) { 2756 con->error_msg = "protocol error, unexpected keepalive2_ack"; 2757 return -EINVAL; 2758 } 2759 2760 ceph_decode_need(&p, end, sizeof(struct ceph_timespec), bad); 2761 ceph_decode_timespec64(&con->last_keepalive_ack, p); 2762 2763 dout("%s con %p timestamp %lld.%09ld\n", __func__, con, 2764 con->last_keepalive_ack.tv_sec, con->last_keepalive_ack.tv_nsec); 2765 2766 return 0; 2767 2768 bad: 2769 pr_err("failed to decode keepalive2_ack\n"); 2770 return -EINVAL; 2771 } 2772 2773 static int process_ack(struct ceph_connection *con, void *p, void *end) 2774 { 2775 u64 seq; 2776 2777 if (con->state != CEPH_CON_S_OPEN) { 2778 con->error_msg = "protocol error, unexpected ack"; 2779 return -EINVAL; 2780 } 2781 2782 ceph_decode_64_safe(&p, end, seq, bad); 2783 2784 dout("%s con %p seq %llu\n", __func__, con, seq); 2785 ceph_con_discard_sent(con, seq); 2786 return 0; 2787 2788 bad: 2789 pr_err("failed to decode ack\n"); 2790 return -EINVAL; 2791 } 2792 2793 static int process_control(struct ceph_connection *con, void *p, void *end) 2794 { 2795 int tag = con->v2.in_desc.fd_tag; 2796 int ret; 2797 2798 dout("%s con %p tag %d len %d\n", __func__, con, tag, (int)(end - p)); 2799 2800 switch (tag) { 2801 case FRAME_TAG_HELLO: 2802 ret = process_hello(con, p, end); 2803 break; 2804 case FRAME_TAG_AUTH_BAD_METHOD: 2805 ret = process_auth_bad_method(con, p, end); 2806 break; 2807 case FRAME_TAG_AUTH_REPLY_MORE: 2808 ret = process_auth_reply_more(con, p, end); 2809 break; 2810 case FRAME_TAG_AUTH_DONE: 2811 ret = process_auth_done(con, p, end); 2812 break; 2813 case FRAME_TAG_AUTH_SIGNATURE: 2814 ret = process_auth_signature(con, p, end); 2815 break; 2816 case FRAME_TAG_SERVER_IDENT: 2817 ret = process_server_ident(con, p, end); 2818 break; 2819 case FRAME_TAG_IDENT_MISSING_FEATURES: 2820 ret = process_ident_missing_features(con, p, end); 2821 break; 2822 case FRAME_TAG_SESSION_RECONNECT_OK: 2823 ret = process_session_reconnect_ok(con, p, end); 2824 break; 2825 case FRAME_TAG_SESSION_RETRY: 2826 ret = process_session_retry(con, p, end); 2827 break; 2828 case FRAME_TAG_SESSION_RETRY_GLOBAL: 2829 ret = process_session_retry_global(con, p, end); 2830 break; 2831 case FRAME_TAG_SESSION_RESET: 2832 ret = process_session_reset(con, p, end); 2833 break; 2834 case FRAME_TAG_KEEPALIVE2_ACK: 2835 ret = process_keepalive2_ack(con, p, end); 2836 break; 2837 case FRAME_TAG_ACK: 2838 ret = process_ack(con, p, end); 2839 break; 2840 default: 2841 pr_err("bad tag %d\n", tag); 2842 con->error_msg = "protocol error, bad tag"; 2843 return -EINVAL; 2844 } 2845 if (ret) { 2846 dout("%s con %p error %d\n", __func__, con, ret); 2847 return ret; 2848 } 2849 2850 prepare_read_preamble(con); 2851 return 0; 2852 } 2853 2854 /* 2855 * Return: 2856 * 1 - con->in_msg set, read message 2857 * 0 - skip message 2858 * <0 - error 2859 */ 2860 static int process_message_header(struct ceph_connection *con, 2861 void *p, void *end) 2862 { 2863 struct ceph_frame_desc *desc = &con->v2.in_desc; 2864 struct ceph_msg_header2 *hdr2 = p; 2865 struct ceph_msg_header hdr; 2866 int skip; 2867 int ret; 2868 u64 seq; 2869 2870 /* verify seq# */ 2871 seq = le64_to_cpu(hdr2->seq); 2872 if ((s64)seq - (s64)con->in_seq < 1) { 2873 pr_info("%s%lld %s skipping old message: seq %llu, expected %llu\n", 2874 ENTITY_NAME(con->peer_name), 2875 ceph_pr_addr(&con->peer_addr), 2876 seq, con->in_seq + 1); 2877 return 0; 2878 } 2879 if ((s64)seq - (s64)con->in_seq > 1) { 2880 pr_err("bad seq %llu, expected %llu\n", seq, con->in_seq + 1); 2881 con->error_msg = "bad message sequence # for incoming message"; 2882 return -EBADE; 2883 } 2884 2885 ceph_con_discard_sent(con, le64_to_cpu(hdr2->ack_seq)); 2886 2887 fill_header(&hdr, hdr2, desc->fd_lens[1], desc->fd_lens[2], 2888 desc->fd_lens[3], &con->peer_name); 2889 ret = ceph_con_in_msg_alloc(con, &hdr, &skip); 2890 if (ret) 2891 return ret; 2892 2893 WARN_ON(!con->in_msg ^ skip); 2894 if (skip) 2895 return 0; 2896 2897 WARN_ON(!con->in_msg); 2898 WARN_ON(con->in_msg->con != con); 2899 return 1; 2900 } 2901 2902 static int process_message(struct ceph_connection *con) 2903 { 2904 ceph_con_process_message(con); 2905 2906 /* 2907 * We could have been closed by ceph_con_close() because 2908 * ceph_con_process_message() temporarily drops con->mutex. 2909 */ 2910 if (con->state != CEPH_CON_S_OPEN) { 2911 dout("%s con %p state changed to %d\n", __func__, con, 2912 con->state); 2913 return -EAGAIN; 2914 } 2915 2916 prepare_read_preamble(con); 2917 return 0; 2918 } 2919 2920 static int __handle_control(struct ceph_connection *con, void *p) 2921 { 2922 void *end = p + con->v2.in_desc.fd_lens[0]; 2923 struct ceph_msg *msg; 2924 int ret; 2925 2926 if (con->v2.in_desc.fd_tag != FRAME_TAG_MESSAGE) 2927 return process_control(con, p, end); 2928 2929 ret = process_message_header(con, p, end); 2930 if (ret < 0) 2931 return ret; 2932 if (ret == 0) { 2933 prepare_skip_message(con); 2934 return 0; 2935 } 2936 2937 msg = con->in_msg; /* set in process_message_header() */ 2938 if (front_len(msg)) { 2939 WARN_ON(front_len(msg) > msg->front_alloc_len); 2940 msg->front.iov_len = front_len(msg); 2941 } else { 2942 msg->front.iov_len = 0; 2943 } 2944 if (middle_len(msg)) { 2945 WARN_ON(middle_len(msg) > msg->middle->alloc_len); 2946 msg->middle->vec.iov_len = middle_len(msg); 2947 } else if (msg->middle) { 2948 msg->middle->vec.iov_len = 0; 2949 } 2950 2951 if (!front_len(msg) && !middle_len(msg) && !data_len(msg)) 2952 return process_message(con); 2953 2954 if (con_secure(con)) 2955 return prepare_read_tail_secure(con); 2956 2957 return prepare_read_tail_plain(con); 2958 } 2959 2960 static int handle_preamble(struct ceph_connection *con) 2961 { 2962 struct ceph_frame_desc *desc = &con->v2.in_desc; 2963 int ret; 2964 2965 if (con_secure(con)) { 2966 ret = decrypt_preamble(con); 2967 if (ret) { 2968 if (ret == -EBADMSG) 2969 con->error_msg = "integrity error, bad preamble auth tag"; 2970 return ret; 2971 } 2972 } 2973 2974 ret = decode_preamble(con->v2.in_buf, desc); 2975 if (ret) { 2976 if (ret == -EBADMSG) 2977 con->error_msg = "integrity error, bad crc"; 2978 else 2979 con->error_msg = "protocol error, bad preamble"; 2980 return ret; 2981 } 2982 2983 dout("%s con %p tag %d seg_cnt %d %d+%d+%d+%d\n", __func__, 2984 con, desc->fd_tag, desc->fd_seg_cnt, desc->fd_lens[0], 2985 desc->fd_lens[1], desc->fd_lens[2], desc->fd_lens[3]); 2986 2987 if (!con_secure(con)) 2988 return prepare_read_control(con); 2989 2990 if (desc->fd_lens[0] > CEPH_PREAMBLE_INLINE_LEN) 2991 return prepare_read_control_remainder(con); 2992 2993 return __handle_control(con, CTRL_BODY(con->v2.in_buf)); 2994 } 2995 2996 static int handle_control(struct ceph_connection *con) 2997 { 2998 int ctrl_len = con->v2.in_desc.fd_lens[0]; 2999 void *buf; 3000 int ret; 3001 3002 WARN_ON(con_secure(con)); 3003 3004 ret = verify_control_crc(con); 3005 if (ret) { 3006 con->error_msg = "integrity error, bad crc"; 3007 return ret; 3008 } 3009 3010 if (con->state == CEPH_CON_S_V2_AUTH) { 3011 buf = alloc_conn_buf(con, ctrl_len); 3012 if (!buf) 3013 return -ENOMEM; 3014 3015 memcpy(buf, con->v2.in_kvecs[0].iov_base, ctrl_len); 3016 return __handle_control(con, buf); 3017 } 3018 3019 return __handle_control(con, con->v2.in_kvecs[0].iov_base); 3020 } 3021 3022 static int handle_control_remainder(struct ceph_connection *con) 3023 { 3024 int ret; 3025 3026 WARN_ON(!con_secure(con)); 3027 3028 ret = decrypt_control_remainder(con); 3029 if (ret) { 3030 if (ret == -EBADMSG) 3031 con->error_msg = "integrity error, bad control remainder auth tag"; 3032 return ret; 3033 } 3034 3035 return __handle_control(con, con->v2.in_kvecs[0].iov_base - 3036 CEPH_PREAMBLE_INLINE_LEN); 3037 } 3038 3039 static int handle_epilogue(struct ceph_connection *con) 3040 { 3041 u32 front_crc, middle_crc, data_crc; 3042 int ret; 3043 3044 if (con_secure(con)) { 3045 ret = decrypt_tail(con); 3046 if (ret) { 3047 if (ret == -EBADMSG) 3048 con->error_msg = "integrity error, bad epilogue auth tag"; 3049 return ret; 3050 } 3051 3052 /* just late_status */ 3053 ret = decode_epilogue(con->v2.in_buf, NULL, NULL, NULL); 3054 if (ret) { 3055 con->error_msg = "protocol error, bad epilogue"; 3056 return ret; 3057 } 3058 } else { 3059 ret = decode_epilogue(con->v2.in_buf, &front_crc, 3060 &middle_crc, &data_crc); 3061 if (ret) { 3062 con->error_msg = "protocol error, bad epilogue"; 3063 return ret; 3064 } 3065 3066 ret = verify_epilogue_crcs(con, front_crc, middle_crc, 3067 data_crc); 3068 if (ret) { 3069 con->error_msg = "integrity error, bad crc"; 3070 return ret; 3071 } 3072 } 3073 3074 return process_message(con); 3075 } 3076 3077 static void finish_skip(struct ceph_connection *con) 3078 { 3079 dout("%s con %p\n", __func__, con); 3080 3081 if (con_secure(con)) 3082 gcm_inc_nonce(&con->v2.in_gcm_nonce); 3083 3084 __finish_skip(con); 3085 } 3086 3087 static int populate_in_iter(struct ceph_connection *con) 3088 { 3089 int ret; 3090 3091 dout("%s con %p state %d in_state %d\n", __func__, con, con->state, 3092 con->v2.in_state); 3093 WARN_ON(iov_iter_count(&con->v2.in_iter)); 3094 3095 if (con->state == CEPH_CON_S_V2_BANNER_PREFIX) { 3096 ret = process_banner_prefix(con); 3097 } else if (con->state == CEPH_CON_S_V2_BANNER_PAYLOAD) { 3098 ret = process_banner_payload(con); 3099 } else if ((con->state >= CEPH_CON_S_V2_HELLO && 3100 con->state <= CEPH_CON_S_V2_SESSION_RECONNECT) || 3101 con->state == CEPH_CON_S_OPEN) { 3102 switch (con->v2.in_state) { 3103 case IN_S_HANDLE_PREAMBLE: 3104 ret = handle_preamble(con); 3105 break; 3106 case IN_S_HANDLE_CONTROL: 3107 ret = handle_control(con); 3108 break; 3109 case IN_S_HANDLE_CONTROL_REMAINDER: 3110 ret = handle_control_remainder(con); 3111 break; 3112 case IN_S_PREPARE_READ_DATA: 3113 ret = prepare_read_data(con); 3114 break; 3115 case IN_S_PREPARE_READ_DATA_CONT: 3116 prepare_read_data_cont(con); 3117 ret = 0; 3118 break; 3119 case IN_S_PREPARE_READ_ENC_PAGE: 3120 prepare_read_enc_page(con); 3121 ret = 0; 3122 break; 3123 case IN_S_PREPARE_SPARSE_DATA: 3124 ret = prepare_sparse_read_data(con); 3125 break; 3126 case IN_S_PREPARE_SPARSE_DATA_CONT: 3127 ret = prepare_sparse_read_cont(con); 3128 break; 3129 case IN_S_HANDLE_EPILOGUE: 3130 ret = handle_epilogue(con); 3131 break; 3132 case IN_S_FINISH_SKIP: 3133 finish_skip(con); 3134 ret = 0; 3135 break; 3136 default: 3137 WARN(1, "bad in_state %d", con->v2.in_state); 3138 return -EINVAL; 3139 } 3140 } else { 3141 WARN(1, "bad state %d", con->state); 3142 return -EINVAL; 3143 } 3144 if (ret) { 3145 dout("%s con %p error %d\n", __func__, con, ret); 3146 return ret; 3147 } 3148 3149 if (WARN_ON(!iov_iter_count(&con->v2.in_iter))) 3150 return -ENODATA; 3151 dout("%s con %p populated %zu\n", __func__, con, 3152 iov_iter_count(&con->v2.in_iter)); 3153 return 1; 3154 } 3155 3156 int ceph_con_v2_try_read(struct ceph_connection *con) 3157 { 3158 int ret; 3159 3160 dout("%s con %p state %d need %zu\n", __func__, con, con->state, 3161 iov_iter_count(&con->v2.in_iter)); 3162 3163 if (con->state == CEPH_CON_S_PREOPEN) 3164 return 0; 3165 3166 /* 3167 * We should always have something pending here. If not, 3168 * avoid calling populate_in_iter() as if we read something 3169 * (ceph_tcp_recv() would immediately return 1). 3170 */ 3171 if (WARN_ON(!iov_iter_count(&con->v2.in_iter))) 3172 return -ENODATA; 3173 3174 for (;;) { 3175 ret = ceph_tcp_recv(con); 3176 if (ret <= 0) 3177 return ret; 3178 3179 ret = populate_in_iter(con); 3180 if (ret <= 0) { 3181 if (ret && ret != -EAGAIN && !con->error_msg) 3182 con->error_msg = "read processing error"; 3183 return ret; 3184 } 3185 } 3186 } 3187 3188 static void queue_data(struct ceph_connection *con) 3189 { 3190 struct bio_vec bv; 3191 3192 con->v2.out_epil.data_crc = -1; 3193 ceph_msg_data_cursor_init(&con->v2.out_cursor, con->out_msg, 3194 data_len(con->out_msg)); 3195 3196 get_bvec_at(&con->v2.out_cursor, &bv); 3197 set_out_bvec(con, &bv, true); 3198 con->v2.out_state = OUT_S_QUEUE_DATA_CONT; 3199 } 3200 3201 static void queue_data_cont(struct ceph_connection *con) 3202 { 3203 struct bio_vec bv; 3204 3205 con->v2.out_epil.data_crc = ceph_crc32c_page( 3206 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page, 3207 con->v2.out_bvec.bv_offset, con->v2.out_bvec.bv_len); 3208 3209 ceph_msg_data_advance(&con->v2.out_cursor, con->v2.out_bvec.bv_len); 3210 if (con->v2.out_cursor.total_resid) { 3211 get_bvec_at(&con->v2.out_cursor, &bv); 3212 set_out_bvec(con, &bv, true); 3213 WARN_ON(con->v2.out_state != OUT_S_QUEUE_DATA_CONT); 3214 return; 3215 } 3216 3217 /* 3218 * We've written all data. Queue epilogue. Once it's written, 3219 * we are done. 3220 */ 3221 reset_out_kvecs(con); 3222 prepare_epilogue_plain(con, false); 3223 con->v2.out_state = OUT_S_FINISH_MESSAGE; 3224 } 3225 3226 static void queue_enc_page(struct ceph_connection *con) 3227 { 3228 struct bio_vec bv; 3229 3230 dout("%s con %p i %d resid %d\n", __func__, con, con->v2.out_enc_i, 3231 con->v2.out_enc_resid); 3232 WARN_ON(!con->v2.out_enc_resid); 3233 3234 bvec_set_page(&bv, con->v2.out_enc_pages[con->v2.out_enc_i], 3235 min(con->v2.out_enc_resid, (int)PAGE_SIZE), 0); 3236 3237 set_out_bvec(con, &bv, false); 3238 con->v2.out_enc_i++; 3239 con->v2.out_enc_resid -= bv.bv_len; 3240 3241 if (con->v2.out_enc_resid) { 3242 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE); 3243 return; 3244 } 3245 3246 /* 3247 * We've queued the last piece of ciphertext (ending with 3248 * epilogue) + auth tag. Once it's written, we are done. 3249 */ 3250 WARN_ON(con->v2.out_enc_i != con->v2.out_enc_page_cnt); 3251 con->v2.out_state = OUT_S_FINISH_MESSAGE; 3252 } 3253 3254 static void queue_zeros(struct ceph_connection *con) 3255 { 3256 dout("%s con %p out_zero %d\n", __func__, con, con->v2.out_zero); 3257 3258 if (con->v2.out_zero) { 3259 set_out_bvec_zero(con); 3260 con->v2.out_zero -= con->v2.out_bvec.bv_len; 3261 con->v2.out_state = OUT_S_QUEUE_ZEROS; 3262 return; 3263 } 3264 3265 /* 3266 * We've zero-filled everything up to epilogue. Queue epilogue 3267 * with late_status set to ABORTED and crcs adjusted for zeros. 3268 * Once it's written, we are done patching up for the revoke. 3269 */ 3270 reset_out_kvecs(con); 3271 prepare_epilogue_plain(con, true); 3272 con->v2.out_state = OUT_S_FINISH_MESSAGE; 3273 } 3274 3275 static void finish_message(struct ceph_connection *con) 3276 { 3277 dout("%s con %p msg %p\n", __func__, con, con->out_msg); 3278 3279 /* we end up here both plain and secure modes */ 3280 if (con->v2.out_enc_pages) { 3281 WARN_ON(!con->v2.out_enc_page_cnt); 3282 ceph_release_page_vector(con->v2.out_enc_pages, 3283 con->v2.out_enc_page_cnt); 3284 con->v2.out_enc_pages = NULL; 3285 con->v2.out_enc_page_cnt = 0; 3286 } 3287 /* message may have been revoked */ 3288 if (con->out_msg) { 3289 ceph_msg_put(con->out_msg); 3290 con->out_msg = NULL; 3291 } 3292 3293 con->v2.out_state = OUT_S_GET_NEXT; 3294 } 3295 3296 static int populate_out_iter(struct ceph_connection *con) 3297 { 3298 int ret; 3299 3300 dout("%s con %p state %d out_state %d\n", __func__, con, con->state, 3301 con->v2.out_state); 3302 WARN_ON(iov_iter_count(&con->v2.out_iter)); 3303 3304 if (con->state != CEPH_CON_S_OPEN) { 3305 WARN_ON(con->state < CEPH_CON_S_V2_BANNER_PREFIX || 3306 con->state > CEPH_CON_S_V2_SESSION_RECONNECT); 3307 goto nothing_pending; 3308 } 3309 3310 switch (con->v2.out_state) { 3311 case OUT_S_QUEUE_DATA: 3312 WARN_ON(!con->out_msg); 3313 queue_data(con); 3314 goto populated; 3315 case OUT_S_QUEUE_DATA_CONT: 3316 WARN_ON(!con->out_msg); 3317 queue_data_cont(con); 3318 goto populated; 3319 case OUT_S_QUEUE_ENC_PAGE: 3320 queue_enc_page(con); 3321 goto populated; 3322 case OUT_S_QUEUE_ZEROS: 3323 WARN_ON(con->out_msg); /* revoked */ 3324 queue_zeros(con); 3325 goto populated; 3326 case OUT_S_FINISH_MESSAGE: 3327 finish_message(con); 3328 break; 3329 case OUT_S_GET_NEXT: 3330 break; 3331 default: 3332 WARN(1, "bad out_state %d", con->v2.out_state); 3333 return -EINVAL; 3334 } 3335 3336 WARN_ON(con->v2.out_state != OUT_S_GET_NEXT); 3337 if (ceph_con_flag_test_and_clear(con, CEPH_CON_F_KEEPALIVE_PENDING)) { 3338 ret = prepare_keepalive2(con); 3339 if (ret) { 3340 pr_err("prepare_keepalive2 failed: %d\n", ret); 3341 return ret; 3342 } 3343 } else if (!list_empty(&con->out_queue)) { 3344 ceph_con_get_out_msg(con); 3345 ret = prepare_message(con); 3346 if (ret) { 3347 pr_err("prepare_message failed: %d\n", ret); 3348 return ret; 3349 } 3350 } else if (con->in_seq > con->in_seq_acked) { 3351 ret = prepare_ack(con); 3352 if (ret) { 3353 pr_err("prepare_ack failed: %d\n", ret); 3354 return ret; 3355 } 3356 } else { 3357 goto nothing_pending; 3358 } 3359 3360 populated: 3361 if (WARN_ON(!iov_iter_count(&con->v2.out_iter))) 3362 return -ENODATA; 3363 dout("%s con %p populated %zu\n", __func__, con, 3364 iov_iter_count(&con->v2.out_iter)); 3365 return 1; 3366 3367 nothing_pending: 3368 WARN_ON(iov_iter_count(&con->v2.out_iter)); 3369 dout("%s con %p nothing pending\n", __func__, con); 3370 ceph_con_flag_clear(con, CEPH_CON_F_WRITE_PENDING); 3371 return 0; 3372 } 3373 3374 int ceph_con_v2_try_write(struct ceph_connection *con) 3375 { 3376 int ret; 3377 3378 dout("%s con %p state %d have %zu\n", __func__, con, con->state, 3379 iov_iter_count(&con->v2.out_iter)); 3380 3381 /* open the socket first? */ 3382 if (con->state == CEPH_CON_S_PREOPEN) { 3383 WARN_ON(con->peer_addr.type != CEPH_ENTITY_ADDR_TYPE_MSGR2); 3384 3385 /* 3386 * Always bump global_seq. Bump connect_seq only if 3387 * there is a session (i.e. we are reconnecting and will 3388 * send session_reconnect instead of client_ident). 3389 */ 3390 con->v2.global_seq = ceph_get_global_seq(con->msgr, 0); 3391 if (con->v2.server_cookie) 3392 con->v2.connect_seq++; 3393 3394 ret = prepare_read_banner_prefix(con); 3395 if (ret) { 3396 pr_err("prepare_read_banner_prefix failed: %d\n", ret); 3397 con->error_msg = "connect error"; 3398 return ret; 3399 } 3400 3401 reset_out_kvecs(con); 3402 ret = prepare_banner(con); 3403 if (ret) { 3404 pr_err("prepare_banner failed: %d\n", ret); 3405 con->error_msg = "connect error"; 3406 return ret; 3407 } 3408 3409 ret = ceph_tcp_connect(con); 3410 if (ret) { 3411 pr_err("ceph_tcp_connect failed: %d\n", ret); 3412 con->error_msg = "connect error"; 3413 return ret; 3414 } 3415 } 3416 3417 if (!iov_iter_count(&con->v2.out_iter)) { 3418 ret = populate_out_iter(con); 3419 if (ret <= 0) { 3420 if (ret && ret != -EAGAIN && !con->error_msg) 3421 con->error_msg = "write processing error"; 3422 return ret; 3423 } 3424 } 3425 3426 tcp_sock_set_cork(con->sock->sk, true); 3427 for (;;) { 3428 ret = ceph_tcp_send(con); 3429 if (ret <= 0) 3430 break; 3431 3432 ret = populate_out_iter(con); 3433 if (ret <= 0) { 3434 if (ret && ret != -EAGAIN && !con->error_msg) 3435 con->error_msg = "write processing error"; 3436 break; 3437 } 3438 } 3439 3440 tcp_sock_set_cork(con->sock->sk, false); 3441 return ret; 3442 } 3443 3444 static u32 crc32c_zeros(u32 crc, int zero_len) 3445 { 3446 int len; 3447 3448 while (zero_len) { 3449 len = min(zero_len, (int)PAGE_SIZE); 3450 crc = crc32c(crc, page_address(ceph_zero_page), len); 3451 zero_len -= len; 3452 } 3453 3454 return crc; 3455 } 3456 3457 static void prepare_zero_front(struct ceph_connection *con, int resid) 3458 { 3459 int sent; 3460 3461 WARN_ON(!resid || resid > front_len(con->out_msg)); 3462 sent = front_len(con->out_msg) - resid; 3463 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid); 3464 3465 if (sent) { 3466 con->v2.out_epil.front_crc = 3467 crc32c(-1, con->out_msg->front.iov_base, sent); 3468 con->v2.out_epil.front_crc = 3469 crc32c_zeros(con->v2.out_epil.front_crc, resid); 3470 } else { 3471 con->v2.out_epil.front_crc = crc32c_zeros(-1, resid); 3472 } 3473 3474 con->v2.out_iter.count -= resid; 3475 out_zero_add(con, resid); 3476 } 3477 3478 static void prepare_zero_middle(struct ceph_connection *con, int resid) 3479 { 3480 int sent; 3481 3482 WARN_ON(!resid || resid > middle_len(con->out_msg)); 3483 sent = middle_len(con->out_msg) - resid; 3484 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid); 3485 3486 if (sent) { 3487 con->v2.out_epil.middle_crc = 3488 crc32c(-1, con->out_msg->middle->vec.iov_base, sent); 3489 con->v2.out_epil.middle_crc = 3490 crc32c_zeros(con->v2.out_epil.middle_crc, resid); 3491 } else { 3492 con->v2.out_epil.middle_crc = crc32c_zeros(-1, resid); 3493 } 3494 3495 con->v2.out_iter.count -= resid; 3496 out_zero_add(con, resid); 3497 } 3498 3499 static void prepare_zero_data(struct ceph_connection *con) 3500 { 3501 dout("%s con %p\n", __func__, con); 3502 con->v2.out_epil.data_crc = crc32c_zeros(-1, data_len(con->out_msg)); 3503 out_zero_add(con, data_len(con->out_msg)); 3504 } 3505 3506 static void revoke_at_queue_data(struct ceph_connection *con) 3507 { 3508 int boundary; 3509 int resid; 3510 3511 WARN_ON(!data_len(con->out_msg)); 3512 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter)); 3513 resid = iov_iter_count(&con->v2.out_iter); 3514 3515 boundary = front_len(con->out_msg) + middle_len(con->out_msg); 3516 if (resid > boundary) { 3517 resid -= boundary; 3518 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN); 3519 dout("%s con %p was sending head\n", __func__, con); 3520 if (front_len(con->out_msg)) 3521 prepare_zero_front(con, front_len(con->out_msg)); 3522 if (middle_len(con->out_msg)) 3523 prepare_zero_middle(con, middle_len(con->out_msg)); 3524 prepare_zero_data(con); 3525 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid); 3526 con->v2.out_state = OUT_S_QUEUE_ZEROS; 3527 return; 3528 } 3529 3530 boundary = middle_len(con->out_msg); 3531 if (resid > boundary) { 3532 resid -= boundary; 3533 dout("%s con %p was sending front\n", __func__, con); 3534 prepare_zero_front(con, resid); 3535 if (middle_len(con->out_msg)) 3536 prepare_zero_middle(con, middle_len(con->out_msg)); 3537 prepare_zero_data(con); 3538 queue_zeros(con); 3539 return; 3540 } 3541 3542 WARN_ON(!resid); 3543 dout("%s con %p was sending middle\n", __func__, con); 3544 prepare_zero_middle(con, resid); 3545 prepare_zero_data(con); 3546 queue_zeros(con); 3547 } 3548 3549 static void revoke_at_queue_data_cont(struct ceph_connection *con) 3550 { 3551 int sent, resid; /* current piece of data */ 3552 3553 WARN_ON(!data_len(con->out_msg)); 3554 WARN_ON(!iov_iter_is_bvec(&con->v2.out_iter)); 3555 resid = iov_iter_count(&con->v2.out_iter); 3556 WARN_ON(!resid || resid > con->v2.out_bvec.bv_len); 3557 sent = con->v2.out_bvec.bv_len - resid; 3558 dout("%s con %p sent %d resid %d\n", __func__, con, sent, resid); 3559 3560 if (sent) { 3561 con->v2.out_epil.data_crc = ceph_crc32c_page( 3562 con->v2.out_epil.data_crc, con->v2.out_bvec.bv_page, 3563 con->v2.out_bvec.bv_offset, sent); 3564 ceph_msg_data_advance(&con->v2.out_cursor, sent); 3565 } 3566 WARN_ON(resid > con->v2.out_cursor.total_resid); 3567 con->v2.out_epil.data_crc = crc32c_zeros(con->v2.out_epil.data_crc, 3568 con->v2.out_cursor.total_resid); 3569 3570 con->v2.out_iter.count -= resid; 3571 out_zero_add(con, con->v2.out_cursor.total_resid); 3572 queue_zeros(con); 3573 } 3574 3575 static void revoke_at_finish_message(struct ceph_connection *con) 3576 { 3577 int boundary; 3578 int resid; 3579 3580 WARN_ON(!iov_iter_is_kvec(&con->v2.out_iter)); 3581 resid = iov_iter_count(&con->v2.out_iter); 3582 3583 if (!front_len(con->out_msg) && !middle_len(con->out_msg) && 3584 !data_len(con->out_msg)) { 3585 WARN_ON(!resid || resid > MESSAGE_HEAD_PLAIN_LEN); 3586 dout("%s con %p was sending head (empty message) - noop\n", 3587 __func__, con); 3588 return; 3589 } 3590 3591 boundary = front_len(con->out_msg) + middle_len(con->out_msg) + 3592 CEPH_EPILOGUE_PLAIN_LEN; 3593 if (resid > boundary) { 3594 resid -= boundary; 3595 WARN_ON(resid > MESSAGE_HEAD_PLAIN_LEN); 3596 dout("%s con %p was sending head\n", __func__, con); 3597 if (front_len(con->out_msg)) 3598 prepare_zero_front(con, front_len(con->out_msg)); 3599 if (middle_len(con->out_msg)) 3600 prepare_zero_middle(con, middle_len(con->out_msg)); 3601 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN; 3602 WARN_ON(iov_iter_count(&con->v2.out_iter) != resid); 3603 con->v2.out_state = OUT_S_QUEUE_ZEROS; 3604 return; 3605 } 3606 3607 boundary = middle_len(con->out_msg) + CEPH_EPILOGUE_PLAIN_LEN; 3608 if (resid > boundary) { 3609 resid -= boundary; 3610 dout("%s con %p was sending front\n", __func__, con); 3611 prepare_zero_front(con, resid); 3612 if (middle_len(con->out_msg)) 3613 prepare_zero_middle(con, middle_len(con->out_msg)); 3614 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN; 3615 queue_zeros(con); 3616 return; 3617 } 3618 3619 boundary = CEPH_EPILOGUE_PLAIN_LEN; 3620 if (resid > boundary) { 3621 resid -= boundary; 3622 dout("%s con %p was sending middle\n", __func__, con); 3623 prepare_zero_middle(con, resid); 3624 con->v2.out_iter.count -= CEPH_EPILOGUE_PLAIN_LEN; 3625 queue_zeros(con); 3626 return; 3627 } 3628 3629 WARN_ON(!resid); 3630 dout("%s con %p was sending epilogue - noop\n", __func__, con); 3631 } 3632 3633 void ceph_con_v2_revoke(struct ceph_connection *con) 3634 { 3635 WARN_ON(con->v2.out_zero); 3636 3637 if (con_secure(con)) { 3638 WARN_ON(con->v2.out_state != OUT_S_QUEUE_ENC_PAGE && 3639 con->v2.out_state != OUT_S_FINISH_MESSAGE); 3640 dout("%s con %p secure - noop\n", __func__, con); 3641 return; 3642 } 3643 3644 switch (con->v2.out_state) { 3645 case OUT_S_QUEUE_DATA: 3646 revoke_at_queue_data(con); 3647 break; 3648 case OUT_S_QUEUE_DATA_CONT: 3649 revoke_at_queue_data_cont(con); 3650 break; 3651 case OUT_S_FINISH_MESSAGE: 3652 revoke_at_finish_message(con); 3653 break; 3654 default: 3655 WARN(1, "bad out_state %d", con->v2.out_state); 3656 break; 3657 } 3658 } 3659 3660 static void revoke_at_prepare_read_data(struct ceph_connection *con) 3661 { 3662 int remaining; 3663 int resid; 3664 3665 WARN_ON(con_secure(con)); 3666 WARN_ON(!data_len(con->in_msg)); 3667 WARN_ON(!iov_iter_is_kvec(&con->v2.in_iter)); 3668 resid = iov_iter_count(&con->v2.in_iter); 3669 WARN_ON(!resid); 3670 3671 remaining = data_len(con->in_msg) + CEPH_EPILOGUE_PLAIN_LEN; 3672 dout("%s con %p resid %d remaining %d\n", __func__, con, resid, 3673 remaining); 3674 con->v2.in_iter.count -= resid; 3675 set_in_skip(con, resid + remaining); 3676 con->v2.in_state = IN_S_FINISH_SKIP; 3677 } 3678 3679 static void revoke_at_prepare_read_data_cont(struct ceph_connection *con) 3680 { 3681 int recved, resid; /* current piece of data */ 3682 int remaining; 3683 3684 WARN_ON(con_secure(con)); 3685 WARN_ON(!data_len(con->in_msg)); 3686 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter)); 3687 resid = iov_iter_count(&con->v2.in_iter); 3688 WARN_ON(!resid || resid > con->v2.in_bvec.bv_len); 3689 recved = con->v2.in_bvec.bv_len - resid; 3690 dout("%s con %p recved %d resid %d\n", __func__, con, recved, resid); 3691 3692 if (recved) 3693 ceph_msg_data_advance(&con->v2.in_cursor, recved); 3694 WARN_ON(resid > con->v2.in_cursor.total_resid); 3695 3696 remaining = CEPH_EPILOGUE_PLAIN_LEN; 3697 dout("%s con %p total_resid %zu remaining %d\n", __func__, con, 3698 con->v2.in_cursor.total_resid, remaining); 3699 con->v2.in_iter.count -= resid; 3700 set_in_skip(con, con->v2.in_cursor.total_resid + remaining); 3701 con->v2.in_state = IN_S_FINISH_SKIP; 3702 } 3703 3704 static void revoke_at_prepare_read_enc_page(struct ceph_connection *con) 3705 { 3706 int resid; /* current enc page (not necessarily data) */ 3707 3708 WARN_ON(!con_secure(con)); 3709 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter)); 3710 resid = iov_iter_count(&con->v2.in_iter); 3711 WARN_ON(!resid || resid > con->v2.in_bvec.bv_len); 3712 3713 dout("%s con %p resid %d enc_resid %d\n", __func__, con, resid, 3714 con->v2.in_enc_resid); 3715 con->v2.in_iter.count -= resid; 3716 set_in_skip(con, resid + con->v2.in_enc_resid); 3717 con->v2.in_state = IN_S_FINISH_SKIP; 3718 } 3719 3720 static void revoke_at_prepare_sparse_data(struct ceph_connection *con) 3721 { 3722 int resid; /* current piece of data */ 3723 int remaining; 3724 3725 WARN_ON(con_secure(con)); 3726 WARN_ON(!data_len(con->in_msg)); 3727 WARN_ON(!iov_iter_is_bvec(&con->v2.in_iter)); 3728 resid = iov_iter_count(&con->v2.in_iter); 3729 dout("%s con %p resid %d\n", __func__, con, resid); 3730 3731 remaining = CEPH_EPILOGUE_PLAIN_LEN + con->v2.data_len_remain; 3732 con->v2.in_iter.count -= resid; 3733 set_in_skip(con, resid + remaining); 3734 con->v2.in_state = IN_S_FINISH_SKIP; 3735 } 3736 3737 static void revoke_at_handle_epilogue(struct ceph_connection *con) 3738 { 3739 int resid; 3740 3741 resid = iov_iter_count(&con->v2.in_iter); 3742 WARN_ON(!resid); 3743 3744 dout("%s con %p resid %d\n", __func__, con, resid); 3745 con->v2.in_iter.count -= resid; 3746 set_in_skip(con, resid); 3747 con->v2.in_state = IN_S_FINISH_SKIP; 3748 } 3749 3750 void ceph_con_v2_revoke_incoming(struct ceph_connection *con) 3751 { 3752 switch (con->v2.in_state) { 3753 case IN_S_PREPARE_SPARSE_DATA: 3754 case IN_S_PREPARE_READ_DATA: 3755 revoke_at_prepare_read_data(con); 3756 break; 3757 case IN_S_PREPARE_READ_DATA_CONT: 3758 revoke_at_prepare_read_data_cont(con); 3759 break; 3760 case IN_S_PREPARE_READ_ENC_PAGE: 3761 revoke_at_prepare_read_enc_page(con); 3762 break; 3763 case IN_S_PREPARE_SPARSE_DATA_CONT: 3764 revoke_at_prepare_sparse_data(con); 3765 break; 3766 case IN_S_HANDLE_EPILOGUE: 3767 revoke_at_handle_epilogue(con); 3768 break; 3769 default: 3770 WARN(1, "bad in_state %d", con->v2.in_state); 3771 break; 3772 } 3773 } 3774 3775 bool ceph_con_v2_opened(struct ceph_connection *con) 3776 { 3777 return con->v2.peer_global_seq; 3778 } 3779 3780 void ceph_con_v2_reset_session(struct ceph_connection *con) 3781 { 3782 con->v2.client_cookie = 0; 3783 con->v2.server_cookie = 0; 3784 con->v2.global_seq = 0; 3785 con->v2.connect_seq = 0; 3786 con->v2.peer_global_seq = 0; 3787 } 3788 3789 void ceph_con_v2_reset_protocol(struct ceph_connection *con) 3790 { 3791 iov_iter_truncate(&con->v2.in_iter, 0); 3792 iov_iter_truncate(&con->v2.out_iter, 0); 3793 con->v2.out_zero = 0; 3794 3795 clear_in_sign_kvecs(con); 3796 clear_out_sign_kvecs(con); 3797 free_conn_bufs(con); 3798 3799 if (con->v2.in_enc_pages) { 3800 WARN_ON(!con->v2.in_enc_page_cnt); 3801 ceph_release_page_vector(con->v2.in_enc_pages, 3802 con->v2.in_enc_page_cnt); 3803 con->v2.in_enc_pages = NULL; 3804 con->v2.in_enc_page_cnt = 0; 3805 } 3806 if (con->v2.out_enc_pages) { 3807 WARN_ON(!con->v2.out_enc_page_cnt); 3808 ceph_release_page_vector(con->v2.out_enc_pages, 3809 con->v2.out_enc_page_cnt); 3810 con->v2.out_enc_pages = NULL; 3811 con->v2.out_enc_page_cnt = 0; 3812 } 3813 3814 con->v2.con_mode = CEPH_CON_MODE_UNKNOWN; 3815 memzero_explicit(&con->v2.in_gcm_nonce, CEPH_GCM_IV_LEN); 3816 memzero_explicit(&con->v2.out_gcm_nonce, CEPH_GCM_IV_LEN); 3817 3818 if (con->v2.hmac_tfm) { 3819 crypto_free_shash(con->v2.hmac_tfm); 3820 con->v2.hmac_tfm = NULL; 3821 } 3822 if (con->v2.gcm_req) { 3823 aead_request_free(con->v2.gcm_req); 3824 con->v2.gcm_req = NULL; 3825 } 3826 if (con->v2.gcm_tfm) { 3827 crypto_free_aead(con->v2.gcm_tfm); 3828 con->v2.gcm_tfm = NULL; 3829 } 3830 } 3831