538 static void ceph_msg_remove(struct ceph_msg *msg)  in ceph_msg_remove()  argument
540 	list_del_init(&msg->list_head);  in ceph_msg_remove()
542 	ceph_msg_put(msg);  in ceph_msg_remove()
548 		struct ceph_msg *msg = list_first_entry(head, struct ceph_msg,  in ceph_msg_remove_list()  local
550 		ceph_msg_remove(msg);  in ceph_msg_remove_list()
673 	struct ceph_msg *msg;  in ceph_con_discard_sent()  local
678 		msg = list_first_entry(&con->out_sent, struct ceph_msg,  in ceph_con_discard_sent()
680 		WARN_ON(msg->needs_out_seq);  in ceph_con_discard_sent()
681 		seq = le64_to_cpu(msg->hdr.seq);  in ceph_con_discard_sent()
686 		     msg, seq);  in ceph_con_discard_sent()
687 		ceph_msg_remove(msg);  in ceph_con_discard_sent()
698 	struct ceph_msg *msg;  in ceph_con_discard_requeued()  local
703 		msg = list_first_entry(&con->out_queue, struct ceph_msg,  in ceph_con_discard_requeued()
705 		if (msg->needs_out_seq)  in ceph_con_discard_requeued()
707 		seq = le64_to_cpu(msg->hdr.seq);  in ceph_con_discard_requeued()
712 		     msg, seq);  in ceph_con_discard_requeued()
713 		ceph_msg_remove(msg);  in ceph_con_discard_requeued()
1067 			       struct ceph_msg *msg, size_t length)  in ceph_msg_data_cursor_init()  argument
1070 	BUG_ON(length > msg->data_length);  in ceph_msg_data_cursor_init()
1071 	BUG_ON(!msg->num_data_items);  in ceph_msg_data_cursor_init()
1074 	cursor->data = msg->data;  in ceph_msg_data_cursor_init()
1403 	struct ceph_msg *msg = con->in_msg;  in ceph_con_process_message()  local
1410 		con->peer_name = msg->hdr.src;  in ceph_con_process_message()
1416 	     msg, le64_to_cpu(msg->hdr.seq),  in ceph_con_process_message()
1417 	     ENTITY_NAME(msg->hdr.src),  in ceph_con_process_message()
1418 	     le16_to_cpu(msg->hdr.type),  in ceph_con_process_message()
1419 	     ceph_msg_type_name(le16_to_cpu(msg->hdr.type)),  in ceph_con_process_message()
1420 	     le32_to_cpu(msg->hdr.front_len),  in ceph_con_process_message()
1421 	     le32_to_cpu(msg->hdr.middle_len),  in ceph_con_process_message()
1422 	     le32_to_cpu(msg->hdr.data_len),  in ceph_con_process_message()
1424 	con->ops->dispatch(con, msg);  in ceph_con_process_message()
1704 static void msg_con_set(struct ceph_msg *msg, struct ceph_connection *con)  in msg_con_set()  argument
1706 	if (msg->con)  in msg_con_set()
1707 		msg->con->ops->put(msg->con);  in msg_con_set()
1709 	msg->con = con ? con->ops->get(con) : NULL;  in msg_con_set()
1710 	BUG_ON(msg->con != con);  in msg_con_set()
1730 void ceph_con_send(struct ceph_connection *con, struct ceph_msg *msg)  in ceph_con_send()  argument
1733 	msg->hdr.src = con->msgr->inst.name;  in ceph_con_send()
1734 	BUG_ON(msg->front.iov_len != le32_to_cpu(msg->hdr.front_len));  in ceph_con_send()
1735 	msg->needs_out_seq = true;  in ceph_con_send()
1740 		dout("con_send %p closed, dropping %p\n", con, msg);  in ceph_con_send()
1741 		ceph_msg_put(msg);  in ceph_con_send()
1746 	msg_con_set(msg, con);  in ceph_con_send()
1748 	BUG_ON(!list_empty(&msg->list_head));  in ceph_con_send()
1749 	list_add_tail(&msg->list_head, &con->out_queue);  in ceph_con_send()
1750 	dout("----- %p to %s%lld %d=%s len %d+%d+%d -----\n", msg,  in ceph_con_send()
1751 	     ENTITY_NAME(con->peer_name), le16_to_cpu(msg->hdr.type),  in ceph_con_send()
1752 	     ceph_msg_type_name(le16_to_cpu(msg->hdr.type)),  in ceph_con_send()
1753 	     le32_to_cpu(msg->hdr.front_len),  in ceph_con_send()
1754 	     le32_to_cpu(msg->hdr.middle_len),  in ceph_con_send()
1755 	     le32_to_cpu(msg->hdr.data_len));  in ceph_con_send()
1770 void ceph_msg_revoke(struct ceph_msg *msg)  in ceph_msg_revoke()  argument
1772 	struct ceph_connection *con = msg->con;  in ceph_msg_revoke()
1775 		dout("%s msg %p null con\n", __func__, msg);  in ceph_msg_revoke()
1780 	if (list_empty(&msg->list_head)) {  in ceph_msg_revoke()
1781 		WARN_ON(con->out_msg == msg);  in ceph_msg_revoke()
1782 		dout("%s con %p msg %p not linked\n", __func__, con, msg);  in ceph_msg_revoke()
1787 	dout("%s con %p msg %p was linked\n", __func__, con, msg);  in ceph_msg_revoke()
1788 	msg->hdr.seq = 0;  in ceph_msg_revoke()
1789 	ceph_msg_remove(msg);  in ceph_msg_revoke()
1791 	if (con->out_msg == msg) {  in ceph_msg_revoke()
1793 		dout("%s con %p msg %p was sending\n", __func__, con, msg);  in ceph_msg_revoke()
1802 		     con, msg, con->out_msg);  in ceph_msg_revoke()
1810 void ceph_msg_revoke_incoming(struct ceph_msg *msg)  in ceph_msg_revoke_incoming()  argument
1812 	struct ceph_connection *con = msg->con;  in ceph_msg_revoke_incoming()
1815 		dout("%s msg %p null con\n", __func__, msg);  in ceph_msg_revoke_incoming()
1820 	if (con->in_msg == msg) {  in ceph_msg_revoke_incoming()
1822 		dout("%s con %p msg %p was recving\n", __func__, con, msg);  in ceph_msg_revoke_incoming()
1831 		     con, msg, con->in_msg);  in ceph_msg_revoke_incoming()
1867 static struct ceph_msg_data *ceph_msg_data_add(struct ceph_msg *msg)  in ceph_msg_data_add()  argument
1869 	BUG_ON(msg->num_data_items >= msg->max_data_items);  in ceph_msg_data_add()
1870 	return &msg->data[msg->num_data_items++];  in ceph_msg_data_add()
1883 void ceph_msg_data_add_pages(struct ceph_msg *msg, struct page **pages,  in ceph_msg_data_add_pages()  argument
1891 	data = ceph_msg_data_add(msg);  in ceph_msg_data_add_pages()
1898 	msg->data_length += length;  in ceph_msg_data_add_pages()
1902 void ceph_msg_data_add_pagelist(struct ceph_msg *msg,  in ceph_msg_data_add_pagelist()  argument
1910 	data = ceph_msg_data_add(msg);  in ceph_msg_data_add_pagelist()
1915 	msg->data_length += pagelist->length;  in ceph_msg_data_add_pagelist()
1920 void ceph_msg_data_add_bio(struct ceph_msg *msg, struct ceph_bio_iter *bio_pos,  in ceph_msg_data_add_bio()  argument
1925 	data = ceph_msg_data_add(msg);  in ceph_msg_data_add_bio()
1930 	msg->data_length += length;  in ceph_msg_data_add_bio()
1935 void ceph_msg_data_add_bvecs(struct ceph_msg *msg,  in ceph_msg_data_add_bvecs()  argument
1940 	data = ceph_msg_data_add(msg);  in ceph_msg_data_add_bvecs()
1944 	msg->data_length += bvec_pos->iter.bi_size;  in ceph_msg_data_add_bvecs()
1948 void ceph_msg_data_add_iter(struct ceph_msg *msg,  in ceph_msg_data_add_iter()  argument
1953 	data = ceph_msg_data_add(msg);  in ceph_msg_data_add_iter()
1957 	msg->data_length += iov_iter_count(&data->iter);  in ceph_msg_data_add_iter()
2034 static int ceph_alloc_middle(struct ceph_connection *con, struct ceph_msg *msg)  in ceph_alloc_middle()  argument
2036 	int type = le16_to_cpu(msg->hdr.type);  in ceph_alloc_middle()
2037 	int middle_len = le32_to_cpu(msg->hdr.middle_len);  in ceph_alloc_middle()
2039 	dout("alloc_middle %p type %d %s middle_len %d\n", msg, type,  in ceph_alloc_middle()
2042 	BUG_ON(msg->middle);  in ceph_alloc_middle()
2044 	msg->middle = ceph_buffer_new(middle_len, GFP_NOFS);  in ceph_alloc_middle()
2045 	if (!msg->middle)  in ceph_alloc_middle()
2069 	struct ceph_msg *msg;  in ceph_con_in_msg_alloc()  local
2076 	msg = con->ops->alloc_msg(con, hdr, skip);  in ceph_con_in_msg_alloc()
2079 		if (msg)  in ceph_con_in_msg_alloc()
2080 			ceph_msg_put(msg);  in ceph_con_in_msg_alloc()
2083 	if (msg) {  in ceph_con_in_msg_alloc()
2085 		msg_con_set(msg, con);  in ceph_con_in_msg_alloc()
2086 		con->in_msg = msg;  in ceph_con_in_msg_alloc()
2114 	struct ceph_msg *msg;  in ceph_con_get_out_msg()  local
2117 	msg = list_first_entry(&con->out_queue, struct ceph_msg, list_head);  in ceph_con_get_out_msg()
2118 	WARN_ON(msg->con != con);  in ceph_con_get_out_msg()
2124 	list_move_tail(&msg->list_head, &con->out_sent);  in ceph_con_get_out_msg()
2130 	if (msg->needs_out_seq) {  in ceph_con_get_out_msg()
2131 		msg->hdr.seq = cpu_to_le64(++con->out_seq);  in ceph_con_get_out_msg()
2132 		msg->needs_out_seq = false;  in ceph_con_get_out_msg()
2135 			con->ops->reencode_message(msg);  in ceph_con_get_out_msg()
2143 	con->out_msg = ceph_msg_get(msg);  in ceph_con_get_out_msg()
2182 struct ceph_msg *ceph_msg_get(struct ceph_msg *msg)  in ceph_msg_get()  argument
2184 	dout("%s %p (was %d)\n", __func__, msg,  in ceph_msg_get()
2185 	     kref_read(&msg->kref));  in ceph_msg_get()
2186 	kref_get(&msg->kref);  in ceph_msg_get()
2187 	return msg;  in ceph_msg_get()
2191 void ceph_msg_put(struct ceph_msg *msg)  in ceph_msg_put()  argument
2193 	dout("%s %p (was %d)\n", __func__, msg,  in ceph_msg_put()
2194 	     kref_read(&msg->kref));  in ceph_msg_put()
2195 	kref_put(&msg->kref, ceph_msg_release);  in ceph_msg_put()
2199 void ceph_msg_dump(struct ceph_msg *msg)  in ceph_msg_dump()  argument
2201 	pr_debug("msg_dump %p (front_alloc_len %d length %zd)\n", msg,  in ceph_msg_dump()
2202 		 msg->front_alloc_len, msg->data_length);  in ceph_msg_dump()
2205 		       &msg->hdr, sizeof(msg->hdr), true);  in ceph_msg_dump()
2208 		       msg->front.iov_base, msg->front.iov_len, true);  in ceph_msg_dump()
2209 	if (msg->middle)  in ceph_msg_dump()
2212 			       msg->middle->vec.iov_base,  in ceph_msg_dump()
2213 			       msg->middle->vec.iov_len, true);  in ceph_msg_dump()
2216 		       &msg->footer, sizeof(msg->footer), true);  in ceph_msg_dump()