Lines Matching full:inc
44 void rds_inc_init(struct rds_incoming *inc, struct rds_connection *conn, in rds_inc_init() argument
47 refcount_set(&inc->i_refcount, 1); in rds_inc_init()
48 INIT_LIST_HEAD(&inc->i_item); in rds_inc_init()
49 inc->i_conn = conn; in rds_inc_init()
50 inc->i_saddr = *saddr; in rds_inc_init()
51 inc->i_usercopy.rdma_cookie = 0; in rds_inc_init()
52 inc->i_usercopy.rx_tstamp = ktime_set(0, 0); in rds_inc_init()
54 memset(inc->i_rx_lat_trace, 0, sizeof(inc->i_rx_lat_trace)); in rds_inc_init()
58 void rds_inc_path_init(struct rds_incoming *inc, struct rds_conn_path *cp, in rds_inc_path_init() argument
61 refcount_set(&inc->i_refcount, 1); in rds_inc_path_init()
62 INIT_LIST_HEAD(&inc->i_item); in rds_inc_path_init()
63 inc->i_conn = cp->cp_conn; in rds_inc_path_init()
64 inc->i_conn_path = cp; in rds_inc_path_init()
65 inc->i_saddr = *saddr; in rds_inc_path_init()
66 inc->i_usercopy.rdma_cookie = 0; in rds_inc_path_init()
67 inc->i_usercopy.rx_tstamp = ktime_set(0, 0); in rds_inc_path_init()
71 static void rds_inc_addref(struct rds_incoming *inc) in rds_inc_addref() argument
73 rdsdebug("addref inc %p ref %d\n", inc, refcount_read(&inc->i_refcount)); in rds_inc_addref()
74 refcount_inc(&inc->i_refcount); in rds_inc_addref()
77 void rds_inc_put(struct rds_incoming *inc) in rds_inc_put() argument
79 rdsdebug("put inc %p ref %d\n", inc, refcount_read(&inc->i_refcount)); in rds_inc_put()
80 if (refcount_dec_and_test(&inc->i_refcount)) { in rds_inc_put()
81 BUG_ON(!list_empty(&inc->i_item)); in rds_inc_put()
83 inc->i_conn->c_trans->inc_free(inc); in rds_inc_put()
166 static void rds_recv_incoming_exthdrs(struct rds_incoming *inc, struct rds_sock *rs) in rds_recv_incoming_exthdrs() argument
168 struct rds_header *hdr = &inc->i_hdr; in rds_recv_incoming_exthdrs()
190 inc->i_usercopy.rdma_cookie = rds_rdma_make_cookie( in rds_recv_incoming_exthdrs()
285 struct rds_incoming *inc, gfp_t gfp) in rds_recv_incoming() argument
292 inc->i_conn = conn; in rds_recv_incoming()
293 inc->i_rx_jiffies = jiffies; in rds_recv_incoming()
295 cp = inc->i_conn_path; in rds_recv_incoming()
299 rdsdebug("conn %p next %llu inc %p seq %llu len %u sport %u dport %u " in rds_recv_incoming()
302 inc, in rds_recv_incoming()
303 (unsigned long long)be64_to_cpu(inc->i_hdr.h_sequence), in rds_recv_incoming()
304 be32_to_cpu(inc->i_hdr.h_len), in rds_recv_incoming()
305 be16_to_cpu(inc->i_hdr.h_sport), in rds_recv_incoming()
306 be16_to_cpu(inc->i_hdr.h_dport), in rds_recv_incoming()
307 inc->i_hdr.h_flags, in rds_recv_incoming()
308 inc->i_rx_jiffies); in rds_recv_incoming()
330 if (be64_to_cpu(inc->i_hdr.h_sequence) < cp->cp_next_rx_seq && in rds_recv_incoming()
331 (inc->i_hdr.h_flags & RDS_FLAG_RETRANSMITTED)) { in rds_recv_incoming()
335 cp->cp_next_rx_seq = be64_to_cpu(inc->i_hdr.h_sequence) + 1; in rds_recv_incoming()
337 if (rds_sysctl_ping_enable && inc->i_hdr.h_dport == 0) { in rds_recv_incoming()
338 if (inc->i_hdr.h_sport == 0) { in rds_recv_incoming()
344 rds_send_pong(cp, inc->i_hdr.h_sport); in rds_recv_incoming()
346 if (RDS_HS_PROBE(be16_to_cpu(inc->i_hdr.h_sport), in rds_recv_incoming()
347 be16_to_cpu(inc->i_hdr.h_dport))) { in rds_recv_incoming()
348 rds_recv_hs_exthdrs(&inc->i_hdr, cp->cp_conn); in rds_recv_incoming()
354 if (be16_to_cpu(inc->i_hdr.h_dport) == RDS_FLAG_PROBE_PORT && in rds_recv_incoming()
355 inc->i_hdr.h_sport == 0) { in rds_recv_incoming()
356 rds_recv_hs_exthdrs(&inc->i_hdr, cp->cp_conn); in rds_recv_incoming()
363 rs = rds_find_bound(daddr, inc->i_hdr.h_dport, conn->c_bound_if); in rds_recv_incoming()
370 rds_recv_incoming_exthdrs(inc, rs); in rds_recv_incoming()
378 rdsdebug("adding inc %p to rs %p's recv queue\n", inc, rs); in rds_recv_incoming()
380 rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong, in rds_recv_incoming()
381 be32_to_cpu(inc->i_hdr.h_len), in rds_recv_incoming()
382 inc->i_hdr.h_dport); in rds_recv_incoming()
384 inc->i_usercopy.rx_tstamp = ktime_get_real(); in rds_recv_incoming()
385 rds_inc_addref(inc); in rds_recv_incoming()
386 inc->i_rx_lat_trace[RDS_MSG_RX_END] = local_clock(); in rds_recv_incoming()
387 list_add_tail(&inc->i_item, &rs->rs_recv_queue); in rds_recv_incoming()
404 static int rds_next_incoming(struct rds_sock *rs, struct rds_incoming **inc) in rds_next_incoming() argument
408 if (!*inc) { in rds_next_incoming()
411 *inc = list_entry(rs->rs_recv_queue.next, in rds_next_incoming()
414 rds_inc_addref(*inc); in rds_next_incoming()
419 return *inc != NULL; in rds_next_incoming()
422 static int rds_still_queued(struct rds_sock *rs, struct rds_incoming *inc, in rds_still_queued() argument
431 if (!list_empty(&inc->i_item)) { in rds_still_queued()
435 rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong, in rds_still_queued()
436 -be32_to_cpu(inc->i_hdr.h_len), in rds_still_queued()
437 inc->i_hdr.h_dport); in rds_still_queued()
438 list_del_init(&inc->i_item); in rds_still_queued()
439 to_drop = inc; in rds_still_queued()
447 rdsdebug("inc %p rs %p still %d dropped %d\n", inc, rs, ret, drop); in rds_still_queued()
544 static int rds_cmsg_recv(struct rds_incoming *inc, struct msghdr *msg, in rds_cmsg_recv() argument
549 if (inc->i_usercopy.rdma_cookie) { in rds_cmsg_recv()
551 sizeof(inc->i_usercopy.rdma_cookie), in rds_cmsg_recv()
552 &inc->i_usercopy.rdma_cookie); in rds_cmsg_recv()
557 if ((inc->i_usercopy.rx_tstamp != 0) && in rds_cmsg_recv()
560 ns_to_kernel_old_timeval(inc->i_usercopy.rx_tstamp); in rds_cmsg_recv()
584 inc->i_rx_lat_trace[RDS_MSG_RX_CMSG] = local_clock(); in rds_cmsg_recv()
589 t.rx_trace[i] = inc->i_rx_lat_trace[j + 1] - in rds_cmsg_recv()
590 inc->i_rx_lat_trace[j]; in rds_cmsg_recv()
647 struct rds_incoming *inc = NULL; in rds_recvmsg() local
671 if (!rds_next_incoming(rs, &inc)) { in rds_recvmsg()
682 rds_next_incoming(rs, &inc)), timeo); in rds_recvmsg()
683 rdsdebug("recvmsg woke inc %p timeo %ld\n", inc, in rds_recvmsg()
694 rdsdebug("copying inc %p from %pI6c:%u to user\n", inc, in rds_recvmsg()
695 &inc->i_conn->c_faddr, in rds_recvmsg()
696 ntohs(inc->i_hdr.h_sport)); in rds_recvmsg()
697 ret = inc->i_conn->c_trans->inc_copy_to_user(inc, &msg->msg_iter); in rds_recvmsg()
706 if (!rds_still_queued(rs, inc, !(msg_flags & MSG_PEEK))) { in rds_recvmsg()
707 rds_inc_put(inc); in rds_recvmsg()
708 inc = NULL; in rds_recvmsg()
714 if (ret < be32_to_cpu(inc->i_hdr.h_len)) { in rds_recvmsg()
716 ret = be32_to_cpu(inc->i_hdr.h_len); in rds_recvmsg()
720 if (rds_cmsg_recv(inc, msg, rs)) { in rds_recvmsg()
729 if (ipv6_addr_v4mapped(&inc->i_saddr)) { in rds_recvmsg()
731 sin->sin_port = inc->i_hdr.h_sport; in rds_recvmsg()
733 inc->i_saddr.s6_addr32[3]; in rds_recvmsg()
738 sin6->sin6_port = inc->i_hdr.h_sport; in rds_recvmsg()
739 sin6->sin6_addr = inc->i_saddr; in rds_recvmsg()
748 if (inc) in rds_recvmsg()
749 rds_inc_put(inc); in rds_recvmsg()
763 struct rds_incoming *inc, *tmp; in rds_clear_recv_queue() local
768 list_for_each_entry_safe(inc, tmp, &rs->rs_recv_queue, i_item) { in rds_clear_recv_queue()
769 rds_recv_rcvbuf_delta(rs, sk, inc->i_conn->c_lcong, in rds_clear_recv_queue()
770 -be32_to_cpu(inc->i_hdr.h_len), in rds_clear_recv_queue()
771 inc->i_hdr.h_dport); in rds_clear_recv_queue()
772 list_move(&inc->i_item, &to_drop); in rds_clear_recv_queue()
776 list_for_each_entry_safe(inc, tmp, &to_drop, i_item) { in rds_clear_recv_queue()
777 list_del_init(&inc->i_item); in rds_clear_recv_queue()
778 rds_inc_put(inc); in rds_clear_recv_queue()
783 * inc->i_saddr isn't used here because it is only set in the receive
786 void rds_inc_info_copy(struct rds_incoming *inc, in rds_inc_info_copy() argument
792 minfo.seq = be64_to_cpu(inc->i_hdr.h_sequence); in rds_inc_info_copy()
793 minfo.len = be32_to_cpu(inc->i_hdr.h_len); in rds_inc_info_copy()
794 minfo.tos = inc->i_conn->c_tos; in rds_inc_info_copy()
799 minfo.lport = inc->i_hdr.h_dport; in rds_inc_info_copy()
800 minfo.fport = inc->i_hdr.h_sport; in rds_inc_info_copy()
804 minfo.lport = inc->i_hdr.h_sport; in rds_inc_info_copy()
805 minfo.fport = inc->i_hdr.h_dport; in rds_inc_info_copy()
814 void rds6_inc_info_copy(struct rds_incoming *inc, in rds6_inc_info_copy() argument
821 minfo6.seq = be64_to_cpu(inc->i_hdr.h_sequence); in rds6_inc_info_copy()
822 minfo6.len = be32_to_cpu(inc->i_hdr.h_len); in rds6_inc_info_copy()
823 minfo6.tos = inc->i_conn->c_tos; in rds6_inc_info_copy()
828 minfo6.lport = inc->i_hdr.h_dport; in rds6_inc_info_copy()
829 minfo6.fport = inc->i_hdr.h_sport; in rds6_inc_info_copy()
833 minfo6.lport = inc->i_hdr.h_sport; in rds6_inc_info_copy()
834 minfo6.fport = inc->i_hdr.h_dport; in rds6_inc_info_copy()