506 __ip_vs_bind_svc(struct ip_vs_dest *dest, struct ip_vs_service *svc)  in __ip_vs_bind_svc()  argument
509 	rcu_assign_pointer(dest->svc, svc);  in __ip_vs_bind_svc()
559 static void ip_vs_rs_hash(struct netns_ipvs *ipvs, struct ip_vs_dest *dest)  in ip_vs_rs_hash()  argument
564 	if (dest->in_rs_table)  in ip_vs_rs_hash()
567 	switch (IP_VS_DFWD_METHOD(dest)) {  in ip_vs_rs_hash()
569 		port = dest->port;  in ip_vs_rs_hash()
572 		switch (dest->tun_type) {  in ip_vs_rs_hash()
574 			port = dest->tun_port;  in ip_vs_rs_hash()
592 	hash = ip_vs_rs_hashkey(dest->af, &dest->addr, port);  in ip_vs_rs_hash()
594 	hlist_add_head_rcu(&dest->d_list, &ipvs->rs_table[hash]);  in ip_vs_rs_hash()
595 	dest->in_rs_table = 1;  in ip_vs_rs_hash()
599 static void ip_vs_rs_unhash(struct ip_vs_dest *dest)  in ip_vs_rs_unhash()  argument
604 	if (dest->in_rs_table) {  in ip_vs_rs_unhash()
605 		hlist_del_rcu(&dest->d_list);  in ip_vs_rs_unhash()
606 		dest->in_rs_table = 0;  in ip_vs_rs_unhash()
615 	struct ip_vs_dest *dest;  in ip_vs_has_real_service()  local
620 	hlist_for_each_entry_rcu(dest, &ipvs->rs_table[hash], d_list) {  in ip_vs_has_real_service()
621 		if (dest->port == dport &&  in ip_vs_has_real_service()
622 		    dest->af == af &&  in ip_vs_has_real_service()
623 		    ip_vs_addr_equal(af, &dest->addr, daddr) &&  in ip_vs_has_real_service()
624 		    (dest->protocol == protocol || dest->vfwmark) &&  in ip_vs_has_real_service()
625 		    IP_VS_DFWD_METHOD(dest) == IP_VS_CONN_F_MASQ) {  in ip_vs_has_real_service()
646 	struct ip_vs_dest *dest;  in ip_vs_find_real_service()  local
651 	hlist_for_each_entry_rcu(dest, &ipvs->rs_table[hash], d_list) {  in ip_vs_find_real_service()
652 		if (dest->port == dport &&  in ip_vs_find_real_service()
653 		    dest->af == af &&  in ip_vs_find_real_service()
654 		    ip_vs_addr_equal(af, &dest->addr, daddr) &&  in ip_vs_find_real_service()
655 		    (dest->protocol == protocol || dest->vfwmark) &&  in ip_vs_find_real_service()
656 		    IP_VS_DFWD_METHOD(dest) == IP_VS_CONN_F_MASQ) {  in ip_vs_find_real_service()
658 			return dest;  in ip_vs_find_real_service()
675 	struct ip_vs_dest *dest;  in ip_vs_find_tunnel()  local
681 	hlist_for_each_entry_rcu(dest, &ipvs->rs_table[hash], d_list) {  in ip_vs_find_tunnel()
682 		if (dest->tun_port == tun_port &&  in ip_vs_find_tunnel()
683 		    dest->af == af &&  in ip_vs_find_tunnel()
684 		    ip_vs_addr_equal(af, &dest->addr, daddr) &&  in ip_vs_find_tunnel()
685 		    IP_VS_DFWD_METHOD(dest) == IP_VS_CONN_F_TUNNEL) {  in ip_vs_find_tunnel()
687 			return dest;  in ip_vs_find_tunnel()
701 	struct ip_vs_dest *dest;  in ip_vs_lookup_dest()  local
706 	list_for_each_entry_rcu(dest, &svc->destinations, n_list) {  in ip_vs_lookup_dest()
707 		if ((dest->af == dest_af) &&  in ip_vs_lookup_dest()
708 		    ip_vs_addr_equal(dest_af, &dest->addr, daddr) &&  in ip_vs_lookup_dest()
709 		    (dest->port == dport)) {  in ip_vs_lookup_dest()
711 			return dest;  in ip_vs_lookup_dest()
733 	struct ip_vs_dest *dest;  in ip_vs_find_dest()  local
742 	dest = ip_vs_lookup_dest(svc, dest_af, daddr, port);  in ip_vs_find_dest()
743 	if (!dest)  in ip_vs_find_dest()
744 		dest = ip_vs_lookup_dest(svc, dest_af, daddr, port ^ dport);  in ip_vs_find_dest()
745 	return dest;  in ip_vs_find_dest()
759 static void __ip_vs_dst_cache_reset(struct ip_vs_dest *dest)  in __ip_vs_dst_cache_reset()  argument
763 	old = rcu_dereference_protected(dest->dest_dst, 1);  in __ip_vs_dst_cache_reset()
765 		RCU_INIT_POINTER(dest->dest_dst, NULL);  in __ip_vs_dst_cache_reset()
784 	struct ip_vs_dest *dest;  in ip_vs_trash_get_dest()  local
791 	list_for_each_entry(dest, &ipvs->dest_trash, t_list) {  in ip_vs_trash_get_dest()
794 			      dest->vfwmark,  in ip_vs_trash_get_dest()
795 			      IP_VS_DBG_ADDR(dest->af, &dest->addr),  in ip_vs_trash_get_dest()
796 			      ntohs(dest->port),  in ip_vs_trash_get_dest()
797 			      refcount_read(&dest->refcnt));  in ip_vs_trash_get_dest()
798 		if (dest->af == dest_af &&  in ip_vs_trash_get_dest()
799 		    ip_vs_addr_equal(dest_af, &dest->addr, daddr) &&  in ip_vs_trash_get_dest()
800 		    dest->port == dport &&  in ip_vs_trash_get_dest()
801 		    dest->vfwmark == svc->fwmark &&  in ip_vs_trash_get_dest()
802 		    dest->protocol == svc->protocol &&  in ip_vs_trash_get_dest()
804 		     (ip_vs_addr_equal(svc->af, &dest->vaddr, &svc->addr) &&  in ip_vs_trash_get_dest()
805 		      dest->vport == svc->port))) {  in ip_vs_trash_get_dest()
807 			list_del(&dest->t_list);  in ip_vs_trash_get_dest()
812 	dest = NULL;  in ip_vs_trash_get_dest()
817 	return dest;  in ip_vs_trash_get_dest()
822 	struct ip_vs_dest *dest;  in ip_vs_dest_rcu_free()  local
824 	dest = container_of(head, struct ip_vs_dest, rcu_head);  in ip_vs_dest_rcu_free()
825 	ip_vs_stats_release(&dest->stats);  in ip_vs_dest_rcu_free()
826 	ip_vs_dest_put_and_free(dest);  in ip_vs_dest_rcu_free()
829 static void ip_vs_dest_free(struct ip_vs_dest *dest)  in ip_vs_dest_free()  argument
831 	struct ip_vs_service *svc = rcu_dereference_protected(dest->svc, 1);  in ip_vs_dest_free()
833 	__ip_vs_dst_cache_reset(dest);  in ip_vs_dest_free()
835 	call_rcu(&dest->rcu_head, ip_vs_dest_rcu_free);  in ip_vs_dest_free()
849 	struct ip_vs_dest *dest, *nxt;  in ip_vs_trash_cleanup()  local
853 	list_for_each_entry_safe(dest, nxt, &ipvs->dest_trash, t_list) {  in ip_vs_trash_cleanup()
854 		list_del(&dest->t_list);  in ip_vs_trash_cleanup()
855 		ip_vs_dest_free(dest);  in ip_vs_trash_cleanup()
967 __ip_vs_update_dest(struct ip_vs_service *svc, struct ip_vs_dest *dest,  in __ip_vs_update_dest()  argument
976 	BUG_ON(!add && udest->af != dest->af);  in __ip_vs_update_dest()
983 		atomic_set(&dest->last_weight, udest->weight);  in __ip_vs_update_dest()
986 	atomic_set(&dest->weight, udest->weight);  in __ip_vs_update_dest()
992 	    IP_VS_DFWD_METHOD(dest) ||  in __ip_vs_update_dest()
993 	    udest->tun_type != dest->tun_type ||  in __ip_vs_update_dest()
994 	    udest->tun_port != dest->tun_port)  in __ip_vs_update_dest()
995 		ip_vs_rs_unhash(dest);  in __ip_vs_update_dest()
998 	dest->tun_type = udest->tun_type;  in __ip_vs_update_dest()
999 	dest->tun_port = udest->tun_port;  in __ip_vs_update_dest()
1000 	dest->tun_flags = udest->tun_flags;  in __ip_vs_update_dest()
1010 	atomic_set(&dest->conn_flags, conn_flags);  in __ip_vs_update_dest()
1012 	ip_vs_rs_hash(ipvs, dest);  in __ip_vs_update_dest()
1015 	old_svc = rcu_dereference_protected(dest->svc, 1);  in __ip_vs_update_dest()
1017 		__ip_vs_bind_svc(dest, svc);  in __ip_vs_update_dest()
1020 			ip_vs_zero_stats(&dest->stats);  in __ip_vs_update_dest()
1021 			__ip_vs_bind_svc(dest, svc);  in __ip_vs_update_dest()
1027 	dest->flags |= IP_VS_DEST_F_AVAILABLE;  in __ip_vs_update_dest()
1029 	if (udest->u_threshold == 0 || udest->u_threshold > dest->u_threshold)  in __ip_vs_update_dest()
1030 		dest->flags &= ~IP_VS_DEST_F_OVERLOAD;  in __ip_vs_update_dest()
1031 	dest->u_threshold = udest->u_threshold;  in __ip_vs_update_dest()
1032 	dest->l_threshold = udest->l_threshold;  in __ip_vs_update_dest()
1034 	dest->af = udest->af;  in __ip_vs_update_dest()
1036 	spin_lock_bh(&dest->dst_lock);  in __ip_vs_update_dest()
1037 	__ip_vs_dst_cache_reset(dest);  in __ip_vs_update_dest()
1038 	spin_unlock_bh(&dest->dst_lock);  in __ip_vs_update_dest()
1041 		list_add_rcu(&dest->n_list, &svc->destinations);  in __ip_vs_update_dest()
1045 			sched->add_dest(svc, dest);  in __ip_vs_update_dest()
1049 			sched->upd_dest(svc, dest);  in __ip_vs_update_dest()
1060 	struct ip_vs_dest *dest;  in ip_vs_new_dest()  local
1083 	dest = kzalloc(sizeof(struct ip_vs_dest), GFP_KERNEL);  in ip_vs_new_dest()
1084 	if (dest == NULL)  in ip_vs_new_dest()
1087 	ret = ip_vs_stats_init_alloc(&dest->stats);  in ip_vs_new_dest()
1091 	ret = ip_vs_start_estimator(svc->ipvs, &dest->stats);  in ip_vs_new_dest()
1095 	dest->af = udest->af;  in ip_vs_new_dest()
1096 	dest->protocol = svc->protocol;  in ip_vs_new_dest()
1097 	dest->vaddr = svc->addr;  in ip_vs_new_dest()
1098 	dest->vport = svc->port;  in ip_vs_new_dest()
1099 	dest->vfwmark = svc->fwmark;  in ip_vs_new_dest()
1100 	ip_vs_addr_copy(udest->af, &dest->addr, &udest->addr);  in ip_vs_new_dest()
1101 	dest->port = udest->port;  in ip_vs_new_dest()
1103 	atomic_set(&dest->activeconns, 0);  in ip_vs_new_dest()
1104 	atomic_set(&dest->inactconns, 0);  in ip_vs_new_dest()
1105 	atomic_set(&dest->persistconns, 0);  in ip_vs_new_dest()
1106 	refcount_set(&dest->refcnt, 1);  in ip_vs_new_dest()
1108 	INIT_HLIST_NODE(&dest->d_list);  in ip_vs_new_dest()
1109 	spin_lock_init(&dest->dst_lock);  in ip_vs_new_dest()
1110 	__ip_vs_update_dest(svc, dest, udest, 1);  in ip_vs_new_dest()
1115 	ip_vs_stats_release(&dest->stats);  in ip_vs_new_dest()
1118 	kfree(dest);  in ip_vs_new_dest()
1129 	struct ip_vs_dest *dest;  in ip_vs_add_dest()  local
1156 	dest = ip_vs_lookup_dest(svc, udest->af, &daddr, dport);  in ip_vs_add_dest()
1159 	if (dest != NULL) {  in ip_vs_add_dest()
1168 	dest = ip_vs_trash_get_dest(svc, udest->af, &daddr, dport);  in ip_vs_add_dest()
1170 	if (dest != NULL) {  in ip_vs_add_dest()
1174 			      refcount_read(&dest->refcnt),  in ip_vs_add_dest()
1175 			      dest->vfwmark,  in ip_vs_add_dest()
1176 			      IP_VS_DBG_ADDR(svc->af, &dest->vaddr),  in ip_vs_add_dest()
1177 			      ntohs(dest->vport));  in ip_vs_add_dest()
1179 		ret = ip_vs_start_estimator(svc->ipvs, &dest->stats);  in ip_vs_add_dest()
1182 		__ip_vs_update_dest(svc, dest, udest, 1);  in ip_vs_add_dest()
1200 	struct ip_vs_dest *dest;  in ip_vs_edit_dest()  local
1226 	dest = ip_vs_lookup_dest(svc, udest->af, &daddr, dport);  in ip_vs_edit_dest()
1229 	if (dest == NULL) {  in ip_vs_edit_dest()
1234 	__ip_vs_update_dest(svc, dest, udest, 0);  in ip_vs_edit_dest()
1242 static void __ip_vs_del_dest(struct netns_ipvs *ipvs, struct ip_vs_dest *dest,  in __ip_vs_del_dest()  argument
1245 	ip_vs_stop_estimator(ipvs, &dest->stats);  in __ip_vs_del_dest()
1250 	ip_vs_rs_unhash(dest);  in __ip_vs_del_dest()
1254 		      IP_VS_DBG_ADDR(dest->af, &dest->addr), ntohs(dest->port),  in __ip_vs_del_dest()
1255 		      refcount_read(&dest->refcnt));  in __ip_vs_del_dest()
1260 	list_add(&dest->t_list, &ipvs->dest_trash);  in __ip_vs_del_dest()
1261 	dest->idle_start = 0;  in __ip_vs_del_dest()
1276 				struct ip_vs_dest *dest,  in __ip_vs_unlink_dest()  argument
1279 	dest->flags &= ~IP_VS_DEST_F_AVAILABLE;  in __ip_vs_unlink_dest()
1284 	list_del_rcu(&dest->n_list);  in __ip_vs_unlink_dest()
1287 	if (dest->af != svc->af)  in __ip_vs_unlink_dest()
1295 			sched->del_dest(svc, dest);  in __ip_vs_unlink_dest()
1306 	struct ip_vs_dest *dest;  in ip_vs_del_dest()  local
1311 	dest = ip_vs_lookup_dest(svc, udest->af, &udest->addr, dport);  in ip_vs_del_dest()
1314 	if (dest == NULL) {  in ip_vs_del_dest()
1322 	__ip_vs_unlink_dest(svc, dest, 1);  in ip_vs_del_dest()
1327 	__ip_vs_del_dest(svc->ipvs, dest, false);  in ip_vs_del_dest()
1335 	struct ip_vs_dest *dest, *next;  in ip_vs_dest_trash_expire()  local
1339 	list_for_each_entry_safe(dest, next, &ipvs->dest_trash, t_list) {  in ip_vs_dest_trash_expire()
1340 		if (refcount_read(&dest->refcnt) > 1)  in ip_vs_dest_trash_expire()
1342 		if (dest->idle_start) {  in ip_vs_dest_trash_expire()
1343 			if (time_before(now, dest->idle_start +  in ip_vs_dest_trash_expire()
1347 			dest->idle_start = max(1UL, now);  in ip_vs_dest_trash_expire()
1351 			      dest->vfwmark,  in ip_vs_dest_trash_expire()
1352 			      IP_VS_DBG_ADDR(dest->af, &dest->addr),  in ip_vs_dest_trash_expire()
1353 			      ntohs(dest->port));  in ip_vs_dest_trash_expire()
1354 		list_del(&dest->t_list);  in ip_vs_dest_trash_expire()
1355 		ip_vs_dest_free(dest);  in ip_vs_dest_trash_expire()
1609 	struct ip_vs_dest *dest, *nxt;  in __ip_vs_del_service()  local
1640 	list_for_each_entry_safe(dest, nxt, &svc->destinations, n_list) {  in __ip_vs_del_service()
1641 		__ip_vs_unlink_dest(svc, dest, 0);  in __ip_vs_del_service()
1642 		__ip_vs_del_dest(svc->ipvs, dest, cleanup);  in __ip_vs_del_service()
1745 ip_vs_forget_dev(struct ip_vs_dest *dest, struct net_device *dev)  in ip_vs_forget_dev()  argument
1749 	spin_lock_bh(&dest->dst_lock);  in ip_vs_forget_dev()
1750 	dest_dst = rcu_dereference_protected(dest->dest_dst, 1);  in ip_vs_forget_dev()
1754 			      IP_VS_DBG_ADDR(dest->af, &dest->addr),  in ip_vs_forget_dev()
1755 			      ntohs(dest->port),  in ip_vs_forget_dev()
1756 			      refcount_read(&dest->refcnt));  in ip_vs_forget_dev()
1757 		__ip_vs_dst_cache_reset(dest);  in ip_vs_forget_dev()
1759 	spin_unlock_bh(&dest->dst_lock);  in ip_vs_forget_dev()
1772 	struct ip_vs_dest *dest;  in ip_vs_dst_event()  local
1782 				list_for_each_entry(dest, &svc->destinations,  in ip_vs_dst_event()
1784 					ip_vs_forget_dev(dest, dev);  in ip_vs_dst_event()
1791 				list_for_each_entry(dest, &svc->destinations,  in ip_vs_dst_event()
1793 					ip_vs_forget_dev(dest, dev);  in ip_vs_dst_event()
1801 	list_for_each_entry(dest, &ipvs->dest_trash, t_list) {  in ip_vs_dst_event()
1802 		ip_vs_forget_dev(dest, dev);  in ip_vs_dst_event()
1814 	struct ip_vs_dest *dest;  in ip_vs_zero_service()  local
1816 	list_for_each_entry(dest, &svc->destinations, n_list) {  in ip_vs_zero_service()
1817 		ip_vs_zero_stats(&dest->stats);  in ip_vs_zero_service()
2411 		const struct ip_vs_dest *dest;  in ip_vs_info_seq_show()  local
2446 		list_for_each_entry_rcu(dest, &svc->destinations, n_list) {  in ip_vs_info_seq_show()
2448 			if (dest->af == AF_INET6)  in ip_vs_info_seq_show()
2452 					   &dest->addr.in6,  in ip_vs_info_seq_show()
2453 					   ntohs(dest->port),  in ip_vs_info_seq_show()
2454 					   ip_vs_fwd_name(atomic_read(&dest->conn_flags)),  in ip_vs_info_seq_show()
2455 					   atomic_read(&dest->weight),  in ip_vs_info_seq_show()
2456 					   atomic_read(&dest->activeconns),  in ip_vs_info_seq_show()
2457 					   atomic_read(&dest->inactconns));  in ip_vs_info_seq_show()
2463 					   ntohl(dest->addr.ip),  in ip_vs_info_seq_show()
2464 					   ntohs(dest->port),  in ip_vs_info_seq_show()
2465 					   ip_vs_fwd_name(atomic_read(&dest->conn_flags)),  in ip_vs_info_seq_show()
2466 					   atomic_read(&dest->weight),  in ip_vs_info_seq_show()
2467 					   atomic_read(&dest->activeconns),  in ip_vs_info_seq_show()
2468 					   atomic_read(&dest->inactconns));  in ip_vs_info_seq_show()
2932 		struct ip_vs_dest *dest;  in __ip_vs_get_dest_entries()  local
2937 		list_for_each_entry(dest, &svc->destinations, n_list) {  in __ip_vs_get_dest_entries()
2944 			if (dest->af != svc->af)  in __ip_vs_get_dest_entries()
2947 			entry.addr = dest->addr.ip;  in __ip_vs_get_dest_entries()
2948 			entry.port = dest->port;  in __ip_vs_get_dest_entries()
2949 			entry.conn_flags = atomic_read(&dest->conn_flags);  in __ip_vs_get_dest_entries()
2950 			entry.weight = atomic_read(&dest->weight);  in __ip_vs_get_dest_entries()
2951 			entry.u_threshold = dest->u_threshold;  in __ip_vs_get_dest_entries()
2952 			entry.l_threshold = dest->l_threshold;  in __ip_vs_get_dest_entries()
2953 			entry.activeconns = atomic_read(&dest->activeconns);  in __ip_vs_get_dest_entries()
2954 			entry.inactconns = atomic_read(&dest->inactconns);  in __ip_vs_get_dest_entries()
2955 			entry.persistconns = atomic_read(&dest->persistconns);  in __ip_vs_get_dest_entries()
2956 			ip_vs_copy_stats(&kstats, &dest->stats);  in __ip_vs_get_dest_entries()
3531 static int ip_vs_genl_fill_dest(struct sk_buff *skb, struct ip_vs_dest *dest)  in ip_vs_genl_fill_dest()  argument
3540 	if (nla_put(skb, IPVS_DEST_ATTR_ADDR, sizeof(dest->addr), &dest->addr) ||  in ip_vs_genl_fill_dest()
3541 	    nla_put_be16(skb, IPVS_DEST_ATTR_PORT, dest->port) ||  in ip_vs_genl_fill_dest()
3543 			(atomic_read(&dest->conn_flags) &  in ip_vs_genl_fill_dest()
3546 			atomic_read(&dest->weight)) ||  in ip_vs_genl_fill_dest()
3548 		       dest->tun_type) ||  in ip_vs_genl_fill_dest()
3550 			 dest->tun_port) ||  in ip_vs_genl_fill_dest()
3552 			dest->tun_flags) ||  in ip_vs_genl_fill_dest()
3553 	    nla_put_u32(skb, IPVS_DEST_ATTR_U_THRESH, dest->u_threshold) ||  in ip_vs_genl_fill_dest()
3554 	    nla_put_u32(skb, IPVS_DEST_ATTR_L_THRESH, dest->l_threshold) ||  in ip_vs_genl_fill_dest()
3556 			atomic_read(&dest->activeconns)) ||  in ip_vs_genl_fill_dest()
3558 			atomic_read(&dest->inactconns)) ||  in ip_vs_genl_fill_dest()
3560 			atomic_read(&dest->persistconns)) ||  in ip_vs_genl_fill_dest()
3561 	    nla_put_u16(skb, IPVS_DEST_ATTR_ADDR_FAMILY, dest->af))  in ip_vs_genl_fill_dest()
3563 	ip_vs_copy_stats(&kstats, &dest->stats);  in ip_vs_genl_fill_dest()
3578 static int ip_vs_genl_dump_dest(struct sk_buff *skb, struct ip_vs_dest *dest,  in ip_vs_genl_dump_dest()  argument
3589 	if (ip_vs_genl_fill_dest(skb, dest) < 0)  in ip_vs_genl_dump_dest()
3606 	struct ip_vs_dest *dest;  in ip_vs_genl_dump_dests()  local
3623 	list_for_each_entry(dest, &svc->destinations, n_list) {  in ip_vs_genl_dump_dests()
3626 		if (ip_vs_genl_dump_dest(skb, dest, cb) < 0) {  in ip_vs_genl_dump_dests()