121 				       struct nh_group *nhg)  in nh_notifier_mpath_info_init()  argument
123 	u16 num_nh = nhg->num_nh;  in nh_notifier_mpath_info_init()
133 	info->nh_grp->is_fdb = nhg->fdb_nh;  in nh_notifier_mpath_info_init()
136 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nh_notifier_mpath_info_init()
150 					   struct nh_group *nhg)  in nh_notifier_res_table_info_init()  argument
152 	struct nh_res_table *res_table = rtnl_dereference(nhg->res_table);  in nh_notifier_res_table_info_init()
183 	struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in nh_notifier_grp_info_init()  local
185 	if (nhg->hash_threshold)  in nh_notifier_grp_info_init()
186 		return nh_notifier_mpath_info_init(info, nhg);  in nh_notifier_grp_info_init()
187 	else if (nhg->resilient)  in nh_notifier_grp_info_init()
188 		return nh_notifier_res_table_info_init(info, nhg);  in nh_notifier_grp_info_init()
195 	struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in nh_notifier_grp_info_fini()  local
197 	if (nhg->hash_threshold)  in nh_notifier_grp_info_fini()
199 	else if (nhg->resilient)  in nh_notifier_grp_info_fini()
257 	struct nh_group *nhg;  in nh_notifier_res_bucket_idle_timer_get()  local
283 	nhg = rcu_dereference(nh->nh_grp);  in nh_notifier_res_bucket_idle_timer_get()
284 	res_table = rcu_dereference(nhg->res_table);  in nh_notifier_res_bucket_idle_timer_get()
397 	struct nh_group *nhg;  in call_nexthop_res_table_notifiers()  local
409 	nhg = rtnl_dereference(nh->nh_grp);  in call_nexthop_res_table_notifiers()
410 	err = nh_notifier_mpath_info_init(&info, nhg);  in call_nexthop_res_table_notifiers()
469 	struct nh_group *nhg;  in nexthop_free_group()  local
472 	nhg = rcu_dereference_raw(nh->nh_grp);  in nexthop_free_group()
473 	for (i = 0; i < nhg->num_nh; ++i) {  in nexthop_free_group()
474 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nexthop_free_group()
480 	WARN_ON(nhg->spare == nhg);  in nexthop_free_group()
482 	if (nhg->resilient)  in nexthop_free_group()
483 		vfree(rcu_dereference_raw(nhg->res_table));  in nexthop_free_group()
485 	kfree(nhg->spare);  in nexthop_free_group()
486 	kfree(nhg);  in nexthop_free_group()
534 	struct nh_group *nhg;  in nexthop_grp_alloc()  local
536 	nhg = kzalloc(struct_size(nhg, nh_entries, num_nh), GFP_KERNEL);  in nexthop_grp_alloc()
537 	if (nhg)  in nexthop_grp_alloc()
538 		nhg->num_nh = num_nh;  in nexthop_grp_alloc()
540 	return nhg;  in nexthop_grp_alloc()
629 static int nla_put_nh_group_res(struct sk_buff *skb, struct nh_group *nhg)  in nla_put_nh_group_res()  argument
631 	struct nh_res_table *res_table = rtnl_dereference(nhg->res_table);  in nla_put_nh_group_res()
657 static int nla_put_nh_group(struct sk_buff *skb, struct nh_group *nhg)  in nla_put_nh_group()  argument
660 	size_t len = nhg->num_nh * sizeof(*p);  in nla_put_nh_group()
665 	if (nhg->hash_threshold)  in nla_put_nh_group()
667 	else if (nhg->resilient)  in nla_put_nh_group()
678 	for (i = 0; i < nhg->num_nh; ++i) {  in nla_put_nh_group()
680 			.id = nhg->nh_entries[i].nh->id,  in nla_put_nh_group()
681 			.weight = nhg->nh_entries[i].weight - 1,  in nla_put_nh_group()
685 	if (nhg->resilient && nla_put_nh_group_res(skb, nhg))  in nla_put_nh_group()
718 		struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in nh_fill_node()  local
720 		if (nhg->fdb_nh && nla_put_flag(skb, NHA_FDB))  in nh_fill_node()
722 		if (nla_put_nh_group(skb, nhg))  in nh_fill_node()
775 static size_t nh_nlmsg_size_grp_res(struct nh_group *nhg)  in nh_nlmsg_size_grp_res()  argument
786 	struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in nh_nlmsg_size_grp()  local
787 	size_t sz = sizeof(struct nexthop_grp) * nhg->num_nh;  in nh_nlmsg_size_grp()
791 	if (nhg->resilient)  in nh_nlmsg_size_grp()
792 		tot += nh_nlmsg_size_grp_res(nhg);  in nh_nlmsg_size_grp()
994 		struct nh_group *nhg = rtnl_dereference(nh->nh_grp);  in valid_group_nh()  local
997 		if (nhg->hash_threshold) {  in valid_group_nh()
1002 		if (nhg->resilient) {  in valid_group_nh()
1007 		*is_fdb = nhg->fdb_nh;  in valid_group_nh()
1050 	struct nexthop_grp *nhg;  in nh_check_attr_group()  local
1061 	len /= sizeof(*nhg);  in nh_check_attr_group()
1063 	nhg = nla_data(tb[NHA_GROUP]);  in nh_check_attr_group()
1065 		if (nhg[i].resvd1 || nhg[i].resvd2) {  in nh_check_attr_group()
1069 		if (nhg[i].weight > 254) {  in nh_check_attr_group()
1074 			if (nhg[i].id == nhg[j].id) {  in nh_check_attr_group()
1083 	nhg = nla_data(tb[NHA_GROUP]);  in nh_check_attr_group()
1088 		nh = nexthop_find_by_id(net, nhg[i].id);  in nh_check_attr_group()
1170 static struct nexthop *nexthop_select_path_fdb(struct nh_group *nhg, int hash)  in nexthop_select_path_fdb()  argument
1174 	for (i = 0; i < nhg->num_nh; i++) {  in nexthop_select_path_fdb()
1175 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nexthop_select_path_fdb()
1187 static struct nexthop *nexthop_select_path_hthr(struct nh_group *nhg, int hash)  in nexthop_select_path_hthr()  argument
1192 	if (nhg->fdb_nh)  in nexthop_select_path_hthr()
1193 		return nexthop_select_path_fdb(nhg, hash);  in nexthop_select_path_hthr()
1195 	for (i = 0; i < nhg->num_nh; ++i) {  in nexthop_select_path_hthr()
1196 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nexthop_select_path_hthr()
1213 	return rc ? : nhg->nh_entries[0].nh;  in nexthop_select_path_hthr()
1216 static struct nexthop *nexthop_select_path_res(struct nh_group *nhg, int hash)  in nexthop_select_path_res()  argument
1218 	struct nh_res_table *res_table = rcu_dereference(nhg->res_table);  in nexthop_select_path_res()
1234 	struct nh_group *nhg;  in nexthop_select_path()  local
1239 	nhg = rcu_dereference(nh->nh_grp);  in nexthop_select_path()
1240 	if (nhg->hash_threshold)  in nexthop_select_path()
1241 		return nexthop_select_path_hthr(nhg, hash);  in nexthop_select_path()
1242 	else if (nhg->resilient)  in nexthop_select_path()
1243 		return nexthop_select_path_res(nhg, hash);  in nexthop_select_path()
1258 		struct nh_group *nhg;  in nexthop_for_each_fib6_nh()  local
1261 		nhg = rcu_dereference_rtnl(nh->nh_grp);  in nexthop_for_each_fib6_nh()
1262 		for (i = 0; i < nhg->num_nh; i++) {  in nexthop_for_each_fib6_nh()
1263 			struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nexthop_for_each_fib6_nh()
1307 		struct nh_group *nhg;  in fib6_check_nexthop()  local
1309 		nhg = rtnl_dereference(nh->nh_grp);  in fib6_check_nexthop()
1310 		if (nhg->has_v4)  in fib6_check_nexthop()
1312 		is_fdb_nh = nhg->fdb_nh;  in fib6_check_nexthop()
1379 		struct nh_group *nhg;  in fib_check_nexthop()  local
1381 		nhg = rtnl_dereference(nh->nh_grp);  in fib_check_nexthop()
1382 		if (nhg->fdb_nh) {  in fib_check_nexthop()
1395 		nhi = rtnl_dereference(nhg->nh_entries[0].nh->nh_info);  in fib_check_nexthop()
1655 static void nh_res_group_rebalance(struct nh_group *nhg,  in nh_res_group_rebalance()  argument
1665 	for (i = 0; i < nhg->num_nh; ++i)  in nh_res_group_rebalance()
1666 		total += nhg->nh_entries[i].weight;  in nh_res_group_rebalance()
1668 	for (i = 0; i < nhg->num_nh; ++i) {  in nh_res_group_rebalance()
1669 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nh_res_group_rebalance()
1692 					 struct nh_group *nhg)  in nh_res_table_migrate_buckets()  argument
1702 		for (j = 0; j < nhg->num_nh; j++) {  in nh_res_table_migrate_buckets()
1703 			struct nh_grp_entry *nhge = &nhg->nh_entries[j];  in nh_res_table_migrate_buckets()
1738 static void nh_hthr_group_rebalance(struct nh_group *nhg)  in nh_hthr_group_rebalance()  argument
1744 	for (i = 0; i < nhg->num_nh; ++i)  in nh_hthr_group_rebalance()
1745 		total += nhg->nh_entries[i].weight;  in nh_hthr_group_rebalance()
1747 	for (i = 0; i < nhg->num_nh; ++i) {  in nh_hthr_group_rebalance()
1748 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nh_hthr_group_rebalance()
1764 	struct nh_group *nhg, *newg;  in remove_nh_grp_entry()  local
1769 	nhg = rtnl_dereference(nhp->nh_grp);  in remove_nh_grp_entry()
1770 	newg = nhg->spare;  in remove_nh_grp_entry()
1773 	if (nhg->num_nh == 1) {  in remove_nh_grp_entry()
1779 	newg->is_multipath = nhg->is_multipath;  in remove_nh_grp_entry()
1780 	newg->hash_threshold = nhg->hash_threshold;  in remove_nh_grp_entry()
1781 	newg->resilient = nhg->resilient;  in remove_nh_grp_entry()
1782 	newg->fdb_nh = nhg->fdb_nh;  in remove_nh_grp_entry()
1783 	newg->num_nh = nhg->num_nh;  in remove_nh_grp_entry()
1786 	nhges = nhg->nh_entries;  in remove_nh_grp_entry()
1788 	for (i = 0, j = 0; i < nhg->num_nh; ++i) {  in remove_nh_grp_entry()
1792 		if (nhg->nh_entries[i].nh == nh) {  in remove_nh_grp_entry()
1812 		replace_nexthop_grp_res(nhg, newg);  in remove_nh_grp_entry()
1847 	struct nh_group *nhg = rcu_dereference_rtnl(nh->nh_grp);  in remove_nexthop_group()  local
1849 	int i, num_nh = nhg->num_nh;  in remove_nexthop_group()
1852 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in remove_nexthop_group()
1860 	if (nhg->resilient) {  in remove_nexthop_group()
1861 		res_table = rtnl_dereference(nhg->res_table);  in remove_nexthop_group()
1931 	struct nh_group *nhg;  in nh_rt_cache_flush()  local
1946 	nhg = rtnl_dereference(replaced_nh->nh_grp);  in nh_rt_cache_flush()
1947 	for (i = 0; i < nhg->num_nh; i++) {  in nh_rt_cache_flush()
1948 		struct nh_grp_entry *nhge = &nhg->nh_entries[i];  in nh_rt_cache_flush()
2043 static void nh_group_v4_update(struct nh_group *nhg)  in nh_group_v4_update()  argument
2049 	nhges = nhg->nh_entries;  in nh_group_v4_update()
2050 	for (i = 0; i < nhg->num_nh; i++) {  in nh_group_v4_update()
2057 	nhg->has_v4 = has_v4;  in nh_group_v4_update()
2109 	struct nh_group *nhg = rtnl_dereference(group_nh->nh_grp);  in replace_nexthop_single_notify()  local
2112 	if (nhg->hash_threshold) {  in replace_nexthop_single_notify()
2115 	} else if (nhg->resilient) {  in replace_nexthop_single_notify()
2116 		res_table = rtnl_dereference(nhg->res_table);  in replace_nexthop_single_notify()
2179 			struct nh_group *nhg;  in replace_nexthop_single()  local
2181 			nhg = rtnl_dereference(nhp->nh_grp);  in replace_nexthop_single()
2182 			nh_group_v4_update(nhg);  in replace_nexthop_single()
2350 		struct nh_group *nhg = rtnl_dereference(new_nh->nh_grp);  in insert_nexthop()  local
2353 		if (nhg->resilient) {  in insert_nexthop()
2354 			res_table = rtnl_dereference(nhg->res_table);  in insert_nexthop()
2365 			nh_res_group_rebalance(nhg, res_table);  in insert_nexthop()
2438 	struct nh_group *nhg;  in nexthop_create_group()  local
2452 	nhg = nexthop_grp_alloc(num_nh);  in nexthop_create_group()
2453 	if (!nhg) {  in nexthop_create_group()
2459 	nhg->spare = nexthop_grp_alloc(num_nh);  in nexthop_create_group()
2460 	if (!nhg->spare) {  in nexthop_create_group()
2461 		kfree(nhg);  in nexthop_create_group()
2465 	nhg->spare->spare = nhg;  in nexthop_create_group()
2467 	for (i = 0; i < nhg->num_nh; ++i) {  in nexthop_create_group()
2479 			nhg->has_v4 = true;  in nexthop_create_group()
2481 		nhg->nh_entries[i].nh = nhe;  in nexthop_create_group()
2482 		nhg->nh_entries[i].weight = entry[i].weight + 1;  in nexthop_create_group()
2483 		list_add(&nhg->nh_entries[i].nh_list, &nhe->grp_list);  in nexthop_create_group()
2484 		nhg->nh_entries[i].nh_parent = nh;  in nexthop_create_group()
2488 		nhg->hash_threshold = 1;  in nexthop_create_group()
2489 		nhg->is_multipath = true;  in nexthop_create_group()
2499 		rcu_assign_pointer(nhg->spare->res_table, res_table);  in nexthop_create_group()
2500 		rcu_assign_pointer(nhg->res_table, res_table);  in nexthop_create_group()
2501 		nhg->resilient = true;  in nexthop_create_group()
2502 		nhg->is_multipath = true;  in nexthop_create_group()
2505 	WARN_ON_ONCE(nhg->hash_threshold + nhg->resilient != 1);  in nexthop_create_group()
2507 	if (nhg->hash_threshold)  in nexthop_create_group()
2508 		nh_hthr_group_rebalance(nhg);  in nexthop_create_group()
2511 		nhg->fdb_nh = 1;  in nexthop_create_group()
2513 	rcu_assign_pointer(nh->nh_grp, nhg);  in nexthop_create_group()
2519 		list_del(&nhg->nh_entries[i].nh_list);  in nexthop_create_group()
2520 		nexthop_put(nhg->nh_entries[i].nh);  in nexthop_create_group()
2523 	kfree(nhg->spare);  in nexthop_create_group()
2524 	kfree(nhg);  in nexthop_create_group()
3259 	struct nh_group *nhg;  in nexthop_find_group_resilient()  local
3271 	nhg = rtnl_dereference(nh->nh_grp);  in nexthop_find_group_resilient()
3272 	if (!nhg->resilient) {  in nexthop_find_group_resilient()
3364 	struct nh_group *nhg;  in rtm_dump_nexthop_bucket_nh()  local
3368 	nhg = rtnl_dereference(nh->nh_grp);  in rtm_dump_nexthop_bucket_nh()
3369 	res_table = rtnl_dereference(nhg->res_table);  in rtm_dump_nexthop_bucket_nh()
3404 	struct nh_group *nhg;  in rtm_dump_nexthop_bucket_cb()  local
3409 	nhg = rtnl_dereference(nh->nh_grp);  in rtm_dump_nexthop_bucket_cb()
3410 	if (!nhg->resilient)  in rtm_dump_nexthop_bucket_cb()
3511 	struct nh_group *nhg;  in rtm_get_nexthop_bucket()  local
3525 	nhg = rtnl_dereference(nh->nh_grp);  in rtm_get_nexthop_bucket()
3526 	res_table = rtnl_dereference(nhg->res_table);  in rtm_get_nexthop_bucket()
3678 	struct nh_group *nhg;  in nexthop_bucket_set_hw_flags()  local
3686 	nhg = rcu_dereference(nexthop->nh_grp);  in nexthop_bucket_set_hw_flags()
3687 	if (!nhg->resilient)  in nexthop_bucket_set_hw_flags()
3690 	if (bucket_index >= nhg->res_table->num_nh_buckets)  in nexthop_bucket_set_hw_flags()
3693 	res_table = rcu_dereference(nhg->res_table);  in nexthop_bucket_set_hw_flags()
3711 	struct nh_group *nhg;  in nexthop_res_grp_activity_update()  local
3720 	nhg = rcu_dereference(nexthop->nh_grp);  in nexthop_res_grp_activity_update()
3721 	if (!nhg->resilient)  in nexthop_res_grp_activity_update()
3727 	res_table = rcu_dereference(nhg->res_table);  in nexthop_res_grp_activity_update()