1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * drivers/net/team/team_mode_loadbalance.c - Load-balancing mode for team
4  * Copyright (c) 2012 Jiri Pirko <jpirko@redhat.com>
5  */
6 
7 #include <linux/kernel.h>
8 #include <linux/types.h>
9 #include <linux/module.h>
10 #include <linux/init.h>
11 #include <linux/errno.h>
12 #include <linux/netdevice.h>
13 #include <linux/etherdevice.h>
14 #include <linux/filter.h>
15 #include <linux/if_team.h>
16 
17 static rx_handler_result_t lb_receive(struct team *team, struct team_port *port,
18 				      struct sk_buff *skb)
19 {
20 	if (unlikely(skb->protocol == htons(ETH_P_SLOW))) {
21 		/* LACPDU packets should go to exact delivery */
22 		const unsigned char *dest = eth_hdr(skb)->h_dest;
23 
24 		if (is_link_local_ether_addr(dest) && dest[5] == 0x02)
25 			return RX_HANDLER_EXACT;
26 	}
27 	return RX_HANDLER_ANOTHER;
28 }
29 
30 struct lb_priv;
31 
32 typedef struct team_port *lb_select_tx_port_func_t(struct team *,
33 						   unsigned char);
34 
35 #define LB_TX_HASHTABLE_SIZE 256 /* hash is a char */
36 
37 struct lb_stats {
38 	u64 tx_bytes;
39 };
40 
41 struct lb_pcpu_stats {
42 	struct lb_stats hash_stats[LB_TX_HASHTABLE_SIZE];
43 	struct u64_stats_sync syncp;
44 };
45 
46 struct lb_stats_info {
47 	struct lb_stats stats;
48 	struct lb_stats last_stats;
49 	struct team_option_inst_info *opt_inst_info;
50 };
51 
52 struct lb_port_mapping {
53 	struct team_port __rcu *port;
54 	struct team_option_inst_info *opt_inst_info;
55 };
56 
57 struct lb_priv_ex {
58 	struct team *team;
59 	struct lb_port_mapping tx_hash_to_port_mapping[LB_TX_HASHTABLE_SIZE];
60 	struct sock_fprog_kern *orig_fprog;
61 	struct {
62 		unsigned int refresh_interval; /* in tenths of second */
63 		struct delayed_work refresh_dw;
64 		struct lb_stats_info info[LB_TX_HASHTABLE_SIZE];
65 	} stats;
66 };
67 
68 struct lb_priv {
69 	struct bpf_prog __rcu *fp;
70 	lb_select_tx_port_func_t __rcu *select_tx_port_func;
71 	struct lb_pcpu_stats __percpu *pcpu_stats;
72 	struct lb_priv_ex *ex; /* priv extension */
73 };
74 
75 static struct lb_priv *get_lb_priv(struct team *team)
76 {
77 	return (struct lb_priv *) &team->mode_priv;
78 }
79 
80 struct lb_port_priv {
81 	struct lb_stats __percpu *pcpu_stats;
82 	struct lb_stats_info stats_info;
83 };
84 
85 static struct lb_port_priv *get_lb_port_priv(struct team_port *port)
86 {
87 	return (struct lb_port_priv *) &port->mode_priv;
88 }
89 
90 #define LB_HTPM_PORT_BY_HASH(lp_priv, hash) \
91 	(lb_priv)->ex->tx_hash_to_port_mapping[hash].port
92 
93 #define LB_HTPM_OPT_INST_INFO_BY_HASH(lp_priv, hash) \
94 	(lb_priv)->ex->tx_hash_to_port_mapping[hash].opt_inst_info
95 
96 static void lb_tx_hash_to_port_mapping_null_port(struct team *team,
97 						 struct team_port *port)
98 {
99 	struct lb_priv *lb_priv = get_lb_priv(team);
100 	bool changed = false;
101 	int i;
102 
103 	for (i = 0; i < LB_TX_HASHTABLE_SIZE; i++) {
104 		struct lb_port_mapping *pm;
105 
106 		pm = &lb_priv->ex->tx_hash_to_port_mapping[i];
107 		if (rcu_access_pointer(pm->port) == port) {
108 			RCU_INIT_POINTER(pm->port, NULL);
109 			team_option_inst_set_change(pm->opt_inst_info);
110 			changed = true;
111 		}
112 	}
113 	if (changed)
114 		team_options_change_check(team);
115 }
116 
117 /* Basic tx selection based solely by hash */
118 static struct team_port *lb_hash_select_tx_port(struct team *team,
119 						unsigned char hash)
120 {
121 	int port_index = team_num_to_port_index(team, hash);
122 
123 	return team_get_port_by_index_rcu(team, port_index);
124 }
125 
126 /* Hash to port mapping select tx port */
127 static struct team_port *lb_htpm_select_tx_port(struct team *team,
128 						unsigned char hash)
129 {
130 	struct lb_priv *lb_priv = get_lb_priv(team);
131 	struct team_port *port;
132 
133 	port = rcu_dereference_bh(LB_HTPM_PORT_BY_HASH(lb_priv, hash));
134 	if (likely(port))
135 		return port;
136 	/* If no valid port in the table, fall back to simple hash */
137 	return lb_hash_select_tx_port(team, hash);
138 }
139 
140 struct lb_select_tx_port {
141 	char *name;
142 	lb_select_tx_port_func_t *func;
143 };
144 
145 static const struct lb_select_tx_port lb_select_tx_port_list[] = {
146 	{
147 		.name = "hash",
148 		.func = lb_hash_select_tx_port,
149 	},
150 	{
151 		.name = "hash_to_port_mapping",
152 		.func = lb_htpm_select_tx_port,
153 	},
154 };
155 #define LB_SELECT_TX_PORT_LIST_COUNT ARRAY_SIZE(lb_select_tx_port_list)
156 
157 static char *lb_select_tx_port_get_name(lb_select_tx_port_func_t *func)
158 {
159 	int i;
160 
161 	for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) {
162 		const struct lb_select_tx_port *item;
163 
164 		item = &lb_select_tx_port_list[i];
165 		if (item->func == func)
166 			return item->name;
167 	}
168 	return NULL;
169 }
170 
171 static lb_select_tx_port_func_t *lb_select_tx_port_get_func(const char *name)
172 {
173 	int i;
174 
175 	for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) {
176 		const struct lb_select_tx_port *item;
177 
178 		item = &lb_select_tx_port_list[i];
179 		if (!strcmp(item->name, name))
180 			return item->func;
181 	}
182 	return NULL;
183 }
184 
185 static unsigned int lb_get_skb_hash(struct lb_priv *lb_priv,
186 				    struct sk_buff *skb)
187 {
188 	struct bpf_prog *fp;
189 	uint32_t lhash;
190 	unsigned char *c;
191 
192 	fp = rcu_dereference_bh(lb_priv->fp);
193 	if (unlikely(!fp))
194 		return 0;
195 	lhash = bpf_prog_run(fp, skb);
196 	c = (char *) &lhash;
197 	return c[0] ^ c[1] ^ c[2] ^ c[3];
198 }
199 
200 static void lb_update_tx_stats(unsigned int tx_bytes, struct lb_priv *lb_priv,
201 			       struct lb_port_priv *lb_port_priv,
202 			       unsigned char hash)
203 {
204 	struct lb_pcpu_stats *pcpu_stats;
205 	struct lb_stats *port_stats;
206 	struct lb_stats *hash_stats;
207 
208 	pcpu_stats = this_cpu_ptr(lb_priv->pcpu_stats);
209 	port_stats = this_cpu_ptr(lb_port_priv->pcpu_stats);
210 	hash_stats = &pcpu_stats->hash_stats[hash];
211 	u64_stats_update_begin(&pcpu_stats->syncp);
212 	port_stats->tx_bytes += tx_bytes;
213 	hash_stats->tx_bytes += tx_bytes;
214 	u64_stats_update_end(&pcpu_stats->syncp);
215 }
216 
217 static bool lb_transmit(struct team *team, struct sk_buff *skb)
218 {
219 	struct lb_priv *lb_priv = get_lb_priv(team);
220 	lb_select_tx_port_func_t *select_tx_port_func;
221 	struct team_port *port;
222 	unsigned char hash;
223 	unsigned int tx_bytes = skb->len;
224 
225 	hash = lb_get_skb_hash(lb_priv, skb);
226 	select_tx_port_func = rcu_dereference_bh(lb_priv->select_tx_port_func);
227 	port = select_tx_port_func(team, hash);
228 	if (unlikely(!port))
229 		goto drop;
230 	if (team_dev_queue_xmit(team, port, skb))
231 		return false;
232 	lb_update_tx_stats(tx_bytes, lb_priv, get_lb_port_priv(port), hash);
233 	return true;
234 
235 drop:
236 	dev_kfree_skb_any(skb);
237 	return false;
238 }
239 
240 static void lb_bpf_func_get(struct team *team, struct team_gsetter_ctx *ctx)
241 {
242 	struct lb_priv *lb_priv = get_lb_priv(team);
243 
244 	if (!lb_priv->ex->orig_fprog) {
245 		ctx->data.bin_val.len = 0;
246 		ctx->data.bin_val.ptr = NULL;
247 		return;
248 	}
249 	ctx->data.bin_val.len = lb_priv->ex->orig_fprog->len *
250 				sizeof(struct sock_filter);
251 	ctx->data.bin_val.ptr = lb_priv->ex->orig_fprog->filter;
252 }
253 
254 static int __fprog_create(struct sock_fprog_kern **pfprog, u32 data_len,
255 			  const void *data)
256 {
257 	struct sock_fprog_kern *fprog;
258 	struct sock_filter *filter = (struct sock_filter *) data;
259 
260 	if (data_len % sizeof(struct sock_filter))
261 		return -EINVAL;
262 	fprog = kmalloc(sizeof(*fprog), GFP_KERNEL);
263 	if (!fprog)
264 		return -ENOMEM;
265 	fprog->filter = kmemdup(filter, data_len, GFP_KERNEL);
266 	if (!fprog->filter) {
267 		kfree(fprog);
268 		return -ENOMEM;
269 	}
270 	fprog->len = data_len / sizeof(struct sock_filter);
271 	*pfprog = fprog;
272 	return 0;
273 }
274 
275 static void __fprog_destroy(struct sock_fprog_kern *fprog)
276 {
277 	kfree(fprog->filter);
278 	kfree(fprog);
279 }
280 
281 static int lb_bpf_func_set(struct team *team, struct team_gsetter_ctx *ctx)
282 {
283 	struct lb_priv *lb_priv = get_lb_priv(team);
284 	struct bpf_prog *fp = NULL;
285 	struct bpf_prog *orig_fp = NULL;
286 	struct sock_fprog_kern *fprog = NULL;
287 	int err;
288 
289 	if (ctx->data.bin_val.len) {
290 		err = __fprog_create(&fprog, ctx->data.bin_val.len,
291 				     ctx->data.bin_val.ptr);
292 		if (err)
293 			return err;
294 		err = bpf_prog_create(&fp, fprog);
295 		if (err) {
296 			__fprog_destroy(fprog);
297 			return err;
298 		}
299 	}
300 
301 	if (lb_priv->ex->orig_fprog) {
302 		/* Clear old filter data */
303 		__fprog_destroy(lb_priv->ex->orig_fprog);
304 		orig_fp = rcu_dereference_protected(lb_priv->fp,
305 						lockdep_is_held(&team->lock));
306 	}
307 
308 	rcu_assign_pointer(lb_priv->fp, fp);
309 	lb_priv->ex->orig_fprog = fprog;
310 
311 	if (orig_fp) {
312 		synchronize_rcu();
313 		bpf_prog_destroy(orig_fp);
314 	}
315 	return 0;
316 }
317 
318 static void lb_bpf_func_free(struct team *team)
319 {
320 	struct lb_priv *lb_priv = get_lb_priv(team);
321 	struct bpf_prog *fp;
322 
323 	if (!lb_priv->ex->orig_fprog)
324 		return;
325 
326 	__fprog_destroy(lb_priv->ex->orig_fprog);
327 	fp = rcu_dereference_protected(lb_priv->fp,
328 				       lockdep_is_held(&team->lock));
329 	bpf_prog_destroy(fp);
330 }
331 
332 static void lb_tx_method_get(struct team *team, struct team_gsetter_ctx *ctx)
333 {
334 	struct lb_priv *lb_priv = get_lb_priv(team);
335 	lb_select_tx_port_func_t *func;
336 	char *name;
337 
338 	func = rcu_dereference_protected(lb_priv->select_tx_port_func,
339 					 lockdep_is_held(&team->lock));
340 	name = lb_select_tx_port_get_name(func);
341 	BUG_ON(!name);
342 	ctx->data.str_val = name;
343 }
344 
345 static int lb_tx_method_set(struct team *team, struct team_gsetter_ctx *ctx)
346 {
347 	struct lb_priv *lb_priv = get_lb_priv(team);
348 	lb_select_tx_port_func_t *func;
349 
350 	func = lb_select_tx_port_get_func(ctx->data.str_val);
351 	if (!func)
352 		return -EINVAL;
353 	rcu_assign_pointer(lb_priv->select_tx_port_func, func);
354 	return 0;
355 }
356 
357 static void lb_tx_hash_to_port_mapping_init(struct team *team,
358 					    struct team_option_inst_info *info)
359 {
360 	struct lb_priv *lb_priv = get_lb_priv(team);
361 	unsigned char hash = info->array_index;
362 
363 	LB_HTPM_OPT_INST_INFO_BY_HASH(lb_priv, hash) = info;
364 }
365 
366 static void lb_tx_hash_to_port_mapping_get(struct team *team,
367 					   struct team_gsetter_ctx *ctx)
368 {
369 	struct lb_priv *lb_priv = get_lb_priv(team);
370 	struct team_port *port;
371 	unsigned char hash = ctx->info->array_index;
372 
373 	port = LB_HTPM_PORT_BY_HASH(lb_priv, hash);
374 	ctx->data.u32_val = port ? port->dev->ifindex : 0;
375 }
376 
377 static int lb_tx_hash_to_port_mapping_set(struct team *team,
378 					  struct team_gsetter_ctx *ctx)
379 {
380 	struct lb_priv *lb_priv = get_lb_priv(team);
381 	struct team_port *port;
382 	unsigned char hash = ctx->info->array_index;
383 
384 	list_for_each_entry(port, &team->port_list, list) {
385 		if (ctx->data.u32_val == port->dev->ifindex &&
386 		    team_port_enabled(port)) {
387 			rcu_assign_pointer(LB_HTPM_PORT_BY_HASH(lb_priv, hash),
388 					   port);
389 			return 0;
390 		}
391 	}
392 	return -ENODEV;
393 }
394 
395 static void lb_hash_stats_init(struct team *team,
396 			       struct team_option_inst_info *info)
397 {
398 	struct lb_priv *lb_priv = get_lb_priv(team);
399 	unsigned char hash = info->array_index;
400 
401 	lb_priv->ex->stats.info[hash].opt_inst_info = info;
402 }
403 
404 static void lb_hash_stats_get(struct team *team, struct team_gsetter_ctx *ctx)
405 {
406 	struct lb_priv *lb_priv = get_lb_priv(team);
407 	unsigned char hash = ctx->info->array_index;
408 
409 	ctx->data.bin_val.ptr = &lb_priv->ex->stats.info[hash].stats;
410 	ctx->data.bin_val.len = sizeof(struct lb_stats);
411 }
412 
413 static void lb_port_stats_init(struct team *team,
414 			       struct team_option_inst_info *info)
415 {
416 	struct team_port *port = info->port;
417 	struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
418 
419 	lb_port_priv->stats_info.opt_inst_info = info;
420 }
421 
422 static void lb_port_stats_get(struct team *team, struct team_gsetter_ctx *ctx)
423 {
424 	struct team_port *port = ctx->info->port;
425 	struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
426 
427 	ctx->data.bin_val.ptr = &lb_port_priv->stats_info.stats;
428 	ctx->data.bin_val.len = sizeof(struct lb_stats);
429 }
430 
431 static void __lb_stats_info_refresh_prepare(struct lb_stats_info *s_info)
432 {
433 	memcpy(&s_info->last_stats, &s_info->stats, sizeof(struct lb_stats));
434 	memset(&s_info->stats, 0, sizeof(struct lb_stats));
435 }
436 
437 static bool __lb_stats_info_refresh_check(struct lb_stats_info *s_info,
438 					  struct team *team)
439 {
440 	if (memcmp(&s_info->last_stats, &s_info->stats,
441 	    sizeof(struct lb_stats))) {
442 		team_option_inst_set_change(s_info->opt_inst_info);
443 		return true;
444 	}
445 	return false;
446 }
447 
448 static void __lb_one_cpu_stats_add(struct lb_stats *acc_stats,
449 				   struct lb_stats *cpu_stats,
450 				   struct u64_stats_sync *syncp)
451 {
452 	unsigned int start;
453 	struct lb_stats tmp;
454 
455 	do {
456 		start = u64_stats_fetch_begin(syncp);
457 		tmp.tx_bytes = cpu_stats->tx_bytes;
458 	} while (u64_stats_fetch_retry(syncp, start));
459 	acc_stats->tx_bytes += tmp.tx_bytes;
460 }
461 
462 static void lb_stats_refresh(struct work_struct *work)
463 {
464 	struct team *team;
465 	struct lb_priv *lb_priv;
466 	struct lb_priv_ex *lb_priv_ex;
467 	struct lb_pcpu_stats *pcpu_stats;
468 	struct lb_stats *stats;
469 	struct lb_stats_info *s_info;
470 	struct team_port *port;
471 	bool changed = false;
472 	int i;
473 	int j;
474 
475 	lb_priv_ex = container_of(work, struct lb_priv_ex,
476 				  stats.refresh_dw.work);
477 
478 	team = lb_priv_ex->team;
479 	lb_priv = get_lb_priv(team);
480 
481 	if (!mutex_trylock(&team->lock)) {
482 		schedule_delayed_work(&lb_priv_ex->stats.refresh_dw, 0);
483 		return;
484 	}
485 
486 	for (j = 0; j < LB_TX_HASHTABLE_SIZE; j++) {
487 		s_info = &lb_priv->ex->stats.info[j];
488 		__lb_stats_info_refresh_prepare(s_info);
489 		for_each_possible_cpu(i) {
490 			pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i);
491 			stats = &pcpu_stats->hash_stats[j];
492 			__lb_one_cpu_stats_add(&s_info->stats, stats,
493 					       &pcpu_stats->syncp);
494 		}
495 		changed |= __lb_stats_info_refresh_check(s_info, team);
496 	}
497 
498 	list_for_each_entry(port, &team->port_list, list) {
499 		struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
500 
501 		s_info = &lb_port_priv->stats_info;
502 		__lb_stats_info_refresh_prepare(s_info);
503 		for_each_possible_cpu(i) {
504 			pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i);
505 			stats = per_cpu_ptr(lb_port_priv->pcpu_stats, i);
506 			__lb_one_cpu_stats_add(&s_info->stats, stats,
507 					       &pcpu_stats->syncp);
508 		}
509 		changed |= __lb_stats_info_refresh_check(s_info, team);
510 	}
511 
512 	if (changed)
513 		team_options_change_check(team);
514 
515 	schedule_delayed_work(&lb_priv_ex->stats.refresh_dw,
516 			      (lb_priv_ex->stats.refresh_interval * HZ) / 10);
517 
518 	mutex_unlock(&team->lock);
519 }
520 
521 static void lb_stats_refresh_interval_get(struct team *team,
522 					  struct team_gsetter_ctx *ctx)
523 {
524 	struct lb_priv *lb_priv = get_lb_priv(team);
525 
526 	ctx->data.u32_val = lb_priv->ex->stats.refresh_interval;
527 }
528 
529 static int lb_stats_refresh_interval_set(struct team *team,
530 					 struct team_gsetter_ctx *ctx)
531 {
532 	struct lb_priv *lb_priv = get_lb_priv(team);
533 	unsigned int interval;
534 
535 	interval = ctx->data.u32_val;
536 	if (lb_priv->ex->stats.refresh_interval == interval)
537 		return 0;
538 	lb_priv->ex->stats.refresh_interval = interval;
539 	if (interval)
540 		schedule_delayed_work(&lb_priv->ex->stats.refresh_dw, 0);
541 	else
542 		cancel_delayed_work(&lb_priv->ex->stats.refresh_dw);
543 	return 0;
544 }
545 
546 static const struct team_option lb_options[] = {
547 	{
548 		.name = "bpf_hash_func",
549 		.type = TEAM_OPTION_TYPE_BINARY,
550 		.getter = lb_bpf_func_get,
551 		.setter = lb_bpf_func_set,
552 	},
553 	{
554 		.name = "lb_tx_method",
555 		.type = TEAM_OPTION_TYPE_STRING,
556 		.getter = lb_tx_method_get,
557 		.setter = lb_tx_method_set,
558 	},
559 	{
560 		.name = "lb_tx_hash_to_port_mapping",
561 		.array_size = LB_TX_HASHTABLE_SIZE,
562 		.type = TEAM_OPTION_TYPE_U32,
563 		.init = lb_tx_hash_to_port_mapping_init,
564 		.getter = lb_tx_hash_to_port_mapping_get,
565 		.setter = lb_tx_hash_to_port_mapping_set,
566 	},
567 	{
568 		.name = "lb_hash_stats",
569 		.array_size = LB_TX_HASHTABLE_SIZE,
570 		.type = TEAM_OPTION_TYPE_BINARY,
571 		.init = lb_hash_stats_init,
572 		.getter = lb_hash_stats_get,
573 	},
574 	{
575 		.name = "lb_port_stats",
576 		.per_port = true,
577 		.type = TEAM_OPTION_TYPE_BINARY,
578 		.init = lb_port_stats_init,
579 		.getter = lb_port_stats_get,
580 	},
581 	{
582 		.name = "lb_stats_refresh_interval",
583 		.type = TEAM_OPTION_TYPE_U32,
584 		.getter = lb_stats_refresh_interval_get,
585 		.setter = lb_stats_refresh_interval_set,
586 	},
587 };
588 
589 static int lb_init(struct team *team)
590 {
591 	struct lb_priv *lb_priv = get_lb_priv(team);
592 	lb_select_tx_port_func_t *func;
593 	int i, err;
594 
595 	/* set default tx port selector */
596 	func = lb_select_tx_port_get_func("hash");
597 	BUG_ON(!func);
598 	rcu_assign_pointer(lb_priv->select_tx_port_func, func);
599 
600 	lb_priv->ex = kzalloc(sizeof(*lb_priv->ex), GFP_KERNEL);
601 	if (!lb_priv->ex)
602 		return -ENOMEM;
603 	lb_priv->ex->team = team;
604 
605 	lb_priv->pcpu_stats = alloc_percpu(struct lb_pcpu_stats);
606 	if (!lb_priv->pcpu_stats) {
607 		err = -ENOMEM;
608 		goto err_alloc_pcpu_stats;
609 	}
610 
611 	for_each_possible_cpu(i) {
612 		struct lb_pcpu_stats *team_lb_stats;
613 		team_lb_stats = per_cpu_ptr(lb_priv->pcpu_stats, i);
614 		u64_stats_init(&team_lb_stats->syncp);
615 	}
616 
617 
618 	INIT_DELAYED_WORK(&lb_priv->ex->stats.refresh_dw, lb_stats_refresh);
619 
620 	err = team_options_register(team, lb_options, ARRAY_SIZE(lb_options));
621 	if (err)
622 		goto err_options_register;
623 	return 0;
624 
625 err_options_register:
626 	free_percpu(lb_priv->pcpu_stats);
627 err_alloc_pcpu_stats:
628 	kfree(lb_priv->ex);
629 	return err;
630 }
631 
632 static void lb_exit(struct team *team)
633 {
634 	struct lb_priv *lb_priv = get_lb_priv(team);
635 
636 	team_options_unregister(team, lb_options,
637 				ARRAY_SIZE(lb_options));
638 	lb_bpf_func_free(team);
639 	cancel_delayed_work_sync(&lb_priv->ex->stats.refresh_dw);
640 	free_percpu(lb_priv->pcpu_stats);
641 	kfree(lb_priv->ex);
642 }
643 
644 static int lb_port_enter(struct team *team, struct team_port *port)
645 {
646 	struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
647 
648 	lb_port_priv->pcpu_stats = alloc_percpu(struct lb_stats);
649 	if (!lb_port_priv->pcpu_stats)
650 		return -ENOMEM;
651 	return 0;
652 }
653 
654 static void lb_port_leave(struct team *team, struct team_port *port)
655 {
656 	struct lb_port_priv *lb_port_priv = get_lb_port_priv(port);
657 
658 	free_percpu(lb_port_priv->pcpu_stats);
659 }
660 
661 static void lb_port_disabled(struct team *team, struct team_port *port)
662 {
663 	lb_tx_hash_to_port_mapping_null_port(team, port);
664 }
665 
666 static const struct team_mode_ops lb_mode_ops = {
667 	.init			= lb_init,
668 	.exit			= lb_exit,
669 	.port_enter		= lb_port_enter,
670 	.port_leave		= lb_port_leave,
671 	.port_disabled		= lb_port_disabled,
672 	.receive		= lb_receive,
673 	.transmit		= lb_transmit,
674 };
675 
676 static const struct team_mode lb_mode = {
677 	.kind		= "loadbalance",
678 	.owner		= THIS_MODULE,
679 	.priv_size	= sizeof(struct lb_priv),
680 	.port_priv_size	= sizeof(struct lb_port_priv),
681 	.ops		= &lb_mode_ops,
682 	.lag_tx_type	= NETDEV_LAG_TX_TYPE_HASH,
683 };
684 
685 static int __init lb_init_module(void)
686 {
687 	return team_mode_register(&lb_mode);
688 }
689 
690 static void __exit lb_cleanup_module(void)
691 {
692 	team_mode_unregister(&lb_mode);
693 }
694 
695 module_init(lb_init_module);
696 module_exit(lb_cleanup_module);
697 
698 MODULE_LICENSE("GPL v2");
699 MODULE_AUTHOR("Jiri Pirko <jpirko@redhat.com>");
700 MODULE_DESCRIPTION("Load-balancing mode for team");
701 MODULE_ALIAS_TEAM_MODE("loadbalance");
702