12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 201d7f30aSJiri Pirko /* 301d7f30aSJiri Pirko * drivers/net/team/team_mode_loadbalance.c - Load-balancing mode for team 401d7f30aSJiri Pirko * Copyright (c) 2012 Jiri Pirko <jpirko@redhat.com> 501d7f30aSJiri Pirko */ 601d7f30aSJiri Pirko 701d7f30aSJiri Pirko #include <linux/kernel.h> 801d7f30aSJiri Pirko #include <linux/types.h> 901d7f30aSJiri Pirko #include <linux/module.h> 1001d7f30aSJiri Pirko #include <linux/init.h> 1101d7f30aSJiri Pirko #include <linux/errno.h> 1201d7f30aSJiri Pirko #include <linux/netdevice.h> 13c15e07b0SJiri Pirko #include <linux/etherdevice.h> 1401d7f30aSJiri Pirko #include <linux/filter.h> 1501d7f30aSJiri Pirko #include <linux/if_team.h> 1601d7f30aSJiri Pirko 17c15e07b0SJiri Pirko static rx_handler_result_t lb_receive(struct team *team, struct team_port *port, 18c15e07b0SJiri Pirko struct sk_buff *skb) 19c15e07b0SJiri Pirko { 20c15e07b0SJiri Pirko if (unlikely(skb->protocol == htons(ETH_P_SLOW))) { 21c15e07b0SJiri Pirko /* LACPDU packets should go to exact delivery */ 22c15e07b0SJiri Pirko const unsigned char *dest = eth_hdr(skb)->h_dest; 23c15e07b0SJiri Pirko 24c15e07b0SJiri Pirko if (is_link_local_ether_addr(dest) && dest[5] == 0x02) 25c15e07b0SJiri Pirko return RX_HANDLER_EXACT; 26c15e07b0SJiri Pirko } 27c15e07b0SJiri Pirko return RX_HANDLER_ANOTHER; 28c15e07b0SJiri Pirko } 29c15e07b0SJiri Pirko 30ab8250d7SJiri Pirko struct lb_priv; 31ab8250d7SJiri Pirko 32ab8250d7SJiri Pirko typedef struct team_port *lb_select_tx_port_func_t(struct team *, 33ab8250d7SJiri Pirko unsigned char); 34ab8250d7SJiri Pirko 35ab8250d7SJiri Pirko #define LB_TX_HASHTABLE_SIZE 256 /* hash is a char */ 36ab8250d7SJiri Pirko 37ab8250d7SJiri Pirko struct lb_stats { 38ab8250d7SJiri Pirko u64 tx_bytes; 3901d7f30aSJiri Pirko }; 4001d7f30aSJiri Pirko 41ab8250d7SJiri Pirko struct lb_pcpu_stats { 42ab8250d7SJiri Pirko struct lb_stats hash_stats[LB_TX_HASHTABLE_SIZE]; 43ab8250d7SJiri Pirko struct u64_stats_sync syncp; 44ab8250d7SJiri Pirko }; 45ab8250d7SJiri Pirko 46ab8250d7SJiri Pirko struct lb_stats_info { 47ab8250d7SJiri Pirko struct lb_stats stats; 48ab8250d7SJiri Pirko struct lb_stats last_stats; 49ab8250d7SJiri Pirko struct team_option_inst_info *opt_inst_info; 50ab8250d7SJiri Pirko }; 51ab8250d7SJiri Pirko 52ab8250d7SJiri Pirko struct lb_port_mapping { 53ab8250d7SJiri Pirko struct team_port __rcu *port; 54ab8250d7SJiri Pirko struct team_option_inst_info *opt_inst_info; 55ab8250d7SJiri Pirko }; 56ab8250d7SJiri Pirko 57ab8250d7SJiri Pirko struct lb_priv_ex { 58ab8250d7SJiri Pirko struct team *team; 59ab8250d7SJiri Pirko struct lb_port_mapping tx_hash_to_port_mapping[LB_TX_HASHTABLE_SIZE]; 60b1fcd35cSDaniel Borkmann struct sock_fprog_kern *orig_fprog; 61ab8250d7SJiri Pirko struct { 62ab8250d7SJiri Pirko unsigned int refresh_interval; /* in tenths of second */ 63ab8250d7SJiri Pirko struct delayed_work refresh_dw; 64ab8250d7SJiri Pirko struct lb_stats_info info[LB_TX_HASHTABLE_SIZE]; 65ab8250d7SJiri Pirko } stats; 66ab8250d7SJiri Pirko }; 67ab8250d7SJiri Pirko 68ab8250d7SJiri Pirko struct lb_priv { 697ae457c1SAlexei Starovoitov struct bpf_prog __rcu *fp; 70ab8250d7SJiri Pirko lb_select_tx_port_func_t __rcu *select_tx_port_func; 71ab8250d7SJiri Pirko struct lb_pcpu_stats __percpu *pcpu_stats; 72ab8250d7SJiri Pirko struct lb_priv_ex *ex; /* priv extension */ 73ab8250d7SJiri Pirko }; 74ab8250d7SJiri Pirko 75ab8250d7SJiri Pirko static struct lb_priv *get_lb_priv(struct team *team) 7601d7f30aSJiri Pirko { 7701d7f30aSJiri Pirko return (struct lb_priv *) &team->mode_priv; 7801d7f30aSJiri Pirko } 7901d7f30aSJiri Pirko 80ab8250d7SJiri Pirko struct lb_port_priv { 81ab8250d7SJiri Pirko struct lb_stats __percpu *pcpu_stats; 82ab8250d7SJiri Pirko struct lb_stats_info stats_info; 83ab8250d7SJiri Pirko }; 84ab8250d7SJiri Pirko 85ab8250d7SJiri Pirko static struct lb_port_priv *get_lb_port_priv(struct team_port *port) 86ab8250d7SJiri Pirko { 87ab8250d7SJiri Pirko return (struct lb_port_priv *) &port->mode_priv; 88ab8250d7SJiri Pirko } 89ab8250d7SJiri Pirko 90ab8250d7SJiri Pirko #define LB_HTPM_PORT_BY_HASH(lp_priv, hash) \ 91ab8250d7SJiri Pirko (lb_priv)->ex->tx_hash_to_port_mapping[hash].port 92ab8250d7SJiri Pirko 93ab8250d7SJiri Pirko #define LB_HTPM_OPT_INST_INFO_BY_HASH(lp_priv, hash) \ 94ab8250d7SJiri Pirko (lb_priv)->ex->tx_hash_to_port_mapping[hash].opt_inst_info 95ab8250d7SJiri Pirko 96ab8250d7SJiri Pirko static void lb_tx_hash_to_port_mapping_null_port(struct team *team, 97ab8250d7SJiri Pirko struct team_port *port) 98ab8250d7SJiri Pirko { 99ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 100ab8250d7SJiri Pirko bool changed = false; 101ab8250d7SJiri Pirko int i; 102ab8250d7SJiri Pirko 103ab8250d7SJiri Pirko for (i = 0; i < LB_TX_HASHTABLE_SIZE; i++) { 104ab8250d7SJiri Pirko struct lb_port_mapping *pm; 105ab8250d7SJiri Pirko 106ab8250d7SJiri Pirko pm = &lb_priv->ex->tx_hash_to_port_mapping[i]; 1076dab015cSJiri Pirko if (rcu_access_pointer(pm->port) == port) { 1086dab015cSJiri Pirko RCU_INIT_POINTER(pm->port, NULL); 109ab8250d7SJiri Pirko team_option_inst_set_change(pm->opt_inst_info); 110ab8250d7SJiri Pirko changed = true; 111ab8250d7SJiri Pirko } 112ab8250d7SJiri Pirko } 113ab8250d7SJiri Pirko if (changed) 114ab8250d7SJiri Pirko team_options_change_check(team); 115ab8250d7SJiri Pirko } 116ab8250d7SJiri Pirko 117ab8250d7SJiri Pirko /* Basic tx selection based solely by hash */ 118ab8250d7SJiri Pirko static struct team_port *lb_hash_select_tx_port(struct team *team, 119ab8250d7SJiri Pirko unsigned char hash) 120ab8250d7SJiri Pirko { 121735d381fSJiri Pirko int port_index = team_num_to_port_index(team, hash); 122ab8250d7SJiri Pirko 123ab8250d7SJiri Pirko return team_get_port_by_index_rcu(team, port_index); 124ab8250d7SJiri Pirko } 125ab8250d7SJiri Pirko 126ab8250d7SJiri Pirko /* Hash to port mapping select tx port */ 127ab8250d7SJiri Pirko static struct team_port *lb_htpm_select_tx_port(struct team *team, 128ab8250d7SJiri Pirko unsigned char hash) 129ab8250d7SJiri Pirko { 130*7790eaebSZhengchao Shao struct lb_priv *lb_priv = get_lb_priv(team); 131bd7d2106SJim Hanko struct team_port *port; 132bd7d2106SJim Hanko 133bd7d2106SJim Hanko port = rcu_dereference_bh(LB_HTPM_PORT_BY_HASH(lb_priv, hash)); 134bd7d2106SJim Hanko if (likely(port)) 135bd7d2106SJim Hanko return port; 136bd7d2106SJim Hanko /* If no valid port in the table, fall back to simple hash */ 137*7790eaebSZhengchao Shao return lb_hash_select_tx_port(team, hash); 138ab8250d7SJiri Pirko } 139ab8250d7SJiri Pirko 140ab8250d7SJiri Pirko struct lb_select_tx_port { 141ab8250d7SJiri Pirko char *name; 142ab8250d7SJiri Pirko lb_select_tx_port_func_t *func; 143ab8250d7SJiri Pirko }; 144ab8250d7SJiri Pirko 145ab8250d7SJiri Pirko static const struct lb_select_tx_port lb_select_tx_port_list[] = { 146ab8250d7SJiri Pirko { 147ab8250d7SJiri Pirko .name = "hash", 148ab8250d7SJiri Pirko .func = lb_hash_select_tx_port, 149ab8250d7SJiri Pirko }, 150ab8250d7SJiri Pirko { 151ab8250d7SJiri Pirko .name = "hash_to_port_mapping", 152ab8250d7SJiri Pirko .func = lb_htpm_select_tx_port, 153ab8250d7SJiri Pirko }, 154ab8250d7SJiri Pirko }; 155ab8250d7SJiri Pirko #define LB_SELECT_TX_PORT_LIST_COUNT ARRAY_SIZE(lb_select_tx_port_list) 156ab8250d7SJiri Pirko 157ab8250d7SJiri Pirko static char *lb_select_tx_port_get_name(lb_select_tx_port_func_t *func) 158ab8250d7SJiri Pirko { 159ab8250d7SJiri Pirko int i; 160ab8250d7SJiri Pirko 161ab8250d7SJiri Pirko for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) { 162ab8250d7SJiri Pirko const struct lb_select_tx_port *item; 163ab8250d7SJiri Pirko 164ab8250d7SJiri Pirko item = &lb_select_tx_port_list[i]; 165ab8250d7SJiri Pirko if (item->func == func) 166ab8250d7SJiri Pirko return item->name; 167ab8250d7SJiri Pirko } 168ab8250d7SJiri Pirko return NULL; 169ab8250d7SJiri Pirko } 170ab8250d7SJiri Pirko 171ab8250d7SJiri Pirko static lb_select_tx_port_func_t *lb_select_tx_port_get_func(const char *name) 172ab8250d7SJiri Pirko { 173ab8250d7SJiri Pirko int i; 174ab8250d7SJiri Pirko 175ab8250d7SJiri Pirko for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) { 176ab8250d7SJiri Pirko const struct lb_select_tx_port *item; 177ab8250d7SJiri Pirko 178ab8250d7SJiri Pirko item = &lb_select_tx_port_list[i]; 179ab8250d7SJiri Pirko if (!strcmp(item->name, name)) 180ab8250d7SJiri Pirko return item->func; 181ab8250d7SJiri Pirko } 182ab8250d7SJiri Pirko return NULL; 183ab8250d7SJiri Pirko } 184ab8250d7SJiri Pirko 185ab8250d7SJiri Pirko static unsigned int lb_get_skb_hash(struct lb_priv *lb_priv, 186596e2024SJiri Pirko struct sk_buff *skb) 18701d7f30aSJiri Pirko { 1887ae457c1SAlexei Starovoitov struct bpf_prog *fp; 189596e2024SJiri Pirko uint32_t lhash; 190596e2024SJiri Pirko unsigned char *c; 191596e2024SJiri Pirko 192d1904fbdSJiri Pirko fp = rcu_dereference_bh(lb_priv->fp); 193596e2024SJiri Pirko if (unlikely(!fp)) 194596e2024SJiri Pirko return 0; 195fb7dd8bcSAndrii Nakryiko lhash = bpf_prog_run(fp, skb); 196596e2024SJiri Pirko c = (char *) &lhash; 197596e2024SJiri Pirko return c[0] ^ c[1] ^ c[2] ^ c[3]; 198596e2024SJiri Pirko } 199596e2024SJiri Pirko 200ab8250d7SJiri Pirko static void lb_update_tx_stats(unsigned int tx_bytes, struct lb_priv *lb_priv, 201ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv, 202ab8250d7SJiri Pirko unsigned char hash) 203ab8250d7SJiri Pirko { 204ab8250d7SJiri Pirko struct lb_pcpu_stats *pcpu_stats; 205ab8250d7SJiri Pirko struct lb_stats *port_stats; 206ab8250d7SJiri Pirko struct lb_stats *hash_stats; 207ab8250d7SJiri Pirko 208ab8250d7SJiri Pirko pcpu_stats = this_cpu_ptr(lb_priv->pcpu_stats); 209ab8250d7SJiri Pirko port_stats = this_cpu_ptr(lb_port_priv->pcpu_stats); 210ab8250d7SJiri Pirko hash_stats = &pcpu_stats->hash_stats[hash]; 211ab8250d7SJiri Pirko u64_stats_update_begin(&pcpu_stats->syncp); 212ab8250d7SJiri Pirko port_stats->tx_bytes += tx_bytes; 213ab8250d7SJiri Pirko hash_stats->tx_bytes += tx_bytes; 214ab8250d7SJiri Pirko u64_stats_update_end(&pcpu_stats->syncp); 215ab8250d7SJiri Pirko } 216ab8250d7SJiri Pirko 217596e2024SJiri Pirko static bool lb_transmit(struct team *team, struct sk_buff *skb) 218596e2024SJiri Pirko { 219ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 220ab8250d7SJiri Pirko lb_select_tx_port_func_t *select_tx_port_func; 22101d7f30aSJiri Pirko struct team_port *port; 222ab8250d7SJiri Pirko unsigned char hash; 223ab8250d7SJiri Pirko unsigned int tx_bytes = skb->len; 22401d7f30aSJiri Pirko 225ab8250d7SJiri Pirko hash = lb_get_skb_hash(lb_priv, skb); 226d1904fbdSJiri Pirko select_tx_port_func = rcu_dereference_bh(lb_priv->select_tx_port_func); 227*7790eaebSZhengchao Shao port = select_tx_port_func(team, hash); 22801d7f30aSJiri Pirko if (unlikely(!port)) 22901d7f30aSJiri Pirko goto drop; 230bd2d0837SJiri Pirko if (team_dev_queue_xmit(team, port, skb)) 23101d7f30aSJiri Pirko return false; 232ab8250d7SJiri Pirko lb_update_tx_stats(tx_bytes, lb_priv, get_lb_port_priv(port), hash); 23301d7f30aSJiri Pirko return true; 23401d7f30aSJiri Pirko 23501d7f30aSJiri Pirko drop: 23601d7f30aSJiri Pirko dev_kfree_skb_any(skb); 23701d7f30aSJiri Pirko return false; 23801d7f30aSJiri Pirko } 23901d7f30aSJiri Pirko 240c3b41f4cSZhengchao Shao static void lb_bpf_func_get(struct team *team, struct team_gsetter_ctx *ctx) 24101d7f30aSJiri Pirko { 242ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 243ab8250d7SJiri Pirko 244ab8250d7SJiri Pirko if (!lb_priv->ex->orig_fprog) { 24580f7c668SJiri Pirko ctx->data.bin_val.len = 0; 24680f7c668SJiri Pirko ctx->data.bin_val.ptr = NULL; 247c3b41f4cSZhengchao Shao return; 24880f7c668SJiri Pirko } 249ab8250d7SJiri Pirko ctx->data.bin_val.len = lb_priv->ex->orig_fprog->len * 25001d7f30aSJiri Pirko sizeof(struct sock_filter); 251ab8250d7SJiri Pirko ctx->data.bin_val.ptr = lb_priv->ex->orig_fprog->filter; 25201d7f30aSJiri Pirko } 25301d7f30aSJiri Pirko 254b1fcd35cSDaniel Borkmann static int __fprog_create(struct sock_fprog_kern **pfprog, u32 data_len, 25580f7c668SJiri Pirko const void *data) 25601d7f30aSJiri Pirko { 257b1fcd35cSDaniel Borkmann struct sock_fprog_kern *fprog; 25801d7f30aSJiri Pirko struct sock_filter *filter = (struct sock_filter *) data; 25901d7f30aSJiri Pirko 26001d7f30aSJiri Pirko if (data_len % sizeof(struct sock_filter)) 26101d7f30aSJiri Pirko return -EINVAL; 262ea5930f4SDaniel Borkmann fprog = kmalloc(sizeof(*fprog), GFP_KERNEL); 26301d7f30aSJiri Pirko if (!fprog) 26401d7f30aSJiri Pirko return -ENOMEM; 26501d7f30aSJiri Pirko fprog->filter = kmemdup(filter, data_len, GFP_KERNEL); 26601d7f30aSJiri Pirko if (!fprog->filter) { 26701d7f30aSJiri Pirko kfree(fprog); 26801d7f30aSJiri Pirko return -ENOMEM; 26901d7f30aSJiri Pirko } 27001d7f30aSJiri Pirko fprog->len = data_len / sizeof(struct sock_filter); 27101d7f30aSJiri Pirko *pfprog = fprog; 27201d7f30aSJiri Pirko return 0; 27301d7f30aSJiri Pirko } 27401d7f30aSJiri Pirko 275b1fcd35cSDaniel Borkmann static void __fprog_destroy(struct sock_fprog_kern *fprog) 27601d7f30aSJiri Pirko { 27701d7f30aSJiri Pirko kfree(fprog->filter); 27801d7f30aSJiri Pirko kfree(fprog); 27901d7f30aSJiri Pirko } 28001d7f30aSJiri Pirko 28180f7c668SJiri Pirko static int lb_bpf_func_set(struct team *team, struct team_gsetter_ctx *ctx) 28201d7f30aSJiri Pirko { 283ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 2847ae457c1SAlexei Starovoitov struct bpf_prog *fp = NULL; 2857ae457c1SAlexei Starovoitov struct bpf_prog *orig_fp = NULL; 286b1fcd35cSDaniel Borkmann struct sock_fprog_kern *fprog = NULL; 28701d7f30aSJiri Pirko int err; 28801d7f30aSJiri Pirko 28980f7c668SJiri Pirko if (ctx->data.bin_val.len) { 29080f7c668SJiri Pirko err = __fprog_create(&fprog, ctx->data.bin_val.len, 29180f7c668SJiri Pirko ctx->data.bin_val.ptr); 29201d7f30aSJiri Pirko if (err) 29301d7f30aSJiri Pirko return err; 2947ae457c1SAlexei Starovoitov err = bpf_prog_create(&fp, fprog); 29501d7f30aSJiri Pirko if (err) { 29601d7f30aSJiri Pirko __fprog_destroy(fprog); 29701d7f30aSJiri Pirko return err; 29801d7f30aSJiri Pirko } 29901d7f30aSJiri Pirko } 30001d7f30aSJiri Pirko 301ab8250d7SJiri Pirko if (lb_priv->ex->orig_fprog) { 30201d7f30aSJiri Pirko /* Clear old filter data */ 303ab8250d7SJiri Pirko __fprog_destroy(lb_priv->ex->orig_fprog); 3046dab015cSJiri Pirko orig_fp = rcu_dereference_protected(lb_priv->fp, 3056dab015cSJiri Pirko lockdep_is_held(&team->lock)); 30601d7f30aSJiri Pirko } 30701d7f30aSJiri Pirko 308ab8250d7SJiri Pirko rcu_assign_pointer(lb_priv->fp, fp); 309ab8250d7SJiri Pirko lb_priv->ex->orig_fprog = fprog; 31034c5bd66SPablo Neira 31134c5bd66SPablo Neira if (orig_fp) { 31234c5bd66SPablo Neira synchronize_rcu(); 3137ae457c1SAlexei Starovoitov bpf_prog_destroy(orig_fp); 31434c5bd66SPablo Neira } 315ab8250d7SJiri Pirko return 0; 316ab8250d7SJiri Pirko } 317ab8250d7SJiri Pirko 318692c31bdSIdo Schimmel static void lb_bpf_func_free(struct team *team) 319692c31bdSIdo Schimmel { 320692c31bdSIdo Schimmel struct lb_priv *lb_priv = get_lb_priv(team); 321692c31bdSIdo Schimmel struct bpf_prog *fp; 322692c31bdSIdo Schimmel 323692c31bdSIdo Schimmel if (!lb_priv->ex->orig_fprog) 324692c31bdSIdo Schimmel return; 325692c31bdSIdo Schimmel 326692c31bdSIdo Schimmel __fprog_destroy(lb_priv->ex->orig_fprog); 327692c31bdSIdo Schimmel fp = rcu_dereference_protected(lb_priv->fp, 328692c31bdSIdo Schimmel lockdep_is_held(&team->lock)); 329692c31bdSIdo Schimmel bpf_prog_destroy(fp); 330692c31bdSIdo Schimmel } 331692c31bdSIdo Schimmel 332c3b41f4cSZhengchao Shao static void lb_tx_method_get(struct team *team, struct team_gsetter_ctx *ctx) 333ab8250d7SJiri Pirko { 334ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 3356dab015cSJiri Pirko lb_select_tx_port_func_t *func; 336ab8250d7SJiri Pirko char *name; 337ab8250d7SJiri Pirko 3386dab015cSJiri Pirko func = rcu_dereference_protected(lb_priv->select_tx_port_func, 3396dab015cSJiri Pirko lockdep_is_held(&team->lock)); 3406dab015cSJiri Pirko name = lb_select_tx_port_get_name(func); 341ab8250d7SJiri Pirko BUG_ON(!name); 342ab8250d7SJiri Pirko ctx->data.str_val = name; 343ab8250d7SJiri Pirko } 344ab8250d7SJiri Pirko 345ab8250d7SJiri Pirko static int lb_tx_method_set(struct team *team, struct team_gsetter_ctx *ctx) 346ab8250d7SJiri Pirko { 347ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 348ab8250d7SJiri Pirko lb_select_tx_port_func_t *func; 349ab8250d7SJiri Pirko 350ab8250d7SJiri Pirko func = lb_select_tx_port_get_func(ctx->data.str_val); 351ab8250d7SJiri Pirko if (!func) 352ab8250d7SJiri Pirko return -EINVAL; 353ab8250d7SJiri Pirko rcu_assign_pointer(lb_priv->select_tx_port_func, func); 354ab8250d7SJiri Pirko return 0; 355ab8250d7SJiri Pirko } 356ab8250d7SJiri Pirko 357de3ecc4fSZhengchao Shao static void lb_tx_hash_to_port_mapping_init(struct team *team, 358ab8250d7SJiri Pirko struct team_option_inst_info *info) 359ab8250d7SJiri Pirko { 360ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 361ab8250d7SJiri Pirko unsigned char hash = info->array_index; 362ab8250d7SJiri Pirko 363ab8250d7SJiri Pirko LB_HTPM_OPT_INST_INFO_BY_HASH(lb_priv, hash) = info; 364ab8250d7SJiri Pirko } 365ab8250d7SJiri Pirko 366c3b41f4cSZhengchao Shao static void lb_tx_hash_to_port_mapping_get(struct team *team, 367ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 368ab8250d7SJiri Pirko { 369ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 370ab8250d7SJiri Pirko struct team_port *port; 371ab8250d7SJiri Pirko unsigned char hash = ctx->info->array_index; 372ab8250d7SJiri Pirko 373ab8250d7SJiri Pirko port = LB_HTPM_PORT_BY_HASH(lb_priv, hash); 374ab8250d7SJiri Pirko ctx->data.u32_val = port ? port->dev->ifindex : 0; 375ab8250d7SJiri Pirko } 376ab8250d7SJiri Pirko 377ab8250d7SJiri Pirko static int lb_tx_hash_to_port_mapping_set(struct team *team, 378ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 379ab8250d7SJiri Pirko { 380ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 381ab8250d7SJiri Pirko struct team_port *port; 382ab8250d7SJiri Pirko unsigned char hash = ctx->info->array_index; 383ab8250d7SJiri Pirko 384ab8250d7SJiri Pirko list_for_each_entry(port, &team->port_list, list) { 38552a4fd77SJiri Pirko if (ctx->data.u32_val == port->dev->ifindex && 38652a4fd77SJiri Pirko team_port_enabled(port)) { 387ab8250d7SJiri Pirko rcu_assign_pointer(LB_HTPM_PORT_BY_HASH(lb_priv, hash), 388ab8250d7SJiri Pirko port); 389ab8250d7SJiri Pirko return 0; 390ab8250d7SJiri Pirko } 391ab8250d7SJiri Pirko } 392ab8250d7SJiri Pirko return -ENODEV; 393ab8250d7SJiri Pirko } 394ab8250d7SJiri Pirko 395de3ecc4fSZhengchao Shao static void lb_hash_stats_init(struct team *team, 396ab8250d7SJiri Pirko struct team_option_inst_info *info) 397ab8250d7SJiri Pirko { 398ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 399ab8250d7SJiri Pirko unsigned char hash = info->array_index; 400ab8250d7SJiri Pirko 401ab8250d7SJiri Pirko lb_priv->ex->stats.info[hash].opt_inst_info = info; 402ab8250d7SJiri Pirko } 403ab8250d7SJiri Pirko 404c3b41f4cSZhengchao Shao static void lb_hash_stats_get(struct team *team, struct team_gsetter_ctx *ctx) 405ab8250d7SJiri Pirko { 406ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 407ab8250d7SJiri Pirko unsigned char hash = ctx->info->array_index; 408ab8250d7SJiri Pirko 409ab8250d7SJiri Pirko ctx->data.bin_val.ptr = &lb_priv->ex->stats.info[hash].stats; 410ab8250d7SJiri Pirko ctx->data.bin_val.len = sizeof(struct lb_stats); 411ab8250d7SJiri Pirko } 412ab8250d7SJiri Pirko 413de3ecc4fSZhengchao Shao static void lb_port_stats_init(struct team *team, 414ab8250d7SJiri Pirko struct team_option_inst_info *info) 415ab8250d7SJiri Pirko { 416ab8250d7SJiri Pirko struct team_port *port = info->port; 417ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 418ab8250d7SJiri Pirko 419ab8250d7SJiri Pirko lb_port_priv->stats_info.opt_inst_info = info; 420ab8250d7SJiri Pirko } 421ab8250d7SJiri Pirko 422c3b41f4cSZhengchao Shao static void lb_port_stats_get(struct team *team, struct team_gsetter_ctx *ctx) 423ab8250d7SJiri Pirko { 424ab8250d7SJiri Pirko struct team_port *port = ctx->info->port; 425ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 426ab8250d7SJiri Pirko 427ab8250d7SJiri Pirko ctx->data.bin_val.ptr = &lb_port_priv->stats_info.stats; 428ab8250d7SJiri Pirko ctx->data.bin_val.len = sizeof(struct lb_stats); 429ab8250d7SJiri Pirko } 430ab8250d7SJiri Pirko 431ab8250d7SJiri Pirko static void __lb_stats_info_refresh_prepare(struct lb_stats_info *s_info) 432ab8250d7SJiri Pirko { 433ab8250d7SJiri Pirko memcpy(&s_info->last_stats, &s_info->stats, sizeof(struct lb_stats)); 434ab8250d7SJiri Pirko memset(&s_info->stats, 0, sizeof(struct lb_stats)); 435ab8250d7SJiri Pirko } 436ab8250d7SJiri Pirko 437ab8250d7SJiri Pirko static bool __lb_stats_info_refresh_check(struct lb_stats_info *s_info, 438ab8250d7SJiri Pirko struct team *team) 439ab8250d7SJiri Pirko { 440ab8250d7SJiri Pirko if (memcmp(&s_info->last_stats, &s_info->stats, 441ab8250d7SJiri Pirko sizeof(struct lb_stats))) { 442ab8250d7SJiri Pirko team_option_inst_set_change(s_info->opt_inst_info); 443ab8250d7SJiri Pirko return true; 444ab8250d7SJiri Pirko } 445ab8250d7SJiri Pirko return false; 446ab8250d7SJiri Pirko } 447ab8250d7SJiri Pirko 448ab8250d7SJiri Pirko static void __lb_one_cpu_stats_add(struct lb_stats *acc_stats, 449ab8250d7SJiri Pirko struct lb_stats *cpu_stats, 450ab8250d7SJiri Pirko struct u64_stats_sync *syncp) 451ab8250d7SJiri Pirko { 452ab8250d7SJiri Pirko unsigned int start; 453ab8250d7SJiri Pirko struct lb_stats tmp; 454ab8250d7SJiri Pirko 455ab8250d7SJiri Pirko do { 456068c38adSThomas Gleixner start = u64_stats_fetch_begin(syncp); 457ab8250d7SJiri Pirko tmp.tx_bytes = cpu_stats->tx_bytes; 458068c38adSThomas Gleixner } while (u64_stats_fetch_retry(syncp, start)); 459ab8250d7SJiri Pirko acc_stats->tx_bytes += tmp.tx_bytes; 460ab8250d7SJiri Pirko } 461ab8250d7SJiri Pirko 462ab8250d7SJiri Pirko static void lb_stats_refresh(struct work_struct *work) 463ab8250d7SJiri Pirko { 464ab8250d7SJiri Pirko struct team *team; 465ab8250d7SJiri Pirko struct lb_priv *lb_priv; 466ab8250d7SJiri Pirko struct lb_priv_ex *lb_priv_ex; 467ab8250d7SJiri Pirko struct lb_pcpu_stats *pcpu_stats; 468ab8250d7SJiri Pirko struct lb_stats *stats; 469ab8250d7SJiri Pirko struct lb_stats_info *s_info; 470ab8250d7SJiri Pirko struct team_port *port; 471ab8250d7SJiri Pirko bool changed = false; 472ab8250d7SJiri Pirko int i; 473ab8250d7SJiri Pirko int j; 474ab8250d7SJiri Pirko 475ab8250d7SJiri Pirko lb_priv_ex = container_of(work, struct lb_priv_ex, 476ab8250d7SJiri Pirko stats.refresh_dw.work); 477ab8250d7SJiri Pirko 478ab8250d7SJiri Pirko team = lb_priv_ex->team; 479ab8250d7SJiri Pirko lb_priv = get_lb_priv(team); 480ab8250d7SJiri Pirko 481ab8250d7SJiri Pirko if (!mutex_trylock(&team->lock)) { 482ab8250d7SJiri Pirko schedule_delayed_work(&lb_priv_ex->stats.refresh_dw, 0); 483ab8250d7SJiri Pirko return; 484ab8250d7SJiri Pirko } 485ab8250d7SJiri Pirko 486ab8250d7SJiri Pirko for (j = 0; j < LB_TX_HASHTABLE_SIZE; j++) { 487ab8250d7SJiri Pirko s_info = &lb_priv->ex->stats.info[j]; 488ab8250d7SJiri Pirko __lb_stats_info_refresh_prepare(s_info); 489ab8250d7SJiri Pirko for_each_possible_cpu(i) { 490ab8250d7SJiri Pirko pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); 491ab8250d7SJiri Pirko stats = &pcpu_stats->hash_stats[j]; 492ab8250d7SJiri Pirko __lb_one_cpu_stats_add(&s_info->stats, stats, 493ab8250d7SJiri Pirko &pcpu_stats->syncp); 494ab8250d7SJiri Pirko } 495ab8250d7SJiri Pirko changed |= __lb_stats_info_refresh_check(s_info, team); 496ab8250d7SJiri Pirko } 497ab8250d7SJiri Pirko 498ab8250d7SJiri Pirko list_for_each_entry(port, &team->port_list, list) { 499ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 500ab8250d7SJiri Pirko 501ab8250d7SJiri Pirko s_info = &lb_port_priv->stats_info; 502ab8250d7SJiri Pirko __lb_stats_info_refresh_prepare(s_info); 503ab8250d7SJiri Pirko for_each_possible_cpu(i) { 504ab8250d7SJiri Pirko pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); 505ab8250d7SJiri Pirko stats = per_cpu_ptr(lb_port_priv->pcpu_stats, i); 506ab8250d7SJiri Pirko __lb_one_cpu_stats_add(&s_info->stats, stats, 507ab8250d7SJiri Pirko &pcpu_stats->syncp); 508ab8250d7SJiri Pirko } 509ab8250d7SJiri Pirko changed |= __lb_stats_info_refresh_check(s_info, team); 510ab8250d7SJiri Pirko } 511ab8250d7SJiri Pirko 512ab8250d7SJiri Pirko if (changed) 513ab8250d7SJiri Pirko team_options_change_check(team); 514ab8250d7SJiri Pirko 515ab8250d7SJiri Pirko schedule_delayed_work(&lb_priv_ex->stats.refresh_dw, 516ab8250d7SJiri Pirko (lb_priv_ex->stats.refresh_interval * HZ) / 10); 517ab8250d7SJiri Pirko 518ab8250d7SJiri Pirko mutex_unlock(&team->lock); 519ab8250d7SJiri Pirko } 520ab8250d7SJiri Pirko 521c3b41f4cSZhengchao Shao static void lb_stats_refresh_interval_get(struct team *team, 522ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 523ab8250d7SJiri Pirko { 524ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 525ab8250d7SJiri Pirko 526ab8250d7SJiri Pirko ctx->data.u32_val = lb_priv->ex->stats.refresh_interval; 527ab8250d7SJiri Pirko } 528ab8250d7SJiri Pirko 529ab8250d7SJiri Pirko static int lb_stats_refresh_interval_set(struct team *team, 530ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 531ab8250d7SJiri Pirko { 532ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 533ab8250d7SJiri Pirko unsigned int interval; 534ab8250d7SJiri Pirko 535ab8250d7SJiri Pirko interval = ctx->data.u32_val; 536ab8250d7SJiri Pirko if (lb_priv->ex->stats.refresh_interval == interval) 537ab8250d7SJiri Pirko return 0; 538ab8250d7SJiri Pirko lb_priv->ex->stats.refresh_interval = interval; 539ab8250d7SJiri Pirko if (interval) 540ab8250d7SJiri Pirko schedule_delayed_work(&lb_priv->ex->stats.refresh_dw, 0); 541ab8250d7SJiri Pirko else 542ab8250d7SJiri Pirko cancel_delayed_work(&lb_priv->ex->stats.refresh_dw); 54301d7f30aSJiri Pirko return 0; 54401d7f30aSJiri Pirko } 54501d7f30aSJiri Pirko 54601d7f30aSJiri Pirko static const struct team_option lb_options[] = { 54701d7f30aSJiri Pirko { 54801d7f30aSJiri Pirko .name = "bpf_hash_func", 54901d7f30aSJiri Pirko .type = TEAM_OPTION_TYPE_BINARY, 55001d7f30aSJiri Pirko .getter = lb_bpf_func_get, 55101d7f30aSJiri Pirko .setter = lb_bpf_func_set, 55201d7f30aSJiri Pirko }, 553ab8250d7SJiri Pirko { 554ab8250d7SJiri Pirko .name = "lb_tx_method", 555ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_STRING, 556ab8250d7SJiri Pirko .getter = lb_tx_method_get, 557ab8250d7SJiri Pirko .setter = lb_tx_method_set, 558ab8250d7SJiri Pirko }, 559ab8250d7SJiri Pirko { 560ab8250d7SJiri Pirko .name = "lb_tx_hash_to_port_mapping", 561ab8250d7SJiri Pirko .array_size = LB_TX_HASHTABLE_SIZE, 562ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_U32, 563ab8250d7SJiri Pirko .init = lb_tx_hash_to_port_mapping_init, 564ab8250d7SJiri Pirko .getter = lb_tx_hash_to_port_mapping_get, 565ab8250d7SJiri Pirko .setter = lb_tx_hash_to_port_mapping_set, 566ab8250d7SJiri Pirko }, 567ab8250d7SJiri Pirko { 568ab8250d7SJiri Pirko .name = "lb_hash_stats", 569ab8250d7SJiri Pirko .array_size = LB_TX_HASHTABLE_SIZE, 570ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_BINARY, 571ab8250d7SJiri Pirko .init = lb_hash_stats_init, 572ab8250d7SJiri Pirko .getter = lb_hash_stats_get, 573ab8250d7SJiri Pirko }, 574ab8250d7SJiri Pirko { 575ab8250d7SJiri Pirko .name = "lb_port_stats", 576ab8250d7SJiri Pirko .per_port = true, 577ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_BINARY, 578ab8250d7SJiri Pirko .init = lb_port_stats_init, 579ab8250d7SJiri Pirko .getter = lb_port_stats_get, 580ab8250d7SJiri Pirko }, 581ab8250d7SJiri Pirko { 582ab8250d7SJiri Pirko .name = "lb_stats_refresh_interval", 583ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_U32, 584ab8250d7SJiri Pirko .getter = lb_stats_refresh_interval_get, 585ab8250d7SJiri Pirko .setter = lb_stats_refresh_interval_set, 586ab8250d7SJiri Pirko }, 58701d7f30aSJiri Pirko }; 58801d7f30aSJiri Pirko 589cade4555SJiri Pirko static int lb_init(struct team *team) 59001d7f30aSJiri Pirko { 591ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 592ab8250d7SJiri Pirko lb_select_tx_port_func_t *func; 593827da44cSJohn Stultz int i, err; 594ab8250d7SJiri Pirko 595ab8250d7SJiri Pirko /* set default tx port selector */ 596ab8250d7SJiri Pirko func = lb_select_tx_port_get_func("hash"); 597ab8250d7SJiri Pirko BUG_ON(!func); 598ab8250d7SJiri Pirko rcu_assign_pointer(lb_priv->select_tx_port_func, func); 599ab8250d7SJiri Pirko 600ab8250d7SJiri Pirko lb_priv->ex = kzalloc(sizeof(*lb_priv->ex), GFP_KERNEL); 601ab8250d7SJiri Pirko if (!lb_priv->ex) 602ab8250d7SJiri Pirko return -ENOMEM; 603ab8250d7SJiri Pirko lb_priv->ex->team = team; 604ab8250d7SJiri Pirko 605ab8250d7SJiri Pirko lb_priv->pcpu_stats = alloc_percpu(struct lb_pcpu_stats); 606ab8250d7SJiri Pirko if (!lb_priv->pcpu_stats) { 607ab8250d7SJiri Pirko err = -ENOMEM; 608ab8250d7SJiri Pirko goto err_alloc_pcpu_stats; 609ab8250d7SJiri Pirko } 610ab8250d7SJiri Pirko 611827da44cSJohn Stultz for_each_possible_cpu(i) { 612827da44cSJohn Stultz struct lb_pcpu_stats *team_lb_stats; 613827da44cSJohn Stultz team_lb_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); 614827da44cSJohn Stultz u64_stats_init(&team_lb_stats->syncp); 615827da44cSJohn Stultz } 616827da44cSJohn Stultz 617827da44cSJohn Stultz 618ab8250d7SJiri Pirko INIT_DELAYED_WORK(&lb_priv->ex->stats.refresh_dw, lb_stats_refresh); 619ab8250d7SJiri Pirko 620ab8250d7SJiri Pirko err = team_options_register(team, lb_options, ARRAY_SIZE(lb_options)); 621ab8250d7SJiri Pirko if (err) 622ab8250d7SJiri Pirko goto err_options_register; 623ab8250d7SJiri Pirko return 0; 624ab8250d7SJiri Pirko 625ab8250d7SJiri Pirko err_options_register: 626ab8250d7SJiri Pirko free_percpu(lb_priv->pcpu_stats); 627ab8250d7SJiri Pirko err_alloc_pcpu_stats: 628ab8250d7SJiri Pirko kfree(lb_priv->ex); 629ab8250d7SJiri Pirko return err; 63001d7f30aSJiri Pirko } 63101d7f30aSJiri Pirko 632cade4555SJiri Pirko static void lb_exit(struct team *team) 63301d7f30aSJiri Pirko { 634ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 635ab8250d7SJiri Pirko 63601d7f30aSJiri Pirko team_options_unregister(team, lb_options, 63701d7f30aSJiri Pirko ARRAY_SIZE(lb_options)); 638692c31bdSIdo Schimmel lb_bpf_func_free(team); 639ab8250d7SJiri Pirko cancel_delayed_work_sync(&lb_priv->ex->stats.refresh_dw); 640ab8250d7SJiri Pirko free_percpu(lb_priv->pcpu_stats); 641ab8250d7SJiri Pirko kfree(lb_priv->ex); 642ab8250d7SJiri Pirko } 643ab8250d7SJiri Pirko 644ab8250d7SJiri Pirko static int lb_port_enter(struct team *team, struct team_port *port) 645ab8250d7SJiri Pirko { 646ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 647ab8250d7SJiri Pirko 648ab8250d7SJiri Pirko lb_port_priv->pcpu_stats = alloc_percpu(struct lb_stats); 649ab8250d7SJiri Pirko if (!lb_port_priv->pcpu_stats) 650ab8250d7SJiri Pirko return -ENOMEM; 651ab8250d7SJiri Pirko return 0; 652ab8250d7SJiri Pirko } 653ab8250d7SJiri Pirko 654ab8250d7SJiri Pirko static void lb_port_leave(struct team *team, struct team_port *port) 655ab8250d7SJiri Pirko { 656ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 657ab8250d7SJiri Pirko 658ab8250d7SJiri Pirko free_percpu(lb_port_priv->pcpu_stats); 659ab8250d7SJiri Pirko } 660ab8250d7SJiri Pirko 661ab8250d7SJiri Pirko static void lb_port_disabled(struct team *team, struct team_port *port) 662ab8250d7SJiri Pirko { 663ab8250d7SJiri Pirko lb_tx_hash_to_port_mapping_null_port(team, port); 66401d7f30aSJiri Pirko } 66501d7f30aSJiri Pirko 66601d7f30aSJiri Pirko static const struct team_mode_ops lb_mode_ops = { 66701d7f30aSJiri Pirko .init = lb_init, 66801d7f30aSJiri Pirko .exit = lb_exit, 669ab8250d7SJiri Pirko .port_enter = lb_port_enter, 670ab8250d7SJiri Pirko .port_leave = lb_port_leave, 671ab8250d7SJiri Pirko .port_disabled = lb_port_disabled, 672c15e07b0SJiri Pirko .receive = lb_receive, 67301d7f30aSJiri Pirko .transmit = lb_transmit, 67401d7f30aSJiri Pirko }; 67501d7f30aSJiri Pirko 6760402788aSJiri Pirko static const struct team_mode lb_mode = { 67701d7f30aSJiri Pirko .kind = "loadbalance", 67801d7f30aSJiri Pirko .owner = THIS_MODULE, 67901d7f30aSJiri Pirko .priv_size = sizeof(struct lb_priv), 680ab8250d7SJiri Pirko .port_priv_size = sizeof(struct lb_port_priv), 68101d7f30aSJiri Pirko .ops = &lb_mode_ops, 6828fd72856SJiri Pirko .lag_tx_type = NETDEV_LAG_TX_TYPE_HASH, 68301d7f30aSJiri Pirko }; 68401d7f30aSJiri Pirko 68501d7f30aSJiri Pirko static int __init lb_init_module(void) 68601d7f30aSJiri Pirko { 68701d7f30aSJiri Pirko return team_mode_register(&lb_mode); 68801d7f30aSJiri Pirko } 68901d7f30aSJiri Pirko 69001d7f30aSJiri Pirko static void __exit lb_cleanup_module(void) 69101d7f30aSJiri Pirko { 69201d7f30aSJiri Pirko team_mode_unregister(&lb_mode); 69301d7f30aSJiri Pirko } 69401d7f30aSJiri Pirko 69501d7f30aSJiri Pirko module_init(lb_init_module); 69601d7f30aSJiri Pirko module_exit(lb_cleanup_module); 69701d7f30aSJiri Pirko 69801d7f30aSJiri Pirko MODULE_LICENSE("GPL v2"); 69901d7f30aSJiri Pirko MODULE_AUTHOR("Jiri Pirko <jpirko@redhat.com>"); 70001d7f30aSJiri Pirko MODULE_DESCRIPTION("Load-balancing mode for team"); 7013a5f8997SZhang Shengju MODULE_ALIAS_TEAM_MODE("loadbalance"); 702