12874c5fdSThomas Gleixner // SPDX-License-Identifier: GPL-2.0-or-later 201d7f30aSJiri Pirko /* 301d7f30aSJiri Pirko * drivers/net/team/team_mode_loadbalance.c - Load-balancing mode for team 401d7f30aSJiri Pirko * Copyright (c) 2012 Jiri Pirko <jpirko@redhat.com> 501d7f30aSJiri Pirko */ 601d7f30aSJiri Pirko 701d7f30aSJiri Pirko #include <linux/kernel.h> 801d7f30aSJiri Pirko #include <linux/types.h> 901d7f30aSJiri Pirko #include <linux/module.h> 1001d7f30aSJiri Pirko #include <linux/init.h> 1101d7f30aSJiri Pirko #include <linux/errno.h> 1201d7f30aSJiri Pirko #include <linux/netdevice.h> 13c15e07b0SJiri Pirko #include <linux/etherdevice.h> 1401d7f30aSJiri Pirko #include <linux/filter.h> 1501d7f30aSJiri Pirko #include <linux/if_team.h> 1601d7f30aSJiri Pirko 17c15e07b0SJiri Pirko static rx_handler_result_t lb_receive(struct team *team, struct team_port *port, 18c15e07b0SJiri Pirko struct sk_buff *skb) 19c15e07b0SJiri Pirko { 20c15e07b0SJiri Pirko if (unlikely(skb->protocol == htons(ETH_P_SLOW))) { 21c15e07b0SJiri Pirko /* LACPDU packets should go to exact delivery */ 22c15e07b0SJiri Pirko const unsigned char *dest = eth_hdr(skb)->h_dest; 23c15e07b0SJiri Pirko 24c15e07b0SJiri Pirko if (is_link_local_ether_addr(dest) && dest[5] == 0x02) 25c15e07b0SJiri Pirko return RX_HANDLER_EXACT; 26c15e07b0SJiri Pirko } 27c15e07b0SJiri Pirko return RX_HANDLER_ANOTHER; 28c15e07b0SJiri Pirko } 29c15e07b0SJiri Pirko 30ab8250d7SJiri Pirko struct lb_priv; 31ab8250d7SJiri Pirko 32ab8250d7SJiri Pirko typedef struct team_port *lb_select_tx_port_func_t(struct team *, 33ab8250d7SJiri Pirko struct lb_priv *, 34ab8250d7SJiri Pirko struct sk_buff *, 35ab8250d7SJiri Pirko unsigned char); 36ab8250d7SJiri Pirko 37ab8250d7SJiri Pirko #define LB_TX_HASHTABLE_SIZE 256 /* hash is a char */ 38ab8250d7SJiri Pirko 39ab8250d7SJiri Pirko struct lb_stats { 40ab8250d7SJiri Pirko u64 tx_bytes; 4101d7f30aSJiri Pirko }; 4201d7f30aSJiri Pirko 43ab8250d7SJiri Pirko struct lb_pcpu_stats { 44ab8250d7SJiri Pirko struct lb_stats hash_stats[LB_TX_HASHTABLE_SIZE]; 45ab8250d7SJiri Pirko struct u64_stats_sync syncp; 46ab8250d7SJiri Pirko }; 47ab8250d7SJiri Pirko 48ab8250d7SJiri Pirko struct lb_stats_info { 49ab8250d7SJiri Pirko struct lb_stats stats; 50ab8250d7SJiri Pirko struct lb_stats last_stats; 51ab8250d7SJiri Pirko struct team_option_inst_info *opt_inst_info; 52ab8250d7SJiri Pirko }; 53ab8250d7SJiri Pirko 54ab8250d7SJiri Pirko struct lb_port_mapping { 55ab8250d7SJiri Pirko struct team_port __rcu *port; 56ab8250d7SJiri Pirko struct team_option_inst_info *opt_inst_info; 57ab8250d7SJiri Pirko }; 58ab8250d7SJiri Pirko 59ab8250d7SJiri Pirko struct lb_priv_ex { 60ab8250d7SJiri Pirko struct team *team; 61ab8250d7SJiri Pirko struct lb_port_mapping tx_hash_to_port_mapping[LB_TX_HASHTABLE_SIZE]; 62b1fcd35cSDaniel Borkmann struct sock_fprog_kern *orig_fprog; 63ab8250d7SJiri Pirko struct { 64ab8250d7SJiri Pirko unsigned int refresh_interval; /* in tenths of second */ 65ab8250d7SJiri Pirko struct delayed_work refresh_dw; 66ab8250d7SJiri Pirko struct lb_stats_info info[LB_TX_HASHTABLE_SIZE]; 67ab8250d7SJiri Pirko } stats; 68ab8250d7SJiri Pirko }; 69ab8250d7SJiri Pirko 70ab8250d7SJiri Pirko struct lb_priv { 717ae457c1SAlexei Starovoitov struct bpf_prog __rcu *fp; 72ab8250d7SJiri Pirko lb_select_tx_port_func_t __rcu *select_tx_port_func; 73ab8250d7SJiri Pirko struct lb_pcpu_stats __percpu *pcpu_stats; 74ab8250d7SJiri Pirko struct lb_priv_ex *ex; /* priv extension */ 75ab8250d7SJiri Pirko }; 76ab8250d7SJiri Pirko 77ab8250d7SJiri Pirko static struct lb_priv *get_lb_priv(struct team *team) 7801d7f30aSJiri Pirko { 7901d7f30aSJiri Pirko return (struct lb_priv *) &team->mode_priv; 8001d7f30aSJiri Pirko } 8101d7f30aSJiri Pirko 82ab8250d7SJiri Pirko struct lb_port_priv { 83ab8250d7SJiri Pirko struct lb_stats __percpu *pcpu_stats; 84ab8250d7SJiri Pirko struct lb_stats_info stats_info; 85ab8250d7SJiri Pirko }; 86ab8250d7SJiri Pirko 87ab8250d7SJiri Pirko static struct lb_port_priv *get_lb_port_priv(struct team_port *port) 88ab8250d7SJiri Pirko { 89ab8250d7SJiri Pirko return (struct lb_port_priv *) &port->mode_priv; 90ab8250d7SJiri Pirko } 91ab8250d7SJiri Pirko 92ab8250d7SJiri Pirko #define LB_HTPM_PORT_BY_HASH(lp_priv, hash) \ 93ab8250d7SJiri Pirko (lb_priv)->ex->tx_hash_to_port_mapping[hash].port 94ab8250d7SJiri Pirko 95ab8250d7SJiri Pirko #define LB_HTPM_OPT_INST_INFO_BY_HASH(lp_priv, hash) \ 96ab8250d7SJiri Pirko (lb_priv)->ex->tx_hash_to_port_mapping[hash].opt_inst_info 97ab8250d7SJiri Pirko 98ab8250d7SJiri Pirko static void lb_tx_hash_to_port_mapping_null_port(struct team *team, 99ab8250d7SJiri Pirko struct team_port *port) 100ab8250d7SJiri Pirko { 101ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 102ab8250d7SJiri Pirko bool changed = false; 103ab8250d7SJiri Pirko int i; 104ab8250d7SJiri Pirko 105ab8250d7SJiri Pirko for (i = 0; i < LB_TX_HASHTABLE_SIZE; i++) { 106ab8250d7SJiri Pirko struct lb_port_mapping *pm; 107ab8250d7SJiri Pirko 108ab8250d7SJiri Pirko pm = &lb_priv->ex->tx_hash_to_port_mapping[i]; 1096dab015cSJiri Pirko if (rcu_access_pointer(pm->port) == port) { 1106dab015cSJiri Pirko RCU_INIT_POINTER(pm->port, NULL); 111ab8250d7SJiri Pirko team_option_inst_set_change(pm->opt_inst_info); 112ab8250d7SJiri Pirko changed = true; 113ab8250d7SJiri Pirko } 114ab8250d7SJiri Pirko } 115ab8250d7SJiri Pirko if (changed) 116ab8250d7SJiri Pirko team_options_change_check(team); 117ab8250d7SJiri Pirko } 118ab8250d7SJiri Pirko 119ab8250d7SJiri Pirko /* Basic tx selection based solely by hash */ 120ab8250d7SJiri Pirko static struct team_port *lb_hash_select_tx_port(struct team *team, 121ab8250d7SJiri Pirko struct lb_priv *lb_priv, 122ab8250d7SJiri Pirko struct sk_buff *skb, 123ab8250d7SJiri Pirko unsigned char hash) 124ab8250d7SJiri Pirko { 125735d381fSJiri Pirko int port_index = team_num_to_port_index(team, hash); 126ab8250d7SJiri Pirko 127ab8250d7SJiri Pirko return team_get_port_by_index_rcu(team, port_index); 128ab8250d7SJiri Pirko } 129ab8250d7SJiri Pirko 130ab8250d7SJiri Pirko /* Hash to port mapping select tx port */ 131ab8250d7SJiri Pirko static struct team_port *lb_htpm_select_tx_port(struct team *team, 132ab8250d7SJiri Pirko struct lb_priv *lb_priv, 133ab8250d7SJiri Pirko struct sk_buff *skb, 134ab8250d7SJiri Pirko unsigned char hash) 135ab8250d7SJiri Pirko { 136bd7d2106SJim Hanko struct team_port *port; 137bd7d2106SJim Hanko 138bd7d2106SJim Hanko port = rcu_dereference_bh(LB_HTPM_PORT_BY_HASH(lb_priv, hash)); 139bd7d2106SJim Hanko if (likely(port)) 140bd7d2106SJim Hanko return port; 141bd7d2106SJim Hanko /* If no valid port in the table, fall back to simple hash */ 142bd7d2106SJim Hanko return lb_hash_select_tx_port(team, lb_priv, skb, hash); 143ab8250d7SJiri Pirko } 144ab8250d7SJiri Pirko 145ab8250d7SJiri Pirko struct lb_select_tx_port { 146ab8250d7SJiri Pirko char *name; 147ab8250d7SJiri Pirko lb_select_tx_port_func_t *func; 148ab8250d7SJiri Pirko }; 149ab8250d7SJiri Pirko 150ab8250d7SJiri Pirko static const struct lb_select_tx_port lb_select_tx_port_list[] = { 151ab8250d7SJiri Pirko { 152ab8250d7SJiri Pirko .name = "hash", 153ab8250d7SJiri Pirko .func = lb_hash_select_tx_port, 154ab8250d7SJiri Pirko }, 155ab8250d7SJiri Pirko { 156ab8250d7SJiri Pirko .name = "hash_to_port_mapping", 157ab8250d7SJiri Pirko .func = lb_htpm_select_tx_port, 158ab8250d7SJiri Pirko }, 159ab8250d7SJiri Pirko }; 160ab8250d7SJiri Pirko #define LB_SELECT_TX_PORT_LIST_COUNT ARRAY_SIZE(lb_select_tx_port_list) 161ab8250d7SJiri Pirko 162ab8250d7SJiri Pirko static char *lb_select_tx_port_get_name(lb_select_tx_port_func_t *func) 163ab8250d7SJiri Pirko { 164ab8250d7SJiri Pirko int i; 165ab8250d7SJiri Pirko 166ab8250d7SJiri Pirko for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) { 167ab8250d7SJiri Pirko const struct lb_select_tx_port *item; 168ab8250d7SJiri Pirko 169ab8250d7SJiri Pirko item = &lb_select_tx_port_list[i]; 170ab8250d7SJiri Pirko if (item->func == func) 171ab8250d7SJiri Pirko return item->name; 172ab8250d7SJiri Pirko } 173ab8250d7SJiri Pirko return NULL; 174ab8250d7SJiri Pirko } 175ab8250d7SJiri Pirko 176ab8250d7SJiri Pirko static lb_select_tx_port_func_t *lb_select_tx_port_get_func(const char *name) 177ab8250d7SJiri Pirko { 178ab8250d7SJiri Pirko int i; 179ab8250d7SJiri Pirko 180ab8250d7SJiri Pirko for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) { 181ab8250d7SJiri Pirko const struct lb_select_tx_port *item; 182ab8250d7SJiri Pirko 183ab8250d7SJiri Pirko item = &lb_select_tx_port_list[i]; 184ab8250d7SJiri Pirko if (!strcmp(item->name, name)) 185ab8250d7SJiri Pirko return item->func; 186ab8250d7SJiri Pirko } 187ab8250d7SJiri Pirko return NULL; 188ab8250d7SJiri Pirko } 189ab8250d7SJiri Pirko 190ab8250d7SJiri Pirko static unsigned int lb_get_skb_hash(struct lb_priv *lb_priv, 191596e2024SJiri Pirko struct sk_buff *skb) 19201d7f30aSJiri Pirko { 1937ae457c1SAlexei Starovoitov struct bpf_prog *fp; 194596e2024SJiri Pirko uint32_t lhash; 195596e2024SJiri Pirko unsigned char *c; 196596e2024SJiri Pirko 197d1904fbdSJiri Pirko fp = rcu_dereference_bh(lb_priv->fp); 198596e2024SJiri Pirko if (unlikely(!fp)) 199596e2024SJiri Pirko return 0; 2007ae457c1SAlexei Starovoitov lhash = BPF_PROG_RUN(fp, skb); 201596e2024SJiri Pirko c = (char *) &lhash; 202596e2024SJiri Pirko return c[0] ^ c[1] ^ c[2] ^ c[3]; 203596e2024SJiri Pirko } 204596e2024SJiri Pirko 205ab8250d7SJiri Pirko static void lb_update_tx_stats(unsigned int tx_bytes, struct lb_priv *lb_priv, 206ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv, 207ab8250d7SJiri Pirko unsigned char hash) 208ab8250d7SJiri Pirko { 209ab8250d7SJiri Pirko struct lb_pcpu_stats *pcpu_stats; 210ab8250d7SJiri Pirko struct lb_stats *port_stats; 211ab8250d7SJiri Pirko struct lb_stats *hash_stats; 212ab8250d7SJiri Pirko 213ab8250d7SJiri Pirko pcpu_stats = this_cpu_ptr(lb_priv->pcpu_stats); 214ab8250d7SJiri Pirko port_stats = this_cpu_ptr(lb_port_priv->pcpu_stats); 215ab8250d7SJiri Pirko hash_stats = &pcpu_stats->hash_stats[hash]; 216ab8250d7SJiri Pirko u64_stats_update_begin(&pcpu_stats->syncp); 217ab8250d7SJiri Pirko port_stats->tx_bytes += tx_bytes; 218ab8250d7SJiri Pirko hash_stats->tx_bytes += tx_bytes; 219ab8250d7SJiri Pirko u64_stats_update_end(&pcpu_stats->syncp); 220ab8250d7SJiri Pirko } 221ab8250d7SJiri Pirko 222596e2024SJiri Pirko static bool lb_transmit(struct team *team, struct sk_buff *skb) 223596e2024SJiri Pirko { 224ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 225ab8250d7SJiri Pirko lb_select_tx_port_func_t *select_tx_port_func; 22601d7f30aSJiri Pirko struct team_port *port; 227ab8250d7SJiri Pirko unsigned char hash; 228ab8250d7SJiri Pirko unsigned int tx_bytes = skb->len; 22901d7f30aSJiri Pirko 230ab8250d7SJiri Pirko hash = lb_get_skb_hash(lb_priv, skb); 231d1904fbdSJiri Pirko select_tx_port_func = rcu_dereference_bh(lb_priv->select_tx_port_func); 232ab8250d7SJiri Pirko port = select_tx_port_func(team, lb_priv, skb, hash); 23301d7f30aSJiri Pirko if (unlikely(!port)) 23401d7f30aSJiri Pirko goto drop; 235bd2d0837SJiri Pirko if (team_dev_queue_xmit(team, port, skb)) 23601d7f30aSJiri Pirko return false; 237ab8250d7SJiri Pirko lb_update_tx_stats(tx_bytes, lb_priv, get_lb_port_priv(port), hash); 23801d7f30aSJiri Pirko return true; 23901d7f30aSJiri Pirko 24001d7f30aSJiri Pirko drop: 24101d7f30aSJiri Pirko dev_kfree_skb_any(skb); 24201d7f30aSJiri Pirko return false; 24301d7f30aSJiri Pirko } 24401d7f30aSJiri Pirko 24580f7c668SJiri Pirko static int lb_bpf_func_get(struct team *team, struct team_gsetter_ctx *ctx) 24601d7f30aSJiri Pirko { 247ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 248ab8250d7SJiri Pirko 249ab8250d7SJiri Pirko if (!lb_priv->ex->orig_fprog) { 25080f7c668SJiri Pirko ctx->data.bin_val.len = 0; 25180f7c668SJiri Pirko ctx->data.bin_val.ptr = NULL; 25201d7f30aSJiri Pirko return 0; 25380f7c668SJiri Pirko } 254ab8250d7SJiri Pirko ctx->data.bin_val.len = lb_priv->ex->orig_fprog->len * 25501d7f30aSJiri Pirko sizeof(struct sock_filter); 256ab8250d7SJiri Pirko ctx->data.bin_val.ptr = lb_priv->ex->orig_fprog->filter; 25701d7f30aSJiri Pirko return 0; 25801d7f30aSJiri Pirko } 25901d7f30aSJiri Pirko 260b1fcd35cSDaniel Borkmann static int __fprog_create(struct sock_fprog_kern **pfprog, u32 data_len, 26180f7c668SJiri Pirko const void *data) 26201d7f30aSJiri Pirko { 263b1fcd35cSDaniel Borkmann struct sock_fprog_kern *fprog; 26401d7f30aSJiri Pirko struct sock_filter *filter = (struct sock_filter *) data; 26501d7f30aSJiri Pirko 26601d7f30aSJiri Pirko if (data_len % sizeof(struct sock_filter)) 26701d7f30aSJiri Pirko return -EINVAL; 268ea5930f4SDaniel Borkmann fprog = kmalloc(sizeof(*fprog), GFP_KERNEL); 26901d7f30aSJiri Pirko if (!fprog) 27001d7f30aSJiri Pirko return -ENOMEM; 27101d7f30aSJiri Pirko fprog->filter = kmemdup(filter, data_len, GFP_KERNEL); 27201d7f30aSJiri Pirko if (!fprog->filter) { 27301d7f30aSJiri Pirko kfree(fprog); 27401d7f30aSJiri Pirko return -ENOMEM; 27501d7f30aSJiri Pirko } 27601d7f30aSJiri Pirko fprog->len = data_len / sizeof(struct sock_filter); 27701d7f30aSJiri Pirko *pfprog = fprog; 27801d7f30aSJiri Pirko return 0; 27901d7f30aSJiri Pirko } 28001d7f30aSJiri Pirko 281b1fcd35cSDaniel Borkmann static void __fprog_destroy(struct sock_fprog_kern *fprog) 28201d7f30aSJiri Pirko { 28301d7f30aSJiri Pirko kfree(fprog->filter); 28401d7f30aSJiri Pirko kfree(fprog); 28501d7f30aSJiri Pirko } 28601d7f30aSJiri Pirko 28780f7c668SJiri Pirko static int lb_bpf_func_set(struct team *team, struct team_gsetter_ctx *ctx) 28801d7f30aSJiri Pirko { 289ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 2907ae457c1SAlexei Starovoitov struct bpf_prog *fp = NULL; 2917ae457c1SAlexei Starovoitov struct bpf_prog *orig_fp = NULL; 292b1fcd35cSDaniel Borkmann struct sock_fprog_kern *fprog = NULL; 29301d7f30aSJiri Pirko int err; 29401d7f30aSJiri Pirko 29580f7c668SJiri Pirko if (ctx->data.bin_val.len) { 29680f7c668SJiri Pirko err = __fprog_create(&fprog, ctx->data.bin_val.len, 29780f7c668SJiri Pirko ctx->data.bin_val.ptr); 29801d7f30aSJiri Pirko if (err) 29901d7f30aSJiri Pirko return err; 3007ae457c1SAlexei Starovoitov err = bpf_prog_create(&fp, fprog); 30101d7f30aSJiri Pirko if (err) { 30201d7f30aSJiri Pirko __fprog_destroy(fprog); 30301d7f30aSJiri Pirko return err; 30401d7f30aSJiri Pirko } 30501d7f30aSJiri Pirko } 30601d7f30aSJiri Pirko 307ab8250d7SJiri Pirko if (lb_priv->ex->orig_fprog) { 30801d7f30aSJiri Pirko /* Clear old filter data */ 309ab8250d7SJiri Pirko __fprog_destroy(lb_priv->ex->orig_fprog); 3106dab015cSJiri Pirko orig_fp = rcu_dereference_protected(lb_priv->fp, 3116dab015cSJiri Pirko lockdep_is_held(&team->lock)); 31201d7f30aSJiri Pirko } 31301d7f30aSJiri Pirko 314ab8250d7SJiri Pirko rcu_assign_pointer(lb_priv->fp, fp); 315ab8250d7SJiri Pirko lb_priv->ex->orig_fprog = fprog; 31634c5bd66SPablo Neira 31734c5bd66SPablo Neira if (orig_fp) { 31834c5bd66SPablo Neira synchronize_rcu(); 3197ae457c1SAlexei Starovoitov bpf_prog_destroy(orig_fp); 32034c5bd66SPablo Neira } 321ab8250d7SJiri Pirko return 0; 322ab8250d7SJiri Pirko } 323ab8250d7SJiri Pirko 324692c31bdSIdo Schimmel static void lb_bpf_func_free(struct team *team) 325692c31bdSIdo Schimmel { 326692c31bdSIdo Schimmel struct lb_priv *lb_priv = get_lb_priv(team); 327692c31bdSIdo Schimmel struct bpf_prog *fp; 328692c31bdSIdo Schimmel 329692c31bdSIdo Schimmel if (!lb_priv->ex->orig_fprog) 330692c31bdSIdo Schimmel return; 331692c31bdSIdo Schimmel 332692c31bdSIdo Schimmel __fprog_destroy(lb_priv->ex->orig_fprog); 333692c31bdSIdo Schimmel fp = rcu_dereference_protected(lb_priv->fp, 334692c31bdSIdo Schimmel lockdep_is_held(&team->lock)); 335692c31bdSIdo Schimmel bpf_prog_destroy(fp); 336692c31bdSIdo Schimmel } 337692c31bdSIdo Schimmel 338ab8250d7SJiri Pirko static int lb_tx_method_get(struct team *team, struct team_gsetter_ctx *ctx) 339ab8250d7SJiri Pirko { 340ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 3416dab015cSJiri Pirko lb_select_tx_port_func_t *func; 342ab8250d7SJiri Pirko char *name; 343ab8250d7SJiri Pirko 3446dab015cSJiri Pirko func = rcu_dereference_protected(lb_priv->select_tx_port_func, 3456dab015cSJiri Pirko lockdep_is_held(&team->lock)); 3466dab015cSJiri Pirko name = lb_select_tx_port_get_name(func); 347ab8250d7SJiri Pirko BUG_ON(!name); 348ab8250d7SJiri Pirko ctx->data.str_val = name; 349ab8250d7SJiri Pirko return 0; 350ab8250d7SJiri Pirko } 351ab8250d7SJiri Pirko 352ab8250d7SJiri Pirko static int lb_tx_method_set(struct team *team, struct team_gsetter_ctx *ctx) 353ab8250d7SJiri Pirko { 354ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 355ab8250d7SJiri Pirko lb_select_tx_port_func_t *func; 356ab8250d7SJiri Pirko 357ab8250d7SJiri Pirko func = lb_select_tx_port_get_func(ctx->data.str_val); 358ab8250d7SJiri Pirko if (!func) 359ab8250d7SJiri Pirko return -EINVAL; 360ab8250d7SJiri Pirko rcu_assign_pointer(lb_priv->select_tx_port_func, func); 361ab8250d7SJiri Pirko return 0; 362ab8250d7SJiri Pirko } 363ab8250d7SJiri Pirko 364ab8250d7SJiri Pirko static int lb_tx_hash_to_port_mapping_init(struct team *team, 365ab8250d7SJiri Pirko struct team_option_inst_info *info) 366ab8250d7SJiri Pirko { 367ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 368ab8250d7SJiri Pirko unsigned char hash = info->array_index; 369ab8250d7SJiri Pirko 370ab8250d7SJiri Pirko LB_HTPM_OPT_INST_INFO_BY_HASH(lb_priv, hash) = info; 371ab8250d7SJiri Pirko return 0; 372ab8250d7SJiri Pirko } 373ab8250d7SJiri Pirko 374ab8250d7SJiri Pirko static int lb_tx_hash_to_port_mapping_get(struct team *team, 375ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 376ab8250d7SJiri Pirko { 377ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 378ab8250d7SJiri Pirko struct team_port *port; 379ab8250d7SJiri Pirko unsigned char hash = ctx->info->array_index; 380ab8250d7SJiri Pirko 381ab8250d7SJiri Pirko port = LB_HTPM_PORT_BY_HASH(lb_priv, hash); 382ab8250d7SJiri Pirko ctx->data.u32_val = port ? port->dev->ifindex : 0; 383ab8250d7SJiri Pirko return 0; 384ab8250d7SJiri Pirko } 385ab8250d7SJiri Pirko 386ab8250d7SJiri Pirko static int lb_tx_hash_to_port_mapping_set(struct team *team, 387ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 388ab8250d7SJiri Pirko { 389ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 390ab8250d7SJiri Pirko struct team_port *port; 391ab8250d7SJiri Pirko unsigned char hash = ctx->info->array_index; 392ab8250d7SJiri Pirko 393ab8250d7SJiri Pirko list_for_each_entry(port, &team->port_list, list) { 39452a4fd77SJiri Pirko if (ctx->data.u32_val == port->dev->ifindex && 39552a4fd77SJiri Pirko team_port_enabled(port)) { 396ab8250d7SJiri Pirko rcu_assign_pointer(LB_HTPM_PORT_BY_HASH(lb_priv, hash), 397ab8250d7SJiri Pirko port); 398ab8250d7SJiri Pirko return 0; 399ab8250d7SJiri Pirko } 400ab8250d7SJiri Pirko } 401ab8250d7SJiri Pirko return -ENODEV; 402ab8250d7SJiri Pirko } 403ab8250d7SJiri Pirko 404ab8250d7SJiri Pirko static int lb_hash_stats_init(struct team *team, 405ab8250d7SJiri Pirko struct team_option_inst_info *info) 406ab8250d7SJiri Pirko { 407ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 408ab8250d7SJiri Pirko unsigned char hash = info->array_index; 409ab8250d7SJiri Pirko 410ab8250d7SJiri Pirko lb_priv->ex->stats.info[hash].opt_inst_info = info; 411ab8250d7SJiri Pirko return 0; 412ab8250d7SJiri Pirko } 413ab8250d7SJiri Pirko 414ab8250d7SJiri Pirko static int lb_hash_stats_get(struct team *team, struct team_gsetter_ctx *ctx) 415ab8250d7SJiri Pirko { 416ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 417ab8250d7SJiri Pirko unsigned char hash = ctx->info->array_index; 418ab8250d7SJiri Pirko 419ab8250d7SJiri Pirko ctx->data.bin_val.ptr = &lb_priv->ex->stats.info[hash].stats; 420ab8250d7SJiri Pirko ctx->data.bin_val.len = sizeof(struct lb_stats); 421ab8250d7SJiri Pirko return 0; 422ab8250d7SJiri Pirko } 423ab8250d7SJiri Pirko 424ab8250d7SJiri Pirko static int lb_port_stats_init(struct team *team, 425ab8250d7SJiri Pirko struct team_option_inst_info *info) 426ab8250d7SJiri Pirko { 427ab8250d7SJiri Pirko struct team_port *port = info->port; 428ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 429ab8250d7SJiri Pirko 430ab8250d7SJiri Pirko lb_port_priv->stats_info.opt_inst_info = info; 431ab8250d7SJiri Pirko return 0; 432ab8250d7SJiri Pirko } 433ab8250d7SJiri Pirko 434ab8250d7SJiri Pirko static int lb_port_stats_get(struct team *team, struct team_gsetter_ctx *ctx) 435ab8250d7SJiri Pirko { 436ab8250d7SJiri Pirko struct team_port *port = ctx->info->port; 437ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 438ab8250d7SJiri Pirko 439ab8250d7SJiri Pirko ctx->data.bin_val.ptr = &lb_port_priv->stats_info.stats; 440ab8250d7SJiri Pirko ctx->data.bin_val.len = sizeof(struct lb_stats); 441ab8250d7SJiri Pirko return 0; 442ab8250d7SJiri Pirko } 443ab8250d7SJiri Pirko 444ab8250d7SJiri Pirko static void __lb_stats_info_refresh_prepare(struct lb_stats_info *s_info) 445ab8250d7SJiri Pirko { 446ab8250d7SJiri Pirko memcpy(&s_info->last_stats, &s_info->stats, sizeof(struct lb_stats)); 447ab8250d7SJiri Pirko memset(&s_info->stats, 0, sizeof(struct lb_stats)); 448ab8250d7SJiri Pirko } 449ab8250d7SJiri Pirko 450ab8250d7SJiri Pirko static bool __lb_stats_info_refresh_check(struct lb_stats_info *s_info, 451ab8250d7SJiri Pirko struct team *team) 452ab8250d7SJiri Pirko { 453ab8250d7SJiri Pirko if (memcmp(&s_info->last_stats, &s_info->stats, 454ab8250d7SJiri Pirko sizeof(struct lb_stats))) { 455ab8250d7SJiri Pirko team_option_inst_set_change(s_info->opt_inst_info); 456ab8250d7SJiri Pirko return true; 457ab8250d7SJiri Pirko } 458ab8250d7SJiri Pirko return false; 459ab8250d7SJiri Pirko } 460ab8250d7SJiri Pirko 461ab8250d7SJiri Pirko static void __lb_one_cpu_stats_add(struct lb_stats *acc_stats, 462ab8250d7SJiri Pirko struct lb_stats *cpu_stats, 463ab8250d7SJiri Pirko struct u64_stats_sync *syncp) 464ab8250d7SJiri Pirko { 465ab8250d7SJiri Pirko unsigned int start; 466ab8250d7SJiri Pirko struct lb_stats tmp; 467ab8250d7SJiri Pirko 468ab8250d7SJiri Pirko do { 46957a7744eSEric W. Biederman start = u64_stats_fetch_begin_irq(syncp); 470ab8250d7SJiri Pirko tmp.tx_bytes = cpu_stats->tx_bytes; 47157a7744eSEric W. Biederman } while (u64_stats_fetch_retry_irq(syncp, start)); 472ab8250d7SJiri Pirko acc_stats->tx_bytes += tmp.tx_bytes; 473ab8250d7SJiri Pirko } 474ab8250d7SJiri Pirko 475ab8250d7SJiri Pirko static void lb_stats_refresh(struct work_struct *work) 476ab8250d7SJiri Pirko { 477ab8250d7SJiri Pirko struct team *team; 478ab8250d7SJiri Pirko struct lb_priv *lb_priv; 479ab8250d7SJiri Pirko struct lb_priv_ex *lb_priv_ex; 480ab8250d7SJiri Pirko struct lb_pcpu_stats *pcpu_stats; 481ab8250d7SJiri Pirko struct lb_stats *stats; 482ab8250d7SJiri Pirko struct lb_stats_info *s_info; 483ab8250d7SJiri Pirko struct team_port *port; 484ab8250d7SJiri Pirko bool changed = false; 485ab8250d7SJiri Pirko int i; 486ab8250d7SJiri Pirko int j; 487ab8250d7SJiri Pirko 488ab8250d7SJiri Pirko lb_priv_ex = container_of(work, struct lb_priv_ex, 489ab8250d7SJiri Pirko stats.refresh_dw.work); 490ab8250d7SJiri Pirko 491ab8250d7SJiri Pirko team = lb_priv_ex->team; 492ab8250d7SJiri Pirko lb_priv = get_lb_priv(team); 493ab8250d7SJiri Pirko 494ab8250d7SJiri Pirko if (!mutex_trylock(&team->lock)) { 495ab8250d7SJiri Pirko schedule_delayed_work(&lb_priv_ex->stats.refresh_dw, 0); 496ab8250d7SJiri Pirko return; 497ab8250d7SJiri Pirko } 498ab8250d7SJiri Pirko 499ab8250d7SJiri Pirko for (j = 0; j < LB_TX_HASHTABLE_SIZE; j++) { 500ab8250d7SJiri Pirko s_info = &lb_priv->ex->stats.info[j]; 501ab8250d7SJiri Pirko __lb_stats_info_refresh_prepare(s_info); 502ab8250d7SJiri Pirko for_each_possible_cpu(i) { 503ab8250d7SJiri Pirko pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); 504ab8250d7SJiri Pirko stats = &pcpu_stats->hash_stats[j]; 505ab8250d7SJiri Pirko __lb_one_cpu_stats_add(&s_info->stats, stats, 506ab8250d7SJiri Pirko &pcpu_stats->syncp); 507ab8250d7SJiri Pirko } 508ab8250d7SJiri Pirko changed |= __lb_stats_info_refresh_check(s_info, team); 509ab8250d7SJiri Pirko } 510ab8250d7SJiri Pirko 511ab8250d7SJiri Pirko list_for_each_entry(port, &team->port_list, list) { 512ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 513ab8250d7SJiri Pirko 514ab8250d7SJiri Pirko s_info = &lb_port_priv->stats_info; 515ab8250d7SJiri Pirko __lb_stats_info_refresh_prepare(s_info); 516ab8250d7SJiri Pirko for_each_possible_cpu(i) { 517ab8250d7SJiri Pirko pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); 518ab8250d7SJiri Pirko stats = per_cpu_ptr(lb_port_priv->pcpu_stats, i); 519ab8250d7SJiri Pirko __lb_one_cpu_stats_add(&s_info->stats, stats, 520ab8250d7SJiri Pirko &pcpu_stats->syncp); 521ab8250d7SJiri Pirko } 522ab8250d7SJiri Pirko changed |= __lb_stats_info_refresh_check(s_info, team); 523ab8250d7SJiri Pirko } 524ab8250d7SJiri Pirko 525ab8250d7SJiri Pirko if (changed) 526ab8250d7SJiri Pirko team_options_change_check(team); 527ab8250d7SJiri Pirko 528ab8250d7SJiri Pirko schedule_delayed_work(&lb_priv_ex->stats.refresh_dw, 529ab8250d7SJiri Pirko (lb_priv_ex->stats.refresh_interval * HZ) / 10); 530ab8250d7SJiri Pirko 531ab8250d7SJiri Pirko mutex_unlock(&team->lock); 532ab8250d7SJiri Pirko } 533ab8250d7SJiri Pirko 534ab8250d7SJiri Pirko static int lb_stats_refresh_interval_get(struct team *team, 535ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 536ab8250d7SJiri Pirko { 537ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 538ab8250d7SJiri Pirko 539ab8250d7SJiri Pirko ctx->data.u32_val = lb_priv->ex->stats.refresh_interval; 540ab8250d7SJiri Pirko return 0; 541ab8250d7SJiri Pirko } 542ab8250d7SJiri Pirko 543ab8250d7SJiri Pirko static int lb_stats_refresh_interval_set(struct team *team, 544ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 545ab8250d7SJiri Pirko { 546ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 547ab8250d7SJiri Pirko unsigned int interval; 548ab8250d7SJiri Pirko 549ab8250d7SJiri Pirko interval = ctx->data.u32_val; 550ab8250d7SJiri Pirko if (lb_priv->ex->stats.refresh_interval == interval) 551ab8250d7SJiri Pirko return 0; 552ab8250d7SJiri Pirko lb_priv->ex->stats.refresh_interval = interval; 553ab8250d7SJiri Pirko if (interval) 554ab8250d7SJiri Pirko schedule_delayed_work(&lb_priv->ex->stats.refresh_dw, 0); 555ab8250d7SJiri Pirko else 556ab8250d7SJiri Pirko cancel_delayed_work(&lb_priv->ex->stats.refresh_dw); 55701d7f30aSJiri Pirko return 0; 55801d7f30aSJiri Pirko } 55901d7f30aSJiri Pirko 56001d7f30aSJiri Pirko static const struct team_option lb_options[] = { 56101d7f30aSJiri Pirko { 56201d7f30aSJiri Pirko .name = "bpf_hash_func", 56301d7f30aSJiri Pirko .type = TEAM_OPTION_TYPE_BINARY, 56401d7f30aSJiri Pirko .getter = lb_bpf_func_get, 56501d7f30aSJiri Pirko .setter = lb_bpf_func_set, 56601d7f30aSJiri Pirko }, 567ab8250d7SJiri Pirko { 568ab8250d7SJiri Pirko .name = "lb_tx_method", 569ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_STRING, 570ab8250d7SJiri Pirko .getter = lb_tx_method_get, 571ab8250d7SJiri Pirko .setter = lb_tx_method_set, 572ab8250d7SJiri Pirko }, 573ab8250d7SJiri Pirko { 574ab8250d7SJiri Pirko .name = "lb_tx_hash_to_port_mapping", 575ab8250d7SJiri Pirko .array_size = LB_TX_HASHTABLE_SIZE, 576ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_U32, 577ab8250d7SJiri Pirko .init = lb_tx_hash_to_port_mapping_init, 578ab8250d7SJiri Pirko .getter = lb_tx_hash_to_port_mapping_get, 579ab8250d7SJiri Pirko .setter = lb_tx_hash_to_port_mapping_set, 580ab8250d7SJiri Pirko }, 581ab8250d7SJiri Pirko { 582ab8250d7SJiri Pirko .name = "lb_hash_stats", 583ab8250d7SJiri Pirko .array_size = LB_TX_HASHTABLE_SIZE, 584ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_BINARY, 585ab8250d7SJiri Pirko .init = lb_hash_stats_init, 586ab8250d7SJiri Pirko .getter = lb_hash_stats_get, 587ab8250d7SJiri Pirko }, 588ab8250d7SJiri Pirko { 589ab8250d7SJiri Pirko .name = "lb_port_stats", 590ab8250d7SJiri Pirko .per_port = true, 591ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_BINARY, 592ab8250d7SJiri Pirko .init = lb_port_stats_init, 593ab8250d7SJiri Pirko .getter = lb_port_stats_get, 594ab8250d7SJiri Pirko }, 595ab8250d7SJiri Pirko { 596ab8250d7SJiri Pirko .name = "lb_stats_refresh_interval", 597ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_U32, 598ab8250d7SJiri Pirko .getter = lb_stats_refresh_interval_get, 599ab8250d7SJiri Pirko .setter = lb_stats_refresh_interval_set, 600ab8250d7SJiri Pirko }, 60101d7f30aSJiri Pirko }; 60201d7f30aSJiri Pirko 603cade4555SJiri Pirko static int lb_init(struct team *team) 60401d7f30aSJiri Pirko { 605ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 606ab8250d7SJiri Pirko lb_select_tx_port_func_t *func; 607827da44cSJohn Stultz int i, err; 608ab8250d7SJiri Pirko 609ab8250d7SJiri Pirko /* set default tx port selector */ 610ab8250d7SJiri Pirko func = lb_select_tx_port_get_func("hash"); 611ab8250d7SJiri Pirko BUG_ON(!func); 612ab8250d7SJiri Pirko rcu_assign_pointer(lb_priv->select_tx_port_func, func); 613ab8250d7SJiri Pirko 614ab8250d7SJiri Pirko lb_priv->ex = kzalloc(sizeof(*lb_priv->ex), GFP_KERNEL); 615ab8250d7SJiri Pirko if (!lb_priv->ex) 616ab8250d7SJiri Pirko return -ENOMEM; 617ab8250d7SJiri Pirko lb_priv->ex->team = team; 618ab8250d7SJiri Pirko 619ab8250d7SJiri Pirko lb_priv->pcpu_stats = alloc_percpu(struct lb_pcpu_stats); 620ab8250d7SJiri Pirko if (!lb_priv->pcpu_stats) { 621ab8250d7SJiri Pirko err = -ENOMEM; 622ab8250d7SJiri Pirko goto err_alloc_pcpu_stats; 623ab8250d7SJiri Pirko } 624ab8250d7SJiri Pirko 625827da44cSJohn Stultz for_each_possible_cpu(i) { 626827da44cSJohn Stultz struct lb_pcpu_stats *team_lb_stats; 627827da44cSJohn Stultz team_lb_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); 628827da44cSJohn Stultz u64_stats_init(&team_lb_stats->syncp); 629827da44cSJohn Stultz } 630827da44cSJohn Stultz 631827da44cSJohn Stultz 632ab8250d7SJiri Pirko INIT_DELAYED_WORK(&lb_priv->ex->stats.refresh_dw, lb_stats_refresh); 633ab8250d7SJiri Pirko 634ab8250d7SJiri Pirko err = team_options_register(team, lb_options, ARRAY_SIZE(lb_options)); 635ab8250d7SJiri Pirko if (err) 636ab8250d7SJiri Pirko goto err_options_register; 637ab8250d7SJiri Pirko return 0; 638ab8250d7SJiri Pirko 639ab8250d7SJiri Pirko err_options_register: 640ab8250d7SJiri Pirko free_percpu(lb_priv->pcpu_stats); 641ab8250d7SJiri Pirko err_alloc_pcpu_stats: 642ab8250d7SJiri Pirko kfree(lb_priv->ex); 643ab8250d7SJiri Pirko return err; 64401d7f30aSJiri Pirko } 64501d7f30aSJiri Pirko 646cade4555SJiri Pirko static void lb_exit(struct team *team) 64701d7f30aSJiri Pirko { 648ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 649ab8250d7SJiri Pirko 65001d7f30aSJiri Pirko team_options_unregister(team, lb_options, 65101d7f30aSJiri Pirko ARRAY_SIZE(lb_options)); 652692c31bdSIdo Schimmel lb_bpf_func_free(team); 653ab8250d7SJiri Pirko cancel_delayed_work_sync(&lb_priv->ex->stats.refresh_dw); 654ab8250d7SJiri Pirko free_percpu(lb_priv->pcpu_stats); 655ab8250d7SJiri Pirko kfree(lb_priv->ex); 656ab8250d7SJiri Pirko } 657ab8250d7SJiri Pirko 658ab8250d7SJiri Pirko static int lb_port_enter(struct team *team, struct team_port *port) 659ab8250d7SJiri Pirko { 660ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 661ab8250d7SJiri Pirko 662ab8250d7SJiri Pirko lb_port_priv->pcpu_stats = alloc_percpu(struct lb_stats); 663ab8250d7SJiri Pirko if (!lb_port_priv->pcpu_stats) 664ab8250d7SJiri Pirko return -ENOMEM; 665ab8250d7SJiri Pirko return 0; 666ab8250d7SJiri Pirko } 667ab8250d7SJiri Pirko 668ab8250d7SJiri Pirko static void lb_port_leave(struct team *team, struct team_port *port) 669ab8250d7SJiri Pirko { 670ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 671ab8250d7SJiri Pirko 672ab8250d7SJiri Pirko free_percpu(lb_port_priv->pcpu_stats); 673ab8250d7SJiri Pirko } 674ab8250d7SJiri Pirko 675ab8250d7SJiri Pirko static void lb_port_disabled(struct team *team, struct team_port *port) 676ab8250d7SJiri Pirko { 677ab8250d7SJiri Pirko lb_tx_hash_to_port_mapping_null_port(team, port); 67801d7f30aSJiri Pirko } 67901d7f30aSJiri Pirko 68001d7f30aSJiri Pirko static const struct team_mode_ops lb_mode_ops = { 68101d7f30aSJiri Pirko .init = lb_init, 68201d7f30aSJiri Pirko .exit = lb_exit, 683ab8250d7SJiri Pirko .port_enter = lb_port_enter, 684ab8250d7SJiri Pirko .port_leave = lb_port_leave, 685ab8250d7SJiri Pirko .port_disabled = lb_port_disabled, 686c15e07b0SJiri Pirko .receive = lb_receive, 68701d7f30aSJiri Pirko .transmit = lb_transmit, 68801d7f30aSJiri Pirko }; 68901d7f30aSJiri Pirko 6900402788aSJiri Pirko static const struct team_mode lb_mode = { 69101d7f30aSJiri Pirko .kind = "loadbalance", 69201d7f30aSJiri Pirko .owner = THIS_MODULE, 69301d7f30aSJiri Pirko .priv_size = sizeof(struct lb_priv), 694ab8250d7SJiri Pirko .port_priv_size = sizeof(struct lb_port_priv), 69501d7f30aSJiri Pirko .ops = &lb_mode_ops, 6968fd72856SJiri Pirko .lag_tx_type = NETDEV_LAG_TX_TYPE_HASH, 69701d7f30aSJiri Pirko }; 69801d7f30aSJiri Pirko 69901d7f30aSJiri Pirko static int __init lb_init_module(void) 70001d7f30aSJiri Pirko { 70101d7f30aSJiri Pirko return team_mode_register(&lb_mode); 70201d7f30aSJiri Pirko } 70301d7f30aSJiri Pirko 70401d7f30aSJiri Pirko static void __exit lb_cleanup_module(void) 70501d7f30aSJiri Pirko { 70601d7f30aSJiri Pirko team_mode_unregister(&lb_mode); 70701d7f30aSJiri Pirko } 70801d7f30aSJiri Pirko 70901d7f30aSJiri Pirko module_init(lb_init_module); 71001d7f30aSJiri Pirko module_exit(lb_cleanup_module); 71101d7f30aSJiri Pirko 71201d7f30aSJiri Pirko MODULE_LICENSE("GPL v2"); 71301d7f30aSJiri Pirko MODULE_AUTHOR("Jiri Pirko <jpirko@redhat.com>"); 71401d7f30aSJiri Pirko MODULE_DESCRIPTION("Load-balancing mode for team"); 7153a5f8997SZhang Shengju MODULE_ALIAS_TEAM_MODE("loadbalance"); 716