101d7f30aSJiri Pirko /* 201d7f30aSJiri Pirko * drivers/net/team/team_mode_loadbalance.c - Load-balancing mode for team 301d7f30aSJiri Pirko * Copyright (c) 2012 Jiri Pirko <jpirko@redhat.com> 401d7f30aSJiri Pirko * 501d7f30aSJiri Pirko * This program is free software; you can redistribute it and/or modify 601d7f30aSJiri Pirko * it under the terms of the GNU General Public License as published by 701d7f30aSJiri Pirko * the Free Software Foundation; either version 2 of the License, or 801d7f30aSJiri Pirko * (at your option) any later version. 901d7f30aSJiri Pirko */ 1001d7f30aSJiri Pirko 1101d7f30aSJiri Pirko #include <linux/kernel.h> 1201d7f30aSJiri Pirko #include <linux/types.h> 1301d7f30aSJiri Pirko #include <linux/module.h> 1401d7f30aSJiri Pirko #include <linux/init.h> 1501d7f30aSJiri Pirko #include <linux/errno.h> 1601d7f30aSJiri Pirko #include <linux/netdevice.h> 17c15e07b0SJiri Pirko #include <linux/etherdevice.h> 1801d7f30aSJiri Pirko #include <linux/filter.h> 1901d7f30aSJiri Pirko #include <linux/if_team.h> 2001d7f30aSJiri Pirko 21c15e07b0SJiri Pirko static rx_handler_result_t lb_receive(struct team *team, struct team_port *port, 22c15e07b0SJiri Pirko struct sk_buff *skb) 23c15e07b0SJiri Pirko { 24c15e07b0SJiri Pirko if (unlikely(skb->protocol == htons(ETH_P_SLOW))) { 25c15e07b0SJiri Pirko /* LACPDU packets should go to exact delivery */ 26c15e07b0SJiri Pirko const unsigned char *dest = eth_hdr(skb)->h_dest; 27c15e07b0SJiri Pirko 28c15e07b0SJiri Pirko if (is_link_local_ether_addr(dest) && dest[5] == 0x02) 29c15e07b0SJiri Pirko return RX_HANDLER_EXACT; 30c15e07b0SJiri Pirko } 31c15e07b0SJiri Pirko return RX_HANDLER_ANOTHER; 32c15e07b0SJiri Pirko } 33c15e07b0SJiri Pirko 34ab8250d7SJiri Pirko struct lb_priv; 35ab8250d7SJiri Pirko 36ab8250d7SJiri Pirko typedef struct team_port *lb_select_tx_port_func_t(struct team *, 37ab8250d7SJiri Pirko struct lb_priv *, 38ab8250d7SJiri Pirko struct sk_buff *, 39ab8250d7SJiri Pirko unsigned char); 40ab8250d7SJiri Pirko 41ab8250d7SJiri Pirko #define LB_TX_HASHTABLE_SIZE 256 /* hash is a char */ 42ab8250d7SJiri Pirko 43ab8250d7SJiri Pirko struct lb_stats { 44ab8250d7SJiri Pirko u64 tx_bytes; 4501d7f30aSJiri Pirko }; 4601d7f30aSJiri Pirko 47ab8250d7SJiri Pirko struct lb_pcpu_stats { 48ab8250d7SJiri Pirko struct lb_stats hash_stats[LB_TX_HASHTABLE_SIZE]; 49ab8250d7SJiri Pirko struct u64_stats_sync syncp; 50ab8250d7SJiri Pirko }; 51ab8250d7SJiri Pirko 52ab8250d7SJiri Pirko struct lb_stats_info { 53ab8250d7SJiri Pirko struct lb_stats stats; 54ab8250d7SJiri Pirko struct lb_stats last_stats; 55ab8250d7SJiri Pirko struct team_option_inst_info *opt_inst_info; 56ab8250d7SJiri Pirko }; 57ab8250d7SJiri Pirko 58ab8250d7SJiri Pirko struct lb_port_mapping { 59ab8250d7SJiri Pirko struct team_port __rcu *port; 60ab8250d7SJiri Pirko struct team_option_inst_info *opt_inst_info; 61ab8250d7SJiri Pirko }; 62ab8250d7SJiri Pirko 63ab8250d7SJiri Pirko struct lb_priv_ex { 64ab8250d7SJiri Pirko struct team *team; 65ab8250d7SJiri Pirko struct lb_port_mapping tx_hash_to_port_mapping[LB_TX_HASHTABLE_SIZE]; 66b1fcd35cSDaniel Borkmann struct sock_fprog_kern *orig_fprog; 67ab8250d7SJiri Pirko struct { 68ab8250d7SJiri Pirko unsigned int refresh_interval; /* in tenths of second */ 69ab8250d7SJiri Pirko struct delayed_work refresh_dw; 70ab8250d7SJiri Pirko struct lb_stats_info info[LB_TX_HASHTABLE_SIZE]; 71ab8250d7SJiri Pirko } stats; 72ab8250d7SJiri Pirko }; 73ab8250d7SJiri Pirko 74ab8250d7SJiri Pirko struct lb_priv { 757ae457c1SAlexei Starovoitov struct bpf_prog __rcu *fp; 76ab8250d7SJiri Pirko lb_select_tx_port_func_t __rcu *select_tx_port_func; 77ab8250d7SJiri Pirko struct lb_pcpu_stats __percpu *pcpu_stats; 78ab8250d7SJiri Pirko struct lb_priv_ex *ex; /* priv extension */ 79ab8250d7SJiri Pirko }; 80ab8250d7SJiri Pirko 81ab8250d7SJiri Pirko static struct lb_priv *get_lb_priv(struct team *team) 8201d7f30aSJiri Pirko { 8301d7f30aSJiri Pirko return (struct lb_priv *) &team->mode_priv; 8401d7f30aSJiri Pirko } 8501d7f30aSJiri Pirko 86ab8250d7SJiri Pirko struct lb_port_priv { 87ab8250d7SJiri Pirko struct lb_stats __percpu *pcpu_stats; 88ab8250d7SJiri Pirko struct lb_stats_info stats_info; 89ab8250d7SJiri Pirko }; 90ab8250d7SJiri Pirko 91ab8250d7SJiri Pirko static struct lb_port_priv *get_lb_port_priv(struct team_port *port) 92ab8250d7SJiri Pirko { 93ab8250d7SJiri Pirko return (struct lb_port_priv *) &port->mode_priv; 94ab8250d7SJiri Pirko } 95ab8250d7SJiri Pirko 96ab8250d7SJiri Pirko #define LB_HTPM_PORT_BY_HASH(lp_priv, hash) \ 97ab8250d7SJiri Pirko (lb_priv)->ex->tx_hash_to_port_mapping[hash].port 98ab8250d7SJiri Pirko 99ab8250d7SJiri Pirko #define LB_HTPM_OPT_INST_INFO_BY_HASH(lp_priv, hash) \ 100ab8250d7SJiri Pirko (lb_priv)->ex->tx_hash_to_port_mapping[hash].opt_inst_info 101ab8250d7SJiri Pirko 102ab8250d7SJiri Pirko static void lb_tx_hash_to_port_mapping_null_port(struct team *team, 103ab8250d7SJiri Pirko struct team_port *port) 104ab8250d7SJiri Pirko { 105ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 106ab8250d7SJiri Pirko bool changed = false; 107ab8250d7SJiri Pirko int i; 108ab8250d7SJiri Pirko 109ab8250d7SJiri Pirko for (i = 0; i < LB_TX_HASHTABLE_SIZE; i++) { 110ab8250d7SJiri Pirko struct lb_port_mapping *pm; 111ab8250d7SJiri Pirko 112ab8250d7SJiri Pirko pm = &lb_priv->ex->tx_hash_to_port_mapping[i]; 1136dab015cSJiri Pirko if (rcu_access_pointer(pm->port) == port) { 1146dab015cSJiri Pirko RCU_INIT_POINTER(pm->port, NULL); 115ab8250d7SJiri Pirko team_option_inst_set_change(pm->opt_inst_info); 116ab8250d7SJiri Pirko changed = true; 117ab8250d7SJiri Pirko } 118ab8250d7SJiri Pirko } 119ab8250d7SJiri Pirko if (changed) 120ab8250d7SJiri Pirko team_options_change_check(team); 121ab8250d7SJiri Pirko } 122ab8250d7SJiri Pirko 123ab8250d7SJiri Pirko /* Basic tx selection based solely by hash */ 124ab8250d7SJiri Pirko static struct team_port *lb_hash_select_tx_port(struct team *team, 125ab8250d7SJiri Pirko struct lb_priv *lb_priv, 126ab8250d7SJiri Pirko struct sk_buff *skb, 127ab8250d7SJiri Pirko unsigned char hash) 128ab8250d7SJiri Pirko { 129735d381fSJiri Pirko int port_index = team_num_to_port_index(team, hash); 130ab8250d7SJiri Pirko 131ab8250d7SJiri Pirko return team_get_port_by_index_rcu(team, port_index); 132ab8250d7SJiri Pirko } 133ab8250d7SJiri Pirko 134ab8250d7SJiri Pirko /* Hash to port mapping select tx port */ 135ab8250d7SJiri Pirko static struct team_port *lb_htpm_select_tx_port(struct team *team, 136ab8250d7SJiri Pirko struct lb_priv *lb_priv, 137ab8250d7SJiri Pirko struct sk_buff *skb, 138ab8250d7SJiri Pirko unsigned char hash) 139ab8250d7SJiri Pirko { 140bd7d2106SJim Hanko struct team_port *port; 141bd7d2106SJim Hanko 142bd7d2106SJim Hanko port = rcu_dereference_bh(LB_HTPM_PORT_BY_HASH(lb_priv, hash)); 143bd7d2106SJim Hanko if (likely(port)) 144bd7d2106SJim Hanko return port; 145bd7d2106SJim Hanko /* If no valid port in the table, fall back to simple hash */ 146bd7d2106SJim Hanko return lb_hash_select_tx_port(team, lb_priv, skb, hash); 147ab8250d7SJiri Pirko } 148ab8250d7SJiri Pirko 149ab8250d7SJiri Pirko struct lb_select_tx_port { 150ab8250d7SJiri Pirko char *name; 151ab8250d7SJiri Pirko lb_select_tx_port_func_t *func; 152ab8250d7SJiri Pirko }; 153ab8250d7SJiri Pirko 154ab8250d7SJiri Pirko static const struct lb_select_tx_port lb_select_tx_port_list[] = { 155ab8250d7SJiri Pirko { 156ab8250d7SJiri Pirko .name = "hash", 157ab8250d7SJiri Pirko .func = lb_hash_select_tx_port, 158ab8250d7SJiri Pirko }, 159ab8250d7SJiri Pirko { 160ab8250d7SJiri Pirko .name = "hash_to_port_mapping", 161ab8250d7SJiri Pirko .func = lb_htpm_select_tx_port, 162ab8250d7SJiri Pirko }, 163ab8250d7SJiri Pirko }; 164ab8250d7SJiri Pirko #define LB_SELECT_TX_PORT_LIST_COUNT ARRAY_SIZE(lb_select_tx_port_list) 165ab8250d7SJiri Pirko 166ab8250d7SJiri Pirko static char *lb_select_tx_port_get_name(lb_select_tx_port_func_t *func) 167ab8250d7SJiri Pirko { 168ab8250d7SJiri Pirko int i; 169ab8250d7SJiri Pirko 170ab8250d7SJiri Pirko for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) { 171ab8250d7SJiri Pirko const struct lb_select_tx_port *item; 172ab8250d7SJiri Pirko 173ab8250d7SJiri Pirko item = &lb_select_tx_port_list[i]; 174ab8250d7SJiri Pirko if (item->func == func) 175ab8250d7SJiri Pirko return item->name; 176ab8250d7SJiri Pirko } 177ab8250d7SJiri Pirko return NULL; 178ab8250d7SJiri Pirko } 179ab8250d7SJiri Pirko 180ab8250d7SJiri Pirko static lb_select_tx_port_func_t *lb_select_tx_port_get_func(const char *name) 181ab8250d7SJiri Pirko { 182ab8250d7SJiri Pirko int i; 183ab8250d7SJiri Pirko 184ab8250d7SJiri Pirko for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) { 185ab8250d7SJiri Pirko const struct lb_select_tx_port *item; 186ab8250d7SJiri Pirko 187ab8250d7SJiri Pirko item = &lb_select_tx_port_list[i]; 188ab8250d7SJiri Pirko if (!strcmp(item->name, name)) 189ab8250d7SJiri Pirko return item->func; 190ab8250d7SJiri Pirko } 191ab8250d7SJiri Pirko return NULL; 192ab8250d7SJiri Pirko } 193ab8250d7SJiri Pirko 194ab8250d7SJiri Pirko static unsigned int lb_get_skb_hash(struct lb_priv *lb_priv, 195596e2024SJiri Pirko struct sk_buff *skb) 19601d7f30aSJiri Pirko { 1977ae457c1SAlexei Starovoitov struct bpf_prog *fp; 198596e2024SJiri Pirko uint32_t lhash; 199596e2024SJiri Pirko unsigned char *c; 200596e2024SJiri Pirko 201d1904fbdSJiri Pirko fp = rcu_dereference_bh(lb_priv->fp); 202596e2024SJiri Pirko if (unlikely(!fp)) 203596e2024SJiri Pirko return 0; 2047ae457c1SAlexei Starovoitov lhash = BPF_PROG_RUN(fp, skb); 205596e2024SJiri Pirko c = (char *) &lhash; 206596e2024SJiri Pirko return c[0] ^ c[1] ^ c[2] ^ c[3]; 207596e2024SJiri Pirko } 208596e2024SJiri Pirko 209ab8250d7SJiri Pirko static void lb_update_tx_stats(unsigned int tx_bytes, struct lb_priv *lb_priv, 210ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv, 211ab8250d7SJiri Pirko unsigned char hash) 212ab8250d7SJiri Pirko { 213ab8250d7SJiri Pirko struct lb_pcpu_stats *pcpu_stats; 214ab8250d7SJiri Pirko struct lb_stats *port_stats; 215ab8250d7SJiri Pirko struct lb_stats *hash_stats; 216ab8250d7SJiri Pirko 217ab8250d7SJiri Pirko pcpu_stats = this_cpu_ptr(lb_priv->pcpu_stats); 218ab8250d7SJiri Pirko port_stats = this_cpu_ptr(lb_port_priv->pcpu_stats); 219ab8250d7SJiri Pirko hash_stats = &pcpu_stats->hash_stats[hash]; 220ab8250d7SJiri Pirko u64_stats_update_begin(&pcpu_stats->syncp); 221ab8250d7SJiri Pirko port_stats->tx_bytes += tx_bytes; 222ab8250d7SJiri Pirko hash_stats->tx_bytes += tx_bytes; 223ab8250d7SJiri Pirko u64_stats_update_end(&pcpu_stats->syncp); 224ab8250d7SJiri Pirko } 225ab8250d7SJiri Pirko 226596e2024SJiri Pirko static bool lb_transmit(struct team *team, struct sk_buff *skb) 227596e2024SJiri Pirko { 228ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 229ab8250d7SJiri Pirko lb_select_tx_port_func_t *select_tx_port_func; 23001d7f30aSJiri Pirko struct team_port *port; 231ab8250d7SJiri Pirko unsigned char hash; 232ab8250d7SJiri Pirko unsigned int tx_bytes = skb->len; 23301d7f30aSJiri Pirko 234ab8250d7SJiri Pirko hash = lb_get_skb_hash(lb_priv, skb); 235d1904fbdSJiri Pirko select_tx_port_func = rcu_dereference_bh(lb_priv->select_tx_port_func); 236ab8250d7SJiri Pirko port = select_tx_port_func(team, lb_priv, skb, hash); 23701d7f30aSJiri Pirko if (unlikely(!port)) 23801d7f30aSJiri Pirko goto drop; 239bd2d0837SJiri Pirko if (team_dev_queue_xmit(team, port, skb)) 24001d7f30aSJiri Pirko return false; 241ab8250d7SJiri Pirko lb_update_tx_stats(tx_bytes, lb_priv, get_lb_port_priv(port), hash); 24201d7f30aSJiri Pirko return true; 24301d7f30aSJiri Pirko 24401d7f30aSJiri Pirko drop: 24501d7f30aSJiri Pirko dev_kfree_skb_any(skb); 24601d7f30aSJiri Pirko return false; 24701d7f30aSJiri Pirko } 24801d7f30aSJiri Pirko 24980f7c668SJiri Pirko static int lb_bpf_func_get(struct team *team, struct team_gsetter_ctx *ctx) 25001d7f30aSJiri Pirko { 251ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 252ab8250d7SJiri Pirko 253ab8250d7SJiri Pirko if (!lb_priv->ex->orig_fprog) { 25480f7c668SJiri Pirko ctx->data.bin_val.len = 0; 25580f7c668SJiri Pirko ctx->data.bin_val.ptr = NULL; 25601d7f30aSJiri Pirko return 0; 25780f7c668SJiri Pirko } 258ab8250d7SJiri Pirko ctx->data.bin_val.len = lb_priv->ex->orig_fprog->len * 25901d7f30aSJiri Pirko sizeof(struct sock_filter); 260ab8250d7SJiri Pirko ctx->data.bin_val.ptr = lb_priv->ex->orig_fprog->filter; 26101d7f30aSJiri Pirko return 0; 26201d7f30aSJiri Pirko } 26301d7f30aSJiri Pirko 264b1fcd35cSDaniel Borkmann static int __fprog_create(struct sock_fprog_kern **pfprog, u32 data_len, 26580f7c668SJiri Pirko const void *data) 26601d7f30aSJiri Pirko { 267b1fcd35cSDaniel Borkmann struct sock_fprog_kern *fprog; 26801d7f30aSJiri Pirko struct sock_filter *filter = (struct sock_filter *) data; 26901d7f30aSJiri Pirko 27001d7f30aSJiri Pirko if (data_len % sizeof(struct sock_filter)) 27101d7f30aSJiri Pirko return -EINVAL; 272ea5930f4SDaniel Borkmann fprog = kmalloc(sizeof(*fprog), GFP_KERNEL); 27301d7f30aSJiri Pirko if (!fprog) 27401d7f30aSJiri Pirko return -ENOMEM; 27501d7f30aSJiri Pirko fprog->filter = kmemdup(filter, data_len, GFP_KERNEL); 27601d7f30aSJiri Pirko if (!fprog->filter) { 27701d7f30aSJiri Pirko kfree(fprog); 27801d7f30aSJiri Pirko return -ENOMEM; 27901d7f30aSJiri Pirko } 28001d7f30aSJiri Pirko fprog->len = data_len / sizeof(struct sock_filter); 28101d7f30aSJiri Pirko *pfprog = fprog; 28201d7f30aSJiri Pirko return 0; 28301d7f30aSJiri Pirko } 28401d7f30aSJiri Pirko 285b1fcd35cSDaniel Borkmann static void __fprog_destroy(struct sock_fprog_kern *fprog) 28601d7f30aSJiri Pirko { 28701d7f30aSJiri Pirko kfree(fprog->filter); 28801d7f30aSJiri Pirko kfree(fprog); 28901d7f30aSJiri Pirko } 29001d7f30aSJiri Pirko 29180f7c668SJiri Pirko static int lb_bpf_func_set(struct team *team, struct team_gsetter_ctx *ctx) 29201d7f30aSJiri Pirko { 293ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 2947ae457c1SAlexei Starovoitov struct bpf_prog *fp = NULL; 2957ae457c1SAlexei Starovoitov struct bpf_prog *orig_fp = NULL; 296b1fcd35cSDaniel Borkmann struct sock_fprog_kern *fprog = NULL; 29701d7f30aSJiri Pirko int err; 29801d7f30aSJiri Pirko 29980f7c668SJiri Pirko if (ctx->data.bin_val.len) { 30080f7c668SJiri Pirko err = __fprog_create(&fprog, ctx->data.bin_val.len, 30180f7c668SJiri Pirko ctx->data.bin_val.ptr); 30201d7f30aSJiri Pirko if (err) 30301d7f30aSJiri Pirko return err; 3047ae457c1SAlexei Starovoitov err = bpf_prog_create(&fp, fprog); 30501d7f30aSJiri Pirko if (err) { 30601d7f30aSJiri Pirko __fprog_destroy(fprog); 30701d7f30aSJiri Pirko return err; 30801d7f30aSJiri Pirko } 30901d7f30aSJiri Pirko } 31001d7f30aSJiri Pirko 311ab8250d7SJiri Pirko if (lb_priv->ex->orig_fprog) { 31201d7f30aSJiri Pirko /* Clear old filter data */ 313ab8250d7SJiri Pirko __fprog_destroy(lb_priv->ex->orig_fprog); 3146dab015cSJiri Pirko orig_fp = rcu_dereference_protected(lb_priv->fp, 3156dab015cSJiri Pirko lockdep_is_held(&team->lock)); 31601d7f30aSJiri Pirko } 31701d7f30aSJiri Pirko 318ab8250d7SJiri Pirko rcu_assign_pointer(lb_priv->fp, fp); 319ab8250d7SJiri Pirko lb_priv->ex->orig_fprog = fprog; 32034c5bd66SPablo Neira 32134c5bd66SPablo Neira if (orig_fp) { 32234c5bd66SPablo Neira synchronize_rcu(); 3237ae457c1SAlexei Starovoitov bpf_prog_destroy(orig_fp); 32434c5bd66SPablo Neira } 325ab8250d7SJiri Pirko return 0; 326ab8250d7SJiri Pirko } 327ab8250d7SJiri Pirko 328ab8250d7SJiri Pirko static int lb_tx_method_get(struct team *team, struct team_gsetter_ctx *ctx) 329ab8250d7SJiri Pirko { 330ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 3316dab015cSJiri Pirko lb_select_tx_port_func_t *func; 332ab8250d7SJiri Pirko char *name; 333ab8250d7SJiri Pirko 3346dab015cSJiri Pirko func = rcu_dereference_protected(lb_priv->select_tx_port_func, 3356dab015cSJiri Pirko lockdep_is_held(&team->lock)); 3366dab015cSJiri Pirko name = lb_select_tx_port_get_name(func); 337ab8250d7SJiri Pirko BUG_ON(!name); 338ab8250d7SJiri Pirko ctx->data.str_val = name; 339ab8250d7SJiri Pirko return 0; 340ab8250d7SJiri Pirko } 341ab8250d7SJiri Pirko 342ab8250d7SJiri Pirko static int lb_tx_method_set(struct team *team, struct team_gsetter_ctx *ctx) 343ab8250d7SJiri Pirko { 344ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 345ab8250d7SJiri Pirko lb_select_tx_port_func_t *func; 346ab8250d7SJiri Pirko 347ab8250d7SJiri Pirko func = lb_select_tx_port_get_func(ctx->data.str_val); 348ab8250d7SJiri Pirko if (!func) 349ab8250d7SJiri Pirko return -EINVAL; 350ab8250d7SJiri Pirko rcu_assign_pointer(lb_priv->select_tx_port_func, func); 351ab8250d7SJiri Pirko return 0; 352ab8250d7SJiri Pirko } 353ab8250d7SJiri Pirko 354ab8250d7SJiri Pirko static int lb_tx_hash_to_port_mapping_init(struct team *team, 355ab8250d7SJiri Pirko struct team_option_inst_info *info) 356ab8250d7SJiri Pirko { 357ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 358ab8250d7SJiri Pirko unsigned char hash = info->array_index; 359ab8250d7SJiri Pirko 360ab8250d7SJiri Pirko LB_HTPM_OPT_INST_INFO_BY_HASH(lb_priv, hash) = info; 361ab8250d7SJiri Pirko return 0; 362ab8250d7SJiri Pirko } 363ab8250d7SJiri Pirko 364ab8250d7SJiri Pirko static int lb_tx_hash_to_port_mapping_get(struct team *team, 365ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 366ab8250d7SJiri Pirko { 367ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 368ab8250d7SJiri Pirko struct team_port *port; 369ab8250d7SJiri Pirko unsigned char hash = ctx->info->array_index; 370ab8250d7SJiri Pirko 371ab8250d7SJiri Pirko port = LB_HTPM_PORT_BY_HASH(lb_priv, hash); 372ab8250d7SJiri Pirko ctx->data.u32_val = port ? port->dev->ifindex : 0; 373ab8250d7SJiri Pirko return 0; 374ab8250d7SJiri Pirko } 375ab8250d7SJiri Pirko 376ab8250d7SJiri Pirko static int lb_tx_hash_to_port_mapping_set(struct team *team, 377ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 378ab8250d7SJiri Pirko { 379ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 380ab8250d7SJiri Pirko struct team_port *port; 381ab8250d7SJiri Pirko unsigned char hash = ctx->info->array_index; 382ab8250d7SJiri Pirko 383ab8250d7SJiri Pirko list_for_each_entry(port, &team->port_list, list) { 38452a4fd77SJiri Pirko if (ctx->data.u32_val == port->dev->ifindex && 38552a4fd77SJiri Pirko team_port_enabled(port)) { 386ab8250d7SJiri Pirko rcu_assign_pointer(LB_HTPM_PORT_BY_HASH(lb_priv, hash), 387ab8250d7SJiri Pirko port); 388ab8250d7SJiri Pirko return 0; 389ab8250d7SJiri Pirko } 390ab8250d7SJiri Pirko } 391ab8250d7SJiri Pirko return -ENODEV; 392ab8250d7SJiri Pirko } 393ab8250d7SJiri Pirko 394ab8250d7SJiri Pirko static int lb_hash_stats_init(struct team *team, 395ab8250d7SJiri Pirko struct team_option_inst_info *info) 396ab8250d7SJiri Pirko { 397ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 398ab8250d7SJiri Pirko unsigned char hash = info->array_index; 399ab8250d7SJiri Pirko 400ab8250d7SJiri Pirko lb_priv->ex->stats.info[hash].opt_inst_info = info; 401ab8250d7SJiri Pirko return 0; 402ab8250d7SJiri Pirko } 403ab8250d7SJiri Pirko 404ab8250d7SJiri Pirko static int lb_hash_stats_get(struct team *team, struct team_gsetter_ctx *ctx) 405ab8250d7SJiri Pirko { 406ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 407ab8250d7SJiri Pirko unsigned char hash = ctx->info->array_index; 408ab8250d7SJiri Pirko 409ab8250d7SJiri Pirko ctx->data.bin_val.ptr = &lb_priv->ex->stats.info[hash].stats; 410ab8250d7SJiri Pirko ctx->data.bin_val.len = sizeof(struct lb_stats); 411ab8250d7SJiri Pirko return 0; 412ab8250d7SJiri Pirko } 413ab8250d7SJiri Pirko 414ab8250d7SJiri Pirko static int lb_port_stats_init(struct team *team, 415ab8250d7SJiri Pirko struct team_option_inst_info *info) 416ab8250d7SJiri Pirko { 417ab8250d7SJiri Pirko struct team_port *port = info->port; 418ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 419ab8250d7SJiri Pirko 420ab8250d7SJiri Pirko lb_port_priv->stats_info.opt_inst_info = info; 421ab8250d7SJiri Pirko return 0; 422ab8250d7SJiri Pirko } 423ab8250d7SJiri Pirko 424ab8250d7SJiri Pirko static int lb_port_stats_get(struct team *team, struct team_gsetter_ctx *ctx) 425ab8250d7SJiri Pirko { 426ab8250d7SJiri Pirko struct team_port *port = ctx->info->port; 427ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 428ab8250d7SJiri Pirko 429ab8250d7SJiri Pirko ctx->data.bin_val.ptr = &lb_port_priv->stats_info.stats; 430ab8250d7SJiri Pirko ctx->data.bin_val.len = sizeof(struct lb_stats); 431ab8250d7SJiri Pirko return 0; 432ab8250d7SJiri Pirko } 433ab8250d7SJiri Pirko 434ab8250d7SJiri Pirko static void __lb_stats_info_refresh_prepare(struct lb_stats_info *s_info) 435ab8250d7SJiri Pirko { 436ab8250d7SJiri Pirko memcpy(&s_info->last_stats, &s_info->stats, sizeof(struct lb_stats)); 437ab8250d7SJiri Pirko memset(&s_info->stats, 0, sizeof(struct lb_stats)); 438ab8250d7SJiri Pirko } 439ab8250d7SJiri Pirko 440ab8250d7SJiri Pirko static bool __lb_stats_info_refresh_check(struct lb_stats_info *s_info, 441ab8250d7SJiri Pirko struct team *team) 442ab8250d7SJiri Pirko { 443ab8250d7SJiri Pirko if (memcmp(&s_info->last_stats, &s_info->stats, 444ab8250d7SJiri Pirko sizeof(struct lb_stats))) { 445ab8250d7SJiri Pirko team_option_inst_set_change(s_info->opt_inst_info); 446ab8250d7SJiri Pirko return true; 447ab8250d7SJiri Pirko } 448ab8250d7SJiri Pirko return false; 449ab8250d7SJiri Pirko } 450ab8250d7SJiri Pirko 451ab8250d7SJiri Pirko static void __lb_one_cpu_stats_add(struct lb_stats *acc_stats, 452ab8250d7SJiri Pirko struct lb_stats *cpu_stats, 453ab8250d7SJiri Pirko struct u64_stats_sync *syncp) 454ab8250d7SJiri Pirko { 455ab8250d7SJiri Pirko unsigned int start; 456ab8250d7SJiri Pirko struct lb_stats tmp; 457ab8250d7SJiri Pirko 458ab8250d7SJiri Pirko do { 45957a7744eSEric W. Biederman start = u64_stats_fetch_begin_irq(syncp); 460ab8250d7SJiri Pirko tmp.tx_bytes = cpu_stats->tx_bytes; 46157a7744eSEric W. Biederman } while (u64_stats_fetch_retry_irq(syncp, start)); 462ab8250d7SJiri Pirko acc_stats->tx_bytes += tmp.tx_bytes; 463ab8250d7SJiri Pirko } 464ab8250d7SJiri Pirko 465ab8250d7SJiri Pirko static void lb_stats_refresh(struct work_struct *work) 466ab8250d7SJiri Pirko { 467ab8250d7SJiri Pirko struct team *team; 468ab8250d7SJiri Pirko struct lb_priv *lb_priv; 469ab8250d7SJiri Pirko struct lb_priv_ex *lb_priv_ex; 470ab8250d7SJiri Pirko struct lb_pcpu_stats *pcpu_stats; 471ab8250d7SJiri Pirko struct lb_stats *stats; 472ab8250d7SJiri Pirko struct lb_stats_info *s_info; 473ab8250d7SJiri Pirko struct team_port *port; 474ab8250d7SJiri Pirko bool changed = false; 475ab8250d7SJiri Pirko int i; 476ab8250d7SJiri Pirko int j; 477ab8250d7SJiri Pirko 478ab8250d7SJiri Pirko lb_priv_ex = container_of(work, struct lb_priv_ex, 479ab8250d7SJiri Pirko stats.refresh_dw.work); 480ab8250d7SJiri Pirko 481ab8250d7SJiri Pirko team = lb_priv_ex->team; 482ab8250d7SJiri Pirko lb_priv = get_lb_priv(team); 483ab8250d7SJiri Pirko 484ab8250d7SJiri Pirko if (!mutex_trylock(&team->lock)) { 485ab8250d7SJiri Pirko schedule_delayed_work(&lb_priv_ex->stats.refresh_dw, 0); 486ab8250d7SJiri Pirko return; 487ab8250d7SJiri Pirko } 488ab8250d7SJiri Pirko 489ab8250d7SJiri Pirko for (j = 0; j < LB_TX_HASHTABLE_SIZE; j++) { 490ab8250d7SJiri Pirko s_info = &lb_priv->ex->stats.info[j]; 491ab8250d7SJiri Pirko __lb_stats_info_refresh_prepare(s_info); 492ab8250d7SJiri Pirko for_each_possible_cpu(i) { 493ab8250d7SJiri Pirko pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); 494ab8250d7SJiri Pirko stats = &pcpu_stats->hash_stats[j]; 495ab8250d7SJiri Pirko __lb_one_cpu_stats_add(&s_info->stats, stats, 496ab8250d7SJiri Pirko &pcpu_stats->syncp); 497ab8250d7SJiri Pirko } 498ab8250d7SJiri Pirko changed |= __lb_stats_info_refresh_check(s_info, team); 499ab8250d7SJiri Pirko } 500ab8250d7SJiri Pirko 501ab8250d7SJiri Pirko list_for_each_entry(port, &team->port_list, list) { 502ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 503ab8250d7SJiri Pirko 504ab8250d7SJiri Pirko s_info = &lb_port_priv->stats_info; 505ab8250d7SJiri Pirko __lb_stats_info_refresh_prepare(s_info); 506ab8250d7SJiri Pirko for_each_possible_cpu(i) { 507ab8250d7SJiri Pirko pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); 508ab8250d7SJiri Pirko stats = per_cpu_ptr(lb_port_priv->pcpu_stats, i); 509ab8250d7SJiri Pirko __lb_one_cpu_stats_add(&s_info->stats, stats, 510ab8250d7SJiri Pirko &pcpu_stats->syncp); 511ab8250d7SJiri Pirko } 512ab8250d7SJiri Pirko changed |= __lb_stats_info_refresh_check(s_info, team); 513ab8250d7SJiri Pirko } 514ab8250d7SJiri Pirko 515ab8250d7SJiri Pirko if (changed) 516ab8250d7SJiri Pirko team_options_change_check(team); 517ab8250d7SJiri Pirko 518ab8250d7SJiri Pirko schedule_delayed_work(&lb_priv_ex->stats.refresh_dw, 519ab8250d7SJiri Pirko (lb_priv_ex->stats.refresh_interval * HZ) / 10); 520ab8250d7SJiri Pirko 521ab8250d7SJiri Pirko mutex_unlock(&team->lock); 522ab8250d7SJiri Pirko } 523ab8250d7SJiri Pirko 524ab8250d7SJiri Pirko static int lb_stats_refresh_interval_get(struct team *team, 525ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 526ab8250d7SJiri Pirko { 527ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 528ab8250d7SJiri Pirko 529ab8250d7SJiri Pirko ctx->data.u32_val = lb_priv->ex->stats.refresh_interval; 530ab8250d7SJiri Pirko return 0; 531ab8250d7SJiri Pirko } 532ab8250d7SJiri Pirko 533ab8250d7SJiri Pirko static int lb_stats_refresh_interval_set(struct team *team, 534ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 535ab8250d7SJiri Pirko { 536ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 537ab8250d7SJiri Pirko unsigned int interval; 538ab8250d7SJiri Pirko 539ab8250d7SJiri Pirko interval = ctx->data.u32_val; 540ab8250d7SJiri Pirko if (lb_priv->ex->stats.refresh_interval == interval) 541ab8250d7SJiri Pirko return 0; 542ab8250d7SJiri Pirko lb_priv->ex->stats.refresh_interval = interval; 543ab8250d7SJiri Pirko if (interval) 544ab8250d7SJiri Pirko schedule_delayed_work(&lb_priv->ex->stats.refresh_dw, 0); 545ab8250d7SJiri Pirko else 546ab8250d7SJiri Pirko cancel_delayed_work(&lb_priv->ex->stats.refresh_dw); 54701d7f30aSJiri Pirko return 0; 54801d7f30aSJiri Pirko } 54901d7f30aSJiri Pirko 55001d7f30aSJiri Pirko static const struct team_option lb_options[] = { 55101d7f30aSJiri Pirko { 55201d7f30aSJiri Pirko .name = "bpf_hash_func", 55301d7f30aSJiri Pirko .type = TEAM_OPTION_TYPE_BINARY, 55401d7f30aSJiri Pirko .getter = lb_bpf_func_get, 55501d7f30aSJiri Pirko .setter = lb_bpf_func_set, 55601d7f30aSJiri Pirko }, 557ab8250d7SJiri Pirko { 558ab8250d7SJiri Pirko .name = "lb_tx_method", 559ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_STRING, 560ab8250d7SJiri Pirko .getter = lb_tx_method_get, 561ab8250d7SJiri Pirko .setter = lb_tx_method_set, 562ab8250d7SJiri Pirko }, 563ab8250d7SJiri Pirko { 564ab8250d7SJiri Pirko .name = "lb_tx_hash_to_port_mapping", 565ab8250d7SJiri Pirko .array_size = LB_TX_HASHTABLE_SIZE, 566ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_U32, 567ab8250d7SJiri Pirko .init = lb_tx_hash_to_port_mapping_init, 568ab8250d7SJiri Pirko .getter = lb_tx_hash_to_port_mapping_get, 569ab8250d7SJiri Pirko .setter = lb_tx_hash_to_port_mapping_set, 570ab8250d7SJiri Pirko }, 571ab8250d7SJiri Pirko { 572ab8250d7SJiri Pirko .name = "lb_hash_stats", 573ab8250d7SJiri Pirko .array_size = LB_TX_HASHTABLE_SIZE, 574ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_BINARY, 575ab8250d7SJiri Pirko .init = lb_hash_stats_init, 576ab8250d7SJiri Pirko .getter = lb_hash_stats_get, 577ab8250d7SJiri Pirko }, 578ab8250d7SJiri Pirko { 579ab8250d7SJiri Pirko .name = "lb_port_stats", 580ab8250d7SJiri Pirko .per_port = true, 581ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_BINARY, 582ab8250d7SJiri Pirko .init = lb_port_stats_init, 583ab8250d7SJiri Pirko .getter = lb_port_stats_get, 584ab8250d7SJiri Pirko }, 585ab8250d7SJiri Pirko { 586ab8250d7SJiri Pirko .name = "lb_stats_refresh_interval", 587ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_U32, 588ab8250d7SJiri Pirko .getter = lb_stats_refresh_interval_get, 589ab8250d7SJiri Pirko .setter = lb_stats_refresh_interval_set, 590ab8250d7SJiri Pirko }, 59101d7f30aSJiri Pirko }; 59201d7f30aSJiri Pirko 593cade4555SJiri Pirko static int lb_init(struct team *team) 59401d7f30aSJiri Pirko { 595ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 596ab8250d7SJiri Pirko lb_select_tx_port_func_t *func; 597827da44cSJohn Stultz int i, err; 598ab8250d7SJiri Pirko 599ab8250d7SJiri Pirko /* set default tx port selector */ 600ab8250d7SJiri Pirko func = lb_select_tx_port_get_func("hash"); 601ab8250d7SJiri Pirko BUG_ON(!func); 602ab8250d7SJiri Pirko rcu_assign_pointer(lb_priv->select_tx_port_func, func); 603ab8250d7SJiri Pirko 604ab8250d7SJiri Pirko lb_priv->ex = kzalloc(sizeof(*lb_priv->ex), GFP_KERNEL); 605ab8250d7SJiri Pirko if (!lb_priv->ex) 606ab8250d7SJiri Pirko return -ENOMEM; 607ab8250d7SJiri Pirko lb_priv->ex->team = team; 608ab8250d7SJiri Pirko 609ab8250d7SJiri Pirko lb_priv->pcpu_stats = alloc_percpu(struct lb_pcpu_stats); 610ab8250d7SJiri Pirko if (!lb_priv->pcpu_stats) { 611ab8250d7SJiri Pirko err = -ENOMEM; 612ab8250d7SJiri Pirko goto err_alloc_pcpu_stats; 613ab8250d7SJiri Pirko } 614ab8250d7SJiri Pirko 615827da44cSJohn Stultz for_each_possible_cpu(i) { 616827da44cSJohn Stultz struct lb_pcpu_stats *team_lb_stats; 617827da44cSJohn Stultz team_lb_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); 618827da44cSJohn Stultz u64_stats_init(&team_lb_stats->syncp); 619827da44cSJohn Stultz } 620827da44cSJohn Stultz 621827da44cSJohn Stultz 622ab8250d7SJiri Pirko INIT_DELAYED_WORK(&lb_priv->ex->stats.refresh_dw, lb_stats_refresh); 623ab8250d7SJiri Pirko 624ab8250d7SJiri Pirko err = team_options_register(team, lb_options, ARRAY_SIZE(lb_options)); 625ab8250d7SJiri Pirko if (err) 626ab8250d7SJiri Pirko goto err_options_register; 627ab8250d7SJiri Pirko return 0; 628ab8250d7SJiri Pirko 629ab8250d7SJiri Pirko err_options_register: 630ab8250d7SJiri Pirko free_percpu(lb_priv->pcpu_stats); 631ab8250d7SJiri Pirko err_alloc_pcpu_stats: 632ab8250d7SJiri Pirko kfree(lb_priv->ex); 633ab8250d7SJiri Pirko return err; 63401d7f30aSJiri Pirko } 63501d7f30aSJiri Pirko 636cade4555SJiri Pirko static void lb_exit(struct team *team) 63701d7f30aSJiri Pirko { 638ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 639ab8250d7SJiri Pirko 64001d7f30aSJiri Pirko team_options_unregister(team, lb_options, 64101d7f30aSJiri Pirko ARRAY_SIZE(lb_options)); 642ab8250d7SJiri Pirko cancel_delayed_work_sync(&lb_priv->ex->stats.refresh_dw); 643ab8250d7SJiri Pirko free_percpu(lb_priv->pcpu_stats); 644ab8250d7SJiri Pirko kfree(lb_priv->ex); 645ab8250d7SJiri Pirko } 646ab8250d7SJiri Pirko 647ab8250d7SJiri Pirko static int lb_port_enter(struct team *team, struct team_port *port) 648ab8250d7SJiri Pirko { 649ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 650ab8250d7SJiri Pirko 651ab8250d7SJiri Pirko lb_port_priv->pcpu_stats = alloc_percpu(struct lb_stats); 652ab8250d7SJiri Pirko if (!lb_port_priv->pcpu_stats) 653ab8250d7SJiri Pirko return -ENOMEM; 654ab8250d7SJiri Pirko return 0; 655ab8250d7SJiri Pirko } 656ab8250d7SJiri Pirko 657ab8250d7SJiri Pirko static void lb_port_leave(struct team *team, struct team_port *port) 658ab8250d7SJiri Pirko { 659ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 660ab8250d7SJiri Pirko 661ab8250d7SJiri Pirko free_percpu(lb_port_priv->pcpu_stats); 662ab8250d7SJiri Pirko } 663ab8250d7SJiri Pirko 664ab8250d7SJiri Pirko static void lb_port_disabled(struct team *team, struct team_port *port) 665ab8250d7SJiri Pirko { 666ab8250d7SJiri Pirko lb_tx_hash_to_port_mapping_null_port(team, port); 66701d7f30aSJiri Pirko } 66801d7f30aSJiri Pirko 66901d7f30aSJiri Pirko static const struct team_mode_ops lb_mode_ops = { 67001d7f30aSJiri Pirko .init = lb_init, 67101d7f30aSJiri Pirko .exit = lb_exit, 672ab8250d7SJiri Pirko .port_enter = lb_port_enter, 673ab8250d7SJiri Pirko .port_leave = lb_port_leave, 674ab8250d7SJiri Pirko .port_disabled = lb_port_disabled, 675c15e07b0SJiri Pirko .receive = lb_receive, 67601d7f30aSJiri Pirko .transmit = lb_transmit, 67701d7f30aSJiri Pirko }; 67801d7f30aSJiri Pirko 6790402788aSJiri Pirko static const struct team_mode lb_mode = { 68001d7f30aSJiri Pirko .kind = "loadbalance", 68101d7f30aSJiri Pirko .owner = THIS_MODULE, 68201d7f30aSJiri Pirko .priv_size = sizeof(struct lb_priv), 683ab8250d7SJiri Pirko .port_priv_size = sizeof(struct lb_port_priv), 68401d7f30aSJiri Pirko .ops = &lb_mode_ops, 6858fd72856SJiri Pirko .lag_tx_type = NETDEV_LAG_TX_TYPE_HASH, 68601d7f30aSJiri Pirko }; 68701d7f30aSJiri Pirko 68801d7f30aSJiri Pirko static int __init lb_init_module(void) 68901d7f30aSJiri Pirko { 69001d7f30aSJiri Pirko return team_mode_register(&lb_mode); 69101d7f30aSJiri Pirko } 69201d7f30aSJiri Pirko 69301d7f30aSJiri Pirko static void __exit lb_cleanup_module(void) 69401d7f30aSJiri Pirko { 69501d7f30aSJiri Pirko team_mode_unregister(&lb_mode); 69601d7f30aSJiri Pirko } 69701d7f30aSJiri Pirko 69801d7f30aSJiri Pirko module_init(lb_init_module); 69901d7f30aSJiri Pirko module_exit(lb_cleanup_module); 70001d7f30aSJiri Pirko 70101d7f30aSJiri Pirko MODULE_LICENSE("GPL v2"); 70201d7f30aSJiri Pirko MODULE_AUTHOR("Jiri Pirko <jpirko@redhat.com>"); 70301d7f30aSJiri Pirko MODULE_DESCRIPTION("Load-balancing mode for team"); 7043a5f8997SZhang Shengju MODULE_ALIAS_TEAM_MODE("loadbalance"); 705