101d7f30aSJiri Pirko /* 201d7f30aSJiri Pirko * drivers/net/team/team_mode_loadbalance.c - Load-balancing mode for team 301d7f30aSJiri Pirko * Copyright (c) 2012 Jiri Pirko <jpirko@redhat.com> 401d7f30aSJiri Pirko * 501d7f30aSJiri Pirko * This program is free software; you can redistribute it and/or modify 601d7f30aSJiri Pirko * it under the terms of the GNU General Public License as published by 701d7f30aSJiri Pirko * the Free Software Foundation; either version 2 of the License, or 801d7f30aSJiri Pirko * (at your option) any later version. 901d7f30aSJiri Pirko */ 1001d7f30aSJiri Pirko 1101d7f30aSJiri Pirko #include <linux/kernel.h> 1201d7f30aSJiri Pirko #include <linux/types.h> 1301d7f30aSJiri Pirko #include <linux/module.h> 1401d7f30aSJiri Pirko #include <linux/init.h> 1501d7f30aSJiri Pirko #include <linux/errno.h> 1601d7f30aSJiri Pirko #include <linux/netdevice.h> 1701d7f30aSJiri Pirko #include <linux/filter.h> 1801d7f30aSJiri Pirko #include <linux/if_team.h> 1901d7f30aSJiri Pirko 20ab8250d7SJiri Pirko struct lb_priv; 21ab8250d7SJiri Pirko 22ab8250d7SJiri Pirko typedef struct team_port *lb_select_tx_port_func_t(struct team *, 23ab8250d7SJiri Pirko struct lb_priv *, 24ab8250d7SJiri Pirko struct sk_buff *, 25ab8250d7SJiri Pirko unsigned char); 26ab8250d7SJiri Pirko 27ab8250d7SJiri Pirko #define LB_TX_HASHTABLE_SIZE 256 /* hash is a char */ 28ab8250d7SJiri Pirko 29ab8250d7SJiri Pirko struct lb_stats { 30ab8250d7SJiri Pirko u64 tx_bytes; 3101d7f30aSJiri Pirko }; 3201d7f30aSJiri Pirko 33ab8250d7SJiri Pirko struct lb_pcpu_stats { 34ab8250d7SJiri Pirko struct lb_stats hash_stats[LB_TX_HASHTABLE_SIZE]; 35ab8250d7SJiri Pirko struct u64_stats_sync syncp; 36ab8250d7SJiri Pirko }; 37ab8250d7SJiri Pirko 38ab8250d7SJiri Pirko struct lb_stats_info { 39ab8250d7SJiri Pirko struct lb_stats stats; 40ab8250d7SJiri Pirko struct lb_stats last_stats; 41ab8250d7SJiri Pirko struct team_option_inst_info *opt_inst_info; 42ab8250d7SJiri Pirko }; 43ab8250d7SJiri Pirko 44ab8250d7SJiri Pirko struct lb_port_mapping { 45ab8250d7SJiri Pirko struct team_port __rcu *port; 46ab8250d7SJiri Pirko struct team_option_inst_info *opt_inst_info; 47ab8250d7SJiri Pirko }; 48ab8250d7SJiri Pirko 49ab8250d7SJiri Pirko struct lb_priv_ex { 50ab8250d7SJiri Pirko struct team *team; 51ab8250d7SJiri Pirko struct lb_port_mapping tx_hash_to_port_mapping[LB_TX_HASHTABLE_SIZE]; 52ab8250d7SJiri Pirko struct sock_fprog *orig_fprog; 53ab8250d7SJiri Pirko struct { 54ab8250d7SJiri Pirko unsigned int refresh_interval; /* in tenths of second */ 55ab8250d7SJiri Pirko struct delayed_work refresh_dw; 56ab8250d7SJiri Pirko struct lb_stats_info info[LB_TX_HASHTABLE_SIZE]; 57ab8250d7SJiri Pirko } stats; 58ab8250d7SJiri Pirko }; 59ab8250d7SJiri Pirko 60ab8250d7SJiri Pirko struct lb_priv { 61ab8250d7SJiri Pirko struct sk_filter __rcu *fp; 62ab8250d7SJiri Pirko lb_select_tx_port_func_t __rcu *select_tx_port_func; 63ab8250d7SJiri Pirko struct lb_pcpu_stats __percpu *pcpu_stats; 64ab8250d7SJiri Pirko struct lb_priv_ex *ex; /* priv extension */ 65ab8250d7SJiri Pirko }; 66ab8250d7SJiri Pirko 67ab8250d7SJiri Pirko static struct lb_priv *get_lb_priv(struct team *team) 6801d7f30aSJiri Pirko { 6901d7f30aSJiri Pirko return (struct lb_priv *) &team->mode_priv; 7001d7f30aSJiri Pirko } 7101d7f30aSJiri Pirko 72ab8250d7SJiri Pirko struct lb_port_priv { 73ab8250d7SJiri Pirko struct lb_stats __percpu *pcpu_stats; 74ab8250d7SJiri Pirko struct lb_stats_info stats_info; 75ab8250d7SJiri Pirko }; 76ab8250d7SJiri Pirko 77ab8250d7SJiri Pirko static struct lb_port_priv *get_lb_port_priv(struct team_port *port) 78ab8250d7SJiri Pirko { 79ab8250d7SJiri Pirko return (struct lb_port_priv *) &port->mode_priv; 80ab8250d7SJiri Pirko } 81ab8250d7SJiri Pirko 82ab8250d7SJiri Pirko #define LB_HTPM_PORT_BY_HASH(lp_priv, hash) \ 83ab8250d7SJiri Pirko (lb_priv)->ex->tx_hash_to_port_mapping[hash].port 84ab8250d7SJiri Pirko 85ab8250d7SJiri Pirko #define LB_HTPM_OPT_INST_INFO_BY_HASH(lp_priv, hash) \ 86ab8250d7SJiri Pirko (lb_priv)->ex->tx_hash_to_port_mapping[hash].opt_inst_info 87ab8250d7SJiri Pirko 88ab8250d7SJiri Pirko static void lb_tx_hash_to_port_mapping_null_port(struct team *team, 89ab8250d7SJiri Pirko struct team_port *port) 90ab8250d7SJiri Pirko { 91ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 92ab8250d7SJiri Pirko bool changed = false; 93ab8250d7SJiri Pirko int i; 94ab8250d7SJiri Pirko 95ab8250d7SJiri Pirko for (i = 0; i < LB_TX_HASHTABLE_SIZE; i++) { 96ab8250d7SJiri Pirko struct lb_port_mapping *pm; 97ab8250d7SJiri Pirko 98ab8250d7SJiri Pirko pm = &lb_priv->ex->tx_hash_to_port_mapping[i]; 996dab015cSJiri Pirko if (rcu_access_pointer(pm->port) == port) { 1006dab015cSJiri Pirko RCU_INIT_POINTER(pm->port, NULL); 101ab8250d7SJiri Pirko team_option_inst_set_change(pm->opt_inst_info); 102ab8250d7SJiri Pirko changed = true; 103ab8250d7SJiri Pirko } 104ab8250d7SJiri Pirko } 105ab8250d7SJiri Pirko if (changed) 106ab8250d7SJiri Pirko team_options_change_check(team); 107ab8250d7SJiri Pirko } 108ab8250d7SJiri Pirko 109ab8250d7SJiri Pirko /* Basic tx selection based solely by hash */ 110ab8250d7SJiri Pirko static struct team_port *lb_hash_select_tx_port(struct team *team, 111ab8250d7SJiri Pirko struct lb_priv *lb_priv, 112ab8250d7SJiri Pirko struct sk_buff *skb, 113ab8250d7SJiri Pirko unsigned char hash) 114ab8250d7SJiri Pirko { 115735d381fSJiri Pirko int port_index = team_num_to_port_index(team, hash); 116ab8250d7SJiri Pirko 117ab8250d7SJiri Pirko return team_get_port_by_index_rcu(team, port_index); 118ab8250d7SJiri Pirko } 119ab8250d7SJiri Pirko 120ab8250d7SJiri Pirko /* Hash to port mapping select tx port */ 121ab8250d7SJiri Pirko static struct team_port *lb_htpm_select_tx_port(struct team *team, 122ab8250d7SJiri Pirko struct lb_priv *lb_priv, 123ab8250d7SJiri Pirko struct sk_buff *skb, 124ab8250d7SJiri Pirko unsigned char hash) 125ab8250d7SJiri Pirko { 126d1904fbdSJiri Pirko return rcu_dereference_bh(LB_HTPM_PORT_BY_HASH(lb_priv, hash)); 127ab8250d7SJiri Pirko } 128ab8250d7SJiri Pirko 129ab8250d7SJiri Pirko struct lb_select_tx_port { 130ab8250d7SJiri Pirko char *name; 131ab8250d7SJiri Pirko lb_select_tx_port_func_t *func; 132ab8250d7SJiri Pirko }; 133ab8250d7SJiri Pirko 134ab8250d7SJiri Pirko static const struct lb_select_tx_port lb_select_tx_port_list[] = { 135ab8250d7SJiri Pirko { 136ab8250d7SJiri Pirko .name = "hash", 137ab8250d7SJiri Pirko .func = lb_hash_select_tx_port, 138ab8250d7SJiri Pirko }, 139ab8250d7SJiri Pirko { 140ab8250d7SJiri Pirko .name = "hash_to_port_mapping", 141ab8250d7SJiri Pirko .func = lb_htpm_select_tx_port, 142ab8250d7SJiri Pirko }, 143ab8250d7SJiri Pirko }; 144ab8250d7SJiri Pirko #define LB_SELECT_TX_PORT_LIST_COUNT ARRAY_SIZE(lb_select_tx_port_list) 145ab8250d7SJiri Pirko 146ab8250d7SJiri Pirko static char *lb_select_tx_port_get_name(lb_select_tx_port_func_t *func) 147ab8250d7SJiri Pirko { 148ab8250d7SJiri Pirko int i; 149ab8250d7SJiri Pirko 150ab8250d7SJiri Pirko for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) { 151ab8250d7SJiri Pirko const struct lb_select_tx_port *item; 152ab8250d7SJiri Pirko 153ab8250d7SJiri Pirko item = &lb_select_tx_port_list[i]; 154ab8250d7SJiri Pirko if (item->func == func) 155ab8250d7SJiri Pirko return item->name; 156ab8250d7SJiri Pirko } 157ab8250d7SJiri Pirko return NULL; 158ab8250d7SJiri Pirko } 159ab8250d7SJiri Pirko 160ab8250d7SJiri Pirko static lb_select_tx_port_func_t *lb_select_tx_port_get_func(const char *name) 161ab8250d7SJiri Pirko { 162ab8250d7SJiri Pirko int i; 163ab8250d7SJiri Pirko 164ab8250d7SJiri Pirko for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) { 165ab8250d7SJiri Pirko const struct lb_select_tx_port *item; 166ab8250d7SJiri Pirko 167ab8250d7SJiri Pirko item = &lb_select_tx_port_list[i]; 168ab8250d7SJiri Pirko if (!strcmp(item->name, name)) 169ab8250d7SJiri Pirko return item->func; 170ab8250d7SJiri Pirko } 171ab8250d7SJiri Pirko return NULL; 172ab8250d7SJiri Pirko } 173ab8250d7SJiri Pirko 174ab8250d7SJiri Pirko static unsigned int lb_get_skb_hash(struct lb_priv *lb_priv, 175596e2024SJiri Pirko struct sk_buff *skb) 17601d7f30aSJiri Pirko { 17701d7f30aSJiri Pirko struct sk_filter *fp; 178596e2024SJiri Pirko uint32_t lhash; 179596e2024SJiri Pirko unsigned char *c; 180596e2024SJiri Pirko 181d1904fbdSJiri Pirko fp = rcu_dereference_bh(lb_priv->fp); 182596e2024SJiri Pirko if (unlikely(!fp)) 183596e2024SJiri Pirko return 0; 184596e2024SJiri Pirko lhash = SK_RUN_FILTER(fp, skb); 185596e2024SJiri Pirko c = (char *) &lhash; 186596e2024SJiri Pirko return c[0] ^ c[1] ^ c[2] ^ c[3]; 187596e2024SJiri Pirko } 188596e2024SJiri Pirko 189ab8250d7SJiri Pirko static void lb_update_tx_stats(unsigned int tx_bytes, struct lb_priv *lb_priv, 190ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv, 191ab8250d7SJiri Pirko unsigned char hash) 192ab8250d7SJiri Pirko { 193ab8250d7SJiri Pirko struct lb_pcpu_stats *pcpu_stats; 194ab8250d7SJiri Pirko struct lb_stats *port_stats; 195ab8250d7SJiri Pirko struct lb_stats *hash_stats; 196ab8250d7SJiri Pirko 197ab8250d7SJiri Pirko pcpu_stats = this_cpu_ptr(lb_priv->pcpu_stats); 198ab8250d7SJiri Pirko port_stats = this_cpu_ptr(lb_port_priv->pcpu_stats); 199ab8250d7SJiri Pirko hash_stats = &pcpu_stats->hash_stats[hash]; 200ab8250d7SJiri Pirko u64_stats_update_begin(&pcpu_stats->syncp); 201ab8250d7SJiri Pirko port_stats->tx_bytes += tx_bytes; 202ab8250d7SJiri Pirko hash_stats->tx_bytes += tx_bytes; 203ab8250d7SJiri Pirko u64_stats_update_end(&pcpu_stats->syncp); 204ab8250d7SJiri Pirko } 205ab8250d7SJiri Pirko 206596e2024SJiri Pirko static bool lb_transmit(struct team *team, struct sk_buff *skb) 207596e2024SJiri Pirko { 208ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 209ab8250d7SJiri Pirko lb_select_tx_port_func_t *select_tx_port_func; 21001d7f30aSJiri Pirko struct team_port *port; 211ab8250d7SJiri Pirko unsigned char hash; 212ab8250d7SJiri Pirko unsigned int tx_bytes = skb->len; 21301d7f30aSJiri Pirko 214ab8250d7SJiri Pirko hash = lb_get_skb_hash(lb_priv, skb); 215d1904fbdSJiri Pirko select_tx_port_func = rcu_dereference_bh(lb_priv->select_tx_port_func); 216ab8250d7SJiri Pirko port = select_tx_port_func(team, lb_priv, skb, hash); 21701d7f30aSJiri Pirko if (unlikely(!port)) 21801d7f30aSJiri Pirko goto drop; 219bd2d0837SJiri Pirko if (team_dev_queue_xmit(team, port, skb)) 22001d7f30aSJiri Pirko return false; 221ab8250d7SJiri Pirko lb_update_tx_stats(tx_bytes, lb_priv, get_lb_port_priv(port), hash); 22201d7f30aSJiri Pirko return true; 22301d7f30aSJiri Pirko 22401d7f30aSJiri Pirko drop: 22501d7f30aSJiri Pirko dev_kfree_skb_any(skb); 22601d7f30aSJiri Pirko return false; 22701d7f30aSJiri Pirko } 22801d7f30aSJiri Pirko 22980f7c668SJiri Pirko static int lb_bpf_func_get(struct team *team, struct team_gsetter_ctx *ctx) 23001d7f30aSJiri Pirko { 231ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 232ab8250d7SJiri Pirko 233ab8250d7SJiri Pirko if (!lb_priv->ex->orig_fprog) { 23480f7c668SJiri Pirko ctx->data.bin_val.len = 0; 23580f7c668SJiri Pirko ctx->data.bin_val.ptr = NULL; 23601d7f30aSJiri Pirko return 0; 23780f7c668SJiri Pirko } 238ab8250d7SJiri Pirko ctx->data.bin_val.len = lb_priv->ex->orig_fprog->len * 23901d7f30aSJiri Pirko sizeof(struct sock_filter); 240ab8250d7SJiri Pirko ctx->data.bin_val.ptr = lb_priv->ex->orig_fprog->filter; 24101d7f30aSJiri Pirko return 0; 24201d7f30aSJiri Pirko } 24301d7f30aSJiri Pirko 24401d7f30aSJiri Pirko static int __fprog_create(struct sock_fprog **pfprog, u32 data_len, 24580f7c668SJiri Pirko const void *data) 24601d7f30aSJiri Pirko { 24701d7f30aSJiri Pirko struct sock_fprog *fprog; 24801d7f30aSJiri Pirko struct sock_filter *filter = (struct sock_filter *) data; 24901d7f30aSJiri Pirko 25001d7f30aSJiri Pirko if (data_len % sizeof(struct sock_filter)) 25101d7f30aSJiri Pirko return -EINVAL; 25201d7f30aSJiri Pirko fprog = kmalloc(sizeof(struct sock_fprog), GFP_KERNEL); 25301d7f30aSJiri Pirko if (!fprog) 25401d7f30aSJiri Pirko return -ENOMEM; 25501d7f30aSJiri Pirko fprog->filter = kmemdup(filter, data_len, GFP_KERNEL); 25601d7f30aSJiri Pirko if (!fprog->filter) { 25701d7f30aSJiri Pirko kfree(fprog); 25801d7f30aSJiri Pirko return -ENOMEM; 25901d7f30aSJiri Pirko } 26001d7f30aSJiri Pirko fprog->len = data_len / sizeof(struct sock_filter); 26101d7f30aSJiri Pirko *pfprog = fprog; 26201d7f30aSJiri Pirko return 0; 26301d7f30aSJiri Pirko } 26401d7f30aSJiri Pirko 26501d7f30aSJiri Pirko static void __fprog_destroy(struct sock_fprog *fprog) 26601d7f30aSJiri Pirko { 26701d7f30aSJiri Pirko kfree(fprog->filter); 26801d7f30aSJiri Pirko kfree(fprog); 26901d7f30aSJiri Pirko } 27001d7f30aSJiri Pirko 27180f7c668SJiri Pirko static int lb_bpf_func_set(struct team *team, struct team_gsetter_ctx *ctx) 27201d7f30aSJiri Pirko { 273ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 27401d7f30aSJiri Pirko struct sk_filter *fp = NULL; 2756dab015cSJiri Pirko struct sk_filter *orig_fp; 27601d7f30aSJiri Pirko struct sock_fprog *fprog = NULL; 27701d7f30aSJiri Pirko int err; 27801d7f30aSJiri Pirko 27980f7c668SJiri Pirko if (ctx->data.bin_val.len) { 28080f7c668SJiri Pirko err = __fprog_create(&fprog, ctx->data.bin_val.len, 28180f7c668SJiri Pirko ctx->data.bin_val.ptr); 28201d7f30aSJiri Pirko if (err) 28301d7f30aSJiri Pirko return err; 28401d7f30aSJiri Pirko err = sk_unattached_filter_create(&fp, fprog); 28501d7f30aSJiri Pirko if (err) { 28601d7f30aSJiri Pirko __fprog_destroy(fprog); 28701d7f30aSJiri Pirko return err; 28801d7f30aSJiri Pirko } 28901d7f30aSJiri Pirko } 29001d7f30aSJiri Pirko 291ab8250d7SJiri Pirko if (lb_priv->ex->orig_fprog) { 29201d7f30aSJiri Pirko /* Clear old filter data */ 293ab8250d7SJiri Pirko __fprog_destroy(lb_priv->ex->orig_fprog); 2946dab015cSJiri Pirko orig_fp = rcu_dereference_protected(lb_priv->fp, 2956dab015cSJiri Pirko lockdep_is_held(&team->lock)); 2966dab015cSJiri Pirko sk_unattached_filter_destroy(orig_fp); 29701d7f30aSJiri Pirko } 29801d7f30aSJiri Pirko 299ab8250d7SJiri Pirko rcu_assign_pointer(lb_priv->fp, fp); 300ab8250d7SJiri Pirko lb_priv->ex->orig_fprog = fprog; 301ab8250d7SJiri Pirko return 0; 302ab8250d7SJiri Pirko } 303ab8250d7SJiri Pirko 304ab8250d7SJiri Pirko static int lb_tx_method_get(struct team *team, struct team_gsetter_ctx *ctx) 305ab8250d7SJiri Pirko { 306ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 3076dab015cSJiri Pirko lb_select_tx_port_func_t *func; 308ab8250d7SJiri Pirko char *name; 309ab8250d7SJiri Pirko 3106dab015cSJiri Pirko func = rcu_dereference_protected(lb_priv->select_tx_port_func, 3116dab015cSJiri Pirko lockdep_is_held(&team->lock)); 3126dab015cSJiri Pirko name = lb_select_tx_port_get_name(func); 313ab8250d7SJiri Pirko BUG_ON(!name); 314ab8250d7SJiri Pirko ctx->data.str_val = name; 315ab8250d7SJiri Pirko return 0; 316ab8250d7SJiri Pirko } 317ab8250d7SJiri Pirko 318ab8250d7SJiri Pirko static int lb_tx_method_set(struct team *team, struct team_gsetter_ctx *ctx) 319ab8250d7SJiri Pirko { 320ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 321ab8250d7SJiri Pirko lb_select_tx_port_func_t *func; 322ab8250d7SJiri Pirko 323ab8250d7SJiri Pirko func = lb_select_tx_port_get_func(ctx->data.str_val); 324ab8250d7SJiri Pirko if (!func) 325ab8250d7SJiri Pirko return -EINVAL; 326ab8250d7SJiri Pirko rcu_assign_pointer(lb_priv->select_tx_port_func, func); 327ab8250d7SJiri Pirko return 0; 328ab8250d7SJiri Pirko } 329ab8250d7SJiri Pirko 330ab8250d7SJiri Pirko static int lb_tx_hash_to_port_mapping_init(struct team *team, 331ab8250d7SJiri Pirko struct team_option_inst_info *info) 332ab8250d7SJiri Pirko { 333ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 334ab8250d7SJiri Pirko unsigned char hash = info->array_index; 335ab8250d7SJiri Pirko 336ab8250d7SJiri Pirko LB_HTPM_OPT_INST_INFO_BY_HASH(lb_priv, hash) = info; 337ab8250d7SJiri Pirko return 0; 338ab8250d7SJiri Pirko } 339ab8250d7SJiri Pirko 340ab8250d7SJiri Pirko static int lb_tx_hash_to_port_mapping_get(struct team *team, 341ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 342ab8250d7SJiri Pirko { 343ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 344ab8250d7SJiri Pirko struct team_port *port; 345ab8250d7SJiri Pirko unsigned char hash = ctx->info->array_index; 346ab8250d7SJiri Pirko 347ab8250d7SJiri Pirko port = LB_HTPM_PORT_BY_HASH(lb_priv, hash); 348ab8250d7SJiri Pirko ctx->data.u32_val = port ? port->dev->ifindex : 0; 349ab8250d7SJiri Pirko return 0; 350ab8250d7SJiri Pirko } 351ab8250d7SJiri Pirko 352ab8250d7SJiri Pirko static int lb_tx_hash_to_port_mapping_set(struct team *team, 353ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 354ab8250d7SJiri Pirko { 355ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 356ab8250d7SJiri Pirko struct team_port *port; 357ab8250d7SJiri Pirko unsigned char hash = ctx->info->array_index; 358ab8250d7SJiri Pirko 359ab8250d7SJiri Pirko list_for_each_entry(port, &team->port_list, list) { 36052a4fd77SJiri Pirko if (ctx->data.u32_val == port->dev->ifindex && 36152a4fd77SJiri Pirko team_port_enabled(port)) { 362ab8250d7SJiri Pirko rcu_assign_pointer(LB_HTPM_PORT_BY_HASH(lb_priv, hash), 363ab8250d7SJiri Pirko port); 364ab8250d7SJiri Pirko return 0; 365ab8250d7SJiri Pirko } 366ab8250d7SJiri Pirko } 367ab8250d7SJiri Pirko return -ENODEV; 368ab8250d7SJiri Pirko } 369ab8250d7SJiri Pirko 370ab8250d7SJiri Pirko static int lb_hash_stats_init(struct team *team, 371ab8250d7SJiri Pirko struct team_option_inst_info *info) 372ab8250d7SJiri Pirko { 373ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 374ab8250d7SJiri Pirko unsigned char hash = info->array_index; 375ab8250d7SJiri Pirko 376ab8250d7SJiri Pirko lb_priv->ex->stats.info[hash].opt_inst_info = info; 377ab8250d7SJiri Pirko return 0; 378ab8250d7SJiri Pirko } 379ab8250d7SJiri Pirko 380ab8250d7SJiri Pirko static int lb_hash_stats_get(struct team *team, struct team_gsetter_ctx *ctx) 381ab8250d7SJiri Pirko { 382ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 383ab8250d7SJiri Pirko unsigned char hash = ctx->info->array_index; 384ab8250d7SJiri Pirko 385ab8250d7SJiri Pirko ctx->data.bin_val.ptr = &lb_priv->ex->stats.info[hash].stats; 386ab8250d7SJiri Pirko ctx->data.bin_val.len = sizeof(struct lb_stats); 387ab8250d7SJiri Pirko return 0; 388ab8250d7SJiri Pirko } 389ab8250d7SJiri Pirko 390ab8250d7SJiri Pirko static int lb_port_stats_init(struct team *team, 391ab8250d7SJiri Pirko struct team_option_inst_info *info) 392ab8250d7SJiri Pirko { 393ab8250d7SJiri Pirko struct team_port *port = info->port; 394ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 395ab8250d7SJiri Pirko 396ab8250d7SJiri Pirko lb_port_priv->stats_info.opt_inst_info = info; 397ab8250d7SJiri Pirko return 0; 398ab8250d7SJiri Pirko } 399ab8250d7SJiri Pirko 400ab8250d7SJiri Pirko static int lb_port_stats_get(struct team *team, struct team_gsetter_ctx *ctx) 401ab8250d7SJiri Pirko { 402ab8250d7SJiri Pirko struct team_port *port = ctx->info->port; 403ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 404ab8250d7SJiri Pirko 405ab8250d7SJiri Pirko ctx->data.bin_val.ptr = &lb_port_priv->stats_info.stats; 406ab8250d7SJiri Pirko ctx->data.bin_val.len = sizeof(struct lb_stats); 407ab8250d7SJiri Pirko return 0; 408ab8250d7SJiri Pirko } 409ab8250d7SJiri Pirko 410ab8250d7SJiri Pirko static void __lb_stats_info_refresh_prepare(struct lb_stats_info *s_info) 411ab8250d7SJiri Pirko { 412ab8250d7SJiri Pirko memcpy(&s_info->last_stats, &s_info->stats, sizeof(struct lb_stats)); 413ab8250d7SJiri Pirko memset(&s_info->stats, 0, sizeof(struct lb_stats)); 414ab8250d7SJiri Pirko } 415ab8250d7SJiri Pirko 416ab8250d7SJiri Pirko static bool __lb_stats_info_refresh_check(struct lb_stats_info *s_info, 417ab8250d7SJiri Pirko struct team *team) 418ab8250d7SJiri Pirko { 419ab8250d7SJiri Pirko if (memcmp(&s_info->last_stats, &s_info->stats, 420ab8250d7SJiri Pirko sizeof(struct lb_stats))) { 421ab8250d7SJiri Pirko team_option_inst_set_change(s_info->opt_inst_info); 422ab8250d7SJiri Pirko return true; 423ab8250d7SJiri Pirko } 424ab8250d7SJiri Pirko return false; 425ab8250d7SJiri Pirko } 426ab8250d7SJiri Pirko 427ab8250d7SJiri Pirko static void __lb_one_cpu_stats_add(struct lb_stats *acc_stats, 428ab8250d7SJiri Pirko struct lb_stats *cpu_stats, 429ab8250d7SJiri Pirko struct u64_stats_sync *syncp) 430ab8250d7SJiri Pirko { 431ab8250d7SJiri Pirko unsigned int start; 432ab8250d7SJiri Pirko struct lb_stats tmp; 433ab8250d7SJiri Pirko 434ab8250d7SJiri Pirko do { 43557a7744eSEric W. Biederman start = u64_stats_fetch_begin_irq(syncp); 436ab8250d7SJiri Pirko tmp.tx_bytes = cpu_stats->tx_bytes; 43757a7744eSEric W. Biederman } while (u64_stats_fetch_retry_irq(syncp, start)); 438ab8250d7SJiri Pirko acc_stats->tx_bytes += tmp.tx_bytes; 439ab8250d7SJiri Pirko } 440ab8250d7SJiri Pirko 441ab8250d7SJiri Pirko static void lb_stats_refresh(struct work_struct *work) 442ab8250d7SJiri Pirko { 443ab8250d7SJiri Pirko struct team *team; 444ab8250d7SJiri Pirko struct lb_priv *lb_priv; 445ab8250d7SJiri Pirko struct lb_priv_ex *lb_priv_ex; 446ab8250d7SJiri Pirko struct lb_pcpu_stats *pcpu_stats; 447ab8250d7SJiri Pirko struct lb_stats *stats; 448ab8250d7SJiri Pirko struct lb_stats_info *s_info; 449ab8250d7SJiri Pirko struct team_port *port; 450ab8250d7SJiri Pirko bool changed = false; 451ab8250d7SJiri Pirko int i; 452ab8250d7SJiri Pirko int j; 453ab8250d7SJiri Pirko 454ab8250d7SJiri Pirko lb_priv_ex = container_of(work, struct lb_priv_ex, 455ab8250d7SJiri Pirko stats.refresh_dw.work); 456ab8250d7SJiri Pirko 457ab8250d7SJiri Pirko team = lb_priv_ex->team; 458ab8250d7SJiri Pirko lb_priv = get_lb_priv(team); 459ab8250d7SJiri Pirko 460ab8250d7SJiri Pirko if (!mutex_trylock(&team->lock)) { 461ab8250d7SJiri Pirko schedule_delayed_work(&lb_priv_ex->stats.refresh_dw, 0); 462ab8250d7SJiri Pirko return; 463ab8250d7SJiri Pirko } 464ab8250d7SJiri Pirko 465ab8250d7SJiri Pirko for (j = 0; j < LB_TX_HASHTABLE_SIZE; j++) { 466ab8250d7SJiri Pirko s_info = &lb_priv->ex->stats.info[j]; 467ab8250d7SJiri Pirko __lb_stats_info_refresh_prepare(s_info); 468ab8250d7SJiri Pirko for_each_possible_cpu(i) { 469ab8250d7SJiri Pirko pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); 470ab8250d7SJiri Pirko stats = &pcpu_stats->hash_stats[j]; 471ab8250d7SJiri Pirko __lb_one_cpu_stats_add(&s_info->stats, stats, 472ab8250d7SJiri Pirko &pcpu_stats->syncp); 473ab8250d7SJiri Pirko } 474ab8250d7SJiri Pirko changed |= __lb_stats_info_refresh_check(s_info, team); 475ab8250d7SJiri Pirko } 476ab8250d7SJiri Pirko 477ab8250d7SJiri Pirko list_for_each_entry(port, &team->port_list, list) { 478ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 479ab8250d7SJiri Pirko 480ab8250d7SJiri Pirko s_info = &lb_port_priv->stats_info; 481ab8250d7SJiri Pirko __lb_stats_info_refresh_prepare(s_info); 482ab8250d7SJiri Pirko for_each_possible_cpu(i) { 483ab8250d7SJiri Pirko pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); 484ab8250d7SJiri Pirko stats = per_cpu_ptr(lb_port_priv->pcpu_stats, i); 485ab8250d7SJiri Pirko __lb_one_cpu_stats_add(&s_info->stats, stats, 486ab8250d7SJiri Pirko &pcpu_stats->syncp); 487ab8250d7SJiri Pirko } 488ab8250d7SJiri Pirko changed |= __lb_stats_info_refresh_check(s_info, team); 489ab8250d7SJiri Pirko } 490ab8250d7SJiri Pirko 491ab8250d7SJiri Pirko if (changed) 492ab8250d7SJiri Pirko team_options_change_check(team); 493ab8250d7SJiri Pirko 494ab8250d7SJiri Pirko schedule_delayed_work(&lb_priv_ex->stats.refresh_dw, 495ab8250d7SJiri Pirko (lb_priv_ex->stats.refresh_interval * HZ) / 10); 496ab8250d7SJiri Pirko 497ab8250d7SJiri Pirko mutex_unlock(&team->lock); 498ab8250d7SJiri Pirko } 499ab8250d7SJiri Pirko 500ab8250d7SJiri Pirko static int lb_stats_refresh_interval_get(struct team *team, 501ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 502ab8250d7SJiri Pirko { 503ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 504ab8250d7SJiri Pirko 505ab8250d7SJiri Pirko ctx->data.u32_val = lb_priv->ex->stats.refresh_interval; 506ab8250d7SJiri Pirko return 0; 507ab8250d7SJiri Pirko } 508ab8250d7SJiri Pirko 509ab8250d7SJiri Pirko static int lb_stats_refresh_interval_set(struct team *team, 510ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 511ab8250d7SJiri Pirko { 512ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 513ab8250d7SJiri Pirko unsigned int interval; 514ab8250d7SJiri Pirko 515ab8250d7SJiri Pirko interval = ctx->data.u32_val; 516ab8250d7SJiri Pirko if (lb_priv->ex->stats.refresh_interval == interval) 517ab8250d7SJiri Pirko return 0; 518ab8250d7SJiri Pirko lb_priv->ex->stats.refresh_interval = interval; 519ab8250d7SJiri Pirko if (interval) 520ab8250d7SJiri Pirko schedule_delayed_work(&lb_priv->ex->stats.refresh_dw, 0); 521ab8250d7SJiri Pirko else 522ab8250d7SJiri Pirko cancel_delayed_work(&lb_priv->ex->stats.refresh_dw); 52301d7f30aSJiri Pirko return 0; 52401d7f30aSJiri Pirko } 52501d7f30aSJiri Pirko 52601d7f30aSJiri Pirko static const struct team_option lb_options[] = { 52701d7f30aSJiri Pirko { 52801d7f30aSJiri Pirko .name = "bpf_hash_func", 52901d7f30aSJiri Pirko .type = TEAM_OPTION_TYPE_BINARY, 53001d7f30aSJiri Pirko .getter = lb_bpf_func_get, 53101d7f30aSJiri Pirko .setter = lb_bpf_func_set, 53201d7f30aSJiri Pirko }, 533ab8250d7SJiri Pirko { 534ab8250d7SJiri Pirko .name = "lb_tx_method", 535ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_STRING, 536ab8250d7SJiri Pirko .getter = lb_tx_method_get, 537ab8250d7SJiri Pirko .setter = lb_tx_method_set, 538ab8250d7SJiri Pirko }, 539ab8250d7SJiri Pirko { 540ab8250d7SJiri Pirko .name = "lb_tx_hash_to_port_mapping", 541ab8250d7SJiri Pirko .array_size = LB_TX_HASHTABLE_SIZE, 542ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_U32, 543ab8250d7SJiri Pirko .init = lb_tx_hash_to_port_mapping_init, 544ab8250d7SJiri Pirko .getter = lb_tx_hash_to_port_mapping_get, 545ab8250d7SJiri Pirko .setter = lb_tx_hash_to_port_mapping_set, 546ab8250d7SJiri Pirko }, 547ab8250d7SJiri Pirko { 548ab8250d7SJiri Pirko .name = "lb_hash_stats", 549ab8250d7SJiri Pirko .array_size = LB_TX_HASHTABLE_SIZE, 550ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_BINARY, 551ab8250d7SJiri Pirko .init = lb_hash_stats_init, 552ab8250d7SJiri Pirko .getter = lb_hash_stats_get, 553ab8250d7SJiri Pirko }, 554ab8250d7SJiri Pirko { 555ab8250d7SJiri Pirko .name = "lb_port_stats", 556ab8250d7SJiri Pirko .per_port = true, 557ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_BINARY, 558ab8250d7SJiri Pirko .init = lb_port_stats_init, 559ab8250d7SJiri Pirko .getter = lb_port_stats_get, 560ab8250d7SJiri Pirko }, 561ab8250d7SJiri Pirko { 562ab8250d7SJiri Pirko .name = "lb_stats_refresh_interval", 563ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_U32, 564ab8250d7SJiri Pirko .getter = lb_stats_refresh_interval_get, 565ab8250d7SJiri Pirko .setter = lb_stats_refresh_interval_set, 566ab8250d7SJiri Pirko }, 56701d7f30aSJiri Pirko }; 56801d7f30aSJiri Pirko 569cade4555SJiri Pirko static int lb_init(struct team *team) 57001d7f30aSJiri Pirko { 571ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 572ab8250d7SJiri Pirko lb_select_tx_port_func_t *func; 573827da44cSJohn Stultz int i, err; 574ab8250d7SJiri Pirko 575ab8250d7SJiri Pirko /* set default tx port selector */ 576ab8250d7SJiri Pirko func = lb_select_tx_port_get_func("hash"); 577ab8250d7SJiri Pirko BUG_ON(!func); 578ab8250d7SJiri Pirko rcu_assign_pointer(lb_priv->select_tx_port_func, func); 579ab8250d7SJiri Pirko 580ab8250d7SJiri Pirko lb_priv->ex = kzalloc(sizeof(*lb_priv->ex), GFP_KERNEL); 581ab8250d7SJiri Pirko if (!lb_priv->ex) 582ab8250d7SJiri Pirko return -ENOMEM; 583ab8250d7SJiri Pirko lb_priv->ex->team = team; 584ab8250d7SJiri Pirko 585ab8250d7SJiri Pirko lb_priv->pcpu_stats = alloc_percpu(struct lb_pcpu_stats); 586ab8250d7SJiri Pirko if (!lb_priv->pcpu_stats) { 587ab8250d7SJiri Pirko err = -ENOMEM; 588ab8250d7SJiri Pirko goto err_alloc_pcpu_stats; 589ab8250d7SJiri Pirko } 590ab8250d7SJiri Pirko 591827da44cSJohn Stultz for_each_possible_cpu(i) { 592827da44cSJohn Stultz struct lb_pcpu_stats *team_lb_stats; 593827da44cSJohn Stultz team_lb_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); 594827da44cSJohn Stultz u64_stats_init(&team_lb_stats->syncp); 595827da44cSJohn Stultz } 596827da44cSJohn Stultz 597827da44cSJohn Stultz 598ab8250d7SJiri Pirko INIT_DELAYED_WORK(&lb_priv->ex->stats.refresh_dw, lb_stats_refresh); 599ab8250d7SJiri Pirko 600ab8250d7SJiri Pirko err = team_options_register(team, lb_options, ARRAY_SIZE(lb_options)); 601ab8250d7SJiri Pirko if (err) 602ab8250d7SJiri Pirko goto err_options_register; 603ab8250d7SJiri Pirko return 0; 604ab8250d7SJiri Pirko 605ab8250d7SJiri Pirko err_options_register: 606ab8250d7SJiri Pirko free_percpu(lb_priv->pcpu_stats); 607ab8250d7SJiri Pirko err_alloc_pcpu_stats: 608ab8250d7SJiri Pirko kfree(lb_priv->ex); 609ab8250d7SJiri Pirko return err; 61001d7f30aSJiri Pirko } 61101d7f30aSJiri Pirko 612cade4555SJiri Pirko static void lb_exit(struct team *team) 61301d7f30aSJiri Pirko { 614ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 615ab8250d7SJiri Pirko 61601d7f30aSJiri Pirko team_options_unregister(team, lb_options, 61701d7f30aSJiri Pirko ARRAY_SIZE(lb_options)); 618ab8250d7SJiri Pirko cancel_delayed_work_sync(&lb_priv->ex->stats.refresh_dw); 619ab8250d7SJiri Pirko free_percpu(lb_priv->pcpu_stats); 620ab8250d7SJiri Pirko kfree(lb_priv->ex); 621ab8250d7SJiri Pirko } 622ab8250d7SJiri Pirko 623ab8250d7SJiri Pirko static int lb_port_enter(struct team *team, struct team_port *port) 624ab8250d7SJiri Pirko { 625ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 626ab8250d7SJiri Pirko 627ab8250d7SJiri Pirko lb_port_priv->pcpu_stats = alloc_percpu(struct lb_stats); 628ab8250d7SJiri Pirko if (!lb_port_priv->pcpu_stats) 629ab8250d7SJiri Pirko return -ENOMEM; 630ab8250d7SJiri Pirko return 0; 631ab8250d7SJiri Pirko } 632ab8250d7SJiri Pirko 633ab8250d7SJiri Pirko static void lb_port_leave(struct team *team, struct team_port *port) 634ab8250d7SJiri Pirko { 635ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 636ab8250d7SJiri Pirko 637ab8250d7SJiri Pirko free_percpu(lb_port_priv->pcpu_stats); 638ab8250d7SJiri Pirko } 639ab8250d7SJiri Pirko 640ab8250d7SJiri Pirko static void lb_port_disabled(struct team *team, struct team_port *port) 641ab8250d7SJiri Pirko { 642ab8250d7SJiri Pirko lb_tx_hash_to_port_mapping_null_port(team, port); 64301d7f30aSJiri Pirko } 64401d7f30aSJiri Pirko 64501d7f30aSJiri Pirko static const struct team_mode_ops lb_mode_ops = { 64601d7f30aSJiri Pirko .init = lb_init, 64701d7f30aSJiri Pirko .exit = lb_exit, 648ab8250d7SJiri Pirko .port_enter = lb_port_enter, 649ab8250d7SJiri Pirko .port_leave = lb_port_leave, 650ab8250d7SJiri Pirko .port_disabled = lb_port_disabled, 65101d7f30aSJiri Pirko .transmit = lb_transmit, 65201d7f30aSJiri Pirko }; 65301d7f30aSJiri Pirko 6540402788aSJiri Pirko static const struct team_mode lb_mode = { 65501d7f30aSJiri Pirko .kind = "loadbalance", 65601d7f30aSJiri Pirko .owner = THIS_MODULE, 65701d7f30aSJiri Pirko .priv_size = sizeof(struct lb_priv), 658ab8250d7SJiri Pirko .port_priv_size = sizeof(struct lb_port_priv), 65901d7f30aSJiri Pirko .ops = &lb_mode_ops, 66001d7f30aSJiri Pirko }; 66101d7f30aSJiri Pirko 66201d7f30aSJiri Pirko static int __init lb_init_module(void) 66301d7f30aSJiri Pirko { 66401d7f30aSJiri Pirko return team_mode_register(&lb_mode); 66501d7f30aSJiri Pirko } 66601d7f30aSJiri Pirko 66701d7f30aSJiri Pirko static void __exit lb_cleanup_module(void) 66801d7f30aSJiri Pirko { 66901d7f30aSJiri Pirko team_mode_unregister(&lb_mode); 67001d7f30aSJiri Pirko } 67101d7f30aSJiri Pirko 67201d7f30aSJiri Pirko module_init(lb_init_module); 67301d7f30aSJiri Pirko module_exit(lb_cleanup_module); 67401d7f30aSJiri Pirko 67501d7f30aSJiri Pirko MODULE_LICENSE("GPL v2"); 67601d7f30aSJiri Pirko MODULE_AUTHOR("Jiri Pirko <jpirko@redhat.com>"); 67701d7f30aSJiri Pirko MODULE_DESCRIPTION("Load-balancing mode for team"); 67801d7f30aSJiri Pirko MODULE_ALIAS("team-mode-loadbalance"); 679