101d7f30aSJiri Pirko /* 201d7f30aSJiri Pirko * drivers/net/team/team_mode_loadbalance.c - Load-balancing mode for team 301d7f30aSJiri Pirko * Copyright (c) 2012 Jiri Pirko <jpirko@redhat.com> 401d7f30aSJiri Pirko * 501d7f30aSJiri Pirko * This program is free software; you can redistribute it and/or modify 601d7f30aSJiri Pirko * it under the terms of the GNU General Public License as published by 701d7f30aSJiri Pirko * the Free Software Foundation; either version 2 of the License, or 801d7f30aSJiri Pirko * (at your option) any later version. 901d7f30aSJiri Pirko */ 1001d7f30aSJiri Pirko 1101d7f30aSJiri Pirko #include <linux/kernel.h> 1201d7f30aSJiri Pirko #include <linux/types.h> 1301d7f30aSJiri Pirko #include <linux/module.h> 1401d7f30aSJiri Pirko #include <linux/init.h> 1501d7f30aSJiri Pirko #include <linux/errno.h> 1601d7f30aSJiri Pirko #include <linux/netdevice.h> 1701d7f30aSJiri Pirko #include <linux/filter.h> 1801d7f30aSJiri Pirko #include <linux/if_team.h> 1901d7f30aSJiri Pirko 20ab8250d7SJiri Pirko struct lb_priv; 21ab8250d7SJiri Pirko 22ab8250d7SJiri Pirko typedef struct team_port *lb_select_tx_port_func_t(struct team *, 23ab8250d7SJiri Pirko struct lb_priv *, 24ab8250d7SJiri Pirko struct sk_buff *, 25ab8250d7SJiri Pirko unsigned char); 26ab8250d7SJiri Pirko 27ab8250d7SJiri Pirko #define LB_TX_HASHTABLE_SIZE 256 /* hash is a char */ 28ab8250d7SJiri Pirko 29ab8250d7SJiri Pirko struct lb_stats { 30ab8250d7SJiri Pirko u64 tx_bytes; 3101d7f30aSJiri Pirko }; 3201d7f30aSJiri Pirko 33ab8250d7SJiri Pirko struct lb_pcpu_stats { 34ab8250d7SJiri Pirko struct lb_stats hash_stats[LB_TX_HASHTABLE_SIZE]; 35ab8250d7SJiri Pirko struct u64_stats_sync syncp; 36ab8250d7SJiri Pirko }; 37ab8250d7SJiri Pirko 38ab8250d7SJiri Pirko struct lb_stats_info { 39ab8250d7SJiri Pirko struct lb_stats stats; 40ab8250d7SJiri Pirko struct lb_stats last_stats; 41ab8250d7SJiri Pirko struct team_option_inst_info *opt_inst_info; 42ab8250d7SJiri Pirko }; 43ab8250d7SJiri Pirko 44ab8250d7SJiri Pirko struct lb_port_mapping { 45ab8250d7SJiri Pirko struct team_port __rcu *port; 46ab8250d7SJiri Pirko struct team_option_inst_info *opt_inst_info; 47ab8250d7SJiri Pirko }; 48ab8250d7SJiri Pirko 49ab8250d7SJiri Pirko struct lb_priv_ex { 50ab8250d7SJiri Pirko struct team *team; 51ab8250d7SJiri Pirko struct lb_port_mapping tx_hash_to_port_mapping[LB_TX_HASHTABLE_SIZE]; 52ab8250d7SJiri Pirko struct sock_fprog *orig_fprog; 53ab8250d7SJiri Pirko struct { 54ab8250d7SJiri Pirko unsigned int refresh_interval; /* in tenths of second */ 55ab8250d7SJiri Pirko struct delayed_work refresh_dw; 56ab8250d7SJiri Pirko struct lb_stats_info info[LB_TX_HASHTABLE_SIZE]; 57ab8250d7SJiri Pirko } stats; 58ab8250d7SJiri Pirko }; 59ab8250d7SJiri Pirko 60ab8250d7SJiri Pirko struct lb_priv { 61ab8250d7SJiri Pirko struct sk_filter __rcu *fp; 62ab8250d7SJiri Pirko lb_select_tx_port_func_t __rcu *select_tx_port_func; 63ab8250d7SJiri Pirko struct lb_pcpu_stats __percpu *pcpu_stats; 64ab8250d7SJiri Pirko struct lb_priv_ex *ex; /* priv extension */ 65ab8250d7SJiri Pirko }; 66ab8250d7SJiri Pirko 67ab8250d7SJiri Pirko static struct lb_priv *get_lb_priv(struct team *team) 6801d7f30aSJiri Pirko { 6901d7f30aSJiri Pirko return (struct lb_priv *) &team->mode_priv; 7001d7f30aSJiri Pirko } 7101d7f30aSJiri Pirko 72ab8250d7SJiri Pirko struct lb_port_priv { 73ab8250d7SJiri Pirko struct lb_stats __percpu *pcpu_stats; 74ab8250d7SJiri Pirko struct lb_stats_info stats_info; 75ab8250d7SJiri Pirko }; 76ab8250d7SJiri Pirko 77ab8250d7SJiri Pirko static struct lb_port_priv *get_lb_port_priv(struct team_port *port) 78ab8250d7SJiri Pirko { 79ab8250d7SJiri Pirko return (struct lb_port_priv *) &port->mode_priv; 80ab8250d7SJiri Pirko } 81ab8250d7SJiri Pirko 82ab8250d7SJiri Pirko #define LB_HTPM_PORT_BY_HASH(lp_priv, hash) \ 83ab8250d7SJiri Pirko (lb_priv)->ex->tx_hash_to_port_mapping[hash].port 84ab8250d7SJiri Pirko 85ab8250d7SJiri Pirko #define LB_HTPM_OPT_INST_INFO_BY_HASH(lp_priv, hash) \ 86ab8250d7SJiri Pirko (lb_priv)->ex->tx_hash_to_port_mapping[hash].opt_inst_info 87ab8250d7SJiri Pirko 88ab8250d7SJiri Pirko static void lb_tx_hash_to_port_mapping_null_port(struct team *team, 89ab8250d7SJiri Pirko struct team_port *port) 90ab8250d7SJiri Pirko { 91ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 92ab8250d7SJiri Pirko bool changed = false; 93ab8250d7SJiri Pirko int i; 94ab8250d7SJiri Pirko 95ab8250d7SJiri Pirko for (i = 0; i < LB_TX_HASHTABLE_SIZE; i++) { 96ab8250d7SJiri Pirko struct lb_port_mapping *pm; 97ab8250d7SJiri Pirko 98ab8250d7SJiri Pirko pm = &lb_priv->ex->tx_hash_to_port_mapping[i]; 99ab8250d7SJiri Pirko if (pm->port == port) { 100ab8250d7SJiri Pirko rcu_assign_pointer(pm->port, NULL); 101ab8250d7SJiri Pirko team_option_inst_set_change(pm->opt_inst_info); 102ab8250d7SJiri Pirko changed = true; 103ab8250d7SJiri Pirko } 104ab8250d7SJiri Pirko } 105ab8250d7SJiri Pirko if (changed) 106ab8250d7SJiri Pirko team_options_change_check(team); 107ab8250d7SJiri Pirko } 108ab8250d7SJiri Pirko 109ab8250d7SJiri Pirko /* Basic tx selection based solely by hash */ 110ab8250d7SJiri Pirko static struct team_port *lb_hash_select_tx_port(struct team *team, 111ab8250d7SJiri Pirko struct lb_priv *lb_priv, 112ab8250d7SJiri Pirko struct sk_buff *skb, 113ab8250d7SJiri Pirko unsigned char hash) 114ab8250d7SJiri Pirko { 115ab8250d7SJiri Pirko int port_index; 116ab8250d7SJiri Pirko 117ab8250d7SJiri Pirko port_index = hash % team->en_port_count; 118ab8250d7SJiri Pirko return team_get_port_by_index_rcu(team, port_index); 119ab8250d7SJiri Pirko } 120ab8250d7SJiri Pirko 121ab8250d7SJiri Pirko /* Hash to port mapping select tx port */ 122ab8250d7SJiri Pirko static struct team_port *lb_htpm_select_tx_port(struct team *team, 123ab8250d7SJiri Pirko struct lb_priv *lb_priv, 124ab8250d7SJiri Pirko struct sk_buff *skb, 125ab8250d7SJiri Pirko unsigned char hash) 126ab8250d7SJiri Pirko { 127ab8250d7SJiri Pirko return rcu_dereference(LB_HTPM_PORT_BY_HASH(lb_priv, hash)); 128ab8250d7SJiri Pirko } 129ab8250d7SJiri Pirko 130ab8250d7SJiri Pirko struct lb_select_tx_port { 131ab8250d7SJiri Pirko char *name; 132ab8250d7SJiri Pirko lb_select_tx_port_func_t *func; 133ab8250d7SJiri Pirko }; 134ab8250d7SJiri Pirko 135ab8250d7SJiri Pirko static const struct lb_select_tx_port lb_select_tx_port_list[] = { 136ab8250d7SJiri Pirko { 137ab8250d7SJiri Pirko .name = "hash", 138ab8250d7SJiri Pirko .func = lb_hash_select_tx_port, 139ab8250d7SJiri Pirko }, 140ab8250d7SJiri Pirko { 141ab8250d7SJiri Pirko .name = "hash_to_port_mapping", 142ab8250d7SJiri Pirko .func = lb_htpm_select_tx_port, 143ab8250d7SJiri Pirko }, 144ab8250d7SJiri Pirko }; 145ab8250d7SJiri Pirko #define LB_SELECT_TX_PORT_LIST_COUNT ARRAY_SIZE(lb_select_tx_port_list) 146ab8250d7SJiri Pirko 147ab8250d7SJiri Pirko static char *lb_select_tx_port_get_name(lb_select_tx_port_func_t *func) 148ab8250d7SJiri Pirko { 149ab8250d7SJiri Pirko int i; 150ab8250d7SJiri Pirko 151ab8250d7SJiri Pirko for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) { 152ab8250d7SJiri Pirko const struct lb_select_tx_port *item; 153ab8250d7SJiri Pirko 154ab8250d7SJiri Pirko item = &lb_select_tx_port_list[i]; 155ab8250d7SJiri Pirko if (item->func == func) 156ab8250d7SJiri Pirko return item->name; 157ab8250d7SJiri Pirko } 158ab8250d7SJiri Pirko return NULL; 159ab8250d7SJiri Pirko } 160ab8250d7SJiri Pirko 161ab8250d7SJiri Pirko static lb_select_tx_port_func_t *lb_select_tx_port_get_func(const char *name) 162ab8250d7SJiri Pirko { 163ab8250d7SJiri Pirko int i; 164ab8250d7SJiri Pirko 165ab8250d7SJiri Pirko for (i = 0; i < LB_SELECT_TX_PORT_LIST_COUNT; i++) { 166ab8250d7SJiri Pirko const struct lb_select_tx_port *item; 167ab8250d7SJiri Pirko 168ab8250d7SJiri Pirko item = &lb_select_tx_port_list[i]; 169ab8250d7SJiri Pirko if (!strcmp(item->name, name)) 170ab8250d7SJiri Pirko return item->func; 171ab8250d7SJiri Pirko } 172ab8250d7SJiri Pirko return NULL; 173ab8250d7SJiri Pirko } 174ab8250d7SJiri Pirko 175ab8250d7SJiri Pirko static unsigned int lb_get_skb_hash(struct lb_priv *lb_priv, 176596e2024SJiri Pirko struct sk_buff *skb) 17701d7f30aSJiri Pirko { 17801d7f30aSJiri Pirko struct sk_filter *fp; 179596e2024SJiri Pirko uint32_t lhash; 180596e2024SJiri Pirko unsigned char *c; 181596e2024SJiri Pirko 182596e2024SJiri Pirko fp = rcu_dereference(lb_priv->fp); 183596e2024SJiri Pirko if (unlikely(!fp)) 184596e2024SJiri Pirko return 0; 185596e2024SJiri Pirko lhash = SK_RUN_FILTER(fp, skb); 186596e2024SJiri Pirko c = (char *) &lhash; 187596e2024SJiri Pirko return c[0] ^ c[1] ^ c[2] ^ c[3]; 188596e2024SJiri Pirko } 189596e2024SJiri Pirko 190ab8250d7SJiri Pirko static void lb_update_tx_stats(unsigned int tx_bytes, struct lb_priv *lb_priv, 191ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv, 192ab8250d7SJiri Pirko unsigned char hash) 193ab8250d7SJiri Pirko { 194ab8250d7SJiri Pirko struct lb_pcpu_stats *pcpu_stats; 195ab8250d7SJiri Pirko struct lb_stats *port_stats; 196ab8250d7SJiri Pirko struct lb_stats *hash_stats; 197ab8250d7SJiri Pirko 198ab8250d7SJiri Pirko pcpu_stats = this_cpu_ptr(lb_priv->pcpu_stats); 199ab8250d7SJiri Pirko port_stats = this_cpu_ptr(lb_port_priv->pcpu_stats); 200ab8250d7SJiri Pirko hash_stats = &pcpu_stats->hash_stats[hash]; 201ab8250d7SJiri Pirko u64_stats_update_begin(&pcpu_stats->syncp); 202ab8250d7SJiri Pirko port_stats->tx_bytes += tx_bytes; 203ab8250d7SJiri Pirko hash_stats->tx_bytes += tx_bytes; 204ab8250d7SJiri Pirko u64_stats_update_end(&pcpu_stats->syncp); 205ab8250d7SJiri Pirko } 206ab8250d7SJiri Pirko 207596e2024SJiri Pirko static bool lb_transmit(struct team *team, struct sk_buff *skb) 208596e2024SJiri Pirko { 209ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 210ab8250d7SJiri Pirko lb_select_tx_port_func_t *select_tx_port_func; 21101d7f30aSJiri Pirko struct team_port *port; 212ab8250d7SJiri Pirko unsigned char hash; 213ab8250d7SJiri Pirko unsigned int tx_bytes = skb->len; 21401d7f30aSJiri Pirko 215ab8250d7SJiri Pirko hash = lb_get_skb_hash(lb_priv, skb); 216ab8250d7SJiri Pirko select_tx_port_func = rcu_dereference(lb_priv->select_tx_port_func); 217ab8250d7SJiri Pirko port = select_tx_port_func(team, lb_priv, skb, hash); 21801d7f30aSJiri Pirko if (unlikely(!port)) 21901d7f30aSJiri Pirko goto drop; 22001d7f30aSJiri Pirko skb->dev = port->dev; 22101d7f30aSJiri Pirko if (dev_queue_xmit(skb)) 22201d7f30aSJiri Pirko return false; 223ab8250d7SJiri Pirko lb_update_tx_stats(tx_bytes, lb_priv, get_lb_port_priv(port), hash); 22401d7f30aSJiri Pirko return true; 22501d7f30aSJiri Pirko 22601d7f30aSJiri Pirko drop: 22701d7f30aSJiri Pirko dev_kfree_skb_any(skb); 22801d7f30aSJiri Pirko return false; 22901d7f30aSJiri Pirko } 23001d7f30aSJiri Pirko 23180f7c668SJiri Pirko static int lb_bpf_func_get(struct team *team, struct team_gsetter_ctx *ctx) 23201d7f30aSJiri Pirko { 233ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 234ab8250d7SJiri Pirko 235ab8250d7SJiri Pirko if (!lb_priv->ex->orig_fprog) { 23680f7c668SJiri Pirko ctx->data.bin_val.len = 0; 23780f7c668SJiri Pirko ctx->data.bin_val.ptr = NULL; 23801d7f30aSJiri Pirko return 0; 23980f7c668SJiri Pirko } 240ab8250d7SJiri Pirko ctx->data.bin_val.len = lb_priv->ex->orig_fprog->len * 24101d7f30aSJiri Pirko sizeof(struct sock_filter); 242ab8250d7SJiri Pirko ctx->data.bin_val.ptr = lb_priv->ex->orig_fprog->filter; 24301d7f30aSJiri Pirko return 0; 24401d7f30aSJiri Pirko } 24501d7f30aSJiri Pirko 24601d7f30aSJiri Pirko static int __fprog_create(struct sock_fprog **pfprog, u32 data_len, 24780f7c668SJiri Pirko const void *data) 24801d7f30aSJiri Pirko { 24901d7f30aSJiri Pirko struct sock_fprog *fprog; 25001d7f30aSJiri Pirko struct sock_filter *filter = (struct sock_filter *) data; 25101d7f30aSJiri Pirko 25201d7f30aSJiri Pirko if (data_len % sizeof(struct sock_filter)) 25301d7f30aSJiri Pirko return -EINVAL; 25401d7f30aSJiri Pirko fprog = kmalloc(sizeof(struct sock_fprog), GFP_KERNEL); 25501d7f30aSJiri Pirko if (!fprog) 25601d7f30aSJiri Pirko return -ENOMEM; 25701d7f30aSJiri Pirko fprog->filter = kmemdup(filter, data_len, GFP_KERNEL); 25801d7f30aSJiri Pirko if (!fprog->filter) { 25901d7f30aSJiri Pirko kfree(fprog); 26001d7f30aSJiri Pirko return -ENOMEM; 26101d7f30aSJiri Pirko } 26201d7f30aSJiri Pirko fprog->len = data_len / sizeof(struct sock_filter); 26301d7f30aSJiri Pirko *pfprog = fprog; 26401d7f30aSJiri Pirko return 0; 26501d7f30aSJiri Pirko } 26601d7f30aSJiri Pirko 26701d7f30aSJiri Pirko static void __fprog_destroy(struct sock_fprog *fprog) 26801d7f30aSJiri Pirko { 26901d7f30aSJiri Pirko kfree(fprog->filter); 27001d7f30aSJiri Pirko kfree(fprog); 27101d7f30aSJiri Pirko } 27201d7f30aSJiri Pirko 27380f7c668SJiri Pirko static int lb_bpf_func_set(struct team *team, struct team_gsetter_ctx *ctx) 27401d7f30aSJiri Pirko { 275ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 27601d7f30aSJiri Pirko struct sk_filter *fp = NULL; 27701d7f30aSJiri Pirko struct sock_fprog *fprog = NULL; 27801d7f30aSJiri Pirko int err; 27901d7f30aSJiri Pirko 28080f7c668SJiri Pirko if (ctx->data.bin_val.len) { 28180f7c668SJiri Pirko err = __fprog_create(&fprog, ctx->data.bin_val.len, 28280f7c668SJiri Pirko ctx->data.bin_val.ptr); 28301d7f30aSJiri Pirko if (err) 28401d7f30aSJiri Pirko return err; 28501d7f30aSJiri Pirko err = sk_unattached_filter_create(&fp, fprog); 28601d7f30aSJiri Pirko if (err) { 28701d7f30aSJiri Pirko __fprog_destroy(fprog); 28801d7f30aSJiri Pirko return err; 28901d7f30aSJiri Pirko } 29001d7f30aSJiri Pirko } 29101d7f30aSJiri Pirko 292ab8250d7SJiri Pirko if (lb_priv->ex->orig_fprog) { 29301d7f30aSJiri Pirko /* Clear old filter data */ 294ab8250d7SJiri Pirko __fprog_destroy(lb_priv->ex->orig_fprog); 295ab8250d7SJiri Pirko sk_unattached_filter_destroy(lb_priv->fp); 29601d7f30aSJiri Pirko } 29701d7f30aSJiri Pirko 298ab8250d7SJiri Pirko rcu_assign_pointer(lb_priv->fp, fp); 299ab8250d7SJiri Pirko lb_priv->ex->orig_fprog = fprog; 300ab8250d7SJiri Pirko return 0; 301ab8250d7SJiri Pirko } 302ab8250d7SJiri Pirko 303ab8250d7SJiri Pirko static int lb_tx_method_get(struct team *team, struct team_gsetter_ctx *ctx) 304ab8250d7SJiri Pirko { 305ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 306ab8250d7SJiri Pirko char *name; 307ab8250d7SJiri Pirko 308ab8250d7SJiri Pirko name = lb_select_tx_port_get_name(lb_priv->select_tx_port_func); 309ab8250d7SJiri Pirko BUG_ON(!name); 310ab8250d7SJiri Pirko ctx->data.str_val = name; 311ab8250d7SJiri Pirko return 0; 312ab8250d7SJiri Pirko } 313ab8250d7SJiri Pirko 314ab8250d7SJiri Pirko static int lb_tx_method_set(struct team *team, struct team_gsetter_ctx *ctx) 315ab8250d7SJiri Pirko { 316ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 317ab8250d7SJiri Pirko lb_select_tx_port_func_t *func; 318ab8250d7SJiri Pirko 319ab8250d7SJiri Pirko func = lb_select_tx_port_get_func(ctx->data.str_val); 320ab8250d7SJiri Pirko if (!func) 321ab8250d7SJiri Pirko return -EINVAL; 322ab8250d7SJiri Pirko rcu_assign_pointer(lb_priv->select_tx_port_func, func); 323ab8250d7SJiri Pirko return 0; 324ab8250d7SJiri Pirko } 325ab8250d7SJiri Pirko 326ab8250d7SJiri Pirko static int lb_tx_hash_to_port_mapping_init(struct team *team, 327ab8250d7SJiri Pirko struct team_option_inst_info *info) 328ab8250d7SJiri Pirko { 329ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 330ab8250d7SJiri Pirko unsigned char hash = info->array_index; 331ab8250d7SJiri Pirko 332ab8250d7SJiri Pirko LB_HTPM_OPT_INST_INFO_BY_HASH(lb_priv, hash) = info; 333ab8250d7SJiri Pirko return 0; 334ab8250d7SJiri Pirko } 335ab8250d7SJiri Pirko 336ab8250d7SJiri Pirko static int lb_tx_hash_to_port_mapping_get(struct team *team, 337ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 338ab8250d7SJiri Pirko { 339ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 340ab8250d7SJiri Pirko struct team_port *port; 341ab8250d7SJiri Pirko unsigned char hash = ctx->info->array_index; 342ab8250d7SJiri Pirko 343ab8250d7SJiri Pirko port = LB_HTPM_PORT_BY_HASH(lb_priv, hash); 344ab8250d7SJiri Pirko ctx->data.u32_val = port ? port->dev->ifindex : 0; 345ab8250d7SJiri Pirko return 0; 346ab8250d7SJiri Pirko } 347ab8250d7SJiri Pirko 348ab8250d7SJiri Pirko static int lb_tx_hash_to_port_mapping_set(struct team *team, 349ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 350ab8250d7SJiri Pirko { 351ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 352ab8250d7SJiri Pirko struct team_port *port; 353ab8250d7SJiri Pirko unsigned char hash = ctx->info->array_index; 354ab8250d7SJiri Pirko 355ab8250d7SJiri Pirko list_for_each_entry(port, &team->port_list, list) { 356ab8250d7SJiri Pirko if (ctx->data.u32_val == port->dev->ifindex) { 357ab8250d7SJiri Pirko rcu_assign_pointer(LB_HTPM_PORT_BY_HASH(lb_priv, hash), 358ab8250d7SJiri Pirko port); 359ab8250d7SJiri Pirko return 0; 360ab8250d7SJiri Pirko } 361ab8250d7SJiri Pirko } 362ab8250d7SJiri Pirko return -ENODEV; 363ab8250d7SJiri Pirko } 364ab8250d7SJiri Pirko 365ab8250d7SJiri Pirko static int lb_hash_stats_init(struct team *team, 366ab8250d7SJiri Pirko struct team_option_inst_info *info) 367ab8250d7SJiri Pirko { 368ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 369ab8250d7SJiri Pirko unsigned char hash = info->array_index; 370ab8250d7SJiri Pirko 371ab8250d7SJiri Pirko lb_priv->ex->stats.info[hash].opt_inst_info = info; 372ab8250d7SJiri Pirko return 0; 373ab8250d7SJiri Pirko } 374ab8250d7SJiri Pirko 375ab8250d7SJiri Pirko static int lb_hash_stats_get(struct team *team, struct team_gsetter_ctx *ctx) 376ab8250d7SJiri Pirko { 377ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 378ab8250d7SJiri Pirko unsigned char hash = ctx->info->array_index; 379ab8250d7SJiri Pirko 380ab8250d7SJiri Pirko ctx->data.bin_val.ptr = &lb_priv->ex->stats.info[hash].stats; 381ab8250d7SJiri Pirko ctx->data.bin_val.len = sizeof(struct lb_stats); 382ab8250d7SJiri Pirko return 0; 383ab8250d7SJiri Pirko } 384ab8250d7SJiri Pirko 385ab8250d7SJiri Pirko static int lb_port_stats_init(struct team *team, 386ab8250d7SJiri Pirko struct team_option_inst_info *info) 387ab8250d7SJiri Pirko { 388ab8250d7SJiri Pirko struct team_port *port = info->port; 389ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 390ab8250d7SJiri Pirko 391ab8250d7SJiri Pirko lb_port_priv->stats_info.opt_inst_info = info; 392ab8250d7SJiri Pirko return 0; 393ab8250d7SJiri Pirko } 394ab8250d7SJiri Pirko 395ab8250d7SJiri Pirko static int lb_port_stats_get(struct team *team, struct team_gsetter_ctx *ctx) 396ab8250d7SJiri Pirko { 397ab8250d7SJiri Pirko struct team_port *port = ctx->info->port; 398ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 399ab8250d7SJiri Pirko 400ab8250d7SJiri Pirko ctx->data.bin_val.ptr = &lb_port_priv->stats_info.stats; 401ab8250d7SJiri Pirko ctx->data.bin_val.len = sizeof(struct lb_stats); 402ab8250d7SJiri Pirko return 0; 403ab8250d7SJiri Pirko } 404ab8250d7SJiri Pirko 405ab8250d7SJiri Pirko static void __lb_stats_info_refresh_prepare(struct lb_stats_info *s_info) 406ab8250d7SJiri Pirko { 407ab8250d7SJiri Pirko memcpy(&s_info->last_stats, &s_info->stats, sizeof(struct lb_stats)); 408ab8250d7SJiri Pirko memset(&s_info->stats, 0, sizeof(struct lb_stats)); 409ab8250d7SJiri Pirko } 410ab8250d7SJiri Pirko 411ab8250d7SJiri Pirko static bool __lb_stats_info_refresh_check(struct lb_stats_info *s_info, 412ab8250d7SJiri Pirko struct team *team) 413ab8250d7SJiri Pirko { 414ab8250d7SJiri Pirko if (memcmp(&s_info->last_stats, &s_info->stats, 415ab8250d7SJiri Pirko sizeof(struct lb_stats))) { 416ab8250d7SJiri Pirko team_option_inst_set_change(s_info->opt_inst_info); 417ab8250d7SJiri Pirko return true; 418ab8250d7SJiri Pirko } 419ab8250d7SJiri Pirko return false; 420ab8250d7SJiri Pirko } 421ab8250d7SJiri Pirko 422ab8250d7SJiri Pirko static void __lb_one_cpu_stats_add(struct lb_stats *acc_stats, 423ab8250d7SJiri Pirko struct lb_stats *cpu_stats, 424ab8250d7SJiri Pirko struct u64_stats_sync *syncp) 425ab8250d7SJiri Pirko { 426ab8250d7SJiri Pirko unsigned int start; 427ab8250d7SJiri Pirko struct lb_stats tmp; 428ab8250d7SJiri Pirko 429ab8250d7SJiri Pirko do { 430ab8250d7SJiri Pirko start = u64_stats_fetch_begin_bh(syncp); 431ab8250d7SJiri Pirko tmp.tx_bytes = cpu_stats->tx_bytes; 432ab8250d7SJiri Pirko } while (u64_stats_fetch_retry_bh(syncp, start)); 433ab8250d7SJiri Pirko acc_stats->tx_bytes += tmp.tx_bytes; 434ab8250d7SJiri Pirko } 435ab8250d7SJiri Pirko 436ab8250d7SJiri Pirko static void lb_stats_refresh(struct work_struct *work) 437ab8250d7SJiri Pirko { 438ab8250d7SJiri Pirko struct team *team; 439ab8250d7SJiri Pirko struct lb_priv *lb_priv; 440ab8250d7SJiri Pirko struct lb_priv_ex *lb_priv_ex; 441ab8250d7SJiri Pirko struct lb_pcpu_stats *pcpu_stats; 442ab8250d7SJiri Pirko struct lb_stats *stats; 443ab8250d7SJiri Pirko struct lb_stats_info *s_info; 444ab8250d7SJiri Pirko struct team_port *port; 445ab8250d7SJiri Pirko bool changed = false; 446ab8250d7SJiri Pirko int i; 447ab8250d7SJiri Pirko int j; 448ab8250d7SJiri Pirko 449ab8250d7SJiri Pirko lb_priv_ex = container_of(work, struct lb_priv_ex, 450ab8250d7SJiri Pirko stats.refresh_dw.work); 451ab8250d7SJiri Pirko 452ab8250d7SJiri Pirko team = lb_priv_ex->team; 453ab8250d7SJiri Pirko lb_priv = get_lb_priv(team); 454ab8250d7SJiri Pirko 455ab8250d7SJiri Pirko if (!mutex_trylock(&team->lock)) { 456ab8250d7SJiri Pirko schedule_delayed_work(&lb_priv_ex->stats.refresh_dw, 0); 457ab8250d7SJiri Pirko return; 458ab8250d7SJiri Pirko } 459ab8250d7SJiri Pirko 460ab8250d7SJiri Pirko for (j = 0; j < LB_TX_HASHTABLE_SIZE; j++) { 461ab8250d7SJiri Pirko s_info = &lb_priv->ex->stats.info[j]; 462ab8250d7SJiri Pirko __lb_stats_info_refresh_prepare(s_info); 463ab8250d7SJiri Pirko for_each_possible_cpu(i) { 464ab8250d7SJiri Pirko pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); 465ab8250d7SJiri Pirko stats = &pcpu_stats->hash_stats[j]; 466ab8250d7SJiri Pirko __lb_one_cpu_stats_add(&s_info->stats, stats, 467ab8250d7SJiri Pirko &pcpu_stats->syncp); 468ab8250d7SJiri Pirko } 469ab8250d7SJiri Pirko changed |= __lb_stats_info_refresh_check(s_info, team); 470ab8250d7SJiri Pirko } 471ab8250d7SJiri Pirko 472ab8250d7SJiri Pirko list_for_each_entry(port, &team->port_list, list) { 473ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 474ab8250d7SJiri Pirko 475ab8250d7SJiri Pirko s_info = &lb_port_priv->stats_info; 476ab8250d7SJiri Pirko __lb_stats_info_refresh_prepare(s_info); 477ab8250d7SJiri Pirko for_each_possible_cpu(i) { 478ab8250d7SJiri Pirko pcpu_stats = per_cpu_ptr(lb_priv->pcpu_stats, i); 479ab8250d7SJiri Pirko stats = per_cpu_ptr(lb_port_priv->pcpu_stats, i); 480ab8250d7SJiri Pirko __lb_one_cpu_stats_add(&s_info->stats, stats, 481ab8250d7SJiri Pirko &pcpu_stats->syncp); 482ab8250d7SJiri Pirko } 483ab8250d7SJiri Pirko changed |= __lb_stats_info_refresh_check(s_info, team); 484ab8250d7SJiri Pirko } 485ab8250d7SJiri Pirko 486ab8250d7SJiri Pirko if (changed) 487ab8250d7SJiri Pirko team_options_change_check(team); 488ab8250d7SJiri Pirko 489ab8250d7SJiri Pirko schedule_delayed_work(&lb_priv_ex->stats.refresh_dw, 490ab8250d7SJiri Pirko (lb_priv_ex->stats.refresh_interval * HZ) / 10); 491ab8250d7SJiri Pirko 492ab8250d7SJiri Pirko mutex_unlock(&team->lock); 493ab8250d7SJiri Pirko } 494ab8250d7SJiri Pirko 495ab8250d7SJiri Pirko static int lb_stats_refresh_interval_get(struct team *team, 496ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 497ab8250d7SJiri Pirko { 498ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 499ab8250d7SJiri Pirko 500ab8250d7SJiri Pirko ctx->data.u32_val = lb_priv->ex->stats.refresh_interval; 501ab8250d7SJiri Pirko return 0; 502ab8250d7SJiri Pirko } 503ab8250d7SJiri Pirko 504ab8250d7SJiri Pirko static int lb_stats_refresh_interval_set(struct team *team, 505ab8250d7SJiri Pirko struct team_gsetter_ctx *ctx) 506ab8250d7SJiri Pirko { 507ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 508ab8250d7SJiri Pirko unsigned int interval; 509ab8250d7SJiri Pirko 510ab8250d7SJiri Pirko interval = ctx->data.u32_val; 511ab8250d7SJiri Pirko if (lb_priv->ex->stats.refresh_interval == interval) 512ab8250d7SJiri Pirko return 0; 513ab8250d7SJiri Pirko lb_priv->ex->stats.refresh_interval = interval; 514ab8250d7SJiri Pirko if (interval) 515ab8250d7SJiri Pirko schedule_delayed_work(&lb_priv->ex->stats.refresh_dw, 0); 516ab8250d7SJiri Pirko else 517ab8250d7SJiri Pirko cancel_delayed_work(&lb_priv->ex->stats.refresh_dw); 51801d7f30aSJiri Pirko return 0; 51901d7f30aSJiri Pirko } 52001d7f30aSJiri Pirko 52101d7f30aSJiri Pirko static const struct team_option lb_options[] = { 52201d7f30aSJiri Pirko { 52301d7f30aSJiri Pirko .name = "bpf_hash_func", 52401d7f30aSJiri Pirko .type = TEAM_OPTION_TYPE_BINARY, 52501d7f30aSJiri Pirko .getter = lb_bpf_func_get, 52601d7f30aSJiri Pirko .setter = lb_bpf_func_set, 52701d7f30aSJiri Pirko }, 528ab8250d7SJiri Pirko { 529ab8250d7SJiri Pirko .name = "lb_tx_method", 530ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_STRING, 531ab8250d7SJiri Pirko .getter = lb_tx_method_get, 532ab8250d7SJiri Pirko .setter = lb_tx_method_set, 533ab8250d7SJiri Pirko }, 534ab8250d7SJiri Pirko { 535ab8250d7SJiri Pirko .name = "lb_tx_hash_to_port_mapping", 536ab8250d7SJiri Pirko .array_size = LB_TX_HASHTABLE_SIZE, 537ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_U32, 538ab8250d7SJiri Pirko .init = lb_tx_hash_to_port_mapping_init, 539ab8250d7SJiri Pirko .getter = lb_tx_hash_to_port_mapping_get, 540ab8250d7SJiri Pirko .setter = lb_tx_hash_to_port_mapping_set, 541ab8250d7SJiri Pirko }, 542ab8250d7SJiri Pirko { 543ab8250d7SJiri Pirko .name = "lb_hash_stats", 544ab8250d7SJiri Pirko .array_size = LB_TX_HASHTABLE_SIZE, 545ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_BINARY, 546ab8250d7SJiri Pirko .init = lb_hash_stats_init, 547ab8250d7SJiri Pirko .getter = lb_hash_stats_get, 548ab8250d7SJiri Pirko }, 549ab8250d7SJiri Pirko { 550ab8250d7SJiri Pirko .name = "lb_port_stats", 551ab8250d7SJiri Pirko .per_port = true, 552ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_BINARY, 553ab8250d7SJiri Pirko .init = lb_port_stats_init, 554ab8250d7SJiri Pirko .getter = lb_port_stats_get, 555ab8250d7SJiri Pirko }, 556ab8250d7SJiri Pirko { 557ab8250d7SJiri Pirko .name = "lb_stats_refresh_interval", 558ab8250d7SJiri Pirko .type = TEAM_OPTION_TYPE_U32, 559ab8250d7SJiri Pirko .getter = lb_stats_refresh_interval_get, 560ab8250d7SJiri Pirko .setter = lb_stats_refresh_interval_set, 561ab8250d7SJiri Pirko }, 56201d7f30aSJiri Pirko }; 56301d7f30aSJiri Pirko 564cade4555SJiri Pirko static int lb_init(struct team *team) 56501d7f30aSJiri Pirko { 566ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 567ab8250d7SJiri Pirko lb_select_tx_port_func_t *func; 568ab8250d7SJiri Pirko int err; 569ab8250d7SJiri Pirko 570ab8250d7SJiri Pirko /* set default tx port selector */ 571ab8250d7SJiri Pirko func = lb_select_tx_port_get_func("hash"); 572ab8250d7SJiri Pirko BUG_ON(!func); 573ab8250d7SJiri Pirko rcu_assign_pointer(lb_priv->select_tx_port_func, func); 574ab8250d7SJiri Pirko 575ab8250d7SJiri Pirko lb_priv->ex = kzalloc(sizeof(*lb_priv->ex), GFP_KERNEL); 576ab8250d7SJiri Pirko if (!lb_priv->ex) 577ab8250d7SJiri Pirko return -ENOMEM; 578ab8250d7SJiri Pirko lb_priv->ex->team = team; 579ab8250d7SJiri Pirko 580ab8250d7SJiri Pirko lb_priv->pcpu_stats = alloc_percpu(struct lb_pcpu_stats); 581ab8250d7SJiri Pirko if (!lb_priv->pcpu_stats) { 582ab8250d7SJiri Pirko err = -ENOMEM; 583ab8250d7SJiri Pirko goto err_alloc_pcpu_stats; 584ab8250d7SJiri Pirko } 585ab8250d7SJiri Pirko 586ab8250d7SJiri Pirko INIT_DELAYED_WORK(&lb_priv->ex->stats.refresh_dw, lb_stats_refresh); 587ab8250d7SJiri Pirko 588ab8250d7SJiri Pirko err = team_options_register(team, lb_options, ARRAY_SIZE(lb_options)); 589ab8250d7SJiri Pirko if (err) 590ab8250d7SJiri Pirko goto err_options_register; 591ab8250d7SJiri Pirko return 0; 592ab8250d7SJiri Pirko 593ab8250d7SJiri Pirko err_options_register: 594ab8250d7SJiri Pirko free_percpu(lb_priv->pcpu_stats); 595ab8250d7SJiri Pirko err_alloc_pcpu_stats: 596ab8250d7SJiri Pirko kfree(lb_priv->ex); 597ab8250d7SJiri Pirko return err; 59801d7f30aSJiri Pirko } 59901d7f30aSJiri Pirko 600cade4555SJiri Pirko static void lb_exit(struct team *team) 60101d7f30aSJiri Pirko { 602ab8250d7SJiri Pirko struct lb_priv *lb_priv = get_lb_priv(team); 603ab8250d7SJiri Pirko 60401d7f30aSJiri Pirko team_options_unregister(team, lb_options, 60501d7f30aSJiri Pirko ARRAY_SIZE(lb_options)); 606ab8250d7SJiri Pirko cancel_delayed_work_sync(&lb_priv->ex->stats.refresh_dw); 607ab8250d7SJiri Pirko free_percpu(lb_priv->pcpu_stats); 608ab8250d7SJiri Pirko kfree(lb_priv->ex); 609ab8250d7SJiri Pirko } 610ab8250d7SJiri Pirko 611ab8250d7SJiri Pirko static int lb_port_enter(struct team *team, struct team_port *port) 612ab8250d7SJiri Pirko { 613ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 614ab8250d7SJiri Pirko 615ab8250d7SJiri Pirko lb_port_priv->pcpu_stats = alloc_percpu(struct lb_stats); 616ab8250d7SJiri Pirko if (!lb_port_priv->pcpu_stats) 617ab8250d7SJiri Pirko return -ENOMEM; 618ab8250d7SJiri Pirko return 0; 619ab8250d7SJiri Pirko } 620ab8250d7SJiri Pirko 621ab8250d7SJiri Pirko static void lb_port_leave(struct team *team, struct team_port *port) 622ab8250d7SJiri Pirko { 623ab8250d7SJiri Pirko struct lb_port_priv *lb_port_priv = get_lb_port_priv(port); 624ab8250d7SJiri Pirko 625ab8250d7SJiri Pirko free_percpu(lb_port_priv->pcpu_stats); 626ab8250d7SJiri Pirko } 627ab8250d7SJiri Pirko 628ab8250d7SJiri Pirko static void lb_port_disabled(struct team *team, struct team_port *port) 629ab8250d7SJiri Pirko { 630ab8250d7SJiri Pirko lb_tx_hash_to_port_mapping_null_port(team, port); 63101d7f30aSJiri Pirko } 63201d7f30aSJiri Pirko 63301d7f30aSJiri Pirko static const struct team_mode_ops lb_mode_ops = { 63401d7f30aSJiri Pirko .init = lb_init, 63501d7f30aSJiri Pirko .exit = lb_exit, 636ab8250d7SJiri Pirko .port_enter = lb_port_enter, 637ab8250d7SJiri Pirko .port_leave = lb_port_leave, 638ab8250d7SJiri Pirko .port_disabled = lb_port_disabled, 63901d7f30aSJiri Pirko .transmit = lb_transmit, 64001d7f30aSJiri Pirko }; 64101d7f30aSJiri Pirko 6420402788aSJiri Pirko static const struct team_mode lb_mode = { 64301d7f30aSJiri Pirko .kind = "loadbalance", 64401d7f30aSJiri Pirko .owner = THIS_MODULE, 64501d7f30aSJiri Pirko .priv_size = sizeof(struct lb_priv), 646ab8250d7SJiri Pirko .port_priv_size = sizeof(struct lb_port_priv), 64701d7f30aSJiri Pirko .ops = &lb_mode_ops, 64801d7f30aSJiri Pirko }; 64901d7f30aSJiri Pirko 65001d7f30aSJiri Pirko static int __init lb_init_module(void) 65101d7f30aSJiri Pirko { 65201d7f30aSJiri Pirko return team_mode_register(&lb_mode); 65301d7f30aSJiri Pirko } 65401d7f30aSJiri Pirko 65501d7f30aSJiri Pirko static void __exit lb_cleanup_module(void) 65601d7f30aSJiri Pirko { 65701d7f30aSJiri Pirko team_mode_unregister(&lb_mode); 65801d7f30aSJiri Pirko } 65901d7f30aSJiri Pirko 66001d7f30aSJiri Pirko module_init(lb_init_module); 66101d7f30aSJiri Pirko module_exit(lb_cleanup_module); 66201d7f30aSJiri Pirko 66301d7f30aSJiri Pirko MODULE_LICENSE("GPL v2"); 66401d7f30aSJiri Pirko MODULE_AUTHOR("Jiri Pirko <jpirko@redhat.com>"); 66501d7f30aSJiri Pirko MODULE_DESCRIPTION("Load-balancing mode for team"); 66601d7f30aSJiri Pirko MODULE_ALIAS("team-mode-loadbalance"); 667