// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ #include #include #include #include #include #include "fs_core.h" #include "lib/macsec_fs.h" #include "mlx5_core.h" /* MACsec TX flow steering */ #define CRYPTO_NUM_MAXSEC_FTE BIT(15) #define CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE 1 #define TX_CRYPTO_TABLE_LEVEL 0 #define TX_CRYPTO_TABLE_NUM_GROUPS 3 #define TX_CRYPTO_TABLE_MKE_GROUP_SIZE 1 #define TX_CRYPTO_TABLE_SA_GROUP_SIZE \ (CRYPTO_NUM_MAXSEC_FTE - (TX_CRYPTO_TABLE_MKE_GROUP_SIZE + \ CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE)) #define TX_CHECK_TABLE_LEVEL 1 #define TX_CHECK_TABLE_NUM_FTE 2 #define RX_CRYPTO_TABLE_LEVEL 0 #define RX_CHECK_TABLE_LEVEL 1 #define RX_ROCE_TABLE_LEVEL 2 #define RX_CHECK_TABLE_NUM_FTE 3 #define RX_ROCE_TABLE_NUM_FTE 2 #define RX_CRYPTO_TABLE_NUM_GROUPS 3 #define RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE \ ((CRYPTO_NUM_MAXSEC_FTE - CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE) / 2) #define RX_CRYPTO_TABLE_SA_RULE_WITHOUT_SCI_GROUP_SIZE \ (CRYPTO_NUM_MAXSEC_FTE - RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE) #define RX_NUM_OF_RULES_PER_SA 2 #define RDMA_RX_ROCE_IP_TABLE_LEVEL 0 #define RDMA_RX_ROCE_MACSEC_OP_TABLE_LEVEL 1 #define MLX5_MACSEC_TAG_LEN 8 /* SecTAG length with ethertype and without the optional SCI */ #define MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK 0x23 #define MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET 0x8 #define MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET 0x5 #define MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT (0x1 << MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET) #define MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI 0x8 #define MLX5_SECTAG_HEADER_SIZE_WITH_SCI (MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI + MACSEC_SCI_LEN) /* MACsec RX flow steering */ #define MLX5_ETH_WQE_FT_META_MACSEC_MASK 0x3E /* MACsec fs_id handling for steering */ #define macsec_fs_set_tx_fs_id(fs_id) (MLX5_ETH_WQE_FT_META_MACSEC | (fs_id) << 2) #define macsec_fs_set_rx_fs_id(fs_id) ((fs_id) | BIT(30)) struct mlx5_sectag_header { __be16 ethertype; u8 tci_an; u8 sl; u32 pn; u8 sci[MACSEC_SCI_LEN]; /* optional */ } __packed; struct mlx5_roce_macsec_tx_rule { u32 fs_id; u16 gid_idx; struct list_head entry; struct mlx5_flow_handle *rule; struct mlx5_modify_hdr *meta_modhdr; }; struct mlx5_macsec_tx_rule { struct mlx5_flow_handle *rule; struct mlx5_pkt_reformat *pkt_reformat; u32 fs_id; }; struct mlx5_macsec_flow_table { int num_groups; struct mlx5_flow_table *t; struct mlx5_flow_group **g; }; struct mlx5_macsec_tables { struct mlx5_macsec_flow_table ft_crypto; struct mlx5_flow_handle *crypto_miss_rule; struct mlx5_flow_table *ft_check; struct mlx5_flow_group *ft_check_group; struct mlx5_fc *check_miss_rule_counter; struct mlx5_flow_handle *check_miss_rule; struct mlx5_fc *check_rule_counter; u32 refcnt; }; struct mlx5_fs_id { u32 id; refcount_t refcnt; sci_t sci; struct rhash_head hash; }; struct mlx5_macsec_device { struct list_head macsec_devices_list_entry; void *macdev; struct xarray tx_id_xa; struct xarray rx_id_xa; }; struct mlx5_macsec_tx { struct mlx5_flow_handle *crypto_mke_rule; struct mlx5_flow_handle *check_rule; struct ida tx_halloc; struct mlx5_macsec_tables tables; struct mlx5_flow_table *ft_rdma_tx; }; struct mlx5_roce_macsec_rx_rule { u32 fs_id; u16 gid_idx; struct mlx5_flow_handle *op; struct mlx5_flow_handle *ip; struct list_head entry; }; struct mlx5_macsec_rx_rule { struct mlx5_flow_handle *rule[RX_NUM_OF_RULES_PER_SA]; struct mlx5_modify_hdr *meta_modhdr; }; struct mlx5_macsec_miss { struct mlx5_flow_group *g; struct mlx5_flow_handle *rule; }; struct mlx5_macsec_rx_roce { /* Flow table/rules in NIC domain, to check if it's a RoCE packet */ struct mlx5_flow_group *g; struct mlx5_flow_table *ft; struct mlx5_flow_handle *rule; struct mlx5_modify_hdr *copy_modify_hdr; struct mlx5_macsec_miss nic_miss; /* Flow table/rule in RDMA domain, to check dgid */ struct mlx5_flow_table *ft_ip_check; struct mlx5_flow_table *ft_macsec_op_check; struct mlx5_macsec_miss miss; }; struct mlx5_macsec_rx { struct mlx5_flow_handle *check_rule[2]; struct mlx5_pkt_reformat *check_rule_pkt_reformat[2]; struct mlx5_macsec_tables tables; struct mlx5_macsec_rx_roce roce; }; union mlx5_macsec_rule { struct mlx5_macsec_tx_rule tx_rule; struct mlx5_macsec_rx_rule rx_rule; }; static const struct rhashtable_params rhash_sci = { .key_len = sizeof_field(struct mlx5_fs_id, sci), .key_offset = offsetof(struct mlx5_fs_id, sci), .head_offset = offsetof(struct mlx5_fs_id, hash), .automatic_shrinking = true, .min_size = 1, }; static const struct rhashtable_params rhash_fs_id = { .key_len = sizeof_field(struct mlx5_fs_id, id), .key_offset = offsetof(struct mlx5_fs_id, id), .head_offset = offsetof(struct mlx5_fs_id, hash), .automatic_shrinking = true, .min_size = 1, }; struct mlx5_macsec_fs { struct mlx5_core_dev *mdev; struct mlx5_macsec_tx *tx_fs; struct mlx5_macsec_rx *rx_fs; /* Stats manage */ struct mlx5_macsec_stats stats; /* Tx sci -> fs id mapping handling */ struct rhashtable sci_hash; /* sci -> mlx5_fs_id */ /* RX fs_id -> mlx5_fs_id mapping handling */ struct rhashtable fs_id_hash; /* fs_id -> mlx5_fs_id */ /* TX & RX fs_id lists per macsec device */ struct list_head macsec_devices_list; }; static void macsec_fs_destroy_groups(struct mlx5_macsec_flow_table *ft) { int i; for (i = ft->num_groups - 1; i >= 0; i--) { if (!IS_ERR_OR_NULL(ft->g[i])) mlx5_destroy_flow_group(ft->g[i]); ft->g[i] = NULL; } ft->num_groups = 0; } static void macsec_fs_destroy_flow_table(struct mlx5_macsec_flow_table *ft) { macsec_fs_destroy_groups(ft); kfree(ft->g); mlx5_destroy_flow_table(ft->t); ft->t = NULL; } static void macsec_fs_tx_destroy(struct mlx5_macsec_fs *macsec_fs) { struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs; struct mlx5_macsec_tables *tx_tables; if (mlx5_is_macsec_roce_supported(macsec_fs->mdev)) mlx5_destroy_flow_table(tx_fs->ft_rdma_tx); tx_tables = &tx_fs->tables; /* Tx check table */ if (tx_fs->check_rule) { mlx5_del_flow_rules(tx_fs->check_rule); tx_fs->check_rule = NULL; } if (tx_tables->check_miss_rule) { mlx5_del_flow_rules(tx_tables->check_miss_rule); tx_tables->check_miss_rule = NULL; } if (tx_tables->ft_check_group) { mlx5_destroy_flow_group(tx_tables->ft_check_group); tx_tables->ft_check_group = NULL; } if (tx_tables->ft_check) { mlx5_destroy_flow_table(tx_tables->ft_check); tx_tables->ft_check = NULL; } /* Tx crypto table */ if (tx_fs->crypto_mke_rule) { mlx5_del_flow_rules(tx_fs->crypto_mke_rule); tx_fs->crypto_mke_rule = NULL; } if (tx_tables->crypto_miss_rule) { mlx5_del_flow_rules(tx_tables->crypto_miss_rule); tx_tables->crypto_miss_rule = NULL; } macsec_fs_destroy_flow_table(&tx_tables->ft_crypto); } static int macsec_fs_tx_create_crypto_table_groups(struct mlx5_macsec_flow_table *ft) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); int mclen = MLX5_ST_SZ_BYTES(fte_match_param); int ix = 0; u32 *in; int err; u8 *mc; ft->g = kcalloc(TX_CRYPTO_TABLE_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL); if (!ft->g) return -ENOMEM; in = kvzalloc(inlen, GFP_KERNEL); if (!in) { kfree(ft->g); ft->g = NULL; return -ENOMEM; } mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); /* Flow Group for MKE match */ MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); MLX5_SET_CFG(in, start_flow_index, ix); ix += TX_CRYPTO_TABLE_MKE_GROUP_SIZE; MLX5_SET_CFG(in, end_flow_index, ix - 1); ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); if (IS_ERR(ft->g[ft->num_groups])) goto err; ft->num_groups++; /* Flow Group for SA rules */ memset(in, 0, inlen); memset(mc, 0, mclen); MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_MISC_PARAMETERS_2); MLX5_SET(fte_match_param, mc, misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_MACSEC_MASK); MLX5_SET_CFG(in, start_flow_index, ix); ix += TX_CRYPTO_TABLE_SA_GROUP_SIZE; MLX5_SET_CFG(in, end_flow_index, ix - 1); ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); if (IS_ERR(ft->g[ft->num_groups])) goto err; ft->num_groups++; /* Flow Group for l2 traps */ memset(in, 0, inlen); memset(mc, 0, mclen); MLX5_SET_CFG(in, start_flow_index, ix); ix += CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE; MLX5_SET_CFG(in, end_flow_index, ix - 1); ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); if (IS_ERR(ft->g[ft->num_groups])) goto err; ft->num_groups++; kvfree(in); return 0; err: err = PTR_ERR(ft->g[ft->num_groups]); ft->g[ft->num_groups] = NULL; kvfree(in); return err; } static struct mlx5_flow_table *macsec_fs_auto_group_table_create(struct mlx5_flow_namespace *ns, int flags, int level, int max_fte) { struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_flow_table *fdb = NULL; /* reserve entry for the match all miss group and rule */ ft_attr.autogroup.num_reserved_entries = 1; ft_attr.autogroup.max_num_groups = 1; ft_attr.prio = 0; ft_attr.flags = flags; ft_attr.level = level; ft_attr.max_fte = max_fte; fdb = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); return fdb; } enum { RDMA_TX_MACSEC_LEVEL = 0, }; static int macsec_fs_tx_roce_create(struct mlx5_macsec_fs *macsec_fs) { struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs; struct mlx5_core_dev *mdev = macsec_fs->mdev; struct mlx5_flow_namespace *ns; struct mlx5_flow_table *ft; int err; if (!mlx5_is_macsec_roce_supported(mdev)) { mlx5_core_dbg(mdev, "Failed to init RoCE MACsec, capabilities not supported\n"); return 0; } ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_MACSEC); if (!ns) return -ENOMEM; /* Tx RoCE crypto table */ ft = macsec_fs_auto_group_table_create(ns, 0, RDMA_TX_MACSEC_LEVEL, CRYPTO_NUM_MAXSEC_FTE); if (IS_ERR(ft)) { err = PTR_ERR(ft); mlx5_core_err(mdev, "Failed to create MACsec RoCE Tx crypto table err(%d)\n", err); return err; } tx_fs->ft_rdma_tx = ft; return 0; } static int macsec_fs_tx_create(struct mlx5_macsec_fs *macsec_fs) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs; struct mlx5_core_dev *mdev = macsec_fs->mdev; struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_flow_destination dest = {}; struct mlx5_macsec_tables *tx_tables; struct mlx5_flow_act flow_act = {}; struct mlx5_macsec_flow_table *ft_crypto; struct mlx5_flow_table *flow_table; struct mlx5_flow_group *flow_group; struct mlx5_flow_namespace *ns; struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; u32 *flow_group_in; int err; ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_EGRESS_MACSEC); if (!ns) return -ENOMEM; spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) return -ENOMEM; flow_group_in = kvzalloc(inlen, GFP_KERNEL); if (!flow_group_in) { err = -ENOMEM; goto out_spec; } tx_tables = &tx_fs->tables; ft_crypto = &tx_tables->ft_crypto; /* Tx crypto table */ ft_attr.flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; ft_attr.level = TX_CRYPTO_TABLE_LEVEL; ft_attr.max_fte = CRYPTO_NUM_MAXSEC_FTE; flow_table = mlx5_create_flow_table(ns, &ft_attr); if (IS_ERR(flow_table)) { err = PTR_ERR(flow_table); mlx5_core_err(mdev, "Failed to create MACsec Tx crypto table err(%d)\n", err); goto out_flow_group; } ft_crypto->t = flow_table; /* Tx crypto table groups */ err = macsec_fs_tx_create_crypto_table_groups(ft_crypto); if (err) { mlx5_core_err(mdev, "Failed to create default flow group for MACsec Tx crypto table err(%d)\n", err); goto err; } /* Tx crypto table MKE rule - MKE packets shouldn't be offloaded */ spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype); MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ETH_P_PAE); flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW; rule = mlx5_add_flow_rules(ft_crypto->t, spec, &flow_act, NULL, 0); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "Failed to add MACsec TX MKE rule, err=%d\n", err); goto err; } tx_fs->crypto_mke_rule = rule; /* Tx crypto table Default miss rule */ memset(&flow_act, 0, sizeof(flow_act)); flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW; rule = mlx5_add_flow_rules(ft_crypto->t, NULL, &flow_act, NULL, 0); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "Failed to add MACsec Tx table default miss rule %d\n", err); goto err; } tx_tables->crypto_miss_rule = rule; /* Tx check table */ flow_table = macsec_fs_auto_group_table_create(ns, 0, TX_CHECK_TABLE_LEVEL, TX_CHECK_TABLE_NUM_FTE); if (IS_ERR(flow_table)) { err = PTR_ERR(flow_table); mlx5_core_err(mdev, "Fail to create MACsec TX check table, err(%d)\n", err); goto err; } tx_tables->ft_check = flow_table; /* Tx check table Default miss group/rule */ memset(flow_group_in, 0, inlen); MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_table->max_fte - 1); MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_table->max_fte - 1); flow_group = mlx5_create_flow_group(tx_tables->ft_check, flow_group_in); if (IS_ERR(flow_group)) { err = PTR_ERR(flow_group); mlx5_core_err(mdev, "Failed to create default flow group for MACsec Tx crypto table err(%d)\n", err); goto err; } tx_tables->ft_check_group = flow_group; /* Tx check table default drop rule */ memset(&dest, 0, sizeof(struct mlx5_flow_destination)); memset(&flow_act, 0, sizeof(flow_act)); dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; dest.counter_id = mlx5_fc_id(tx_tables->check_miss_rule_counter); flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; rule = mlx5_add_flow_rules(tx_tables->ft_check, NULL, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "Failed to added MACsec tx check drop rule, err(%d)\n", err); goto err; } tx_tables->check_miss_rule = rule; /* Tx check table rule */ memset(spec, 0, sizeof(struct mlx5_flow_spec)); memset(&dest, 0, sizeof(struct mlx5_flow_destination)); memset(&flow_act, 0, sizeof(flow_act)); MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_4); MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_4, 0); spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; flow_act.flags = FLOW_ACT_NO_APPEND; flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW | MLX5_FLOW_CONTEXT_ACTION_COUNT; dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; dest.counter_id = mlx5_fc_id(tx_tables->check_rule_counter); rule = mlx5_add_flow_rules(tx_tables->ft_check, spec, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "Failed to add MACsec check rule, err=%d\n", err); goto err; } tx_fs->check_rule = rule; err = macsec_fs_tx_roce_create(macsec_fs); if (err) goto err; kvfree(flow_group_in); kvfree(spec); return 0; err: macsec_fs_tx_destroy(macsec_fs); out_flow_group: kvfree(flow_group_in); out_spec: kvfree(spec); return err; } static int macsec_fs_tx_ft_get(struct mlx5_macsec_fs *macsec_fs) { struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs; struct mlx5_macsec_tables *tx_tables; int err = 0; tx_tables = &tx_fs->tables; if (tx_tables->refcnt) goto out; err = macsec_fs_tx_create(macsec_fs); if (err) return err; out: tx_tables->refcnt++; return err; } static void macsec_fs_tx_ft_put(struct mlx5_macsec_fs *macsec_fs) { struct mlx5_macsec_tables *tx_tables = &macsec_fs->tx_fs->tables; if (--tx_tables->refcnt) return; macsec_fs_tx_destroy(macsec_fs); } static int macsec_fs_tx_setup_fte(struct mlx5_macsec_fs *macsec_fs, struct mlx5_flow_spec *spec, struct mlx5_flow_act *flow_act, u32 macsec_obj_id, u32 *fs_id) { struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs; int err = 0; u32 id; err = ida_alloc_range(&tx_fs->tx_halloc, 1, MLX5_MACSEC_NUM_OF_SUPPORTED_INTERFACES, GFP_KERNEL); if (err < 0) return err; id = err; spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; /* Metadata match */ MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_MACSEC_MASK); MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_a, macsec_fs_set_tx_fs_id(id)); *fs_id = id; flow_act->crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_MACSEC; flow_act->crypto.obj_id = macsec_obj_id; mlx5_core_dbg(macsec_fs->mdev, "Tx fte: macsec obj_id %u, fs_id %u\n", macsec_obj_id, id); return 0; } static void macsec_fs_tx_create_sectag_header(const struct macsec_context *ctx, char *reformatbf, size_t *reformat_size) { const struct macsec_secy *secy = ctx->secy; bool sci_present = macsec_send_sci(secy); struct mlx5_sectag_header sectag = {}; const struct macsec_tx_sc *tx_sc; tx_sc = &secy->tx_sc; sectag.ethertype = htons(ETH_P_MACSEC); if (sci_present) { sectag.tci_an |= MACSEC_TCI_SC; memcpy(§ag.sci, &secy->sci, sizeof(sectag.sci)); } else { if (tx_sc->end_station) sectag.tci_an |= MACSEC_TCI_ES; if (tx_sc->scb) sectag.tci_an |= MACSEC_TCI_SCB; } /* With GCM, C/E clear for !encrypt, both set for encrypt */ if (tx_sc->encrypt) sectag.tci_an |= MACSEC_TCI_CONFID; else if (secy->icv_len != MACSEC_DEFAULT_ICV_LEN) sectag.tci_an |= MACSEC_TCI_C; sectag.tci_an |= tx_sc->encoding_sa; *reformat_size = MLX5_MACSEC_TAG_LEN + (sci_present ? MACSEC_SCI_LEN : 0); memcpy(reformatbf, §ag, *reformat_size); } static bool macsec_fs_is_macsec_device_empty(struct mlx5_macsec_device *macsec_device) { if (xa_empty(&macsec_device->tx_id_xa) && xa_empty(&macsec_device->rx_id_xa)) return true; return false; } static void macsec_fs_id_del(struct list_head *macsec_devices_list, u32 fs_id, void *macdev, struct rhashtable *hash_table, bool is_tx) { const struct rhashtable_params *rhash = (is_tx) ? &rhash_sci : &rhash_fs_id; struct mlx5_macsec_device *iter, *macsec_device = NULL; struct mlx5_fs_id *fs_id_found; struct xarray *fs_id_xa; list_for_each_entry(iter, macsec_devices_list, macsec_devices_list_entry) { if (iter->macdev == macdev) { macsec_device = iter; break; } } WARN_ON(!macsec_device); fs_id_xa = (is_tx) ? &macsec_device->tx_id_xa : &macsec_device->rx_id_xa; xa_lock(fs_id_xa); fs_id_found = xa_load(fs_id_xa, fs_id); WARN_ON(!fs_id_found); if (!refcount_dec_and_test(&fs_id_found->refcnt)) { xa_unlock(fs_id_xa); return; } if (fs_id_found->id) { /* Make sure ongoing datapath readers sees a valid SA */ rhashtable_remove_fast(hash_table, &fs_id_found->hash, *rhash); fs_id_found->id = 0; } xa_unlock(fs_id_xa); xa_erase(fs_id_xa, fs_id); kfree(fs_id_found); if (macsec_fs_is_macsec_device_empty(macsec_device)) { list_del(&macsec_device->macsec_devices_list_entry); kfree(macsec_device); } } static int macsec_fs_id_add(struct list_head *macsec_devices_list, u32 fs_id, void *macdev, struct rhashtable *hash_table, sci_t sci, bool is_tx) { const struct rhashtable_params *rhash = (is_tx) ? &rhash_sci : &rhash_fs_id; struct mlx5_macsec_device *iter, *macsec_device = NULL; struct mlx5_fs_id *fs_id_iter; struct xarray *fs_id_xa; int err; if (!is_tx) { rcu_read_lock(); fs_id_iter = rhashtable_lookup(hash_table, &fs_id, rhash_fs_id); if (fs_id_iter) { refcount_inc(&fs_id_iter->refcnt); rcu_read_unlock(); return 0; } rcu_read_unlock(); } fs_id_iter = kzalloc(sizeof(*fs_id_iter), GFP_KERNEL); if (!fs_id_iter) return -ENOMEM; list_for_each_entry(iter, macsec_devices_list, macsec_devices_list_entry) { if (iter->macdev == macdev) { macsec_device = iter; break; } } if (!macsec_device) { /* first time adding a SA to that device */ macsec_device = kzalloc(sizeof(*macsec_device), GFP_KERNEL); if (!macsec_device) { err = -ENOMEM; goto err_alloc_dev; } macsec_device->macdev = macdev; xa_init(&macsec_device->tx_id_xa); xa_init(&macsec_device->rx_id_xa); list_add(&macsec_device->macsec_devices_list_entry, macsec_devices_list); } fs_id_xa = (is_tx) ? &macsec_device->tx_id_xa : &macsec_device->rx_id_xa; fs_id_iter->id = fs_id; refcount_set(&fs_id_iter->refcnt, 1); fs_id_iter->sci = sci; err = xa_err(xa_store(fs_id_xa, fs_id, fs_id_iter, GFP_KERNEL)); if (err) goto err_store_id; err = rhashtable_insert_fast(hash_table, &fs_id_iter->hash, *rhash); if (err) goto err_hash_insert; return 0; err_hash_insert: xa_erase(fs_id_xa, fs_id); err_store_id: if (macsec_fs_is_macsec_device_empty(macsec_device)) { list_del(&macsec_device->macsec_devices_list_entry); kfree(macsec_device); } err_alloc_dev: kfree(fs_id_iter); return err; } static void macsec_fs_tx_del_rule(struct mlx5_macsec_fs *macsec_fs, struct mlx5_macsec_tx_rule *tx_rule, void *macdev) { macsec_fs_id_del(&macsec_fs->macsec_devices_list, tx_rule->fs_id, macdev, &macsec_fs->sci_hash, true); if (tx_rule->rule) { mlx5_del_flow_rules(tx_rule->rule); tx_rule->rule = NULL; } if (tx_rule->pkt_reformat) { mlx5_packet_reformat_dealloc(macsec_fs->mdev, tx_rule->pkt_reformat); tx_rule->pkt_reformat = NULL; } if (tx_rule->fs_id) { ida_free(&macsec_fs->tx_fs->tx_halloc, tx_rule->fs_id); tx_rule->fs_id = 0; } kfree(tx_rule); macsec_fs_tx_ft_put(macsec_fs); } #define MLX5_REFORMAT_PARAM_ADD_MACSEC_OFFSET_4_BYTES 1 static union mlx5_macsec_rule * macsec_fs_tx_add_rule(struct mlx5_macsec_fs *macsec_fs, const struct macsec_context *macsec_ctx, struct mlx5_macsec_rule_attrs *attrs, u32 *fs_id) { char reformatbf[MLX5_MACSEC_TAG_LEN + MACSEC_SCI_LEN]; struct mlx5_pkt_reformat_params reformat_params = {}; struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs; struct mlx5_core_dev *mdev = macsec_fs->mdev; union mlx5_macsec_rule *macsec_rule = NULL; struct mlx5_flow_destination dest = {}; struct mlx5_macsec_tables *tx_tables; struct mlx5_macsec_tx_rule *tx_rule; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; size_t reformat_size; int err = 0; tx_tables = &tx_fs->tables; spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) return NULL; err = macsec_fs_tx_ft_get(macsec_fs); if (err) goto out_spec; macsec_rule = kzalloc(sizeof(*macsec_rule), GFP_KERNEL); if (!macsec_rule) { macsec_fs_tx_ft_put(macsec_fs); goto out_spec; } tx_rule = &macsec_rule->tx_rule; /* Tx crypto table crypto rule */ macsec_fs_tx_create_sectag_header(macsec_ctx, reformatbf, &reformat_size); reformat_params.type = MLX5_REFORMAT_TYPE_ADD_MACSEC; reformat_params.size = reformat_size; reformat_params.data = reformatbf; if (is_vlan_dev(macsec_ctx->netdev)) reformat_params.param_0 = MLX5_REFORMAT_PARAM_ADD_MACSEC_OFFSET_4_BYTES; flow_act.pkt_reformat = mlx5_packet_reformat_alloc(mdev, &reformat_params, MLX5_FLOW_NAMESPACE_EGRESS_MACSEC); if (IS_ERR(flow_act.pkt_reformat)) { err = PTR_ERR(flow_act.pkt_reformat); mlx5_core_err(mdev, "Failed to allocate MACsec Tx reformat context err=%d\n", err); goto err; } tx_rule->pkt_reformat = flow_act.pkt_reformat; err = macsec_fs_tx_setup_fte(macsec_fs, spec, &flow_act, attrs->macsec_obj_id, fs_id); if (err) { mlx5_core_err(mdev, "Failed to add packet reformat for MACsec TX crypto rule, err=%d\n", err); goto err; } tx_rule->fs_id = *fs_id; flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT | MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; dest.ft = tx_tables->ft_check; rule = mlx5_add_flow_rules(tx_tables->ft_crypto.t, spec, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "Failed to add MACsec TX crypto rule, err=%d\n", err); goto err; } tx_rule->rule = rule; err = macsec_fs_id_add(&macsec_fs->macsec_devices_list, *fs_id, macsec_ctx->secy->netdev, &macsec_fs->sci_hash, attrs->sci, true); if (err) { mlx5_core_err(mdev, "Failed to save fs_id, err=%d\n", err); goto err; } goto out_spec; err: macsec_fs_tx_del_rule(macsec_fs, tx_rule, macsec_ctx->secy->netdev); macsec_rule = NULL; out_spec: kvfree(spec); return macsec_rule; } static void macsec_fs_tx_cleanup(struct mlx5_macsec_fs *macsec_fs) { struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs; struct mlx5_core_dev *mdev = macsec_fs->mdev; struct mlx5_macsec_tables *tx_tables; if (!tx_fs) return; tx_tables = &tx_fs->tables; if (tx_tables->refcnt) { mlx5_core_err(mdev, "Can't destroy MACsec offload tx_fs, refcnt(%u) isn't 0\n", tx_tables->refcnt); return; } ida_destroy(&tx_fs->tx_halloc); if (tx_tables->check_miss_rule_counter) { mlx5_fc_destroy(mdev, tx_tables->check_miss_rule_counter); tx_tables->check_miss_rule_counter = NULL; } if (tx_tables->check_rule_counter) { mlx5_fc_destroy(mdev, tx_tables->check_rule_counter); tx_tables->check_rule_counter = NULL; } kfree(tx_fs); macsec_fs->tx_fs = NULL; } static int macsec_fs_tx_init(struct mlx5_macsec_fs *macsec_fs) { struct mlx5_core_dev *mdev = macsec_fs->mdev; struct mlx5_macsec_tables *tx_tables; struct mlx5_macsec_tx *tx_fs; struct mlx5_fc *flow_counter; int err; tx_fs = kzalloc(sizeof(*tx_fs), GFP_KERNEL); if (!tx_fs) return -ENOMEM; tx_tables = &tx_fs->tables; flow_counter = mlx5_fc_create(mdev, false); if (IS_ERR(flow_counter)) { err = PTR_ERR(flow_counter); mlx5_core_err(mdev, "Failed to create MACsec Tx encrypt flow counter, err(%d)\n", err); goto err_encrypt_counter; } tx_tables->check_rule_counter = flow_counter; flow_counter = mlx5_fc_create(mdev, false); if (IS_ERR(flow_counter)) { err = PTR_ERR(flow_counter); mlx5_core_err(mdev, "Failed to create MACsec Tx drop flow counter, err(%d)\n", err); goto err_drop_counter; } tx_tables->check_miss_rule_counter = flow_counter; ida_init(&tx_fs->tx_halloc); INIT_LIST_HEAD(&macsec_fs->macsec_devices_list); macsec_fs->tx_fs = tx_fs; return 0; err_drop_counter: mlx5_fc_destroy(mdev, tx_tables->check_rule_counter); tx_tables->check_rule_counter = NULL; err_encrypt_counter: kfree(tx_fs); macsec_fs->tx_fs = NULL; return err; } static void macsec_fs_rx_roce_miss_destroy(struct mlx5_macsec_miss *miss) { mlx5_del_flow_rules(miss->rule); mlx5_destroy_flow_group(miss->g); } static void macsec_fs_rdma_rx_destroy(struct mlx5_macsec_rx_roce *roce, struct mlx5_core_dev *mdev) { if (!mlx5_is_macsec_roce_supported(mdev)) return; mlx5_del_flow_rules(roce->nic_miss.rule); mlx5_del_flow_rules(roce->rule); mlx5_modify_header_dealloc(mdev, roce->copy_modify_hdr); mlx5_destroy_flow_group(roce->nic_miss.g); mlx5_destroy_flow_group(roce->g); mlx5_destroy_flow_table(roce->ft); macsec_fs_rx_roce_miss_destroy(&roce->miss); mlx5_destroy_flow_table(roce->ft_macsec_op_check); mlx5_destroy_flow_table(roce->ft_ip_check); } static void macsec_fs_rx_destroy(struct mlx5_macsec_fs *macsec_fs) { struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs; struct mlx5_macsec_tables *rx_tables; int i; /* Rx check table */ for (i = 1; i >= 0; --i) { if (rx_fs->check_rule[i]) { mlx5_del_flow_rules(rx_fs->check_rule[i]); rx_fs->check_rule[i] = NULL; } if (rx_fs->check_rule_pkt_reformat[i]) { mlx5_packet_reformat_dealloc(macsec_fs->mdev, rx_fs->check_rule_pkt_reformat[i]); rx_fs->check_rule_pkt_reformat[i] = NULL; } } rx_tables = &rx_fs->tables; if (rx_tables->check_miss_rule) { mlx5_del_flow_rules(rx_tables->check_miss_rule); rx_tables->check_miss_rule = NULL; } if (rx_tables->ft_check_group) { mlx5_destroy_flow_group(rx_tables->ft_check_group); rx_tables->ft_check_group = NULL; } if (rx_tables->ft_check) { mlx5_destroy_flow_table(rx_tables->ft_check); rx_tables->ft_check = NULL; } /* Rx crypto table */ if (rx_tables->crypto_miss_rule) { mlx5_del_flow_rules(rx_tables->crypto_miss_rule); rx_tables->crypto_miss_rule = NULL; } macsec_fs_destroy_flow_table(&rx_tables->ft_crypto); macsec_fs_rdma_rx_destroy(&macsec_fs->rx_fs->roce, macsec_fs->mdev); } static int macsec_fs_rx_create_crypto_table_groups(struct mlx5_macsec_flow_table *ft) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); int mclen = MLX5_ST_SZ_BYTES(fte_match_param); int ix = 0; u32 *in; int err; u8 *mc; ft->g = kcalloc(RX_CRYPTO_TABLE_NUM_GROUPS, sizeof(*ft->g), GFP_KERNEL); if (!ft->g) return -ENOMEM; in = kvzalloc(inlen, GFP_KERNEL); if (!in) { kfree(ft->g); return -ENOMEM; } mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); /* Flow group for SA rule with SCI */ MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_5); MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); MLX5_SET(fte_match_param, mc, misc_parameters_5.macsec_tag_0, MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); MLX5_SET_TO_ONES(fte_match_param, mc, misc_parameters_5.macsec_tag_2); MLX5_SET_TO_ONES(fte_match_param, mc, misc_parameters_5.macsec_tag_3); MLX5_SET_CFG(in, start_flow_index, ix); ix += RX_CRYPTO_TABLE_SA_RULE_WITH_SCI_GROUP_SIZE; MLX5_SET_CFG(in, end_flow_index, ix - 1); ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); if (IS_ERR(ft->g[ft->num_groups])) goto err; ft->num_groups++; /* Flow group for SA rule without SCI */ memset(in, 0, inlen); memset(mc, 0, mclen); MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS_5); MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.smac_47_16); MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.smac_15_0); MLX5_SET_TO_ONES(fte_match_param, mc, outer_headers.ethertype); MLX5_SET(fte_match_param, mc, misc_parameters_5.macsec_tag_0, MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); MLX5_SET_CFG(in, start_flow_index, ix); ix += RX_CRYPTO_TABLE_SA_RULE_WITHOUT_SCI_GROUP_SIZE; MLX5_SET_CFG(in, end_flow_index, ix - 1); ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); if (IS_ERR(ft->g[ft->num_groups])) goto err; ft->num_groups++; /* Flow Group for l2 traps */ memset(in, 0, inlen); memset(mc, 0, mclen); MLX5_SET_CFG(in, start_flow_index, ix); ix += CRYPTO_TABLE_DEFAULT_RULE_GROUP_SIZE; MLX5_SET_CFG(in, end_flow_index, ix - 1); ft->g[ft->num_groups] = mlx5_create_flow_group(ft->t, in); if (IS_ERR(ft->g[ft->num_groups])) goto err; ft->num_groups++; kvfree(in); return 0; err: err = PTR_ERR(ft->g[ft->num_groups]); ft->g[ft->num_groups] = NULL; kvfree(in); return err; } static int macsec_fs_rx_create_check_decap_rule(struct mlx5_macsec_fs *macsec_fs, struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, struct mlx5_flow_spec *spec, int reformat_param_size) { int rule_index = (reformat_param_size == MLX5_SECTAG_HEADER_SIZE_WITH_SCI) ? 0 : 1; u8 mlx5_reformat_buf[MLX5_SECTAG_HEADER_SIZE_WITH_SCI]; struct mlx5_pkt_reformat_params reformat_params = {}; struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs; struct mlx5_core_dev *mdev = macsec_fs->mdev; struct mlx5_flow_destination roce_dest[2]; struct mlx5_macsec_tables *rx_tables; struct mlx5_flow_handle *rule; int err = 0, dstn = 0; rx_tables = &rx_fs->tables; /* Rx check table decap 16B rule */ memset(dest, 0, sizeof(*dest)); memset(flow_act, 0, sizeof(*flow_act)); memset(spec, 0, sizeof(*spec)); reformat_params.type = MLX5_REFORMAT_TYPE_DEL_MACSEC; reformat_params.size = reformat_param_size; reformat_params.data = mlx5_reformat_buf; flow_act->pkt_reformat = mlx5_packet_reformat_alloc(mdev, &reformat_params, MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC); if (IS_ERR(flow_act->pkt_reformat)) { err = PTR_ERR(flow_act->pkt_reformat); mlx5_core_err(mdev, "Failed to allocate MACsec Rx reformat context err=%d\n", err); return err; } rx_fs->check_rule_pkt_reformat[rule_index] = flow_act->pkt_reformat; spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; /* MACsec syndrome match */ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.macsec_syndrome); MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.macsec_syndrome, 0); /* ASO return reg syndrome match */ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_4); MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_4, 0); spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_5; /* Sectag TCI SC present bit*/ MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_5.macsec_tag_0, MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); if (reformat_param_size == MLX5_SECTAG_HEADER_SIZE_WITH_SCI) MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_0, MLX5_MACSEC_SECTAG_TCI_SC_FIELD_BIT << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); flow_act->flags = FLOW_ACT_NO_APPEND; if (rx_fs->roce.ft) { flow_act->action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; roce_dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; roce_dest[dstn].ft = rx_fs->roce.ft; dstn++; } else { flow_act->action = MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; } flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT | MLX5_FLOW_CONTEXT_ACTION_COUNT; roce_dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; roce_dest[dstn].counter_id = mlx5_fc_id(rx_tables->check_rule_counter); rule = mlx5_add_flow_rules(rx_tables->ft_check, spec, flow_act, roce_dest, dstn + 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "Failed to add MACsec Rx check rule, err=%d\n", err); return err; } rx_fs->check_rule[rule_index] = rule; return 0; } static int macsec_fs_rx_roce_miss_create(struct mlx5_core_dev *mdev, struct mlx5_macsec_rx_roce *roce) { struct mlx5_flow_act flow_act = {}; struct mlx5_flow_group *flow_group; struct mlx5_flow_handle *rule; u32 *flow_group_in; int err; flow_group_in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL); if (!flow_group_in) return -ENOMEM; /* IP check ft has no miss rule since we use default miss action which is go to next PRIO */ MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, roce->ft_macsec_op_check->max_fte - 1); MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, roce->ft_macsec_op_check->max_fte - 1); flow_group = mlx5_create_flow_group(roce->ft_macsec_op_check, flow_group_in); if (IS_ERR(flow_group)) { err = PTR_ERR(flow_group); mlx5_core_err(mdev, "Failed to create miss flow group for MACsec RoCE operation check table err(%d)\n", err); goto err_macsec_op_miss_group; } roce->miss.g = flow_group; flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; rule = mlx5_add_flow_rules(roce->ft_macsec_op_check, NULL, &flow_act, NULL, 0); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "Failed to add miss rule to MACsec RoCE operation check table err(%d)\n", err); goto err_macsec_op_rule; } roce->miss.rule = rule; kvfree(flow_group_in); return 0; err_macsec_op_rule: mlx5_destroy_flow_group(roce->miss.g); err_macsec_op_miss_group: kvfree(flow_group_in); return err; } #define MLX5_RX_ROCE_GROUP_SIZE BIT(0) static int macsec_fs_rx_roce_jump_to_rdma_groups_create(struct mlx5_core_dev *mdev, struct mlx5_macsec_rx_roce *roce) { struct mlx5_flow_group *g; void *outer_headers_c; int ix = 0; u32 *in; int err; u8 *mc; in = kvzalloc(MLX5_ST_SZ_BYTES(create_flow_group_in), GFP_KERNEL); if (!in) return -ENOMEM; mc = MLX5_ADDR_OF(create_flow_group_in, in, match_criteria); outer_headers_c = MLX5_ADDR_OF(fte_match_param, mc, outer_headers); MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, ip_protocol); MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, outer_headers_c, udp_dport); MLX5_SET_CFG(in, match_criteria_enable, MLX5_MATCH_OUTER_HEADERS); MLX5_SET_CFG(in, start_flow_index, ix); ix += MLX5_RX_ROCE_GROUP_SIZE; MLX5_SET_CFG(in, end_flow_index, ix - 1); g = mlx5_create_flow_group(roce->ft, in); if (IS_ERR(g)) { err = PTR_ERR(g); mlx5_core_err(mdev, "Failed to create main flow group for MACsec RoCE NIC UDP table err(%d)\n", err); goto err_udp_group; } roce->g = g; memset(in, 0, MLX5_ST_SZ_BYTES(create_flow_group_in)); MLX5_SET_CFG(in, start_flow_index, ix); ix += MLX5_RX_ROCE_GROUP_SIZE; MLX5_SET_CFG(in, end_flow_index, ix - 1); g = mlx5_create_flow_group(roce->ft, in); if (IS_ERR(g)) { err = PTR_ERR(g); mlx5_core_err(mdev, "Failed to create miss flow group for MACsec RoCE NIC UDP table err(%d)\n", err); goto err_udp_miss_group; } roce->nic_miss.g = g; kvfree(in); return 0; err_udp_miss_group: mlx5_destroy_flow_group(roce->g); err_udp_group: kvfree(in); return err; } static int macsec_fs_rx_roce_jump_to_rdma_rules_create(struct mlx5_macsec_fs *macsec_fs, struct mlx5_macsec_rx_roce *roce) { u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; struct mlx5_core_dev *mdev = macsec_fs->mdev; struct mlx5_flow_destination dst = {}; struct mlx5_modify_hdr *modify_hdr; MLX5_DECLARE_FLOW_ACT(flow_act); struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; int err; spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) return -ENOMEM; spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_UDP); MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.udp_dport); MLX5_SET(fte_match_param, spec->match_value, outer_headers.udp_dport, ROCE_V2_UDP_DPORT); MLX5_SET(copy_action_in, action, action_type, MLX5_ACTION_TYPE_COPY); MLX5_SET(copy_action_in, action, src_field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); MLX5_SET(copy_action_in, action, src_offset, 0); MLX5_SET(copy_action_in, action, length, 32); MLX5_SET(copy_action_in, action, dst_field, MLX5_ACTION_IN_FIELD_METADATA_REG_C_5); MLX5_SET(copy_action_in, action, dst_offset, 0); modify_hdr = mlx5_modify_header_alloc(macsec_fs->mdev, MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC, 1, action); if (IS_ERR(modify_hdr)) { err = PTR_ERR(modify_hdr); mlx5_core_err(mdev, "Failed to alloc macsec copy modify_header_id err(%d)\n", err); goto err_alloc_hdr; } flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; flow_act.modify_hdr = modify_hdr; dst.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; dst.ft = roce->ft_ip_check; rule = mlx5_add_flow_rules(roce->ft, spec, &flow_act, &dst, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "Failed to add rule to MACsec RoCE NIC UDP table err(%d)\n", err); goto err_add_rule; } roce->rule = rule; roce->copy_modify_hdr = modify_hdr; memset(&flow_act, 0, sizeof(flow_act)); flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; rule = mlx5_add_flow_rules(roce->ft, NULL, &flow_act, NULL, 0); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "Failed to add miss rule to MACsec RoCE NIC UDP table err(%d)\n", err); goto err_add_rule2; } roce->nic_miss.rule = rule; kvfree(spec); return 0; err_add_rule2: mlx5_del_flow_rules(roce->rule); err_add_rule: mlx5_modify_header_dealloc(macsec_fs->mdev, modify_hdr); err_alloc_hdr: kvfree(spec); return err; } static int macsec_fs_rx_roce_jump_to_rdma_create(struct mlx5_macsec_fs *macsec_fs, struct mlx5_macsec_rx_roce *roce) { int err; err = macsec_fs_rx_roce_jump_to_rdma_groups_create(macsec_fs->mdev, roce); if (err) return err; err = macsec_fs_rx_roce_jump_to_rdma_rules_create(macsec_fs, roce); if (err) goto err; return 0; err: mlx5_destroy_flow_group(roce->nic_miss.g); mlx5_destroy_flow_group(roce->g); return err; } static int macsec_fs_rx_roce_create(struct mlx5_macsec_fs *macsec_fs) { struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs; struct mlx5_core_dev *mdev = macsec_fs->mdev; struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_flow_namespace *ns; struct mlx5_flow_table *ft; int err = 0; if (!mlx5_is_macsec_roce_supported(macsec_fs->mdev)) { mlx5_core_dbg(mdev, "Failed to init RoCE MACsec, capabilities not supported\n"); return 0; } ns = mlx5_get_flow_namespace(macsec_fs->mdev, MLX5_FLOW_NAMESPACE_RDMA_RX_MACSEC); if (!ns) return -ENOMEM; ft = macsec_fs_auto_group_table_create(ns, 0, RDMA_RX_ROCE_IP_TABLE_LEVEL, CRYPTO_NUM_MAXSEC_FTE); if (IS_ERR(ft)) { err = PTR_ERR(ft); mlx5_core_err(mdev, "Failed to create MACsec IP check RoCE table err(%d)\n", err); return err; } rx_fs->roce.ft_ip_check = ft; ft = macsec_fs_auto_group_table_create(ns, 0, RDMA_RX_ROCE_MACSEC_OP_TABLE_LEVEL, CRYPTO_NUM_MAXSEC_FTE); if (IS_ERR(ft)) { err = PTR_ERR(ft); mlx5_core_err(mdev, "Failed to create MACsec operation check RoCE table err(%d)\n", err); goto err_macsec_op; } rx_fs->roce.ft_macsec_op_check = ft; err = macsec_fs_rx_roce_miss_create(mdev, &rx_fs->roce); if (err) goto err_miss_create; ns = mlx5_get_flow_namespace(macsec_fs->mdev, MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC); if (!ns) { err = -EOPNOTSUPP; goto err_ns; } ft_attr.level = RX_ROCE_TABLE_LEVEL; ft_attr.max_fte = RX_ROCE_TABLE_NUM_FTE; ft = mlx5_create_flow_table(ns, &ft_attr); if (IS_ERR(ft)) { err = PTR_ERR(ft); mlx5_core_err(mdev, "Failed to create MACsec jump to RX RoCE, NIC table err(%d)\n", err); goto err_ns; } rx_fs->roce.ft = ft; err = macsec_fs_rx_roce_jump_to_rdma_create(macsec_fs, &rx_fs->roce); if (err) goto err_udp_ft; return 0; err_udp_ft: mlx5_destroy_flow_table(rx_fs->roce.ft); err_ns: macsec_fs_rx_roce_miss_destroy(&rx_fs->roce.miss); err_miss_create: mlx5_destroy_flow_table(rx_fs->roce.ft_macsec_op_check); err_macsec_op: mlx5_destroy_flow_table(rx_fs->roce.ft_ip_check); return err; } static int macsec_fs_rx_create(struct mlx5_macsec_fs *macsec_fs) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs; struct mlx5_core_dev *mdev = macsec_fs->mdev; struct mlx5_macsec_flow_table *ft_crypto; struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_flow_destination dest = {}; struct mlx5_macsec_tables *rx_tables; struct mlx5_flow_table *flow_table; struct mlx5_flow_group *flow_group; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_namespace *ns; struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; u32 *flow_group_in; int err; ns = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC); if (!ns) return -ENOMEM; spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) return -ENOMEM; flow_group_in = kvzalloc(inlen, GFP_KERNEL); if (!flow_group_in) { err = -ENOMEM; goto free_spec; } rx_tables = &rx_fs->tables; ft_crypto = &rx_tables->ft_crypto; err = macsec_fs_rx_roce_create(macsec_fs); if (err) goto out_flow_group; /* Rx crypto table */ ft_attr.level = RX_CRYPTO_TABLE_LEVEL; ft_attr.max_fte = CRYPTO_NUM_MAXSEC_FTE; flow_table = mlx5_create_flow_table(ns, &ft_attr); if (IS_ERR(flow_table)) { err = PTR_ERR(flow_table); mlx5_core_err(mdev, "Failed to create MACsec Rx crypto table err(%d)\n", err); goto err; } ft_crypto->t = flow_table; /* Rx crypto table groups */ err = macsec_fs_rx_create_crypto_table_groups(ft_crypto); if (err) { mlx5_core_err(mdev, "Failed to create default flow group for MACsec Tx crypto table err(%d)\n", err); goto err; } flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_NEXT_PRIO; rule = mlx5_add_flow_rules(ft_crypto->t, NULL, &flow_act, NULL, 0); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "Failed to add MACsec Rx crypto table default miss rule %d\n", err); goto err; } rx_tables->crypto_miss_rule = rule; /* Rx check table */ flow_table = macsec_fs_auto_group_table_create(ns, MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT, RX_CHECK_TABLE_LEVEL, RX_CHECK_TABLE_NUM_FTE); if (IS_ERR(flow_table)) { err = PTR_ERR(flow_table); mlx5_core_err(mdev, "Fail to create MACsec RX check table, err(%d)\n", err); goto err; } rx_tables->ft_check = flow_table; /* Rx check table Default miss group/rule */ MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_table->max_fte - 1); MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_table->max_fte - 1); flow_group = mlx5_create_flow_group(rx_tables->ft_check, flow_group_in); if (IS_ERR(flow_group)) { err = PTR_ERR(flow_group); mlx5_core_err(mdev, "Failed to create default flow group for MACsec Rx check table err(%d)\n", err); goto err; } rx_tables->ft_check_group = flow_group; /* Rx check table default drop rule */ memset(&flow_act, 0, sizeof(flow_act)); dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; dest.counter_id = mlx5_fc_id(rx_tables->check_miss_rule_counter); flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; rule = mlx5_add_flow_rules(rx_tables->ft_check, NULL, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "Failed to added MACsec Rx check drop rule, err(%d)\n", err); goto err; } rx_tables->check_miss_rule = rule; /* Rx check table decap rules */ err = macsec_fs_rx_create_check_decap_rule(macsec_fs, &dest, &flow_act, spec, MLX5_SECTAG_HEADER_SIZE_WITH_SCI); if (err) goto err; err = macsec_fs_rx_create_check_decap_rule(macsec_fs, &dest, &flow_act, spec, MLX5_SECTAG_HEADER_SIZE_WITHOUT_SCI); if (err) goto err; goto out_flow_group; err: macsec_fs_rx_destroy(macsec_fs); out_flow_group: kvfree(flow_group_in); free_spec: kvfree(spec); return err; } static int macsec_fs_rx_ft_get(struct mlx5_macsec_fs *macsec_fs) { struct mlx5_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables; int err = 0; if (rx_tables->refcnt) goto out; err = macsec_fs_rx_create(macsec_fs); if (err) return err; out: rx_tables->refcnt++; return err; } static void macsec_fs_rx_ft_put(struct mlx5_macsec_fs *macsec_fs) { struct mlx5_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables; if (--rx_tables->refcnt) return; macsec_fs_rx_destroy(macsec_fs); } static void macsec_fs_rx_del_rule(struct mlx5_macsec_fs *macsec_fs, struct mlx5_macsec_rx_rule *rx_rule, void *macdev, u32 fs_id) { int i; macsec_fs_id_del(&macsec_fs->macsec_devices_list, fs_id, macdev, &macsec_fs->fs_id_hash, false); for (i = 0; i < RX_NUM_OF_RULES_PER_SA; ++i) { if (rx_rule->rule[i]) { mlx5_del_flow_rules(rx_rule->rule[i]); rx_rule->rule[i] = NULL; } } if (rx_rule->meta_modhdr) { mlx5_modify_header_dealloc(macsec_fs->mdev, rx_rule->meta_modhdr); rx_rule->meta_modhdr = NULL; } kfree(rx_rule); macsec_fs_rx_ft_put(macsec_fs); } static void macsec_fs_rx_setup_fte(struct mlx5_flow_spec *spec, struct mlx5_flow_act *flow_act, struct mlx5_macsec_rule_attrs *attrs, bool sci_present) { u8 tci_an = (sci_present << MLX5_MACSEC_SECTAG_TCI_SC_FIELD_OFFSET) | attrs->assoc_num; struct mlx5_flow_act_crypto_params *crypto_params = &flow_act->crypto; __be32 *sci_p = (__be32 *)(&attrs->sci); spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; /* MACsec ethertype */ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ethertype); MLX5_SET(fte_match_param, spec->match_value, outer_headers.ethertype, ETH_P_MACSEC); spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_5; /* Sectag AN + TCI SC present bit*/ MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_5.macsec_tag_0, MLX5_MACSEC_SECTAG_TCI_AN_FIELD_BITMASK << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_0, tci_an << MLX5_MACSEC_SECTAG_TCI_AN_FIELD_OFFSET); if (sci_present) { MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_5.macsec_tag_2); MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_2, be32_to_cpu(sci_p[0])); MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_5.macsec_tag_3); MLX5_SET(fte_match_param, spec->match_value, misc_parameters_5.macsec_tag_3, be32_to_cpu(sci_p[1])); } else { /* When SCI isn't present in the Sectag, need to match the source */ /* MAC address only if the SCI contains the default MACsec PORT */ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.smac_47_16); MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.smac_15_0); memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers.smac_47_16), sci_p, ETH_ALEN); } crypto_params->type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_MACSEC; crypto_params->obj_id = attrs->macsec_obj_id; } static union mlx5_macsec_rule * macsec_fs_rx_add_rule(struct mlx5_macsec_fs *macsec_fs, const struct macsec_context *macsec_ctx, struct mlx5_macsec_rule_attrs *attrs, u32 fs_id) { u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs; struct mlx5_core_dev *mdev = macsec_fs->mdev; union mlx5_macsec_rule *macsec_rule = NULL; struct mlx5_modify_hdr *modify_hdr = NULL; struct mlx5_macsec_flow_table *ft_crypto; struct mlx5_flow_destination dest = {}; struct mlx5_macsec_tables *rx_tables; struct mlx5_macsec_rx_rule *rx_rule; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; int err = 0; spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) return NULL; err = macsec_fs_rx_ft_get(macsec_fs); if (err) goto out_spec; macsec_rule = kzalloc(sizeof(*macsec_rule), GFP_KERNEL); if (!macsec_rule) { macsec_fs_rx_ft_put(macsec_fs); goto out_spec; } rx_rule = &macsec_rule->rx_rule; rx_tables = &rx_fs->tables; ft_crypto = &rx_tables->ft_crypto; /* Set bit[31 - 30] macsec marker - 0x01 */ /* Set bit[15-0] fs id */ MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); MLX5_SET(set_action_in, action, data, macsec_fs_set_rx_fs_id(fs_id)); MLX5_SET(set_action_in, action, offset, 0); MLX5_SET(set_action_in, action, length, 32); modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL_RX_MACSEC, 1, action); if (IS_ERR(modify_hdr)) { err = PTR_ERR(modify_hdr); mlx5_core_err(mdev, "Fail to alloc MACsec set modify_header_id err=%d\n", err); modify_hdr = NULL; goto err; } rx_rule->meta_modhdr = modify_hdr; /* Rx crypto table with SCI rule */ macsec_fs_rx_setup_fte(spec, &flow_act, attrs, true); flow_act.modify_hdr = modify_hdr; flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; dest.ft = rx_tables->ft_check; rule = mlx5_add_flow_rules(ft_crypto->t, spec, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "Failed to add SA with SCI rule to Rx crypto rule, err=%d\n", err); goto err; } rx_rule->rule[0] = rule; /* Rx crypto table without SCI rule */ if ((cpu_to_be64((__force u64)attrs->sci) & 0xFFFF) == ntohs(MACSEC_PORT_ES)) { memset(spec, 0, sizeof(struct mlx5_flow_spec)); memset(&dest, 0, sizeof(struct mlx5_flow_destination)); memset(&flow_act, 0, sizeof(flow_act)); macsec_fs_rx_setup_fte(spec, &flow_act, attrs, false); flow_act.modify_hdr = modify_hdr; flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; dest.ft = rx_tables->ft_check; rule = mlx5_add_flow_rules(ft_crypto->t, spec, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); mlx5_core_err(mdev, "Failed to add SA without SCI rule to Rx crypto rule, err=%d\n", err); goto err; } rx_rule->rule[1] = rule; } err = macsec_fs_id_add(&macsec_fs->macsec_devices_list, fs_id, macsec_ctx->secy->netdev, &macsec_fs->fs_id_hash, attrs->sci, false); if (err) { mlx5_core_err(mdev, "Failed to save fs_id, err=%d\n", err); goto err; } kvfree(spec); return macsec_rule; err: macsec_fs_rx_del_rule(macsec_fs, rx_rule, macsec_ctx->secy->netdev, fs_id); macsec_rule = NULL; out_spec: kvfree(spec); return macsec_rule; } static int macsec_fs_rx_init(struct mlx5_macsec_fs *macsec_fs) { struct mlx5_core_dev *mdev = macsec_fs->mdev; struct mlx5_macsec_tables *rx_tables; struct mlx5_macsec_rx *rx_fs; struct mlx5_fc *flow_counter; int err; rx_fs = kzalloc(sizeof(*rx_fs), GFP_KERNEL); if (!rx_fs) return -ENOMEM; flow_counter = mlx5_fc_create(mdev, false); if (IS_ERR(flow_counter)) { err = PTR_ERR(flow_counter); mlx5_core_err(mdev, "Failed to create MACsec Rx encrypt flow counter, err(%d)\n", err); goto err_encrypt_counter; } rx_tables = &rx_fs->tables; rx_tables->check_rule_counter = flow_counter; flow_counter = mlx5_fc_create(mdev, false); if (IS_ERR(flow_counter)) { err = PTR_ERR(flow_counter); mlx5_core_err(mdev, "Failed to create MACsec Rx drop flow counter, err(%d)\n", err); goto err_drop_counter; } rx_tables->check_miss_rule_counter = flow_counter; macsec_fs->rx_fs = rx_fs; return 0; err_drop_counter: mlx5_fc_destroy(mdev, rx_tables->check_rule_counter); rx_tables->check_rule_counter = NULL; err_encrypt_counter: kfree(rx_fs); macsec_fs->rx_fs = NULL; return err; } static void macsec_fs_rx_cleanup(struct mlx5_macsec_fs *macsec_fs) { struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs; struct mlx5_core_dev *mdev = macsec_fs->mdev; struct mlx5_macsec_tables *rx_tables; if (!rx_fs) return; rx_tables = &rx_fs->tables; if (rx_tables->refcnt) { mlx5_core_err(mdev, "Can't destroy MACsec offload rx_fs, refcnt(%u) isn't 0\n", rx_tables->refcnt); return; } if (rx_tables->check_miss_rule_counter) { mlx5_fc_destroy(mdev, rx_tables->check_miss_rule_counter); rx_tables->check_miss_rule_counter = NULL; } if (rx_tables->check_rule_counter) { mlx5_fc_destroy(mdev, rx_tables->check_rule_counter); rx_tables->check_rule_counter = NULL; } kfree(rx_fs); macsec_fs->rx_fs = NULL; } static void set_ipaddr_spec_v4(struct sockaddr_in *in, struct mlx5_flow_spec *spec, bool is_dst_ip) { MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, MLX5_FS_IPV4_VERSION); if (is_dst_ip) { MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), &in->sin_addr.s_addr, 4); } else { MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), &in->sin_addr.s_addr, 4); } } static void set_ipaddr_spec_v6(struct sockaddr_in6 *in6, struct mlx5_flow_spec *spec, bool is_dst_ip) { MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, MLX5_FS_IPV6_VERSION); if (is_dst_ip) { memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 0xff, 16); memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), &in6->sin6_addr, 16); } else { memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 0xff, 16); memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), &in6->sin6_addr, 16); } } static void set_ipaddr_spec(const struct sockaddr *addr, struct mlx5_flow_spec *spec, bool is_dst_ip) { struct sockaddr_in6 *in6; spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); if (addr->sa_family == AF_INET) { struct sockaddr_in *in = (struct sockaddr_in *)addr; set_ipaddr_spec_v4(in, spec, is_dst_ip); return; } in6 = (struct sockaddr_in6 *)addr; set_ipaddr_spec_v6(in6, spec, is_dst_ip); } static void macsec_fs_del_roce_rule_rx(struct mlx5_roce_macsec_rx_rule *rx_rule) { mlx5_del_flow_rules(rx_rule->op); mlx5_del_flow_rules(rx_rule->ip); list_del(&rx_rule->entry); kfree(rx_rule); } static void macsec_fs_del_roce_rules_rx(struct mlx5_macsec_fs *macsec_fs, u32 fs_id, struct list_head *rx_rules_list) { struct mlx5_roce_macsec_rx_rule *rx_rule, *next; if (!mlx5_is_macsec_roce_supported(macsec_fs->mdev)) return; list_for_each_entry_safe(rx_rule, next, rx_rules_list, entry) { if (rx_rule->fs_id == fs_id) macsec_fs_del_roce_rule_rx(rx_rule); } } static void macsec_fs_del_roce_rule_tx(struct mlx5_core_dev *mdev, struct mlx5_roce_macsec_tx_rule *tx_rule) { mlx5_del_flow_rules(tx_rule->rule); mlx5_modify_header_dealloc(mdev, tx_rule->meta_modhdr); list_del(&tx_rule->entry); kfree(tx_rule); } static void macsec_fs_del_roce_rules_tx(struct mlx5_macsec_fs *macsec_fs, u32 fs_id, struct list_head *tx_rules_list) { struct mlx5_roce_macsec_tx_rule *tx_rule, *next; if (!mlx5_is_macsec_roce_supported(macsec_fs->mdev)) return; list_for_each_entry_safe(tx_rule, next, tx_rules_list, entry) { if (tx_rule->fs_id == fs_id) macsec_fs_del_roce_rule_tx(macsec_fs->mdev, tx_rule); } } void mlx5_macsec_fs_get_stats_fill(struct mlx5_macsec_fs *macsec_fs, void *macsec_stats) { struct mlx5_macsec_stats *stats = (struct mlx5_macsec_stats *)macsec_stats; struct mlx5_macsec_tables *tx_tables = &macsec_fs->tx_fs->tables; struct mlx5_macsec_tables *rx_tables = &macsec_fs->rx_fs->tables; struct mlx5_core_dev *mdev = macsec_fs->mdev; if (tx_tables->check_rule_counter) mlx5_fc_query(mdev, tx_tables->check_rule_counter, &stats->macsec_tx_pkts, &stats->macsec_tx_bytes); if (tx_tables->check_miss_rule_counter) mlx5_fc_query(mdev, tx_tables->check_miss_rule_counter, &stats->macsec_tx_pkts_drop, &stats->macsec_tx_bytes_drop); if (rx_tables->check_rule_counter) mlx5_fc_query(mdev, rx_tables->check_rule_counter, &stats->macsec_rx_pkts, &stats->macsec_rx_bytes); if (rx_tables->check_miss_rule_counter) mlx5_fc_query(mdev, rx_tables->check_miss_rule_counter, &stats->macsec_rx_pkts_drop, &stats->macsec_rx_bytes_drop); } struct mlx5_macsec_stats *mlx5_macsec_fs_get_stats(struct mlx5_macsec_fs *macsec_fs) { if (!macsec_fs) return NULL; return &macsec_fs->stats; } u32 mlx5_macsec_fs_get_fs_id_from_hashtable(struct mlx5_macsec_fs *macsec_fs, sci_t *sci) { struct mlx5_fs_id *mlx5_fs_id; u32 fs_id = 0; rcu_read_lock(); mlx5_fs_id = rhashtable_lookup(&macsec_fs->sci_hash, sci, rhash_sci); if (mlx5_fs_id) fs_id = mlx5_fs_id->id; rcu_read_unlock(); return fs_id; } union mlx5_macsec_rule * mlx5_macsec_fs_add_rule(struct mlx5_macsec_fs *macsec_fs, const struct macsec_context *macsec_ctx, struct mlx5_macsec_rule_attrs *attrs, u32 *sa_fs_id) { struct mlx5_macsec_event_data data = {.macsec_fs = macsec_fs, .macdev = macsec_ctx->secy->netdev, .is_tx = (attrs->action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) }; union mlx5_macsec_rule *macsec_rule; u32 tx_new_fs_id; macsec_rule = (attrs->action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) ? macsec_fs_tx_add_rule(macsec_fs, macsec_ctx, attrs, &tx_new_fs_id) : macsec_fs_rx_add_rule(macsec_fs, macsec_ctx, attrs, *sa_fs_id); data.fs_id = (data.is_tx) ? tx_new_fs_id : *sa_fs_id; if (macsec_rule) blocking_notifier_call_chain(&macsec_fs->mdev->macsec_nh, MLX5_DRIVER_EVENT_MACSEC_SA_ADDED, &data); return macsec_rule; } void mlx5_macsec_fs_del_rule(struct mlx5_macsec_fs *macsec_fs, union mlx5_macsec_rule *macsec_rule, int action, void *macdev, u32 sa_fs_id) { struct mlx5_macsec_event_data data = {.macsec_fs = macsec_fs, .macdev = macdev, .is_tx = (action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) }; data.fs_id = (data.is_tx) ? macsec_rule->tx_rule.fs_id : sa_fs_id; blocking_notifier_call_chain(&macsec_fs->mdev->macsec_nh, MLX5_DRIVER_EVENT_MACSEC_SA_DELETED, &data); (action == MLX5_ACCEL_MACSEC_ACTION_ENCRYPT) ? macsec_fs_tx_del_rule(macsec_fs, &macsec_rule->tx_rule, macdev) : macsec_fs_rx_del_rule(macsec_fs, &macsec_rule->rx_rule, macdev, sa_fs_id); } static int mlx5_macsec_fs_add_roce_rule_rx(struct mlx5_macsec_fs *macsec_fs, u32 fs_id, u16 gid_idx, const struct sockaddr *addr, struct list_head *rx_rules_list) { struct mlx5_macsec_rx *rx_fs = macsec_fs->rx_fs; struct mlx5_roce_macsec_rx_rule *rx_rule; struct mlx5_flow_destination dest = {}; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *new_rule; struct mlx5_flow_spec *spec; int err = 0; spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) return -ENOMEM; rx_rule = kzalloc(sizeof(*rx_rule), GFP_KERNEL); if (!rx_rule) { err = -ENOMEM; goto out; } set_ipaddr_spec(addr, spec, true); flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; dest.ft = rx_fs->roce.ft_macsec_op_check; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; new_rule = mlx5_add_flow_rules(rx_fs->roce.ft_ip_check, spec, &flow_act, &dest, 1); if (IS_ERR(new_rule)) { err = PTR_ERR(new_rule); goto ip_rule_err; } rx_rule->ip = new_rule; memset(&flow_act, 0, sizeof(flow_act)); memset(spec, 0, sizeof(*spec)); spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_c_5); MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_5, macsec_fs_set_rx_fs_id(fs_id)); flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW; new_rule = mlx5_add_flow_rules(rx_fs->roce.ft_macsec_op_check, spec, &flow_act, NULL, 0); if (IS_ERR(new_rule)) { err = PTR_ERR(new_rule); goto op_rule_err; } rx_rule->op = new_rule; rx_rule->gid_idx = gid_idx; rx_rule->fs_id = fs_id; list_add_tail(&rx_rule->entry, rx_rules_list); goto out; op_rule_err: mlx5_del_flow_rules(rx_rule->ip); rx_rule->ip = NULL; ip_rule_err: kfree(rx_rule); out: kvfree(spec); return err; } static int mlx5_macsec_fs_add_roce_rule_tx(struct mlx5_macsec_fs *macsec_fs, u32 fs_id, u16 gid_idx, const struct sockaddr *addr, struct list_head *tx_rules_list) { u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; struct mlx5_macsec_tx *tx_fs = macsec_fs->tx_fs; struct mlx5_core_dev *mdev = macsec_fs->mdev; struct mlx5_modify_hdr *modify_hdr = NULL; struct mlx5_roce_macsec_tx_rule *tx_rule; struct mlx5_flow_destination dest = {}; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *new_rule; struct mlx5_flow_spec *spec; int err = 0; spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) return -ENOMEM; tx_rule = kzalloc(sizeof(*tx_rule), GFP_KERNEL); if (!tx_rule) { err = -ENOMEM; goto out; } set_ipaddr_spec(addr, spec, false); MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_A); MLX5_SET(set_action_in, action, data, macsec_fs_set_tx_fs_id(fs_id)); MLX5_SET(set_action_in, action, offset, 0); MLX5_SET(set_action_in, action, length, 32); modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_RDMA_TX_MACSEC, 1, action); if (IS_ERR(modify_hdr)) { err = PTR_ERR(modify_hdr); mlx5_core_err(mdev, "Fail to alloc ROCE MACsec set modify_header_id err=%d\n", err); modify_hdr = NULL; goto modify_hdr_err; } tx_rule->meta_modhdr = modify_hdr; flow_act.modify_hdr = modify_hdr; flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; dest.type = MLX5_FLOW_DESTINATION_TYPE_TABLE_TYPE; dest.ft = tx_fs->tables.ft_crypto.t; new_rule = mlx5_add_flow_rules(tx_fs->ft_rdma_tx, spec, &flow_act, &dest, 1); if (IS_ERR(new_rule)) { err = PTR_ERR(new_rule); mlx5_core_err(mdev, "Failed to add ROCE TX rule, err=%d\n", err); goto rule_err; } tx_rule->rule = new_rule; tx_rule->gid_idx = gid_idx; tx_rule->fs_id = fs_id; list_add_tail(&tx_rule->entry, tx_rules_list); goto out; rule_err: mlx5_modify_header_dealloc(mdev, tx_rule->meta_modhdr); modify_hdr_err: kfree(tx_rule); out: kvfree(spec); return err; } void mlx5_macsec_del_roce_rule(u16 gid_idx, struct mlx5_macsec_fs *macsec_fs, struct list_head *tx_rules_list, struct list_head *rx_rules_list) { struct mlx5_roce_macsec_rx_rule *rx_rule, *next_rx; struct mlx5_roce_macsec_tx_rule *tx_rule, *next_tx; list_for_each_entry_safe(tx_rule, next_tx, tx_rules_list, entry) { if (tx_rule->gid_idx == gid_idx) macsec_fs_del_roce_rule_tx(macsec_fs->mdev, tx_rule); } list_for_each_entry_safe(rx_rule, next_rx, rx_rules_list, entry) { if (rx_rule->gid_idx == gid_idx) macsec_fs_del_roce_rule_rx(rx_rule); } } EXPORT_SYMBOL_GPL(mlx5_macsec_del_roce_rule); int mlx5_macsec_add_roce_rule(void *macdev, const struct sockaddr *addr, u16 gid_idx, struct list_head *tx_rules_list, struct list_head *rx_rules_list, struct mlx5_macsec_fs *macsec_fs) { struct mlx5_macsec_device *iter, *macsec_device = NULL; struct mlx5_core_dev *mdev = macsec_fs->mdev; struct mlx5_fs_id *fs_id_iter; unsigned long index = 0; int err; list_for_each_entry(iter, &macsec_fs->macsec_devices_list, macsec_devices_list_entry) { if (iter->macdev == macdev) { macsec_device = iter; break; } } if (!macsec_device) return 0; xa_for_each(&macsec_device->tx_id_xa, index, fs_id_iter) { err = mlx5_macsec_fs_add_roce_rule_tx(macsec_fs, fs_id_iter->id, gid_idx, addr, tx_rules_list); if (err) { mlx5_core_err(mdev, "MACsec offload: Failed to add roce TX rule\n"); goto out; } } index = 0; xa_for_each(&macsec_device->rx_id_xa, index, fs_id_iter) { err = mlx5_macsec_fs_add_roce_rule_rx(macsec_fs, fs_id_iter->id, gid_idx, addr, rx_rules_list); if (err) { mlx5_core_err(mdev, "MACsec offload: Failed to add roce TX rule\n"); goto out; } } return 0; out: mlx5_macsec_del_roce_rule(gid_idx, macsec_fs, tx_rules_list, rx_rules_list); return err; } EXPORT_SYMBOL_GPL(mlx5_macsec_add_roce_rule); void mlx5_macsec_add_roce_sa_rules(u32 fs_id, const struct sockaddr *addr, u16 gid_idx, struct list_head *tx_rules_list, struct list_head *rx_rules_list, struct mlx5_macsec_fs *macsec_fs, bool is_tx) { (is_tx) ? mlx5_macsec_fs_add_roce_rule_tx(macsec_fs, fs_id, gid_idx, addr, tx_rules_list) : mlx5_macsec_fs_add_roce_rule_rx(macsec_fs, fs_id, gid_idx, addr, rx_rules_list); } EXPORT_SYMBOL_GPL(mlx5_macsec_add_roce_sa_rules); void mlx5_macsec_del_roce_sa_rules(u32 fs_id, struct mlx5_macsec_fs *macsec_fs, struct list_head *tx_rules_list, struct list_head *rx_rules_list, bool is_tx) { (is_tx) ? macsec_fs_del_roce_rules_tx(macsec_fs, fs_id, tx_rules_list) : macsec_fs_del_roce_rules_rx(macsec_fs, fs_id, rx_rules_list); } EXPORT_SYMBOL_GPL(mlx5_macsec_del_roce_sa_rules); void mlx5_macsec_fs_cleanup(struct mlx5_macsec_fs *macsec_fs) { macsec_fs_rx_cleanup(macsec_fs); macsec_fs_tx_cleanup(macsec_fs); rhashtable_destroy(&macsec_fs->fs_id_hash); rhashtable_destroy(&macsec_fs->sci_hash); kfree(macsec_fs); } struct mlx5_macsec_fs * mlx5_macsec_fs_init(struct mlx5_core_dev *mdev) { struct mlx5_macsec_fs *macsec_fs; int err; macsec_fs = kzalloc(sizeof(*macsec_fs), GFP_KERNEL); if (!macsec_fs) return NULL; macsec_fs->mdev = mdev; err = rhashtable_init(&macsec_fs->sci_hash, &rhash_sci); if (err) { mlx5_core_err(mdev, "MACsec offload: Failed to init SCI hash table, err=%d\n", err); goto err_hash; } err = rhashtable_init(&macsec_fs->fs_id_hash, &rhash_fs_id); if (err) { mlx5_core_err(mdev, "MACsec offload: Failed to init FS_ID hash table, err=%d\n", err); goto sci_hash_cleanup; } err = macsec_fs_tx_init(macsec_fs); if (err) { mlx5_core_err(mdev, "MACsec offload: Failed to init tx_fs, err=%d\n", err); goto fs_id_hash_cleanup; } err = macsec_fs_rx_init(macsec_fs); if (err) { mlx5_core_err(mdev, "MACsec offload: Failed to init tx_fs, err=%d\n", err); goto tx_cleanup; } BLOCKING_INIT_NOTIFIER_HEAD(&mdev->macsec_nh); return macsec_fs; tx_cleanup: macsec_fs_tx_cleanup(macsec_fs); fs_id_hash_cleanup: rhashtable_destroy(&macsec_fs->fs_id_hash); sci_hash_cleanup: rhashtable_destroy(&macsec_fs->sci_hash); err_hash: kfree(macsec_fs); return NULL; }