1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/bitmap.h> 5 #include <linux/errno.h> 6 #include <linux/genalloc.h> 7 #include <linux/gfp.h> 8 #include <linux/kernel.h> 9 #include <linux/list.h> 10 #include <linux/objagg.h> 11 #include <linux/rtnetlink.h> 12 #include <linux/slab.h> 13 14 #include "core.h" 15 #include "reg.h" 16 #include "spectrum.h" 17 #include "spectrum_acl_tcam.h" 18 19 /* gen_pool_alloc() returns 0 when allocation fails, so use an offset */ 20 #define MLXSW_SP_ACL_ERP_GENALLOC_OFFSET 0x100 21 #define MLXSW_SP_ACL_ERP_MAX_PER_REGION 16 22 23 struct mlxsw_sp_acl_erp_core { 24 unsigned int erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_MAX + 1]; 25 struct gen_pool *erp_tables; 26 struct mlxsw_sp *mlxsw_sp; 27 struct mlxsw_sp_acl_bf *bf; 28 unsigned int num_erp_banks; 29 }; 30 31 struct mlxsw_sp_acl_erp_key { 32 char mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN]; 33 #define __MASK_LEN 0x38 34 #define __MASK_IDX(i) (__MASK_LEN - (i) - 1) 35 bool ctcam; 36 }; 37 38 struct mlxsw_sp_acl_erp { 39 struct mlxsw_sp_acl_erp_key key; 40 u8 id; 41 u8 index; 42 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 43 struct list_head list; 44 struct mlxsw_sp_acl_erp_table *erp_table; 45 }; 46 47 struct mlxsw_sp_acl_erp_master_mask { 48 DECLARE_BITMAP(bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 49 unsigned int count[MLXSW_SP_ACL_TCAM_MASK_LEN]; 50 }; 51 52 struct mlxsw_sp_acl_erp_table { 53 struct mlxsw_sp_acl_erp_master_mask master_mask; 54 DECLARE_BITMAP(erp_id_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION); 55 DECLARE_BITMAP(erp_index_bitmap, MLXSW_SP_ACL_ERP_MAX_PER_REGION); 56 struct list_head atcam_erps_list; 57 struct mlxsw_sp_acl_erp_core *erp_core; 58 struct mlxsw_sp_acl_atcam_region *aregion; 59 const struct mlxsw_sp_acl_erp_table_ops *ops; 60 unsigned long base_index; 61 unsigned int num_atcam_erps; 62 unsigned int num_max_atcam_erps; 63 unsigned int num_ctcam_erps; 64 unsigned int num_deltas; 65 struct objagg *objagg; 66 }; 67 68 struct mlxsw_sp_acl_erp_table_ops { 69 struct mlxsw_sp_acl_erp * 70 (*erp_create)(struct mlxsw_sp_acl_erp_table *erp_table, 71 struct mlxsw_sp_acl_erp_key *key); 72 void (*erp_destroy)(struct mlxsw_sp_acl_erp_table *erp_table, 73 struct mlxsw_sp_acl_erp *erp); 74 }; 75 76 static struct mlxsw_sp_acl_erp * 77 mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 78 struct mlxsw_sp_acl_erp_key *key); 79 static void 80 mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 81 struct mlxsw_sp_acl_erp *erp); 82 static struct mlxsw_sp_acl_erp * 83 mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 84 struct mlxsw_sp_acl_erp_key *key); 85 static void 86 mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 87 struct mlxsw_sp_acl_erp *erp); 88 static struct mlxsw_sp_acl_erp * 89 mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 90 struct mlxsw_sp_acl_erp_key *key); 91 static void 92 mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 93 struct mlxsw_sp_acl_erp *erp); 94 static void 95 mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 96 struct mlxsw_sp_acl_erp *erp); 97 98 static const struct mlxsw_sp_acl_erp_table_ops erp_multiple_masks_ops = { 99 .erp_create = mlxsw_sp_acl_erp_mask_create, 100 .erp_destroy = mlxsw_sp_acl_erp_mask_destroy, 101 }; 102 103 static const struct mlxsw_sp_acl_erp_table_ops erp_two_masks_ops = { 104 .erp_create = mlxsw_sp_acl_erp_mask_create, 105 .erp_destroy = mlxsw_sp_acl_erp_second_mask_destroy, 106 }; 107 108 static const struct mlxsw_sp_acl_erp_table_ops erp_single_mask_ops = { 109 .erp_create = mlxsw_sp_acl_erp_second_mask_create, 110 .erp_destroy = mlxsw_sp_acl_erp_first_mask_destroy, 111 }; 112 113 static const struct mlxsw_sp_acl_erp_table_ops erp_no_mask_ops = { 114 .erp_create = mlxsw_sp_acl_erp_first_mask_create, 115 .erp_destroy = mlxsw_sp_acl_erp_no_mask_destroy, 116 }; 117 118 static bool 119 mlxsw_sp_acl_erp_table_is_used(const struct mlxsw_sp_acl_erp_table *erp_table) 120 { 121 return erp_table->ops != &erp_single_mask_ops && 122 erp_table->ops != &erp_no_mask_ops; 123 } 124 125 static unsigned int 126 mlxsw_sp_acl_erp_bank_get(const struct mlxsw_sp_acl_erp *erp) 127 { 128 return erp->index % erp->erp_table->erp_core->num_erp_banks; 129 } 130 131 static unsigned int 132 mlxsw_sp_acl_erp_table_entry_size(const struct mlxsw_sp_acl_erp_table *erp_table) 133 { 134 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion; 135 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 136 137 return erp_core->erpt_entries_size[aregion->type]; 138 } 139 140 static int mlxsw_sp_acl_erp_id_get(struct mlxsw_sp_acl_erp_table *erp_table, 141 u8 *p_id) 142 { 143 u8 id; 144 145 id = find_first_zero_bit(erp_table->erp_id_bitmap, 146 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 147 if (id < MLXSW_SP_ACL_ERP_MAX_PER_REGION) { 148 __set_bit(id, erp_table->erp_id_bitmap); 149 *p_id = id; 150 return 0; 151 } 152 153 return -ENOBUFS; 154 } 155 156 static void mlxsw_sp_acl_erp_id_put(struct mlxsw_sp_acl_erp_table *erp_table, 157 u8 id) 158 { 159 __clear_bit(id, erp_table->erp_id_bitmap); 160 } 161 162 static void 163 mlxsw_sp_acl_erp_master_mask_bit_set(unsigned long bit, 164 struct mlxsw_sp_acl_erp_master_mask *mask) 165 { 166 if (mask->count[bit]++ == 0) 167 __set_bit(bit, mask->bitmap); 168 } 169 170 static void 171 mlxsw_sp_acl_erp_master_mask_bit_clear(unsigned long bit, 172 struct mlxsw_sp_acl_erp_master_mask *mask) 173 { 174 if (--mask->count[bit] == 0) 175 __clear_bit(bit, mask->bitmap); 176 } 177 178 static int 179 mlxsw_sp_acl_erp_master_mask_update(struct mlxsw_sp_acl_erp_table *erp_table) 180 { 181 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 182 struct mlxsw_sp *mlxsw_sp = region->mlxsw_sp; 183 char percr_pl[MLXSW_REG_PERCR_LEN]; 184 char *master_mask; 185 186 mlxsw_reg_percr_pack(percr_pl, region->id); 187 master_mask = mlxsw_reg_percr_master_mask_data(percr_pl); 188 bitmap_to_arr32((u32 *) master_mask, erp_table->master_mask.bitmap, 189 MLXSW_SP_ACL_TCAM_MASK_LEN); 190 191 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl); 192 } 193 194 static int 195 mlxsw_sp_acl_erp_master_mask_set(struct mlxsw_sp_acl_erp_table *erp_table, 196 struct mlxsw_sp_acl_erp_key *key) 197 { 198 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 199 unsigned long bit; 200 int err; 201 202 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask, 203 MLXSW_SP_ACL_TCAM_MASK_LEN); 204 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 205 mlxsw_sp_acl_erp_master_mask_bit_set(bit, 206 &erp_table->master_mask); 207 208 err = mlxsw_sp_acl_erp_master_mask_update(erp_table); 209 if (err) 210 goto err_master_mask_update; 211 212 return 0; 213 214 err_master_mask_update: 215 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 216 mlxsw_sp_acl_erp_master_mask_bit_clear(bit, 217 &erp_table->master_mask); 218 return err; 219 } 220 221 static int 222 mlxsw_sp_acl_erp_master_mask_clear(struct mlxsw_sp_acl_erp_table *erp_table, 223 struct mlxsw_sp_acl_erp_key *key) 224 { 225 DECLARE_BITMAP(mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN); 226 unsigned long bit; 227 int err; 228 229 bitmap_from_arr32(mask_bitmap, (u32 *) key->mask, 230 MLXSW_SP_ACL_TCAM_MASK_LEN); 231 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 232 mlxsw_sp_acl_erp_master_mask_bit_clear(bit, 233 &erp_table->master_mask); 234 235 err = mlxsw_sp_acl_erp_master_mask_update(erp_table); 236 if (err) 237 goto err_master_mask_update; 238 239 return 0; 240 241 err_master_mask_update: 242 for_each_set_bit(bit, mask_bitmap, MLXSW_SP_ACL_TCAM_MASK_LEN) 243 mlxsw_sp_acl_erp_master_mask_bit_set(bit, 244 &erp_table->master_mask); 245 return err; 246 } 247 248 static struct mlxsw_sp_acl_erp * 249 mlxsw_sp_acl_erp_generic_create(struct mlxsw_sp_acl_erp_table *erp_table, 250 struct mlxsw_sp_acl_erp_key *key) 251 { 252 struct mlxsw_sp_acl_erp *erp; 253 int err; 254 255 erp = kzalloc(sizeof(*erp), GFP_KERNEL); 256 if (!erp) 257 return ERR_PTR(-ENOMEM); 258 259 err = mlxsw_sp_acl_erp_id_get(erp_table, &erp->id); 260 if (err) 261 goto err_erp_id_get; 262 263 memcpy(&erp->key, key, sizeof(*key)); 264 list_add(&erp->list, &erp_table->atcam_erps_list); 265 erp_table->num_atcam_erps++; 266 erp->erp_table = erp_table; 267 268 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key); 269 if (err) 270 goto err_master_mask_set; 271 272 return erp; 273 274 err_master_mask_set: 275 erp_table->num_atcam_erps--; 276 list_del(&erp->list); 277 mlxsw_sp_acl_erp_id_put(erp_table, erp->id); 278 err_erp_id_get: 279 kfree(erp); 280 return ERR_PTR(err); 281 } 282 283 static void 284 mlxsw_sp_acl_erp_generic_destroy(struct mlxsw_sp_acl_erp *erp) 285 { 286 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 287 288 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key); 289 erp_table->num_atcam_erps--; 290 list_del(&erp->list); 291 mlxsw_sp_acl_erp_id_put(erp_table, erp->id); 292 kfree(erp); 293 } 294 295 static int 296 mlxsw_sp_acl_erp_table_alloc(struct mlxsw_sp_acl_erp_core *erp_core, 297 unsigned int num_erps, 298 enum mlxsw_sp_acl_atcam_region_type region_type, 299 unsigned long *p_index) 300 { 301 unsigned int num_rows, entry_size; 302 303 /* We only allow allocations of entire rows */ 304 if (num_erps % erp_core->num_erp_banks != 0) 305 return -EINVAL; 306 307 entry_size = erp_core->erpt_entries_size[region_type]; 308 num_rows = num_erps / erp_core->num_erp_banks; 309 310 *p_index = gen_pool_alloc(erp_core->erp_tables, num_rows * entry_size); 311 if (*p_index == 0) 312 return -ENOBUFS; 313 *p_index -= MLXSW_SP_ACL_ERP_GENALLOC_OFFSET; 314 315 return 0; 316 } 317 318 static void 319 mlxsw_sp_acl_erp_table_free(struct mlxsw_sp_acl_erp_core *erp_core, 320 unsigned int num_erps, 321 enum mlxsw_sp_acl_atcam_region_type region_type, 322 unsigned long index) 323 { 324 unsigned long base_index; 325 unsigned int entry_size; 326 size_t size; 327 328 entry_size = erp_core->erpt_entries_size[region_type]; 329 base_index = index + MLXSW_SP_ACL_ERP_GENALLOC_OFFSET; 330 size = num_erps / erp_core->num_erp_banks * entry_size; 331 gen_pool_free(erp_core->erp_tables, base_index, size); 332 } 333 334 static struct mlxsw_sp_acl_erp * 335 mlxsw_sp_acl_erp_table_master_rp(struct mlxsw_sp_acl_erp_table *erp_table) 336 { 337 if (!list_is_singular(&erp_table->atcam_erps_list)) 338 return NULL; 339 340 return list_first_entry(&erp_table->atcam_erps_list, 341 struct mlxsw_sp_acl_erp, list); 342 } 343 344 static int mlxsw_sp_acl_erp_index_get(struct mlxsw_sp_acl_erp_table *erp_table, 345 u8 *p_index) 346 { 347 u8 index; 348 349 index = find_first_zero_bit(erp_table->erp_index_bitmap, 350 erp_table->num_max_atcam_erps); 351 if (index < erp_table->num_max_atcam_erps) { 352 __set_bit(index, erp_table->erp_index_bitmap); 353 *p_index = index; 354 return 0; 355 } 356 357 return -ENOBUFS; 358 } 359 360 static void mlxsw_sp_acl_erp_index_put(struct mlxsw_sp_acl_erp_table *erp_table, 361 u8 index) 362 { 363 __clear_bit(index, erp_table->erp_index_bitmap); 364 } 365 366 static void 367 mlxsw_sp_acl_erp_table_locate(const struct mlxsw_sp_acl_erp_table *erp_table, 368 const struct mlxsw_sp_acl_erp *erp, 369 u8 *p_erpt_bank, u8 *p_erpt_index) 370 { 371 unsigned int entry_size = mlxsw_sp_acl_erp_table_entry_size(erp_table); 372 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 373 unsigned int row; 374 375 *p_erpt_bank = erp->index % erp_core->num_erp_banks; 376 row = erp->index / erp_core->num_erp_banks; 377 *p_erpt_index = erp_table->base_index + row * entry_size; 378 } 379 380 static int 381 mlxsw_sp_acl_erp_table_erp_add(struct mlxsw_sp_acl_erp_table *erp_table, 382 struct mlxsw_sp_acl_erp *erp) 383 { 384 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 385 enum mlxsw_reg_perpt_key_size key_size; 386 char perpt_pl[MLXSW_REG_PERPT_LEN]; 387 u8 erpt_bank, erpt_index; 388 389 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index); 390 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type; 391 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id, 392 0, erp_table->base_index, erp->index, 393 erp->key.mask); 394 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap, 395 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 396 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, true); 397 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl); 398 } 399 400 static void mlxsw_sp_acl_erp_table_erp_del(struct mlxsw_sp_acl_erp *erp) 401 { 402 char empty_mask[MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN] = { 0 }; 403 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 404 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 405 enum mlxsw_reg_perpt_key_size key_size; 406 char perpt_pl[MLXSW_REG_PERPT_LEN]; 407 u8 erpt_bank, erpt_index; 408 409 mlxsw_sp_acl_erp_table_locate(erp_table, erp, &erpt_bank, &erpt_index); 410 key_size = (enum mlxsw_reg_perpt_key_size) erp_table->aregion->type; 411 mlxsw_reg_perpt_pack(perpt_pl, erpt_bank, erpt_index, key_size, erp->id, 412 0, erp_table->base_index, erp->index, empty_mask); 413 mlxsw_reg_perpt_erp_vector_pack(perpt_pl, erp_table->erp_index_bitmap, 414 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 415 mlxsw_reg_perpt_erp_vector_set(perpt_pl, erp->index, false); 416 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(perpt), perpt_pl); 417 } 418 419 static int 420 mlxsw_sp_acl_erp_table_enable(struct mlxsw_sp_acl_erp_table *erp_table, 421 bool ctcam_le) 422 { 423 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 424 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 425 char pererp_pl[MLXSW_REG_PERERP_LEN]; 426 427 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0, 428 erp_table->base_index, 0); 429 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap, 430 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 431 432 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 433 } 434 435 static void 436 mlxsw_sp_acl_erp_table_disable(struct mlxsw_sp_acl_erp_table *erp_table) 437 { 438 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 439 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 440 char pererp_pl[MLXSW_REG_PERERP_LEN]; 441 struct mlxsw_sp_acl_erp *master_rp; 442 443 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table); 444 /* It is possible we do not have a master RP when we disable the 445 * table when there are no rules in the A-TCAM and the last C-TCAM 446 * rule is deleted 447 */ 448 mlxsw_reg_pererp_pack(pererp_pl, region->id, false, false, 0, 0, 449 master_rp ? master_rp->id : 0); 450 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 451 } 452 453 static int 454 mlxsw_sp_acl_erp_table_relocate(struct mlxsw_sp_acl_erp_table *erp_table) 455 { 456 struct mlxsw_sp_acl_erp *erp; 457 int err; 458 459 list_for_each_entry(erp, &erp_table->atcam_erps_list, list) { 460 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp); 461 if (err) 462 goto err_table_erp_add; 463 } 464 465 return 0; 466 467 err_table_erp_add: 468 list_for_each_entry_continue_reverse(erp, &erp_table->atcam_erps_list, 469 list) 470 mlxsw_sp_acl_erp_table_erp_del(erp); 471 return err; 472 } 473 474 static int 475 mlxsw_sp_acl_erp_table_expand(struct mlxsw_sp_acl_erp_table *erp_table) 476 { 477 unsigned int num_erps, old_num_erps = erp_table->num_max_atcam_erps; 478 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 479 unsigned long old_base_index = erp_table->base_index; 480 bool ctcam_le = erp_table->num_ctcam_erps > 0; 481 int err; 482 483 if (erp_table->num_atcam_erps < erp_table->num_max_atcam_erps) 484 return 0; 485 486 if (erp_table->num_max_atcam_erps == MLXSW_SP_ACL_ERP_MAX_PER_REGION) 487 return -ENOBUFS; 488 489 num_erps = old_num_erps + erp_core->num_erp_banks; 490 err = mlxsw_sp_acl_erp_table_alloc(erp_core, num_erps, 491 erp_table->aregion->type, 492 &erp_table->base_index); 493 if (err) 494 return err; 495 erp_table->num_max_atcam_erps = num_erps; 496 497 err = mlxsw_sp_acl_erp_table_relocate(erp_table); 498 if (err) 499 goto err_table_relocate; 500 501 err = mlxsw_sp_acl_erp_table_enable(erp_table, ctcam_le); 502 if (err) 503 goto err_table_enable; 504 505 mlxsw_sp_acl_erp_table_free(erp_core, old_num_erps, 506 erp_table->aregion->type, old_base_index); 507 508 return 0; 509 510 err_table_enable: 511 err_table_relocate: 512 erp_table->num_max_atcam_erps = old_num_erps; 513 mlxsw_sp_acl_erp_table_free(erp_core, num_erps, 514 erp_table->aregion->type, 515 erp_table->base_index); 516 erp_table->base_index = old_base_index; 517 return err; 518 } 519 520 static int 521 mlxsw_acl_erp_table_bf_add(struct mlxsw_sp_acl_erp_table *erp_table, 522 struct mlxsw_sp_acl_erp *erp) 523 { 524 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion; 525 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 526 struct mlxsw_sp_acl_atcam_entry *aentry; 527 int err; 528 529 list_for_each_entry(aentry, &aregion->entries_list, list) { 530 err = mlxsw_sp_acl_bf_entry_add(aregion->region->mlxsw_sp, 531 erp_table->erp_core->bf, 532 aregion, erp_bank, aentry); 533 if (err) 534 goto bf_entry_add_err; 535 } 536 537 return 0; 538 539 bf_entry_add_err: 540 list_for_each_entry_continue_reverse(aentry, &aregion->entries_list, 541 list) 542 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp, 543 erp_table->erp_core->bf, 544 aregion, erp_bank, aentry); 545 return err; 546 } 547 548 static void 549 mlxsw_acl_erp_table_bf_del(struct mlxsw_sp_acl_erp_table *erp_table, 550 struct mlxsw_sp_acl_erp *erp) 551 { 552 struct mlxsw_sp_acl_atcam_region *aregion = erp_table->aregion; 553 unsigned int erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 554 struct mlxsw_sp_acl_atcam_entry *aentry; 555 556 list_for_each_entry_reverse(aentry, &aregion->entries_list, list) 557 mlxsw_sp_acl_bf_entry_del(aregion->region->mlxsw_sp, 558 erp_table->erp_core->bf, 559 aregion, erp_bank, aentry); 560 } 561 562 static int 563 mlxsw_sp_acl_erp_region_table_trans(struct mlxsw_sp_acl_erp_table *erp_table) 564 { 565 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 566 struct mlxsw_sp_acl_erp *master_rp; 567 int err; 568 569 /* Initially, allocate a single eRP row. Expand later as needed */ 570 err = mlxsw_sp_acl_erp_table_alloc(erp_core, erp_core->num_erp_banks, 571 erp_table->aregion->type, 572 &erp_table->base_index); 573 if (err) 574 return err; 575 erp_table->num_max_atcam_erps = erp_core->num_erp_banks; 576 577 /* Transition the sole RP currently configured (the master RP) 578 * to the eRP table 579 */ 580 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table); 581 if (!master_rp) { 582 err = -EINVAL; 583 goto err_table_master_rp; 584 } 585 586 /* Make sure the master RP is using a valid index, as 587 * only a single eRP row is currently allocated. 588 */ 589 master_rp->index = 0; 590 __set_bit(master_rp->index, erp_table->erp_index_bitmap); 591 592 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, master_rp); 593 if (err) 594 goto err_table_master_rp_add; 595 596 /* Update Bloom filter before enabling eRP table, as rules 597 * on the master RP were not set to Bloom filter up to this 598 * point. 599 */ 600 err = mlxsw_acl_erp_table_bf_add(erp_table, master_rp); 601 if (err) 602 goto err_table_bf_add; 603 604 err = mlxsw_sp_acl_erp_table_enable(erp_table, false); 605 if (err) 606 goto err_table_enable; 607 608 return 0; 609 610 err_table_enable: 611 mlxsw_acl_erp_table_bf_del(erp_table, master_rp); 612 err_table_bf_add: 613 mlxsw_sp_acl_erp_table_erp_del(master_rp); 614 err_table_master_rp_add: 615 __clear_bit(master_rp->index, erp_table->erp_index_bitmap); 616 err_table_master_rp: 617 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps, 618 erp_table->aregion->type, 619 erp_table->base_index); 620 return err; 621 } 622 623 static void 624 mlxsw_sp_acl_erp_region_master_mask_trans(struct mlxsw_sp_acl_erp_table *erp_table) 625 { 626 struct mlxsw_sp_acl_erp_core *erp_core = erp_table->erp_core; 627 struct mlxsw_sp_acl_erp *master_rp; 628 629 mlxsw_sp_acl_erp_table_disable(erp_table); 630 master_rp = mlxsw_sp_acl_erp_table_master_rp(erp_table); 631 if (!master_rp) 632 return; 633 mlxsw_acl_erp_table_bf_del(erp_table, master_rp); 634 mlxsw_sp_acl_erp_table_erp_del(master_rp); 635 __clear_bit(master_rp->index, erp_table->erp_index_bitmap); 636 mlxsw_sp_acl_erp_table_free(erp_core, erp_table->num_max_atcam_erps, 637 erp_table->aregion->type, 638 erp_table->base_index); 639 } 640 641 static int 642 mlxsw_sp_acl_erp_region_erp_add(struct mlxsw_sp_acl_erp_table *erp_table, 643 struct mlxsw_sp_acl_erp *erp) 644 { 645 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 646 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 647 bool ctcam_le = erp_table->num_ctcam_erps > 0; 648 char pererp_pl[MLXSW_REG_PERERP_LEN]; 649 650 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0, 651 erp_table->base_index, 0); 652 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap, 653 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 654 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, true); 655 656 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 657 } 658 659 static void mlxsw_sp_acl_erp_region_erp_del(struct mlxsw_sp_acl_erp *erp) 660 { 661 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 662 struct mlxsw_sp_acl_tcam_region *region = erp_table->aregion->region; 663 struct mlxsw_sp *mlxsw_sp = erp_table->erp_core->mlxsw_sp; 664 bool ctcam_le = erp_table->num_ctcam_erps > 0; 665 char pererp_pl[MLXSW_REG_PERERP_LEN]; 666 667 mlxsw_reg_pererp_pack(pererp_pl, region->id, ctcam_le, true, 0, 668 erp_table->base_index, 0); 669 mlxsw_reg_pererp_erp_vector_pack(pererp_pl, erp_table->erp_index_bitmap, 670 MLXSW_SP_ACL_ERP_MAX_PER_REGION); 671 mlxsw_reg_pererp_erpt_vector_set(pererp_pl, erp->index, false); 672 673 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 674 } 675 676 static int 677 mlxsw_sp_acl_erp_region_ctcam_enable(struct mlxsw_sp_acl_erp_table *erp_table) 678 { 679 /* No need to re-enable lookup in the C-TCAM */ 680 if (erp_table->num_ctcam_erps > 1) 681 return 0; 682 683 return mlxsw_sp_acl_erp_table_enable(erp_table, true); 684 } 685 686 static void 687 mlxsw_sp_acl_erp_region_ctcam_disable(struct mlxsw_sp_acl_erp_table *erp_table) 688 { 689 /* Only disable C-TCAM lookup when last C-TCAM eRP is deleted */ 690 if (erp_table->num_ctcam_erps > 1) 691 return; 692 693 mlxsw_sp_acl_erp_table_enable(erp_table, false); 694 } 695 696 static int 697 __mlxsw_sp_acl_erp_table_other_inc(struct mlxsw_sp_acl_erp_table *erp_table, 698 unsigned int *inc_num) 699 { 700 int err; 701 702 /* If there are C-TCAM eRP or deltas in use we need to transition 703 * the region to use eRP table, if it is not already done 704 */ 705 if (!mlxsw_sp_acl_erp_table_is_used(erp_table)) { 706 err = mlxsw_sp_acl_erp_region_table_trans(erp_table); 707 if (err) 708 return err; 709 } 710 711 /* When C-TCAM or deltas are used, the eRP table must be used */ 712 if (erp_table->ops != &erp_multiple_masks_ops) 713 erp_table->ops = &erp_multiple_masks_ops; 714 715 (*inc_num)++; 716 717 return 0; 718 } 719 720 static int mlxsw_sp_acl_erp_ctcam_inc(struct mlxsw_sp_acl_erp_table *erp_table) 721 { 722 return __mlxsw_sp_acl_erp_table_other_inc(erp_table, 723 &erp_table->num_ctcam_erps); 724 } 725 726 static int mlxsw_sp_acl_erp_delta_inc(struct mlxsw_sp_acl_erp_table *erp_table) 727 { 728 return __mlxsw_sp_acl_erp_table_other_inc(erp_table, 729 &erp_table->num_deltas); 730 } 731 732 static void 733 __mlxsw_sp_acl_erp_table_other_dec(struct mlxsw_sp_acl_erp_table *erp_table, 734 unsigned int *dec_num) 735 { 736 (*dec_num)--; 737 738 /* If there are no C-TCAM eRP or deltas in use, the state we 739 * transition to depends on the number of A-TCAM eRPs currently 740 * in use. 741 */ 742 if (erp_table->num_ctcam_erps > 0 || erp_table->num_deltas > 0) 743 return; 744 745 switch (erp_table->num_atcam_erps) { 746 case 2: 747 /* Keep using the eRP table, but correctly set the 748 * operations pointer so that when an A-TCAM eRP is 749 * deleted we will transition to use the master mask 750 */ 751 erp_table->ops = &erp_two_masks_ops; 752 break; 753 case 1: 754 /* We only kept the eRP table because we had C-TCAM 755 * eRPs in use. Now that the last C-TCAM eRP is gone we 756 * can stop using the table and transition to use the 757 * master mask 758 */ 759 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table); 760 erp_table->ops = &erp_single_mask_ops; 761 break; 762 case 0: 763 /* There are no more eRPs of any kind used by the region 764 * so free its eRP table and transition to initial state 765 */ 766 mlxsw_sp_acl_erp_table_disable(erp_table); 767 mlxsw_sp_acl_erp_table_free(erp_table->erp_core, 768 erp_table->num_max_atcam_erps, 769 erp_table->aregion->type, 770 erp_table->base_index); 771 erp_table->ops = &erp_no_mask_ops; 772 break; 773 default: 774 break; 775 } 776 } 777 778 static void mlxsw_sp_acl_erp_ctcam_dec(struct mlxsw_sp_acl_erp_table *erp_table) 779 { 780 __mlxsw_sp_acl_erp_table_other_dec(erp_table, 781 &erp_table->num_ctcam_erps); 782 } 783 784 static void mlxsw_sp_acl_erp_delta_dec(struct mlxsw_sp_acl_erp_table *erp_table) 785 { 786 __mlxsw_sp_acl_erp_table_other_dec(erp_table, 787 &erp_table->num_deltas); 788 } 789 790 static struct mlxsw_sp_acl_erp * 791 mlxsw_sp_acl_erp_ctcam_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 792 struct mlxsw_sp_acl_erp_key *key) 793 { 794 struct mlxsw_sp_acl_erp *erp; 795 int err; 796 797 erp = kzalloc(sizeof(*erp), GFP_KERNEL); 798 if (!erp) 799 return ERR_PTR(-ENOMEM); 800 801 memcpy(&erp->key, key, sizeof(*key)); 802 bitmap_from_arr32(erp->mask_bitmap, (u32 *) key->mask, 803 MLXSW_SP_ACL_TCAM_MASK_LEN); 804 805 err = mlxsw_sp_acl_erp_ctcam_inc(erp_table); 806 if (err) 807 goto err_erp_ctcam_inc; 808 809 erp->erp_table = erp_table; 810 811 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &erp->key); 812 if (err) 813 goto err_master_mask_set; 814 815 err = mlxsw_sp_acl_erp_region_ctcam_enable(erp_table); 816 if (err) 817 goto err_erp_region_ctcam_enable; 818 819 return erp; 820 821 err_erp_region_ctcam_enable: 822 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key); 823 err_master_mask_set: 824 mlxsw_sp_acl_erp_ctcam_dec(erp_table); 825 err_erp_ctcam_inc: 826 kfree(erp); 827 return ERR_PTR(err); 828 } 829 830 static void 831 mlxsw_sp_acl_erp_ctcam_mask_destroy(struct mlxsw_sp_acl_erp *erp) 832 { 833 struct mlxsw_sp_acl_erp_table *erp_table = erp->erp_table; 834 835 mlxsw_sp_acl_erp_region_ctcam_disable(erp_table); 836 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &erp->key); 837 mlxsw_sp_acl_erp_ctcam_dec(erp_table); 838 kfree(erp); 839 } 840 841 static struct mlxsw_sp_acl_erp * 842 mlxsw_sp_acl_erp_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 843 struct mlxsw_sp_acl_erp_key *key) 844 { 845 struct mlxsw_sp_acl_erp *erp; 846 int err; 847 848 if (key->ctcam) 849 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key); 850 851 /* Expand the eRP table for the new eRP, if needed */ 852 err = mlxsw_sp_acl_erp_table_expand(erp_table); 853 if (err) 854 return ERR_PTR(err); 855 856 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key); 857 if (IS_ERR(erp)) 858 return erp; 859 860 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index); 861 if (err) 862 goto err_erp_index_get; 863 864 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp); 865 if (err) 866 goto err_table_erp_add; 867 868 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp); 869 if (err) 870 goto err_region_erp_add; 871 872 erp_table->ops = &erp_multiple_masks_ops; 873 874 return erp; 875 876 err_region_erp_add: 877 mlxsw_sp_acl_erp_table_erp_del(erp); 878 err_table_erp_add: 879 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 880 err_erp_index_get: 881 mlxsw_sp_acl_erp_generic_destroy(erp); 882 return ERR_PTR(err); 883 } 884 885 static void 886 mlxsw_sp_acl_erp_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 887 struct mlxsw_sp_acl_erp *erp) 888 { 889 if (erp->key.ctcam) 890 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp); 891 892 mlxsw_sp_acl_erp_region_erp_del(erp); 893 mlxsw_sp_acl_erp_table_erp_del(erp); 894 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 895 mlxsw_sp_acl_erp_generic_destroy(erp); 896 897 if (erp_table->num_atcam_erps == 2 && erp_table->num_ctcam_erps == 0 && 898 erp_table->num_deltas == 0) 899 erp_table->ops = &erp_two_masks_ops; 900 } 901 902 static struct mlxsw_sp_acl_erp * 903 mlxsw_sp_acl_erp_second_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 904 struct mlxsw_sp_acl_erp_key *key) 905 { 906 struct mlxsw_sp_acl_erp *erp; 907 int err; 908 909 if (key->ctcam) 910 return mlxsw_sp_acl_erp_ctcam_mask_create(erp_table, key); 911 912 /* Transition to use eRP table instead of master mask */ 913 err = mlxsw_sp_acl_erp_region_table_trans(erp_table); 914 if (err) 915 return ERR_PTR(err); 916 917 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key); 918 if (IS_ERR(erp)) { 919 err = PTR_ERR(erp); 920 goto err_erp_create; 921 } 922 923 err = mlxsw_sp_acl_erp_index_get(erp_table, &erp->index); 924 if (err) 925 goto err_erp_index_get; 926 927 err = mlxsw_sp_acl_erp_table_erp_add(erp_table, erp); 928 if (err) 929 goto err_table_erp_add; 930 931 err = mlxsw_sp_acl_erp_region_erp_add(erp_table, erp); 932 if (err) 933 goto err_region_erp_add; 934 935 erp_table->ops = &erp_two_masks_ops; 936 937 return erp; 938 939 err_region_erp_add: 940 mlxsw_sp_acl_erp_table_erp_del(erp); 941 err_table_erp_add: 942 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 943 err_erp_index_get: 944 mlxsw_sp_acl_erp_generic_destroy(erp); 945 err_erp_create: 946 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table); 947 return ERR_PTR(err); 948 } 949 950 static void 951 mlxsw_sp_acl_erp_second_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 952 struct mlxsw_sp_acl_erp *erp) 953 { 954 if (erp->key.ctcam) 955 return mlxsw_sp_acl_erp_ctcam_mask_destroy(erp); 956 957 mlxsw_sp_acl_erp_region_erp_del(erp); 958 mlxsw_sp_acl_erp_table_erp_del(erp); 959 mlxsw_sp_acl_erp_index_put(erp_table, erp->index); 960 mlxsw_sp_acl_erp_generic_destroy(erp); 961 /* Transition to use master mask instead of eRP table */ 962 mlxsw_sp_acl_erp_region_master_mask_trans(erp_table); 963 964 erp_table->ops = &erp_single_mask_ops; 965 } 966 967 static struct mlxsw_sp_acl_erp * 968 mlxsw_sp_acl_erp_first_mask_create(struct mlxsw_sp_acl_erp_table *erp_table, 969 struct mlxsw_sp_acl_erp_key *key) 970 { 971 struct mlxsw_sp_acl_erp *erp; 972 973 if (key->ctcam) 974 return ERR_PTR(-EINVAL); 975 976 erp = mlxsw_sp_acl_erp_generic_create(erp_table, key); 977 if (IS_ERR(erp)) 978 return erp; 979 980 erp_table->ops = &erp_single_mask_ops; 981 982 return erp; 983 } 984 985 static void 986 mlxsw_sp_acl_erp_first_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 987 struct mlxsw_sp_acl_erp *erp) 988 { 989 mlxsw_sp_acl_erp_generic_destroy(erp); 990 erp_table->ops = &erp_no_mask_ops; 991 } 992 993 static void 994 mlxsw_sp_acl_erp_no_mask_destroy(struct mlxsw_sp_acl_erp_table *erp_table, 995 struct mlxsw_sp_acl_erp *erp) 996 { 997 WARN_ON(1); 998 } 999 1000 struct mlxsw_sp_acl_erp_mask * 1001 mlxsw_sp_acl_erp_mask_get(struct mlxsw_sp_acl_atcam_region *aregion, 1002 const char *mask, bool ctcam) 1003 { 1004 struct mlxsw_sp_acl_erp_key key; 1005 struct objagg_obj *objagg_obj; 1006 1007 /* eRPs are allocated from a shared resource, but currently all 1008 * allocations are done under RTNL. 1009 */ 1010 ASSERT_RTNL(); 1011 1012 memcpy(key.mask, mask, MLXSW_REG_PTCEX_FLEX_KEY_BLOCKS_LEN); 1013 key.ctcam = ctcam; 1014 objagg_obj = objagg_obj_get(aregion->erp_table->objagg, &key); 1015 if (IS_ERR(objagg_obj)) 1016 return ERR_CAST(objagg_obj); 1017 return (struct mlxsw_sp_acl_erp_mask *) objagg_obj; 1018 } 1019 1020 void mlxsw_sp_acl_erp_mask_put(struct mlxsw_sp_acl_atcam_region *aregion, 1021 struct mlxsw_sp_acl_erp_mask *erp_mask) 1022 { 1023 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1024 1025 objagg_obj_put(aregion->erp_table->objagg, objagg_obj); 1026 } 1027 1028 int mlxsw_sp_acl_erp_bf_insert(struct mlxsw_sp *mlxsw_sp, 1029 struct mlxsw_sp_acl_atcam_region *aregion, 1030 struct mlxsw_sp_acl_erp_mask *erp_mask, 1031 struct mlxsw_sp_acl_atcam_entry *aentry) 1032 { 1033 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1034 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj); 1035 unsigned int erp_bank; 1036 1037 ASSERT_RTNL(); 1038 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table)) 1039 return 0; 1040 1041 erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 1042 return mlxsw_sp_acl_bf_entry_add(mlxsw_sp, 1043 erp->erp_table->erp_core->bf, 1044 aregion, erp_bank, aentry); 1045 } 1046 1047 void mlxsw_sp_acl_erp_bf_remove(struct mlxsw_sp *mlxsw_sp, 1048 struct mlxsw_sp_acl_atcam_region *aregion, 1049 struct mlxsw_sp_acl_erp_mask *erp_mask, 1050 struct mlxsw_sp_acl_atcam_entry *aentry) 1051 { 1052 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1053 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj); 1054 unsigned int erp_bank; 1055 1056 if (!mlxsw_sp_acl_erp_table_is_used(erp->erp_table)) 1057 return; 1058 1059 erp_bank = mlxsw_sp_acl_erp_bank_get(erp); 1060 mlxsw_sp_acl_bf_entry_del(mlxsw_sp, 1061 erp->erp_table->erp_core->bf, 1062 aregion, erp_bank, aentry); 1063 } 1064 1065 bool 1066 mlxsw_sp_acl_erp_mask_is_ctcam(const struct mlxsw_sp_acl_erp_mask *erp_mask) 1067 { 1068 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1069 const struct mlxsw_sp_acl_erp_key *key = objagg_obj_raw(objagg_obj); 1070 1071 return key->ctcam; 1072 } 1073 1074 u8 mlxsw_sp_acl_erp_mask_erp_id(const struct mlxsw_sp_acl_erp_mask *erp_mask) 1075 { 1076 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1077 const struct mlxsw_sp_acl_erp *erp = objagg_obj_root_priv(objagg_obj); 1078 1079 return erp->id; 1080 } 1081 1082 struct mlxsw_sp_acl_erp_delta { 1083 struct mlxsw_sp_acl_erp_key key; 1084 u16 start; 1085 u8 mask; 1086 }; 1087 1088 u16 mlxsw_sp_acl_erp_delta_start(const struct mlxsw_sp_acl_erp_delta *delta) 1089 { 1090 return delta->start; 1091 } 1092 1093 u8 mlxsw_sp_acl_erp_delta_mask(const struct mlxsw_sp_acl_erp_delta *delta) 1094 { 1095 return delta->mask; 1096 } 1097 1098 u8 mlxsw_sp_acl_erp_delta_value(const struct mlxsw_sp_acl_erp_delta *delta, 1099 const char *enc_key) 1100 { 1101 u16 start = delta->start; 1102 u8 mask = delta->mask; 1103 u16 tmp; 1104 1105 if (!mask) 1106 return 0; 1107 1108 tmp = (unsigned char) enc_key[__MASK_IDX(start / 8)]; 1109 if (start / 8 + 1 < __MASK_LEN) 1110 tmp |= (unsigned char) enc_key[__MASK_IDX(start / 8 + 1)] << 8; 1111 tmp >>= start % 8; 1112 tmp &= mask; 1113 return tmp; 1114 } 1115 1116 void mlxsw_sp_acl_erp_delta_clear(const struct mlxsw_sp_acl_erp_delta *delta, 1117 const char *enc_key) 1118 { 1119 u16 start = delta->start; 1120 u8 mask = delta->mask; 1121 unsigned char *byte; 1122 u16 tmp; 1123 1124 tmp = mask; 1125 tmp <<= start % 8; 1126 tmp = ~tmp; 1127 1128 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8)]; 1129 *byte &= tmp & 0xff; 1130 if (start / 8 + 1 < __MASK_LEN) { 1131 byte = (unsigned char *) &enc_key[__MASK_IDX(start / 8 + 1)]; 1132 *byte &= (tmp >> 8) & 0xff; 1133 } 1134 } 1135 1136 static const struct mlxsw_sp_acl_erp_delta 1137 mlxsw_sp_acl_erp_delta_default = {}; 1138 1139 const struct mlxsw_sp_acl_erp_delta * 1140 mlxsw_sp_acl_erp_delta(const struct mlxsw_sp_acl_erp_mask *erp_mask) 1141 { 1142 struct objagg_obj *objagg_obj = (struct objagg_obj *) erp_mask; 1143 const struct mlxsw_sp_acl_erp_delta *delta; 1144 1145 delta = objagg_obj_delta_priv(objagg_obj); 1146 if (!delta) 1147 delta = &mlxsw_sp_acl_erp_delta_default; 1148 return delta; 1149 } 1150 1151 static int 1152 mlxsw_sp_acl_erp_delta_fill(const struct mlxsw_sp_acl_erp_key *parent_key, 1153 const struct mlxsw_sp_acl_erp_key *key, 1154 u16 *delta_start, u8 *delta_mask) 1155 { 1156 int offset = 0; 1157 int si = -1; 1158 u16 pmask; 1159 u16 mask; 1160 int i; 1161 1162 /* The difference between 2 masks can be up to 8 consecutive bits. */ 1163 for (i = 0; i < __MASK_LEN; i++) { 1164 if (parent_key->mask[__MASK_IDX(i)] == key->mask[__MASK_IDX(i)]) 1165 continue; 1166 if (si == -1) 1167 si = i; 1168 else if (si != i - 1) 1169 return -EINVAL; 1170 } 1171 if (si == -1) { 1172 /* The masks are the same, this cannot happen. 1173 * That means the caller is broken. 1174 */ 1175 WARN_ON(1); 1176 *delta_start = 0; 1177 *delta_mask = 0; 1178 return 0; 1179 } 1180 pmask = (unsigned char) parent_key->mask[__MASK_IDX(si)]; 1181 mask = (unsigned char) key->mask[__MASK_IDX(si)]; 1182 if (si + 1 < __MASK_LEN) { 1183 pmask |= (unsigned char) parent_key->mask[__MASK_IDX(si + 1)] << 8; 1184 mask |= (unsigned char) key->mask[__MASK_IDX(si + 1)] << 8; 1185 } 1186 1187 if ((pmask ^ mask) & pmask) 1188 return -EINVAL; 1189 mask &= ~pmask; 1190 while (!(mask & (1 << offset))) 1191 offset++; 1192 while (!(mask & 1)) 1193 mask >>= 1; 1194 if (mask & 0xff00) 1195 return -EINVAL; 1196 1197 *delta_start = si * 8 + offset; 1198 *delta_mask = mask; 1199 1200 return 0; 1201 } 1202 1203 static void *mlxsw_sp_acl_erp_delta_create(void *priv, void *parent_obj, 1204 void *obj) 1205 { 1206 struct mlxsw_sp_acl_erp_key *parent_key = parent_obj; 1207 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1208 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1209 struct mlxsw_sp_acl_erp_key *key = obj; 1210 struct mlxsw_sp_acl_erp_delta *delta; 1211 u16 delta_start; 1212 u8 delta_mask; 1213 int err; 1214 1215 if (parent_key->ctcam || key->ctcam) 1216 return ERR_PTR(-EINVAL); 1217 err = mlxsw_sp_acl_erp_delta_fill(parent_key, key, 1218 &delta_start, &delta_mask); 1219 if (err) 1220 return ERR_PTR(-EINVAL); 1221 1222 delta = kzalloc(sizeof(*delta), GFP_KERNEL); 1223 if (!delta) 1224 return ERR_PTR(-ENOMEM); 1225 delta->start = delta_start; 1226 delta->mask = delta_mask; 1227 1228 err = mlxsw_sp_acl_erp_delta_inc(erp_table); 1229 if (err) 1230 goto err_erp_delta_inc; 1231 1232 memcpy(&delta->key, key, sizeof(*key)); 1233 err = mlxsw_sp_acl_erp_master_mask_set(erp_table, &delta->key); 1234 if (err) 1235 goto err_master_mask_set; 1236 1237 return delta; 1238 1239 err_master_mask_set: 1240 mlxsw_sp_acl_erp_delta_dec(erp_table); 1241 err_erp_delta_inc: 1242 kfree(delta); 1243 return ERR_PTR(err); 1244 } 1245 1246 static void mlxsw_sp_acl_erp_delta_destroy(void *priv, void *delta_priv) 1247 { 1248 struct mlxsw_sp_acl_erp_delta *delta = delta_priv; 1249 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1250 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1251 1252 mlxsw_sp_acl_erp_master_mask_clear(erp_table, &delta->key); 1253 mlxsw_sp_acl_erp_delta_dec(erp_table); 1254 kfree(delta); 1255 } 1256 1257 static void *mlxsw_sp_acl_erp_root_create(void *priv, void *obj) 1258 { 1259 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1260 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1261 struct mlxsw_sp_acl_erp_key *key = obj; 1262 1263 return erp_table->ops->erp_create(erp_table, key); 1264 } 1265 1266 static void mlxsw_sp_acl_erp_root_destroy(void *priv, void *root_priv) 1267 { 1268 struct mlxsw_sp_acl_atcam_region *aregion = priv; 1269 struct mlxsw_sp_acl_erp_table *erp_table = aregion->erp_table; 1270 1271 erp_table->ops->erp_destroy(erp_table, root_priv); 1272 } 1273 1274 static const struct objagg_ops mlxsw_sp_acl_erp_objagg_ops = { 1275 .obj_size = sizeof(struct mlxsw_sp_acl_erp_key), 1276 .delta_create = mlxsw_sp_acl_erp_delta_create, 1277 .delta_destroy = mlxsw_sp_acl_erp_delta_destroy, 1278 .root_create = mlxsw_sp_acl_erp_root_create, 1279 .root_destroy = mlxsw_sp_acl_erp_root_destroy, 1280 }; 1281 1282 static struct mlxsw_sp_acl_erp_table * 1283 mlxsw_sp_acl_erp_table_create(struct mlxsw_sp_acl_atcam_region *aregion) 1284 { 1285 struct mlxsw_sp_acl_erp_table *erp_table; 1286 int err; 1287 1288 erp_table = kzalloc(sizeof(*erp_table), GFP_KERNEL); 1289 if (!erp_table) 1290 return ERR_PTR(-ENOMEM); 1291 1292 erp_table->objagg = objagg_create(&mlxsw_sp_acl_erp_objagg_ops, 1293 aregion); 1294 if (IS_ERR(erp_table->objagg)) { 1295 err = PTR_ERR(erp_table->objagg); 1296 goto err_objagg_create; 1297 } 1298 1299 erp_table->erp_core = aregion->atcam->erp_core; 1300 erp_table->ops = &erp_no_mask_ops; 1301 INIT_LIST_HEAD(&erp_table->atcam_erps_list); 1302 erp_table->aregion = aregion; 1303 1304 return erp_table; 1305 1306 err_objagg_create: 1307 kfree(erp_table); 1308 return ERR_PTR(err); 1309 } 1310 1311 static void 1312 mlxsw_sp_acl_erp_table_destroy(struct mlxsw_sp_acl_erp_table *erp_table) 1313 { 1314 WARN_ON(!list_empty(&erp_table->atcam_erps_list)); 1315 objagg_destroy(erp_table->objagg); 1316 kfree(erp_table); 1317 } 1318 1319 static int 1320 mlxsw_sp_acl_erp_master_mask_init(struct mlxsw_sp_acl_atcam_region *aregion) 1321 { 1322 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp; 1323 char percr_pl[MLXSW_REG_PERCR_LEN]; 1324 1325 mlxsw_reg_percr_pack(percr_pl, aregion->region->id); 1326 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(percr), percr_pl); 1327 } 1328 1329 static int 1330 mlxsw_sp_acl_erp_region_param_init(struct mlxsw_sp_acl_atcam_region *aregion) 1331 { 1332 struct mlxsw_sp *mlxsw_sp = aregion->region->mlxsw_sp; 1333 char pererp_pl[MLXSW_REG_PERERP_LEN]; 1334 1335 mlxsw_reg_pererp_pack(pererp_pl, aregion->region->id, false, false, 0, 1336 0, 0); 1337 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(pererp), pererp_pl); 1338 } 1339 1340 int mlxsw_sp_acl_erp_region_init(struct mlxsw_sp_acl_atcam_region *aregion) 1341 { 1342 struct mlxsw_sp_acl_erp_table *erp_table; 1343 int err; 1344 1345 erp_table = mlxsw_sp_acl_erp_table_create(aregion); 1346 if (IS_ERR(erp_table)) 1347 return PTR_ERR(erp_table); 1348 aregion->erp_table = erp_table; 1349 1350 /* Initialize the region's master mask to all zeroes */ 1351 err = mlxsw_sp_acl_erp_master_mask_init(aregion); 1352 if (err) 1353 goto err_erp_master_mask_init; 1354 1355 /* Initialize the region to not use the eRP table */ 1356 err = mlxsw_sp_acl_erp_region_param_init(aregion); 1357 if (err) 1358 goto err_erp_region_param_init; 1359 1360 return 0; 1361 1362 err_erp_region_param_init: 1363 err_erp_master_mask_init: 1364 mlxsw_sp_acl_erp_table_destroy(erp_table); 1365 return err; 1366 } 1367 1368 void mlxsw_sp_acl_erp_region_fini(struct mlxsw_sp_acl_atcam_region *aregion) 1369 { 1370 mlxsw_sp_acl_erp_table_destroy(aregion->erp_table); 1371 } 1372 1373 static int 1374 mlxsw_sp_acl_erp_tables_sizes_query(struct mlxsw_sp *mlxsw_sp, 1375 struct mlxsw_sp_acl_erp_core *erp_core) 1376 { 1377 unsigned int size; 1378 1379 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB) || 1380 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB) || 1381 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB) || 1382 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB)) 1383 return -EIO; 1384 1385 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_2KB); 1386 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_2KB] = size; 1387 1388 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_4KB); 1389 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_4KB] = size; 1390 1391 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_8KB); 1392 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_8KB] = size; 1393 1394 size = MLXSW_CORE_RES_GET(mlxsw_sp->core, ACL_ERPT_ENTRIES_12KB); 1395 erp_core->erpt_entries_size[MLXSW_SP_ACL_ATCAM_REGION_TYPE_12KB] = size; 1396 1397 return 0; 1398 } 1399 1400 static int mlxsw_sp_acl_erp_tables_init(struct mlxsw_sp *mlxsw_sp, 1401 struct mlxsw_sp_acl_erp_core *erp_core) 1402 { 1403 unsigned int erpt_bank_size; 1404 int err; 1405 1406 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANK_SIZE) || 1407 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, ACL_MAX_ERPT_BANKS)) 1408 return -EIO; 1409 erpt_bank_size = MLXSW_CORE_RES_GET(mlxsw_sp->core, 1410 ACL_MAX_ERPT_BANK_SIZE); 1411 erp_core->num_erp_banks = MLXSW_CORE_RES_GET(mlxsw_sp->core, 1412 ACL_MAX_ERPT_BANKS); 1413 1414 erp_core->erp_tables = gen_pool_create(0, -1); 1415 if (!erp_core->erp_tables) 1416 return -ENOMEM; 1417 gen_pool_set_algo(erp_core->erp_tables, gen_pool_best_fit, NULL); 1418 1419 err = gen_pool_add(erp_core->erp_tables, 1420 MLXSW_SP_ACL_ERP_GENALLOC_OFFSET, erpt_bank_size, 1421 -1); 1422 if (err) 1423 goto err_gen_pool_add; 1424 1425 erp_core->bf = mlxsw_sp_acl_bf_init(mlxsw_sp, erp_core->num_erp_banks); 1426 if (IS_ERR(erp_core->bf)) { 1427 err = PTR_ERR(erp_core->bf); 1428 goto err_bf_init; 1429 } 1430 1431 /* Different regions require masks of different sizes */ 1432 err = mlxsw_sp_acl_erp_tables_sizes_query(mlxsw_sp, erp_core); 1433 if (err) 1434 goto err_erp_tables_sizes_query; 1435 1436 return 0; 1437 1438 err_erp_tables_sizes_query: 1439 mlxsw_sp_acl_bf_fini(erp_core->bf); 1440 err_bf_init: 1441 err_gen_pool_add: 1442 gen_pool_destroy(erp_core->erp_tables); 1443 return err; 1444 } 1445 1446 static void mlxsw_sp_acl_erp_tables_fini(struct mlxsw_sp *mlxsw_sp, 1447 struct mlxsw_sp_acl_erp_core *erp_core) 1448 { 1449 mlxsw_sp_acl_bf_fini(erp_core->bf); 1450 gen_pool_destroy(erp_core->erp_tables); 1451 } 1452 1453 int mlxsw_sp_acl_erps_init(struct mlxsw_sp *mlxsw_sp, 1454 struct mlxsw_sp_acl_atcam *atcam) 1455 { 1456 struct mlxsw_sp_acl_erp_core *erp_core; 1457 int err; 1458 1459 erp_core = kzalloc(sizeof(*erp_core), GFP_KERNEL); 1460 if (!erp_core) 1461 return -ENOMEM; 1462 erp_core->mlxsw_sp = mlxsw_sp; 1463 atcam->erp_core = erp_core; 1464 1465 err = mlxsw_sp_acl_erp_tables_init(mlxsw_sp, erp_core); 1466 if (err) 1467 goto err_erp_tables_init; 1468 1469 return 0; 1470 1471 err_erp_tables_init: 1472 kfree(erp_core); 1473 return err; 1474 } 1475 1476 void mlxsw_sp_acl_erps_fini(struct mlxsw_sp *mlxsw_sp, 1477 struct mlxsw_sp_acl_atcam *atcam) 1478 { 1479 mlxsw_sp_acl_erp_tables_fini(mlxsw_sp, atcam->erp_core); 1480 kfree(atcam->erp_core); 1481 } 1482