1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/slab.h> 6 #include <linux/list.h> 7 #include <linux/errno.h> 8 #include <linux/refcount.h> 9 10 #include "item.h" 11 #include "core_acl_flex_keys.h" 12 13 /* For the purpose of the driver, define an internal storage scratchpad 14 * that will be used to store key/mask values. For each defined element type 15 * define an internal storage geometry. 16 * 17 * When adding new elements, MLXSW_AFK_ELEMENT_STORAGE_SIZE must be increased 18 * accordingly. 19 */ 20 static const struct mlxsw_afk_element_info mlxsw_afk_element_infos[] = { 21 MLXSW_AFK_ELEMENT_INFO_U32(SRC_SYS_PORT, 0x00, 16, 16), 22 MLXSW_AFK_ELEMENT_INFO_BUF(DMAC_32_47, 0x04, 2), 23 MLXSW_AFK_ELEMENT_INFO_BUF(DMAC_0_31, 0x06, 4), 24 MLXSW_AFK_ELEMENT_INFO_BUF(SMAC_32_47, 0x0A, 2), 25 MLXSW_AFK_ELEMENT_INFO_BUF(SMAC_0_31, 0x0C, 4), 26 MLXSW_AFK_ELEMENT_INFO_U32(ETHERTYPE, 0x00, 0, 16), 27 MLXSW_AFK_ELEMENT_INFO_U32(IP_PROTO, 0x10, 0, 8), 28 MLXSW_AFK_ELEMENT_INFO_U32(VID, 0x10, 8, 12), 29 MLXSW_AFK_ELEMENT_INFO_U32(PCP, 0x10, 20, 3), 30 MLXSW_AFK_ELEMENT_INFO_U32(TCP_FLAGS, 0x10, 23, 9), 31 MLXSW_AFK_ELEMENT_INFO_U32(DST_L4_PORT, 0x14, 0, 16), 32 MLXSW_AFK_ELEMENT_INFO_U32(SRC_L4_PORT, 0x14, 16, 16), 33 MLXSW_AFK_ELEMENT_INFO_U32(IP_TTL_, 0x18, 0, 8), 34 MLXSW_AFK_ELEMENT_INFO_U32(IP_ECN, 0x18, 9, 2), 35 MLXSW_AFK_ELEMENT_INFO_U32(IP_DSCP, 0x18, 11, 6), 36 MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_MSB, 0x18, 17, 4), 37 MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_LSB, 0x18, 21, 8), 38 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_96_127, 0x20, 4), 39 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_64_95, 0x24, 4), 40 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_32_63, 0x28, 4), 41 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_0_31, 0x2C, 4), 42 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_96_127, 0x30, 4), 43 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_64_95, 0x34, 4), 44 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_32_63, 0x38, 4), 45 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_0_31, 0x3C, 4), 46 MLXSW_AFK_ELEMENT_INFO_U32(FDB_MISS, 0x40, 0, 1), 47 MLXSW_AFK_ELEMENT_INFO_U32(L4_PORT_RANGE, 0x40, 1, 16), 48 }; 49 50 struct mlxsw_afk { 51 struct list_head key_info_list; 52 unsigned int max_blocks; 53 const struct mlxsw_afk_ops *ops; 54 const struct mlxsw_afk_block *blocks; 55 unsigned int blocks_count; 56 }; 57 58 static bool mlxsw_afk_blocks_check(struct mlxsw_afk *mlxsw_afk) 59 { 60 int i; 61 int j; 62 63 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 64 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i]; 65 66 for (j = 0; j < block->instances_count; j++) { 67 const struct mlxsw_afk_element_info *elinfo; 68 struct mlxsw_afk_element_inst *elinst; 69 70 elinst = &block->instances[j]; 71 elinfo = &mlxsw_afk_element_infos[elinst->element]; 72 if (elinst->type != elinfo->type || 73 (!elinst->avoid_size_check && 74 elinst->item.size.bits != 75 elinfo->item.size.bits)) 76 return false; 77 } 78 } 79 return true; 80 } 81 82 struct mlxsw_afk *mlxsw_afk_create(unsigned int max_blocks, 83 const struct mlxsw_afk_ops *ops) 84 { 85 struct mlxsw_afk *mlxsw_afk; 86 87 mlxsw_afk = kzalloc(sizeof(*mlxsw_afk), GFP_KERNEL); 88 if (!mlxsw_afk) 89 return NULL; 90 INIT_LIST_HEAD(&mlxsw_afk->key_info_list); 91 mlxsw_afk->max_blocks = max_blocks; 92 mlxsw_afk->ops = ops; 93 mlxsw_afk->blocks = ops->blocks; 94 mlxsw_afk->blocks_count = ops->blocks_count; 95 WARN_ON(!mlxsw_afk_blocks_check(mlxsw_afk)); 96 return mlxsw_afk; 97 } 98 EXPORT_SYMBOL(mlxsw_afk_create); 99 100 void mlxsw_afk_destroy(struct mlxsw_afk *mlxsw_afk) 101 { 102 WARN_ON(!list_empty(&mlxsw_afk->key_info_list)); 103 kfree(mlxsw_afk); 104 } 105 EXPORT_SYMBOL(mlxsw_afk_destroy); 106 107 struct mlxsw_afk_key_info { 108 struct list_head list; 109 refcount_t ref_count; 110 unsigned int blocks_count; 111 int element_to_block[MLXSW_AFK_ELEMENT_MAX]; /* index is element, value 112 * is index inside "blocks" 113 */ 114 struct mlxsw_afk_element_usage elusage; 115 const struct mlxsw_afk_block *blocks[]; 116 }; 117 118 static bool 119 mlxsw_afk_key_info_elements_eq(struct mlxsw_afk_key_info *key_info, 120 struct mlxsw_afk_element_usage *elusage) 121 { 122 return memcmp(&key_info->elusage, elusage, sizeof(*elusage)) == 0; 123 } 124 125 static struct mlxsw_afk_key_info * 126 mlxsw_afk_key_info_find(struct mlxsw_afk *mlxsw_afk, 127 struct mlxsw_afk_element_usage *elusage) 128 { 129 struct mlxsw_afk_key_info *key_info; 130 131 list_for_each_entry(key_info, &mlxsw_afk->key_info_list, list) { 132 if (mlxsw_afk_key_info_elements_eq(key_info, elusage)) 133 return key_info; 134 } 135 return NULL; 136 } 137 138 struct mlxsw_afk_picker { 139 DECLARE_BITMAP(element, MLXSW_AFK_ELEMENT_MAX); 140 unsigned int total; 141 }; 142 143 static void mlxsw_afk_picker_count_hits(struct mlxsw_afk *mlxsw_afk, 144 struct mlxsw_afk_picker *picker, 145 enum mlxsw_afk_element element) 146 { 147 int i; 148 int j; 149 150 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 151 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i]; 152 153 for (j = 0; j < block->instances_count; j++) { 154 struct mlxsw_afk_element_inst *elinst; 155 156 elinst = &block->instances[j]; 157 if (elinst->element == element) { 158 __set_bit(element, picker[i].element); 159 picker[i].total++; 160 } 161 } 162 } 163 } 164 165 static void mlxsw_afk_picker_subtract_hits(struct mlxsw_afk *mlxsw_afk, 166 struct mlxsw_afk_picker *picker, 167 int block_index) 168 { 169 DECLARE_BITMAP(hits_element, MLXSW_AFK_ELEMENT_MAX); 170 int i; 171 int j; 172 173 memcpy(&hits_element, &picker[block_index].element, 174 sizeof(hits_element)); 175 176 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 177 for_each_set_bit(j, hits_element, MLXSW_AFK_ELEMENT_MAX) { 178 if (__test_and_clear_bit(j, picker[i].element)) 179 picker[i].total--; 180 } 181 } 182 } 183 184 static int mlxsw_afk_picker_most_hits_get(struct mlxsw_afk *mlxsw_afk, 185 struct mlxsw_afk_picker *picker) 186 { 187 int most_index = -EINVAL; /* Should never happen to return this */ 188 int most_hits = 0; 189 int i; 190 191 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 192 if (picker[i].total > most_hits) { 193 most_hits = picker[i].total; 194 most_index = i; 195 } 196 } 197 return most_index; 198 } 199 200 static int mlxsw_afk_picker_key_info_add(struct mlxsw_afk *mlxsw_afk, 201 struct mlxsw_afk_picker *picker, 202 int block_index, 203 struct mlxsw_afk_key_info *key_info) 204 { 205 enum mlxsw_afk_element element; 206 207 if (key_info->blocks_count == mlxsw_afk->max_blocks) 208 return -EINVAL; 209 210 for_each_set_bit(element, picker[block_index].element, 211 MLXSW_AFK_ELEMENT_MAX) { 212 key_info->element_to_block[element] = key_info->blocks_count; 213 mlxsw_afk_element_usage_add(&key_info->elusage, element); 214 } 215 216 key_info->blocks[key_info->blocks_count] = 217 &mlxsw_afk->blocks[block_index]; 218 key_info->blocks_count++; 219 return 0; 220 } 221 222 static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk, 223 struct mlxsw_afk_key_info *key_info, 224 struct mlxsw_afk_element_usage *elusage) 225 { 226 struct mlxsw_afk_picker *picker; 227 enum mlxsw_afk_element element; 228 int err; 229 230 picker = kcalloc(mlxsw_afk->blocks_count, sizeof(*picker), GFP_KERNEL); 231 if (!picker) 232 return -ENOMEM; 233 234 /* Since the same elements could be present in multiple blocks, 235 * we must find out optimal block list in order to make the 236 * block count as low as possible. 237 * 238 * First, we count hits. We go over all available blocks and count 239 * how many of requested elements are covered by each. 240 * 241 * Then in loop, we find block with most hits and add it to 242 * output key_info. Then we have to subtract this block hits so 243 * the next iteration will find most suitable block for 244 * the rest of requested elements. 245 */ 246 247 mlxsw_afk_element_usage_for_each(element, elusage) 248 mlxsw_afk_picker_count_hits(mlxsw_afk, picker, element); 249 250 do { 251 int block_index; 252 253 block_index = mlxsw_afk_picker_most_hits_get(mlxsw_afk, picker); 254 if (block_index < 0) { 255 err = block_index; 256 goto out; 257 } 258 err = mlxsw_afk_picker_key_info_add(mlxsw_afk, picker, 259 block_index, key_info); 260 if (err) 261 goto out; 262 mlxsw_afk_picker_subtract_hits(mlxsw_afk, picker, block_index); 263 } while (!mlxsw_afk_key_info_elements_eq(key_info, elusage)); 264 265 err = 0; 266 out: 267 kfree(picker); 268 return err; 269 } 270 271 static struct mlxsw_afk_key_info * 272 mlxsw_afk_key_info_create(struct mlxsw_afk *mlxsw_afk, 273 struct mlxsw_afk_element_usage *elusage) 274 { 275 struct mlxsw_afk_key_info *key_info; 276 int err; 277 278 key_info = kzalloc(struct_size(key_info, blocks, mlxsw_afk->max_blocks), 279 GFP_KERNEL); 280 if (!key_info) 281 return ERR_PTR(-ENOMEM); 282 err = mlxsw_afk_picker(mlxsw_afk, key_info, elusage); 283 if (err) 284 goto err_picker; 285 list_add(&key_info->list, &mlxsw_afk->key_info_list); 286 refcount_set(&key_info->ref_count, 1); 287 return key_info; 288 289 err_picker: 290 kfree(key_info); 291 return ERR_PTR(err); 292 } 293 294 static void mlxsw_afk_key_info_destroy(struct mlxsw_afk_key_info *key_info) 295 { 296 list_del(&key_info->list); 297 kfree(key_info); 298 } 299 300 struct mlxsw_afk_key_info * 301 mlxsw_afk_key_info_get(struct mlxsw_afk *mlxsw_afk, 302 struct mlxsw_afk_element_usage *elusage) 303 { 304 struct mlxsw_afk_key_info *key_info; 305 306 key_info = mlxsw_afk_key_info_find(mlxsw_afk, elusage); 307 if (key_info) { 308 refcount_inc(&key_info->ref_count); 309 return key_info; 310 } 311 return mlxsw_afk_key_info_create(mlxsw_afk, elusage); 312 } 313 EXPORT_SYMBOL(mlxsw_afk_key_info_get); 314 315 void mlxsw_afk_key_info_put(struct mlxsw_afk_key_info *key_info) 316 { 317 if (!refcount_dec_and_test(&key_info->ref_count)) 318 return; 319 mlxsw_afk_key_info_destroy(key_info); 320 } 321 EXPORT_SYMBOL(mlxsw_afk_key_info_put); 322 323 bool mlxsw_afk_key_info_subset(struct mlxsw_afk_key_info *key_info, 324 struct mlxsw_afk_element_usage *elusage) 325 { 326 return mlxsw_afk_element_usage_subset(elusage, &key_info->elusage); 327 } 328 EXPORT_SYMBOL(mlxsw_afk_key_info_subset); 329 330 static const struct mlxsw_afk_element_inst * 331 mlxsw_afk_block_elinst_get(const struct mlxsw_afk_block *block, 332 enum mlxsw_afk_element element) 333 { 334 int i; 335 336 for (i = 0; i < block->instances_count; i++) { 337 struct mlxsw_afk_element_inst *elinst; 338 339 elinst = &block->instances[i]; 340 if (elinst->element == element) 341 return elinst; 342 } 343 return NULL; 344 } 345 346 static const struct mlxsw_afk_element_inst * 347 mlxsw_afk_key_info_elinst_get(struct mlxsw_afk_key_info *key_info, 348 enum mlxsw_afk_element element, 349 int *p_block_index) 350 { 351 const struct mlxsw_afk_element_inst *elinst; 352 const struct mlxsw_afk_block *block; 353 int block_index; 354 355 if (WARN_ON(!test_bit(element, key_info->elusage.usage))) 356 return NULL; 357 block_index = key_info->element_to_block[element]; 358 block = key_info->blocks[block_index]; 359 360 elinst = mlxsw_afk_block_elinst_get(block, element); 361 if (WARN_ON(!elinst)) 362 return NULL; 363 364 *p_block_index = block_index; 365 return elinst; 366 } 367 368 u16 369 mlxsw_afk_key_info_block_encoding_get(const struct mlxsw_afk_key_info *key_info, 370 int block_index) 371 { 372 return key_info->blocks[block_index]->encoding; 373 } 374 EXPORT_SYMBOL(mlxsw_afk_key_info_block_encoding_get); 375 376 unsigned int 377 mlxsw_afk_key_info_blocks_count_get(const struct mlxsw_afk_key_info *key_info) 378 { 379 return key_info->blocks_count; 380 } 381 EXPORT_SYMBOL(mlxsw_afk_key_info_blocks_count_get); 382 383 void mlxsw_afk_values_add_u32(struct mlxsw_afk_element_values *values, 384 enum mlxsw_afk_element element, 385 u32 key_value, u32 mask_value) 386 { 387 const struct mlxsw_afk_element_info *elinfo = 388 &mlxsw_afk_element_infos[element]; 389 const struct mlxsw_item *storage_item = &elinfo->item; 390 391 if (!mask_value) 392 return; 393 if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_U32)) 394 return; 395 __mlxsw_item_set32(values->storage.key, storage_item, 0, key_value); 396 __mlxsw_item_set32(values->storage.mask, storage_item, 0, mask_value); 397 mlxsw_afk_element_usage_add(&values->elusage, element); 398 } 399 EXPORT_SYMBOL(mlxsw_afk_values_add_u32); 400 401 void mlxsw_afk_values_add_buf(struct mlxsw_afk_element_values *values, 402 enum mlxsw_afk_element element, 403 const char *key_value, const char *mask_value, 404 unsigned int len) 405 { 406 const struct mlxsw_afk_element_info *elinfo = 407 &mlxsw_afk_element_infos[element]; 408 const struct mlxsw_item *storage_item = &elinfo->item; 409 410 if (!memchr_inv(mask_value, 0, len)) /* If mask is zero */ 411 return; 412 if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_BUF) || 413 WARN_ON(elinfo->item.size.bytes != len)) 414 return; 415 __mlxsw_item_memcpy_to(values->storage.key, key_value, 416 storage_item, 0); 417 __mlxsw_item_memcpy_to(values->storage.mask, mask_value, 418 storage_item, 0); 419 mlxsw_afk_element_usage_add(&values->elusage, element); 420 } 421 EXPORT_SYMBOL(mlxsw_afk_values_add_buf); 422 423 static void mlxsw_sp_afk_encode_u32(const struct mlxsw_item *storage_item, 424 const struct mlxsw_item *output_item, 425 char *storage, char *output, int diff) 426 { 427 u32 value; 428 429 value = __mlxsw_item_get32(storage, storage_item, 0); 430 __mlxsw_item_set32(output, output_item, 0, value + diff); 431 } 432 433 static void mlxsw_sp_afk_encode_buf(const struct mlxsw_item *storage_item, 434 const struct mlxsw_item *output_item, 435 char *storage, char *output) 436 { 437 char *storage_data = __mlxsw_item_data(storage, storage_item, 0); 438 char *output_data = __mlxsw_item_data(output, output_item, 0); 439 size_t len = output_item->size.bytes; 440 441 memcpy(output_data, storage_data, len); 442 } 443 444 static void 445 mlxsw_sp_afk_encode_one(const struct mlxsw_afk_element_inst *elinst, 446 char *output, char *storage, int u32_diff) 447 { 448 const struct mlxsw_item *output_item = &elinst->item; 449 const struct mlxsw_afk_element_info *elinfo; 450 const struct mlxsw_item *storage_item; 451 452 elinfo = &mlxsw_afk_element_infos[elinst->element]; 453 storage_item = &elinfo->item; 454 if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_U32) 455 mlxsw_sp_afk_encode_u32(storage_item, output_item, 456 storage, output, u32_diff); 457 else if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_BUF) 458 mlxsw_sp_afk_encode_buf(storage_item, output_item, 459 storage, output); 460 } 461 462 #define MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE 16 463 464 void mlxsw_afk_encode(struct mlxsw_afk *mlxsw_afk, 465 struct mlxsw_afk_key_info *key_info, 466 struct mlxsw_afk_element_values *values, 467 char *key, char *mask) 468 { 469 unsigned int blocks_count = 470 mlxsw_afk_key_info_blocks_count_get(key_info); 471 char block_mask[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE]; 472 char block_key[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE]; 473 const struct mlxsw_afk_element_inst *elinst; 474 enum mlxsw_afk_element element; 475 int block_index, i; 476 477 for (i = 0; i < blocks_count; i++) { 478 memset(block_key, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE); 479 memset(block_mask, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE); 480 481 mlxsw_afk_element_usage_for_each(element, &values->elusage) { 482 elinst = mlxsw_afk_key_info_elinst_get(key_info, 483 element, 484 &block_index); 485 if (!elinst || block_index != i) 486 continue; 487 488 mlxsw_sp_afk_encode_one(elinst, block_key, 489 values->storage.key, 490 elinst->u32_key_diff); 491 mlxsw_sp_afk_encode_one(elinst, block_mask, 492 values->storage.mask, 0); 493 } 494 495 mlxsw_afk->ops->encode_block(key, i, block_key); 496 mlxsw_afk->ops->encode_block(mask, i, block_mask); 497 } 498 } 499 EXPORT_SYMBOL(mlxsw_afk_encode); 500 501 void mlxsw_afk_clear(struct mlxsw_afk *mlxsw_afk, char *key, 502 int block_start, int block_end) 503 { 504 int i; 505 506 for (i = block_start; i <= block_end; i++) 507 mlxsw_afk->ops->clear_block(key, i); 508 } 509 EXPORT_SYMBOL(mlxsw_afk_clear); 510