1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/slab.h> 6 #include <linux/list.h> 7 #include <linux/errno.h> 8 9 #include "item.h" 10 #include "core_acl_flex_keys.h" 11 12 /* For the purpose of the driver, define an internal storage scratchpad 13 * that will be used to store key/mask values. For each defined element type 14 * define an internal storage geometry. 15 * 16 * When adding new elements, MLXSW_AFK_ELEMENT_STORAGE_SIZE must be increased 17 * accordingly. 18 */ 19 static const struct mlxsw_afk_element_info mlxsw_afk_element_infos[] = { 20 MLXSW_AFK_ELEMENT_INFO_U32(SRC_SYS_PORT, 0x00, 16, 16), 21 MLXSW_AFK_ELEMENT_INFO_BUF(DMAC_32_47, 0x04, 2), 22 MLXSW_AFK_ELEMENT_INFO_BUF(DMAC_0_31, 0x06, 4), 23 MLXSW_AFK_ELEMENT_INFO_BUF(SMAC_32_47, 0x0A, 2), 24 MLXSW_AFK_ELEMENT_INFO_BUF(SMAC_0_31, 0x0C, 4), 25 MLXSW_AFK_ELEMENT_INFO_U32(ETHERTYPE, 0x00, 0, 16), 26 MLXSW_AFK_ELEMENT_INFO_U32(IP_PROTO, 0x10, 0, 8), 27 MLXSW_AFK_ELEMENT_INFO_U32(VID, 0x10, 8, 12), 28 MLXSW_AFK_ELEMENT_INFO_U32(PCP, 0x10, 20, 3), 29 MLXSW_AFK_ELEMENT_INFO_U32(TCP_FLAGS, 0x10, 23, 9), 30 MLXSW_AFK_ELEMENT_INFO_U32(DST_L4_PORT, 0x14, 0, 16), 31 MLXSW_AFK_ELEMENT_INFO_U32(SRC_L4_PORT, 0x14, 16, 16), 32 MLXSW_AFK_ELEMENT_INFO_U32(IP_TTL_, 0x18, 0, 8), 33 MLXSW_AFK_ELEMENT_INFO_U32(IP_ECN, 0x18, 9, 2), 34 MLXSW_AFK_ELEMENT_INFO_U32(IP_DSCP, 0x18, 11, 6), 35 MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_MSB, 0x18, 17, 3), 36 MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_LSB, 0x18, 20, 8), 37 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_96_127, 0x20, 4), 38 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_64_95, 0x24, 4), 39 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_32_63, 0x28, 4), 40 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_0_31, 0x2C, 4), 41 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_96_127, 0x30, 4), 42 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_64_95, 0x34, 4), 43 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_32_63, 0x38, 4), 44 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_0_31, 0x3C, 4), 45 MLXSW_AFK_ELEMENT_INFO_U32(FDB_MISS, 0x40, 0, 1), 46 }; 47 48 struct mlxsw_afk { 49 struct list_head key_info_list; 50 unsigned int max_blocks; 51 const struct mlxsw_afk_ops *ops; 52 const struct mlxsw_afk_block *blocks; 53 unsigned int blocks_count; 54 }; 55 56 static bool mlxsw_afk_blocks_check(struct mlxsw_afk *mlxsw_afk) 57 { 58 int i; 59 int j; 60 61 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 62 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i]; 63 64 for (j = 0; j < block->instances_count; j++) { 65 const struct mlxsw_afk_element_info *elinfo; 66 struct mlxsw_afk_element_inst *elinst; 67 68 elinst = &block->instances[j]; 69 elinfo = &mlxsw_afk_element_infos[elinst->element]; 70 if (elinst->type != elinfo->type || 71 (!elinst->avoid_size_check && 72 elinst->item.size.bits != 73 elinfo->item.size.bits)) 74 return false; 75 } 76 } 77 return true; 78 } 79 80 struct mlxsw_afk *mlxsw_afk_create(unsigned int max_blocks, 81 const struct mlxsw_afk_ops *ops) 82 { 83 struct mlxsw_afk *mlxsw_afk; 84 85 mlxsw_afk = kzalloc(sizeof(*mlxsw_afk), GFP_KERNEL); 86 if (!mlxsw_afk) 87 return NULL; 88 INIT_LIST_HEAD(&mlxsw_afk->key_info_list); 89 mlxsw_afk->max_blocks = max_blocks; 90 mlxsw_afk->ops = ops; 91 mlxsw_afk->blocks = ops->blocks; 92 mlxsw_afk->blocks_count = ops->blocks_count; 93 WARN_ON(!mlxsw_afk_blocks_check(mlxsw_afk)); 94 return mlxsw_afk; 95 } 96 EXPORT_SYMBOL(mlxsw_afk_create); 97 98 void mlxsw_afk_destroy(struct mlxsw_afk *mlxsw_afk) 99 { 100 WARN_ON(!list_empty(&mlxsw_afk->key_info_list)); 101 kfree(mlxsw_afk); 102 } 103 EXPORT_SYMBOL(mlxsw_afk_destroy); 104 105 struct mlxsw_afk_key_info { 106 struct list_head list; 107 unsigned int ref_count; 108 unsigned int blocks_count; 109 int element_to_block[MLXSW_AFK_ELEMENT_MAX]; /* index is element, value 110 * is index inside "blocks" 111 */ 112 struct mlxsw_afk_element_usage elusage; 113 const struct mlxsw_afk_block *blocks[]; 114 }; 115 116 static bool 117 mlxsw_afk_key_info_elements_eq(struct mlxsw_afk_key_info *key_info, 118 struct mlxsw_afk_element_usage *elusage) 119 { 120 return memcmp(&key_info->elusage, elusage, sizeof(*elusage)) == 0; 121 } 122 123 static struct mlxsw_afk_key_info * 124 mlxsw_afk_key_info_find(struct mlxsw_afk *mlxsw_afk, 125 struct mlxsw_afk_element_usage *elusage) 126 { 127 struct mlxsw_afk_key_info *key_info; 128 129 list_for_each_entry(key_info, &mlxsw_afk->key_info_list, list) { 130 if (mlxsw_afk_key_info_elements_eq(key_info, elusage)) 131 return key_info; 132 } 133 return NULL; 134 } 135 136 struct mlxsw_afk_picker { 137 DECLARE_BITMAP(element, MLXSW_AFK_ELEMENT_MAX); 138 unsigned int total; 139 }; 140 141 static void mlxsw_afk_picker_count_hits(struct mlxsw_afk *mlxsw_afk, 142 struct mlxsw_afk_picker *picker, 143 enum mlxsw_afk_element element) 144 { 145 int i; 146 int j; 147 148 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 149 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i]; 150 151 for (j = 0; j < block->instances_count; j++) { 152 struct mlxsw_afk_element_inst *elinst; 153 154 elinst = &block->instances[j]; 155 if (elinst->element == element) { 156 __set_bit(element, picker[i].element); 157 picker[i].total++; 158 } 159 } 160 } 161 } 162 163 static void mlxsw_afk_picker_subtract_hits(struct mlxsw_afk *mlxsw_afk, 164 struct mlxsw_afk_picker *picker, 165 int block_index) 166 { 167 DECLARE_BITMAP(hits_element, MLXSW_AFK_ELEMENT_MAX); 168 int i; 169 int j; 170 171 memcpy(&hits_element, &picker[block_index].element, 172 sizeof(hits_element)); 173 174 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 175 for_each_set_bit(j, hits_element, MLXSW_AFK_ELEMENT_MAX) { 176 if (__test_and_clear_bit(j, picker[i].element)) 177 picker[i].total--; 178 } 179 } 180 } 181 182 static int mlxsw_afk_picker_most_hits_get(struct mlxsw_afk *mlxsw_afk, 183 struct mlxsw_afk_picker *picker) 184 { 185 int most_index = -EINVAL; /* Should never happen to return this */ 186 int most_hits = 0; 187 int i; 188 189 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 190 if (picker[i].total > most_hits) { 191 most_hits = picker[i].total; 192 most_index = i; 193 } 194 } 195 return most_index; 196 } 197 198 static int mlxsw_afk_picker_key_info_add(struct mlxsw_afk *mlxsw_afk, 199 struct mlxsw_afk_picker *picker, 200 int block_index, 201 struct mlxsw_afk_key_info *key_info) 202 { 203 enum mlxsw_afk_element element; 204 205 if (key_info->blocks_count == mlxsw_afk->max_blocks) 206 return -EINVAL; 207 208 for_each_set_bit(element, picker[block_index].element, 209 MLXSW_AFK_ELEMENT_MAX) { 210 key_info->element_to_block[element] = key_info->blocks_count; 211 mlxsw_afk_element_usage_add(&key_info->elusage, element); 212 } 213 214 key_info->blocks[key_info->blocks_count] = 215 &mlxsw_afk->blocks[block_index]; 216 key_info->blocks_count++; 217 return 0; 218 } 219 220 static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk, 221 struct mlxsw_afk_key_info *key_info, 222 struct mlxsw_afk_element_usage *elusage) 223 { 224 struct mlxsw_afk_picker *picker; 225 enum mlxsw_afk_element element; 226 int err; 227 228 picker = kcalloc(mlxsw_afk->blocks_count, sizeof(*picker), GFP_KERNEL); 229 if (!picker) 230 return -ENOMEM; 231 232 /* Since the same elements could be present in multiple blocks, 233 * we must find out optimal block list in order to make the 234 * block count as low as possible. 235 * 236 * First, we count hits. We go over all available blocks and count 237 * how many of requested elements are covered by each. 238 * 239 * Then in loop, we find block with most hits and add it to 240 * output key_info. Then we have to subtract this block hits so 241 * the next iteration will find most suitable block for 242 * the rest of requested elements. 243 */ 244 245 mlxsw_afk_element_usage_for_each(element, elusage) 246 mlxsw_afk_picker_count_hits(mlxsw_afk, picker, element); 247 248 do { 249 int block_index; 250 251 block_index = mlxsw_afk_picker_most_hits_get(mlxsw_afk, picker); 252 if (block_index < 0) { 253 err = block_index; 254 goto out; 255 } 256 err = mlxsw_afk_picker_key_info_add(mlxsw_afk, picker, 257 block_index, key_info); 258 if (err) 259 goto out; 260 mlxsw_afk_picker_subtract_hits(mlxsw_afk, picker, block_index); 261 } while (!mlxsw_afk_key_info_elements_eq(key_info, elusage)); 262 263 err = 0; 264 out: 265 kfree(picker); 266 return err; 267 } 268 269 static struct mlxsw_afk_key_info * 270 mlxsw_afk_key_info_create(struct mlxsw_afk *mlxsw_afk, 271 struct mlxsw_afk_element_usage *elusage) 272 { 273 struct mlxsw_afk_key_info *key_info; 274 int err; 275 276 key_info = kzalloc(struct_size(key_info, blocks, mlxsw_afk->max_blocks), 277 GFP_KERNEL); 278 if (!key_info) 279 return ERR_PTR(-ENOMEM); 280 err = mlxsw_afk_picker(mlxsw_afk, key_info, elusage); 281 if (err) 282 goto err_picker; 283 list_add(&key_info->list, &mlxsw_afk->key_info_list); 284 key_info->ref_count = 1; 285 return key_info; 286 287 err_picker: 288 kfree(key_info); 289 return ERR_PTR(err); 290 } 291 292 static void mlxsw_afk_key_info_destroy(struct mlxsw_afk_key_info *key_info) 293 { 294 list_del(&key_info->list); 295 kfree(key_info); 296 } 297 298 struct mlxsw_afk_key_info * 299 mlxsw_afk_key_info_get(struct mlxsw_afk *mlxsw_afk, 300 struct mlxsw_afk_element_usage *elusage) 301 { 302 struct mlxsw_afk_key_info *key_info; 303 304 key_info = mlxsw_afk_key_info_find(mlxsw_afk, elusage); 305 if (key_info) { 306 key_info->ref_count++; 307 return key_info; 308 } 309 return mlxsw_afk_key_info_create(mlxsw_afk, elusage); 310 } 311 EXPORT_SYMBOL(mlxsw_afk_key_info_get); 312 313 void mlxsw_afk_key_info_put(struct mlxsw_afk_key_info *key_info) 314 { 315 if (--key_info->ref_count) 316 return; 317 mlxsw_afk_key_info_destroy(key_info); 318 } 319 EXPORT_SYMBOL(mlxsw_afk_key_info_put); 320 321 bool mlxsw_afk_key_info_subset(struct mlxsw_afk_key_info *key_info, 322 struct mlxsw_afk_element_usage *elusage) 323 { 324 return mlxsw_afk_element_usage_subset(elusage, &key_info->elusage); 325 } 326 EXPORT_SYMBOL(mlxsw_afk_key_info_subset); 327 328 static const struct mlxsw_afk_element_inst * 329 mlxsw_afk_block_elinst_get(const struct mlxsw_afk_block *block, 330 enum mlxsw_afk_element element) 331 { 332 int i; 333 334 for (i = 0; i < block->instances_count; i++) { 335 struct mlxsw_afk_element_inst *elinst; 336 337 elinst = &block->instances[i]; 338 if (elinst->element == element) 339 return elinst; 340 } 341 return NULL; 342 } 343 344 static const struct mlxsw_afk_element_inst * 345 mlxsw_afk_key_info_elinst_get(struct mlxsw_afk_key_info *key_info, 346 enum mlxsw_afk_element element, 347 int *p_block_index) 348 { 349 const struct mlxsw_afk_element_inst *elinst; 350 const struct mlxsw_afk_block *block; 351 int block_index; 352 353 if (WARN_ON(!test_bit(element, key_info->elusage.usage))) 354 return NULL; 355 block_index = key_info->element_to_block[element]; 356 block = key_info->blocks[block_index]; 357 358 elinst = mlxsw_afk_block_elinst_get(block, element); 359 if (WARN_ON(!elinst)) 360 return NULL; 361 362 *p_block_index = block_index; 363 return elinst; 364 } 365 366 u16 367 mlxsw_afk_key_info_block_encoding_get(const struct mlxsw_afk_key_info *key_info, 368 int block_index) 369 { 370 return key_info->blocks[block_index]->encoding; 371 } 372 EXPORT_SYMBOL(mlxsw_afk_key_info_block_encoding_get); 373 374 unsigned int 375 mlxsw_afk_key_info_blocks_count_get(const struct mlxsw_afk_key_info *key_info) 376 { 377 return key_info->blocks_count; 378 } 379 EXPORT_SYMBOL(mlxsw_afk_key_info_blocks_count_get); 380 381 void mlxsw_afk_values_add_u32(struct mlxsw_afk_element_values *values, 382 enum mlxsw_afk_element element, 383 u32 key_value, u32 mask_value) 384 { 385 const struct mlxsw_afk_element_info *elinfo = 386 &mlxsw_afk_element_infos[element]; 387 const struct mlxsw_item *storage_item = &elinfo->item; 388 389 if (!mask_value) 390 return; 391 if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_U32)) 392 return; 393 __mlxsw_item_set32(values->storage.key, storage_item, 0, key_value); 394 __mlxsw_item_set32(values->storage.mask, storage_item, 0, mask_value); 395 mlxsw_afk_element_usage_add(&values->elusage, element); 396 } 397 EXPORT_SYMBOL(mlxsw_afk_values_add_u32); 398 399 void mlxsw_afk_values_add_buf(struct mlxsw_afk_element_values *values, 400 enum mlxsw_afk_element element, 401 const char *key_value, const char *mask_value, 402 unsigned int len) 403 { 404 const struct mlxsw_afk_element_info *elinfo = 405 &mlxsw_afk_element_infos[element]; 406 const struct mlxsw_item *storage_item = &elinfo->item; 407 408 if (!memchr_inv(mask_value, 0, len)) /* If mask is zero */ 409 return; 410 if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_BUF) || 411 WARN_ON(elinfo->item.size.bytes != len)) 412 return; 413 __mlxsw_item_memcpy_to(values->storage.key, key_value, 414 storage_item, 0); 415 __mlxsw_item_memcpy_to(values->storage.mask, mask_value, 416 storage_item, 0); 417 mlxsw_afk_element_usage_add(&values->elusage, element); 418 } 419 EXPORT_SYMBOL(mlxsw_afk_values_add_buf); 420 421 static void mlxsw_sp_afk_encode_u32(const struct mlxsw_item *storage_item, 422 const struct mlxsw_item *output_item, 423 char *storage, char *output, int diff) 424 { 425 u32 value; 426 427 value = __mlxsw_item_get32(storage, storage_item, 0); 428 __mlxsw_item_set32(output, output_item, 0, value + diff); 429 } 430 431 static void mlxsw_sp_afk_encode_buf(const struct mlxsw_item *storage_item, 432 const struct mlxsw_item *output_item, 433 char *storage, char *output) 434 { 435 char *storage_data = __mlxsw_item_data(storage, storage_item, 0); 436 char *output_data = __mlxsw_item_data(output, output_item, 0); 437 size_t len = output_item->size.bytes; 438 439 memcpy(output_data, storage_data, len); 440 } 441 442 static void 443 mlxsw_sp_afk_encode_one(const struct mlxsw_afk_element_inst *elinst, 444 char *output, char *storage, int u32_diff) 445 { 446 const struct mlxsw_item *output_item = &elinst->item; 447 const struct mlxsw_afk_element_info *elinfo; 448 const struct mlxsw_item *storage_item; 449 450 elinfo = &mlxsw_afk_element_infos[elinst->element]; 451 storage_item = &elinfo->item; 452 if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_U32) 453 mlxsw_sp_afk_encode_u32(storage_item, output_item, 454 storage, output, u32_diff); 455 else if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_BUF) 456 mlxsw_sp_afk_encode_buf(storage_item, output_item, 457 storage, output); 458 } 459 460 #define MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE 16 461 462 void mlxsw_afk_encode(struct mlxsw_afk *mlxsw_afk, 463 struct mlxsw_afk_key_info *key_info, 464 struct mlxsw_afk_element_values *values, 465 char *key, char *mask) 466 { 467 unsigned int blocks_count = 468 mlxsw_afk_key_info_blocks_count_get(key_info); 469 char block_mask[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE]; 470 char block_key[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE]; 471 const struct mlxsw_afk_element_inst *elinst; 472 enum mlxsw_afk_element element; 473 int block_index, i; 474 475 for (i = 0; i < blocks_count; i++) { 476 memset(block_key, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE); 477 memset(block_mask, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE); 478 479 mlxsw_afk_element_usage_for_each(element, &values->elusage) { 480 elinst = mlxsw_afk_key_info_elinst_get(key_info, 481 element, 482 &block_index); 483 if (!elinst || block_index != i) 484 continue; 485 486 mlxsw_sp_afk_encode_one(elinst, block_key, 487 values->storage.key, 488 elinst->u32_key_diff); 489 mlxsw_sp_afk_encode_one(elinst, block_mask, 490 values->storage.mask, 0); 491 } 492 493 mlxsw_afk->ops->encode_block(key, i, block_key); 494 mlxsw_afk->ops->encode_block(mask, i, block_mask); 495 } 496 } 497 EXPORT_SYMBOL(mlxsw_afk_encode); 498 499 void mlxsw_afk_clear(struct mlxsw_afk *mlxsw_afk, char *key, 500 int block_start, int block_end) 501 { 502 int i; 503 504 for (i = block_start; i <= block_end; i++) 505 mlxsw_afk->ops->clear_block(key, i); 506 } 507 EXPORT_SYMBOL(mlxsw_afk_clear); 508