1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/slab.h> 6 #include <linux/list.h> 7 #include <linux/errno.h> 8 9 #include "item.h" 10 #include "core_acl_flex_keys.h" 11 12 /* For the purpose of the driver, define an internal storage scratchpad 13 * that will be used to store key/mask values. For each defined element type 14 * define an internal storage geometry. 15 * 16 * When adding new elements, MLXSW_AFK_ELEMENT_STORAGE_SIZE must be increased 17 * accordingly. 18 */ 19 static const struct mlxsw_afk_element_info mlxsw_afk_element_infos[] = { 20 MLXSW_AFK_ELEMENT_INFO_U32(SRC_SYS_PORT, 0x00, 16, 16), 21 MLXSW_AFK_ELEMENT_INFO_BUF(DMAC_32_47, 0x04, 2), 22 MLXSW_AFK_ELEMENT_INFO_BUF(DMAC_0_31, 0x06, 4), 23 MLXSW_AFK_ELEMENT_INFO_BUF(SMAC_32_47, 0x0A, 2), 24 MLXSW_AFK_ELEMENT_INFO_BUF(SMAC_0_31, 0x0C, 4), 25 MLXSW_AFK_ELEMENT_INFO_U32(ETHERTYPE, 0x00, 0, 16), 26 MLXSW_AFK_ELEMENT_INFO_U32(IP_PROTO, 0x10, 0, 8), 27 MLXSW_AFK_ELEMENT_INFO_U32(VID, 0x10, 8, 12), 28 MLXSW_AFK_ELEMENT_INFO_U32(PCP, 0x10, 20, 3), 29 MLXSW_AFK_ELEMENT_INFO_U32(TCP_FLAGS, 0x10, 23, 9), 30 MLXSW_AFK_ELEMENT_INFO_U32(DST_L4_PORT, 0x14, 0, 16), 31 MLXSW_AFK_ELEMENT_INFO_U32(SRC_L4_PORT, 0x14, 16, 16), 32 MLXSW_AFK_ELEMENT_INFO_U32(IP_TTL_, 0x18, 0, 8), 33 MLXSW_AFK_ELEMENT_INFO_U32(IP_ECN, 0x18, 9, 2), 34 MLXSW_AFK_ELEMENT_INFO_U32(IP_DSCP, 0x18, 11, 6), 35 MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_MSB, 0x18, 17, 4), 36 MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_LSB, 0x18, 21, 8), 37 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_96_127, 0x20, 4), 38 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_64_95, 0x24, 4), 39 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_32_63, 0x28, 4), 40 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_0_31, 0x2C, 4), 41 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_96_127, 0x30, 4), 42 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_64_95, 0x34, 4), 43 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_32_63, 0x38, 4), 44 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_0_31, 0x3C, 4), 45 MLXSW_AFK_ELEMENT_INFO_U32(FDB_MISS, 0x40, 0, 1), 46 MLXSW_AFK_ELEMENT_INFO_U32(L4_PORT_RANGE, 0x40, 1, 16), 47 }; 48 49 struct mlxsw_afk { 50 struct list_head key_info_list; 51 unsigned int max_blocks; 52 const struct mlxsw_afk_ops *ops; 53 const struct mlxsw_afk_block *blocks; 54 unsigned int blocks_count; 55 }; 56 57 static bool mlxsw_afk_blocks_check(struct mlxsw_afk *mlxsw_afk) 58 { 59 int i; 60 int j; 61 62 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 63 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i]; 64 65 for (j = 0; j < block->instances_count; j++) { 66 const struct mlxsw_afk_element_info *elinfo; 67 struct mlxsw_afk_element_inst *elinst; 68 69 elinst = &block->instances[j]; 70 elinfo = &mlxsw_afk_element_infos[elinst->element]; 71 if (elinst->type != elinfo->type || 72 (!elinst->avoid_size_check && 73 elinst->item.size.bits != 74 elinfo->item.size.bits)) 75 return false; 76 } 77 } 78 return true; 79 } 80 81 struct mlxsw_afk *mlxsw_afk_create(unsigned int max_blocks, 82 const struct mlxsw_afk_ops *ops) 83 { 84 struct mlxsw_afk *mlxsw_afk; 85 86 mlxsw_afk = kzalloc(sizeof(*mlxsw_afk), GFP_KERNEL); 87 if (!mlxsw_afk) 88 return NULL; 89 INIT_LIST_HEAD(&mlxsw_afk->key_info_list); 90 mlxsw_afk->max_blocks = max_blocks; 91 mlxsw_afk->ops = ops; 92 mlxsw_afk->blocks = ops->blocks; 93 mlxsw_afk->blocks_count = ops->blocks_count; 94 WARN_ON(!mlxsw_afk_blocks_check(mlxsw_afk)); 95 return mlxsw_afk; 96 } 97 EXPORT_SYMBOL(mlxsw_afk_create); 98 99 void mlxsw_afk_destroy(struct mlxsw_afk *mlxsw_afk) 100 { 101 WARN_ON(!list_empty(&mlxsw_afk->key_info_list)); 102 kfree(mlxsw_afk); 103 } 104 EXPORT_SYMBOL(mlxsw_afk_destroy); 105 106 struct mlxsw_afk_key_info { 107 struct list_head list; 108 unsigned int ref_count; 109 unsigned int blocks_count; 110 int element_to_block[MLXSW_AFK_ELEMENT_MAX]; /* index is element, value 111 * is index inside "blocks" 112 */ 113 struct mlxsw_afk_element_usage elusage; 114 const struct mlxsw_afk_block *blocks[]; 115 }; 116 117 static bool 118 mlxsw_afk_key_info_elements_eq(struct mlxsw_afk_key_info *key_info, 119 struct mlxsw_afk_element_usage *elusage) 120 { 121 return memcmp(&key_info->elusage, elusage, sizeof(*elusage)) == 0; 122 } 123 124 static struct mlxsw_afk_key_info * 125 mlxsw_afk_key_info_find(struct mlxsw_afk *mlxsw_afk, 126 struct mlxsw_afk_element_usage *elusage) 127 { 128 struct mlxsw_afk_key_info *key_info; 129 130 list_for_each_entry(key_info, &mlxsw_afk->key_info_list, list) { 131 if (mlxsw_afk_key_info_elements_eq(key_info, elusage)) 132 return key_info; 133 } 134 return NULL; 135 } 136 137 struct mlxsw_afk_picker { 138 DECLARE_BITMAP(element, MLXSW_AFK_ELEMENT_MAX); 139 unsigned int total; 140 }; 141 142 static void mlxsw_afk_picker_count_hits(struct mlxsw_afk *mlxsw_afk, 143 struct mlxsw_afk_picker *picker, 144 enum mlxsw_afk_element element) 145 { 146 int i; 147 int j; 148 149 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 150 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i]; 151 152 for (j = 0; j < block->instances_count; j++) { 153 struct mlxsw_afk_element_inst *elinst; 154 155 elinst = &block->instances[j]; 156 if (elinst->element == element) { 157 __set_bit(element, picker[i].element); 158 picker[i].total++; 159 } 160 } 161 } 162 } 163 164 static void mlxsw_afk_picker_subtract_hits(struct mlxsw_afk *mlxsw_afk, 165 struct mlxsw_afk_picker *picker, 166 int block_index) 167 { 168 DECLARE_BITMAP(hits_element, MLXSW_AFK_ELEMENT_MAX); 169 int i; 170 int j; 171 172 memcpy(&hits_element, &picker[block_index].element, 173 sizeof(hits_element)); 174 175 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 176 for_each_set_bit(j, hits_element, MLXSW_AFK_ELEMENT_MAX) { 177 if (__test_and_clear_bit(j, picker[i].element)) 178 picker[i].total--; 179 } 180 } 181 } 182 183 static int mlxsw_afk_picker_most_hits_get(struct mlxsw_afk *mlxsw_afk, 184 struct mlxsw_afk_picker *picker) 185 { 186 int most_index = -EINVAL; /* Should never happen to return this */ 187 int most_hits = 0; 188 int i; 189 190 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 191 if (picker[i].total > most_hits) { 192 most_hits = picker[i].total; 193 most_index = i; 194 } 195 } 196 return most_index; 197 } 198 199 static int mlxsw_afk_picker_key_info_add(struct mlxsw_afk *mlxsw_afk, 200 struct mlxsw_afk_picker *picker, 201 int block_index, 202 struct mlxsw_afk_key_info *key_info) 203 { 204 enum mlxsw_afk_element element; 205 206 if (key_info->blocks_count == mlxsw_afk->max_blocks) 207 return -EINVAL; 208 209 for_each_set_bit(element, picker[block_index].element, 210 MLXSW_AFK_ELEMENT_MAX) { 211 key_info->element_to_block[element] = key_info->blocks_count; 212 mlxsw_afk_element_usage_add(&key_info->elusage, element); 213 } 214 215 key_info->blocks[key_info->blocks_count] = 216 &mlxsw_afk->blocks[block_index]; 217 key_info->blocks_count++; 218 return 0; 219 } 220 221 static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk, 222 struct mlxsw_afk_key_info *key_info, 223 struct mlxsw_afk_element_usage *elusage) 224 { 225 struct mlxsw_afk_picker *picker; 226 enum mlxsw_afk_element element; 227 int err; 228 229 picker = kcalloc(mlxsw_afk->blocks_count, sizeof(*picker), GFP_KERNEL); 230 if (!picker) 231 return -ENOMEM; 232 233 /* Since the same elements could be present in multiple blocks, 234 * we must find out optimal block list in order to make the 235 * block count as low as possible. 236 * 237 * First, we count hits. We go over all available blocks and count 238 * how many of requested elements are covered by each. 239 * 240 * Then in loop, we find block with most hits and add it to 241 * output key_info. Then we have to subtract this block hits so 242 * the next iteration will find most suitable block for 243 * the rest of requested elements. 244 */ 245 246 mlxsw_afk_element_usage_for_each(element, elusage) 247 mlxsw_afk_picker_count_hits(mlxsw_afk, picker, element); 248 249 do { 250 int block_index; 251 252 block_index = mlxsw_afk_picker_most_hits_get(mlxsw_afk, picker); 253 if (block_index < 0) { 254 err = block_index; 255 goto out; 256 } 257 err = mlxsw_afk_picker_key_info_add(mlxsw_afk, picker, 258 block_index, key_info); 259 if (err) 260 goto out; 261 mlxsw_afk_picker_subtract_hits(mlxsw_afk, picker, block_index); 262 } while (!mlxsw_afk_key_info_elements_eq(key_info, elusage)); 263 264 err = 0; 265 out: 266 kfree(picker); 267 return err; 268 } 269 270 static struct mlxsw_afk_key_info * 271 mlxsw_afk_key_info_create(struct mlxsw_afk *mlxsw_afk, 272 struct mlxsw_afk_element_usage *elusage) 273 { 274 struct mlxsw_afk_key_info *key_info; 275 int err; 276 277 key_info = kzalloc(struct_size(key_info, blocks, mlxsw_afk->max_blocks), 278 GFP_KERNEL); 279 if (!key_info) 280 return ERR_PTR(-ENOMEM); 281 err = mlxsw_afk_picker(mlxsw_afk, key_info, elusage); 282 if (err) 283 goto err_picker; 284 list_add(&key_info->list, &mlxsw_afk->key_info_list); 285 key_info->ref_count = 1; 286 return key_info; 287 288 err_picker: 289 kfree(key_info); 290 return ERR_PTR(err); 291 } 292 293 static void mlxsw_afk_key_info_destroy(struct mlxsw_afk_key_info *key_info) 294 { 295 list_del(&key_info->list); 296 kfree(key_info); 297 } 298 299 struct mlxsw_afk_key_info * 300 mlxsw_afk_key_info_get(struct mlxsw_afk *mlxsw_afk, 301 struct mlxsw_afk_element_usage *elusage) 302 { 303 struct mlxsw_afk_key_info *key_info; 304 305 key_info = mlxsw_afk_key_info_find(mlxsw_afk, elusage); 306 if (key_info) { 307 key_info->ref_count++; 308 return key_info; 309 } 310 return mlxsw_afk_key_info_create(mlxsw_afk, elusage); 311 } 312 EXPORT_SYMBOL(mlxsw_afk_key_info_get); 313 314 void mlxsw_afk_key_info_put(struct mlxsw_afk_key_info *key_info) 315 { 316 if (--key_info->ref_count) 317 return; 318 mlxsw_afk_key_info_destroy(key_info); 319 } 320 EXPORT_SYMBOL(mlxsw_afk_key_info_put); 321 322 bool mlxsw_afk_key_info_subset(struct mlxsw_afk_key_info *key_info, 323 struct mlxsw_afk_element_usage *elusage) 324 { 325 return mlxsw_afk_element_usage_subset(elusage, &key_info->elusage); 326 } 327 EXPORT_SYMBOL(mlxsw_afk_key_info_subset); 328 329 static const struct mlxsw_afk_element_inst * 330 mlxsw_afk_block_elinst_get(const struct mlxsw_afk_block *block, 331 enum mlxsw_afk_element element) 332 { 333 int i; 334 335 for (i = 0; i < block->instances_count; i++) { 336 struct mlxsw_afk_element_inst *elinst; 337 338 elinst = &block->instances[i]; 339 if (elinst->element == element) 340 return elinst; 341 } 342 return NULL; 343 } 344 345 static const struct mlxsw_afk_element_inst * 346 mlxsw_afk_key_info_elinst_get(struct mlxsw_afk_key_info *key_info, 347 enum mlxsw_afk_element element, 348 int *p_block_index) 349 { 350 const struct mlxsw_afk_element_inst *elinst; 351 const struct mlxsw_afk_block *block; 352 int block_index; 353 354 if (WARN_ON(!test_bit(element, key_info->elusage.usage))) 355 return NULL; 356 block_index = key_info->element_to_block[element]; 357 block = key_info->blocks[block_index]; 358 359 elinst = mlxsw_afk_block_elinst_get(block, element); 360 if (WARN_ON(!elinst)) 361 return NULL; 362 363 *p_block_index = block_index; 364 return elinst; 365 } 366 367 u16 368 mlxsw_afk_key_info_block_encoding_get(const struct mlxsw_afk_key_info *key_info, 369 int block_index) 370 { 371 return key_info->blocks[block_index]->encoding; 372 } 373 EXPORT_SYMBOL(mlxsw_afk_key_info_block_encoding_get); 374 375 unsigned int 376 mlxsw_afk_key_info_blocks_count_get(const struct mlxsw_afk_key_info *key_info) 377 { 378 return key_info->blocks_count; 379 } 380 EXPORT_SYMBOL(mlxsw_afk_key_info_blocks_count_get); 381 382 void mlxsw_afk_values_add_u32(struct mlxsw_afk_element_values *values, 383 enum mlxsw_afk_element element, 384 u32 key_value, u32 mask_value) 385 { 386 const struct mlxsw_afk_element_info *elinfo = 387 &mlxsw_afk_element_infos[element]; 388 const struct mlxsw_item *storage_item = &elinfo->item; 389 390 if (!mask_value) 391 return; 392 if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_U32)) 393 return; 394 __mlxsw_item_set32(values->storage.key, storage_item, 0, key_value); 395 __mlxsw_item_set32(values->storage.mask, storage_item, 0, mask_value); 396 mlxsw_afk_element_usage_add(&values->elusage, element); 397 } 398 EXPORT_SYMBOL(mlxsw_afk_values_add_u32); 399 400 void mlxsw_afk_values_add_buf(struct mlxsw_afk_element_values *values, 401 enum mlxsw_afk_element element, 402 const char *key_value, const char *mask_value, 403 unsigned int len) 404 { 405 const struct mlxsw_afk_element_info *elinfo = 406 &mlxsw_afk_element_infos[element]; 407 const struct mlxsw_item *storage_item = &elinfo->item; 408 409 if (!memchr_inv(mask_value, 0, len)) /* If mask is zero */ 410 return; 411 if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_BUF) || 412 WARN_ON(elinfo->item.size.bytes != len)) 413 return; 414 __mlxsw_item_memcpy_to(values->storage.key, key_value, 415 storage_item, 0); 416 __mlxsw_item_memcpy_to(values->storage.mask, mask_value, 417 storage_item, 0); 418 mlxsw_afk_element_usage_add(&values->elusage, element); 419 } 420 EXPORT_SYMBOL(mlxsw_afk_values_add_buf); 421 422 static void mlxsw_sp_afk_encode_u32(const struct mlxsw_item *storage_item, 423 const struct mlxsw_item *output_item, 424 char *storage, char *output, int diff) 425 { 426 u32 value; 427 428 value = __mlxsw_item_get32(storage, storage_item, 0); 429 __mlxsw_item_set32(output, output_item, 0, value + diff); 430 } 431 432 static void mlxsw_sp_afk_encode_buf(const struct mlxsw_item *storage_item, 433 const struct mlxsw_item *output_item, 434 char *storage, char *output) 435 { 436 char *storage_data = __mlxsw_item_data(storage, storage_item, 0); 437 char *output_data = __mlxsw_item_data(output, output_item, 0); 438 size_t len = output_item->size.bytes; 439 440 memcpy(output_data, storage_data, len); 441 } 442 443 static void 444 mlxsw_sp_afk_encode_one(const struct mlxsw_afk_element_inst *elinst, 445 char *output, char *storage, int u32_diff) 446 { 447 const struct mlxsw_item *output_item = &elinst->item; 448 const struct mlxsw_afk_element_info *elinfo; 449 const struct mlxsw_item *storage_item; 450 451 elinfo = &mlxsw_afk_element_infos[elinst->element]; 452 storage_item = &elinfo->item; 453 if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_U32) 454 mlxsw_sp_afk_encode_u32(storage_item, output_item, 455 storage, output, u32_diff); 456 else if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_BUF) 457 mlxsw_sp_afk_encode_buf(storage_item, output_item, 458 storage, output); 459 } 460 461 #define MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE 16 462 463 void mlxsw_afk_encode(struct mlxsw_afk *mlxsw_afk, 464 struct mlxsw_afk_key_info *key_info, 465 struct mlxsw_afk_element_values *values, 466 char *key, char *mask) 467 { 468 unsigned int blocks_count = 469 mlxsw_afk_key_info_blocks_count_get(key_info); 470 char block_mask[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE]; 471 char block_key[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE]; 472 const struct mlxsw_afk_element_inst *elinst; 473 enum mlxsw_afk_element element; 474 int block_index, i; 475 476 for (i = 0; i < blocks_count; i++) { 477 memset(block_key, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE); 478 memset(block_mask, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE); 479 480 mlxsw_afk_element_usage_for_each(element, &values->elusage) { 481 elinst = mlxsw_afk_key_info_elinst_get(key_info, 482 element, 483 &block_index); 484 if (!elinst || block_index != i) 485 continue; 486 487 mlxsw_sp_afk_encode_one(elinst, block_key, 488 values->storage.key, 489 elinst->u32_key_diff); 490 mlxsw_sp_afk_encode_one(elinst, block_mask, 491 values->storage.mask, 0); 492 } 493 494 mlxsw_afk->ops->encode_block(key, i, block_key); 495 mlxsw_afk->ops->encode_block(mask, i, block_mask); 496 } 497 } 498 EXPORT_SYMBOL(mlxsw_afk_encode); 499 500 void mlxsw_afk_clear(struct mlxsw_afk *mlxsw_afk, char *key, 501 int block_start, int block_end) 502 { 503 int i; 504 505 for (i = block_start; i <= block_end; i++) 506 mlxsw_afk->ops->clear_block(key, i); 507 } 508 EXPORT_SYMBOL(mlxsw_afk_clear); 509