1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2017-2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/kernel.h> 5 #include <linux/slab.h> 6 #include <linux/list.h> 7 #include <linux/errno.h> 8 9 #include "item.h" 10 #include "core_acl_flex_keys.h" 11 12 /* For the purpose of the driver, define an internal storage scratchpad 13 * that will be used to store key/mask values. For each defined element type 14 * define an internal storage geometry. 15 * 16 * When adding new elements, MLXSW_AFK_ELEMENT_STORAGE_SIZE must be increased 17 * accordingly. 18 */ 19 static const struct mlxsw_afk_element_info mlxsw_afk_element_infos[] = { 20 MLXSW_AFK_ELEMENT_INFO_U32(SRC_SYS_PORT, 0x00, 16, 16), 21 MLXSW_AFK_ELEMENT_INFO_BUF(DMAC_32_47, 0x04, 2), 22 MLXSW_AFK_ELEMENT_INFO_BUF(DMAC_0_31, 0x06, 4), 23 MLXSW_AFK_ELEMENT_INFO_BUF(SMAC_32_47, 0x0A, 2), 24 MLXSW_AFK_ELEMENT_INFO_BUF(SMAC_0_31, 0x0C, 4), 25 MLXSW_AFK_ELEMENT_INFO_U32(ETHERTYPE, 0x00, 0, 16), 26 MLXSW_AFK_ELEMENT_INFO_U32(IP_PROTO, 0x10, 0, 8), 27 MLXSW_AFK_ELEMENT_INFO_U32(VID, 0x10, 8, 12), 28 MLXSW_AFK_ELEMENT_INFO_U32(PCP, 0x10, 20, 3), 29 MLXSW_AFK_ELEMENT_INFO_U32(TCP_FLAGS, 0x10, 23, 9), 30 MLXSW_AFK_ELEMENT_INFO_U32(DST_L4_PORT, 0x14, 0, 16), 31 MLXSW_AFK_ELEMENT_INFO_U32(SRC_L4_PORT, 0x14, 16, 16), 32 MLXSW_AFK_ELEMENT_INFO_U32(IP_TTL_, 0x18, 0, 8), 33 MLXSW_AFK_ELEMENT_INFO_U32(IP_ECN, 0x18, 9, 2), 34 MLXSW_AFK_ELEMENT_INFO_U32(IP_DSCP, 0x18, 11, 6), 35 MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_MSB, 0x18, 17, 3), 36 MLXSW_AFK_ELEMENT_INFO_U32(VIRT_ROUTER_LSB, 0x18, 20, 8), 37 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_96_127, 0x20, 4), 38 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_64_95, 0x24, 4), 39 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_32_63, 0x28, 4), 40 MLXSW_AFK_ELEMENT_INFO_BUF(SRC_IP_0_31, 0x2C, 4), 41 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_96_127, 0x30, 4), 42 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_64_95, 0x34, 4), 43 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_32_63, 0x38, 4), 44 MLXSW_AFK_ELEMENT_INFO_BUF(DST_IP_0_31, 0x3C, 4), 45 }; 46 47 struct mlxsw_afk { 48 struct list_head key_info_list; 49 unsigned int max_blocks; 50 const struct mlxsw_afk_ops *ops; 51 const struct mlxsw_afk_block *blocks; 52 unsigned int blocks_count; 53 }; 54 55 static bool mlxsw_afk_blocks_check(struct mlxsw_afk *mlxsw_afk) 56 { 57 int i; 58 int j; 59 60 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 61 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i]; 62 63 for (j = 0; j < block->instances_count; j++) { 64 const struct mlxsw_afk_element_info *elinfo; 65 struct mlxsw_afk_element_inst *elinst; 66 67 elinst = &block->instances[j]; 68 elinfo = &mlxsw_afk_element_infos[elinst->element]; 69 if (elinst->type != elinfo->type || 70 (!elinst->avoid_size_check && 71 elinst->item.size.bits != 72 elinfo->item.size.bits)) 73 return false; 74 } 75 } 76 return true; 77 } 78 79 struct mlxsw_afk *mlxsw_afk_create(unsigned int max_blocks, 80 const struct mlxsw_afk_ops *ops) 81 { 82 struct mlxsw_afk *mlxsw_afk; 83 84 mlxsw_afk = kzalloc(sizeof(*mlxsw_afk), GFP_KERNEL); 85 if (!mlxsw_afk) 86 return NULL; 87 INIT_LIST_HEAD(&mlxsw_afk->key_info_list); 88 mlxsw_afk->max_blocks = max_blocks; 89 mlxsw_afk->ops = ops; 90 mlxsw_afk->blocks = ops->blocks; 91 mlxsw_afk->blocks_count = ops->blocks_count; 92 WARN_ON(!mlxsw_afk_blocks_check(mlxsw_afk)); 93 return mlxsw_afk; 94 } 95 EXPORT_SYMBOL(mlxsw_afk_create); 96 97 void mlxsw_afk_destroy(struct mlxsw_afk *mlxsw_afk) 98 { 99 WARN_ON(!list_empty(&mlxsw_afk->key_info_list)); 100 kfree(mlxsw_afk); 101 } 102 EXPORT_SYMBOL(mlxsw_afk_destroy); 103 104 struct mlxsw_afk_key_info { 105 struct list_head list; 106 unsigned int ref_count; 107 unsigned int blocks_count; 108 int element_to_block[MLXSW_AFK_ELEMENT_MAX]; /* index is element, value 109 * is index inside "blocks" 110 */ 111 struct mlxsw_afk_element_usage elusage; 112 const struct mlxsw_afk_block *blocks[]; 113 }; 114 115 static bool 116 mlxsw_afk_key_info_elements_eq(struct mlxsw_afk_key_info *key_info, 117 struct mlxsw_afk_element_usage *elusage) 118 { 119 return memcmp(&key_info->elusage, elusage, sizeof(*elusage)) == 0; 120 } 121 122 static struct mlxsw_afk_key_info * 123 mlxsw_afk_key_info_find(struct mlxsw_afk *mlxsw_afk, 124 struct mlxsw_afk_element_usage *elusage) 125 { 126 struct mlxsw_afk_key_info *key_info; 127 128 list_for_each_entry(key_info, &mlxsw_afk->key_info_list, list) { 129 if (mlxsw_afk_key_info_elements_eq(key_info, elusage)) 130 return key_info; 131 } 132 return NULL; 133 } 134 135 struct mlxsw_afk_picker { 136 DECLARE_BITMAP(element, MLXSW_AFK_ELEMENT_MAX); 137 unsigned int total; 138 }; 139 140 static void mlxsw_afk_picker_count_hits(struct mlxsw_afk *mlxsw_afk, 141 struct mlxsw_afk_picker *picker, 142 enum mlxsw_afk_element element) 143 { 144 int i; 145 int j; 146 147 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 148 const struct mlxsw_afk_block *block = &mlxsw_afk->blocks[i]; 149 150 for (j = 0; j < block->instances_count; j++) { 151 struct mlxsw_afk_element_inst *elinst; 152 153 elinst = &block->instances[j]; 154 if (elinst->element == element) { 155 __set_bit(element, picker[i].element); 156 picker[i].total++; 157 } 158 } 159 } 160 } 161 162 static void mlxsw_afk_picker_subtract_hits(struct mlxsw_afk *mlxsw_afk, 163 struct mlxsw_afk_picker *picker, 164 int block_index) 165 { 166 DECLARE_BITMAP(hits_element, MLXSW_AFK_ELEMENT_MAX); 167 int i; 168 int j; 169 170 memcpy(&hits_element, &picker[block_index].element, 171 sizeof(hits_element)); 172 173 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 174 for_each_set_bit(j, hits_element, MLXSW_AFK_ELEMENT_MAX) { 175 if (__test_and_clear_bit(j, picker[i].element)) 176 picker[i].total--; 177 } 178 } 179 } 180 181 static int mlxsw_afk_picker_most_hits_get(struct mlxsw_afk *mlxsw_afk, 182 struct mlxsw_afk_picker *picker) 183 { 184 int most_index = -EINVAL; /* Should never happen to return this */ 185 int most_hits = 0; 186 int i; 187 188 for (i = 0; i < mlxsw_afk->blocks_count; i++) { 189 if (picker[i].total > most_hits) { 190 most_hits = picker[i].total; 191 most_index = i; 192 } 193 } 194 return most_index; 195 } 196 197 static int mlxsw_afk_picker_key_info_add(struct mlxsw_afk *mlxsw_afk, 198 struct mlxsw_afk_picker *picker, 199 int block_index, 200 struct mlxsw_afk_key_info *key_info) 201 { 202 enum mlxsw_afk_element element; 203 204 if (key_info->blocks_count == mlxsw_afk->max_blocks) 205 return -EINVAL; 206 207 for_each_set_bit(element, picker[block_index].element, 208 MLXSW_AFK_ELEMENT_MAX) { 209 key_info->element_to_block[element] = key_info->blocks_count; 210 mlxsw_afk_element_usage_add(&key_info->elusage, element); 211 } 212 213 key_info->blocks[key_info->blocks_count] = 214 &mlxsw_afk->blocks[block_index]; 215 key_info->blocks_count++; 216 return 0; 217 } 218 219 static int mlxsw_afk_picker(struct mlxsw_afk *mlxsw_afk, 220 struct mlxsw_afk_key_info *key_info, 221 struct mlxsw_afk_element_usage *elusage) 222 { 223 struct mlxsw_afk_picker *picker; 224 enum mlxsw_afk_element element; 225 int err; 226 227 picker = kcalloc(mlxsw_afk->blocks_count, sizeof(*picker), GFP_KERNEL); 228 if (!picker) 229 return -ENOMEM; 230 231 /* Since the same elements could be present in multiple blocks, 232 * we must find out optimal block list in order to make the 233 * block count as low as possible. 234 * 235 * First, we count hits. We go over all available blocks and count 236 * how many of requested elements are covered by each. 237 * 238 * Then in loop, we find block with most hits and add it to 239 * output key_info. Then we have to subtract this block hits so 240 * the next iteration will find most suitable block for 241 * the rest of requested elements. 242 */ 243 244 mlxsw_afk_element_usage_for_each(element, elusage) 245 mlxsw_afk_picker_count_hits(mlxsw_afk, picker, element); 246 247 do { 248 int block_index; 249 250 block_index = mlxsw_afk_picker_most_hits_get(mlxsw_afk, picker); 251 if (block_index < 0) { 252 err = block_index; 253 goto out; 254 } 255 err = mlxsw_afk_picker_key_info_add(mlxsw_afk, picker, 256 block_index, key_info); 257 if (err) 258 goto out; 259 mlxsw_afk_picker_subtract_hits(mlxsw_afk, picker, block_index); 260 } while (!mlxsw_afk_key_info_elements_eq(key_info, elusage)); 261 262 err = 0; 263 out: 264 kfree(picker); 265 return err; 266 } 267 268 static struct mlxsw_afk_key_info * 269 mlxsw_afk_key_info_create(struct mlxsw_afk *mlxsw_afk, 270 struct mlxsw_afk_element_usage *elusage) 271 { 272 struct mlxsw_afk_key_info *key_info; 273 int err; 274 275 key_info = kzalloc(struct_size(key_info, blocks, mlxsw_afk->max_blocks), 276 GFP_KERNEL); 277 if (!key_info) 278 return ERR_PTR(-ENOMEM); 279 err = mlxsw_afk_picker(mlxsw_afk, key_info, elusage); 280 if (err) 281 goto err_picker; 282 list_add(&key_info->list, &mlxsw_afk->key_info_list); 283 key_info->ref_count = 1; 284 return key_info; 285 286 err_picker: 287 kfree(key_info); 288 return ERR_PTR(err); 289 } 290 291 static void mlxsw_afk_key_info_destroy(struct mlxsw_afk_key_info *key_info) 292 { 293 list_del(&key_info->list); 294 kfree(key_info); 295 } 296 297 struct mlxsw_afk_key_info * 298 mlxsw_afk_key_info_get(struct mlxsw_afk *mlxsw_afk, 299 struct mlxsw_afk_element_usage *elusage) 300 { 301 struct mlxsw_afk_key_info *key_info; 302 303 key_info = mlxsw_afk_key_info_find(mlxsw_afk, elusage); 304 if (key_info) { 305 key_info->ref_count++; 306 return key_info; 307 } 308 return mlxsw_afk_key_info_create(mlxsw_afk, elusage); 309 } 310 EXPORT_SYMBOL(mlxsw_afk_key_info_get); 311 312 void mlxsw_afk_key_info_put(struct mlxsw_afk_key_info *key_info) 313 { 314 if (--key_info->ref_count) 315 return; 316 mlxsw_afk_key_info_destroy(key_info); 317 } 318 EXPORT_SYMBOL(mlxsw_afk_key_info_put); 319 320 bool mlxsw_afk_key_info_subset(struct mlxsw_afk_key_info *key_info, 321 struct mlxsw_afk_element_usage *elusage) 322 { 323 return mlxsw_afk_element_usage_subset(elusage, &key_info->elusage); 324 } 325 EXPORT_SYMBOL(mlxsw_afk_key_info_subset); 326 327 static const struct mlxsw_afk_element_inst * 328 mlxsw_afk_block_elinst_get(const struct mlxsw_afk_block *block, 329 enum mlxsw_afk_element element) 330 { 331 int i; 332 333 for (i = 0; i < block->instances_count; i++) { 334 struct mlxsw_afk_element_inst *elinst; 335 336 elinst = &block->instances[i]; 337 if (elinst->element == element) 338 return elinst; 339 } 340 return NULL; 341 } 342 343 static const struct mlxsw_afk_element_inst * 344 mlxsw_afk_key_info_elinst_get(struct mlxsw_afk_key_info *key_info, 345 enum mlxsw_afk_element element, 346 int *p_block_index) 347 { 348 const struct mlxsw_afk_element_inst *elinst; 349 const struct mlxsw_afk_block *block; 350 int block_index; 351 352 if (WARN_ON(!test_bit(element, key_info->elusage.usage))) 353 return NULL; 354 block_index = key_info->element_to_block[element]; 355 block = key_info->blocks[block_index]; 356 357 elinst = mlxsw_afk_block_elinst_get(block, element); 358 if (WARN_ON(!elinst)) 359 return NULL; 360 361 *p_block_index = block_index; 362 return elinst; 363 } 364 365 u16 366 mlxsw_afk_key_info_block_encoding_get(const struct mlxsw_afk_key_info *key_info, 367 int block_index) 368 { 369 return key_info->blocks[block_index]->encoding; 370 } 371 EXPORT_SYMBOL(mlxsw_afk_key_info_block_encoding_get); 372 373 unsigned int 374 mlxsw_afk_key_info_blocks_count_get(const struct mlxsw_afk_key_info *key_info) 375 { 376 return key_info->blocks_count; 377 } 378 EXPORT_SYMBOL(mlxsw_afk_key_info_blocks_count_get); 379 380 void mlxsw_afk_values_add_u32(struct mlxsw_afk_element_values *values, 381 enum mlxsw_afk_element element, 382 u32 key_value, u32 mask_value) 383 { 384 const struct mlxsw_afk_element_info *elinfo = 385 &mlxsw_afk_element_infos[element]; 386 const struct mlxsw_item *storage_item = &elinfo->item; 387 388 if (!mask_value) 389 return; 390 if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_U32)) 391 return; 392 __mlxsw_item_set32(values->storage.key, storage_item, 0, key_value); 393 __mlxsw_item_set32(values->storage.mask, storage_item, 0, mask_value); 394 mlxsw_afk_element_usage_add(&values->elusage, element); 395 } 396 EXPORT_SYMBOL(mlxsw_afk_values_add_u32); 397 398 void mlxsw_afk_values_add_buf(struct mlxsw_afk_element_values *values, 399 enum mlxsw_afk_element element, 400 const char *key_value, const char *mask_value, 401 unsigned int len) 402 { 403 const struct mlxsw_afk_element_info *elinfo = 404 &mlxsw_afk_element_infos[element]; 405 const struct mlxsw_item *storage_item = &elinfo->item; 406 407 if (!memchr_inv(mask_value, 0, len)) /* If mask is zero */ 408 return; 409 if (WARN_ON(elinfo->type != MLXSW_AFK_ELEMENT_TYPE_BUF) || 410 WARN_ON(elinfo->item.size.bytes != len)) 411 return; 412 __mlxsw_item_memcpy_to(values->storage.key, key_value, 413 storage_item, 0); 414 __mlxsw_item_memcpy_to(values->storage.mask, mask_value, 415 storage_item, 0); 416 mlxsw_afk_element_usage_add(&values->elusage, element); 417 } 418 EXPORT_SYMBOL(mlxsw_afk_values_add_buf); 419 420 static void mlxsw_sp_afk_encode_u32(const struct mlxsw_item *storage_item, 421 const struct mlxsw_item *output_item, 422 char *storage, char *output, int diff) 423 { 424 u32 value; 425 426 value = __mlxsw_item_get32(storage, storage_item, 0); 427 __mlxsw_item_set32(output, output_item, 0, value + diff); 428 } 429 430 static void mlxsw_sp_afk_encode_buf(const struct mlxsw_item *storage_item, 431 const struct mlxsw_item *output_item, 432 char *storage, char *output) 433 { 434 char *storage_data = __mlxsw_item_data(storage, storage_item, 0); 435 char *output_data = __mlxsw_item_data(output, output_item, 0); 436 size_t len = output_item->size.bytes; 437 438 memcpy(output_data, storage_data, len); 439 } 440 441 static void 442 mlxsw_sp_afk_encode_one(const struct mlxsw_afk_element_inst *elinst, 443 char *output, char *storage, int u32_diff) 444 { 445 const struct mlxsw_item *output_item = &elinst->item; 446 const struct mlxsw_afk_element_info *elinfo; 447 const struct mlxsw_item *storage_item; 448 449 elinfo = &mlxsw_afk_element_infos[elinst->element]; 450 storage_item = &elinfo->item; 451 if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_U32) 452 mlxsw_sp_afk_encode_u32(storage_item, output_item, 453 storage, output, u32_diff); 454 else if (elinst->type == MLXSW_AFK_ELEMENT_TYPE_BUF) 455 mlxsw_sp_afk_encode_buf(storage_item, output_item, 456 storage, output); 457 } 458 459 #define MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE 16 460 461 void mlxsw_afk_encode(struct mlxsw_afk *mlxsw_afk, 462 struct mlxsw_afk_key_info *key_info, 463 struct mlxsw_afk_element_values *values, 464 char *key, char *mask) 465 { 466 unsigned int blocks_count = 467 mlxsw_afk_key_info_blocks_count_get(key_info); 468 char block_mask[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE]; 469 char block_key[MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE]; 470 const struct mlxsw_afk_element_inst *elinst; 471 enum mlxsw_afk_element element; 472 int block_index, i; 473 474 for (i = 0; i < blocks_count; i++) { 475 memset(block_key, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE); 476 memset(block_mask, 0, MLXSW_SP_AFK_KEY_BLOCK_MAX_SIZE); 477 478 mlxsw_afk_element_usage_for_each(element, &values->elusage) { 479 elinst = mlxsw_afk_key_info_elinst_get(key_info, 480 element, 481 &block_index); 482 if (!elinst || block_index != i) 483 continue; 484 485 mlxsw_sp_afk_encode_one(elinst, block_key, 486 values->storage.key, 487 elinst->u32_key_diff); 488 mlxsw_sp_afk_encode_one(elinst, block_mask, 489 values->storage.mask, 0); 490 } 491 492 mlxsw_afk->ops->encode_block(key, i, block_key); 493 mlxsw_afk->ops->encode_block(mask, i, block_mask); 494 } 495 } 496 EXPORT_SYMBOL(mlxsw_afk_encode); 497 498 void mlxsw_afk_clear(struct mlxsw_afk *mlxsw_afk, char *key, 499 int block_start, int block_end) 500 { 501 int i; 502 503 for (i = block_start; i <= block_end; i++) 504 mlxsw_afk->ops->clear_block(key, i); 505 } 506 EXPORT_SYMBOL(mlxsw_afk_clear); 507