1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/err.h> 5 #include <linux/gfp.h> 6 #include <linux/kernel.h> 7 #include <linux/list.h> 8 #include <linux/netlink.h> 9 #include <linux/rtnetlink.h> 10 #include <linux/slab.h> 11 #include <net/inet_ecn.h> 12 #include <net/ipv6.h> 13 14 #include "reg.h" 15 #include "spectrum.h" 16 #include "spectrum_nve.h" 17 18 const struct mlxsw_sp_nve_ops *mlxsw_sp1_nve_ops_arr[] = { 19 [MLXSW_SP_NVE_TYPE_VXLAN] = &mlxsw_sp1_nve_vxlan_ops, 20 }; 21 22 const struct mlxsw_sp_nve_ops *mlxsw_sp2_nve_ops_arr[] = { 23 [MLXSW_SP_NVE_TYPE_VXLAN] = &mlxsw_sp2_nve_vxlan_ops, 24 }; 25 26 struct mlxsw_sp_nve_mc_entry; 27 struct mlxsw_sp_nve_mc_record; 28 struct mlxsw_sp_nve_mc_list; 29 30 struct mlxsw_sp_nve_mc_record_ops { 31 enum mlxsw_reg_tnumt_record_type type; 32 int (*entry_add)(struct mlxsw_sp_nve_mc_record *mc_record, 33 struct mlxsw_sp_nve_mc_entry *mc_entry, 34 const union mlxsw_sp_l3addr *addr); 35 void (*entry_del)(const struct mlxsw_sp_nve_mc_record *mc_record, 36 const struct mlxsw_sp_nve_mc_entry *mc_entry); 37 void (*entry_set)(const struct mlxsw_sp_nve_mc_record *mc_record, 38 const struct mlxsw_sp_nve_mc_entry *mc_entry, 39 char *tnumt_pl, unsigned int entry_index); 40 bool (*entry_compare)(const struct mlxsw_sp_nve_mc_record *mc_record, 41 const struct mlxsw_sp_nve_mc_entry *mc_entry, 42 const union mlxsw_sp_l3addr *addr); 43 }; 44 45 struct mlxsw_sp_nve_mc_list_key { 46 u16 fid_index; 47 }; 48 49 struct mlxsw_sp_nve_mc_ipv6_entry { 50 struct in6_addr addr6; 51 u32 addr6_kvdl_index; 52 }; 53 54 struct mlxsw_sp_nve_mc_entry { 55 union { 56 __be32 addr4; 57 struct mlxsw_sp_nve_mc_ipv6_entry ipv6_entry; 58 }; 59 u8 valid:1; 60 }; 61 62 struct mlxsw_sp_nve_mc_record { 63 struct list_head list; 64 enum mlxsw_sp_l3proto proto; 65 unsigned int num_entries; 66 struct mlxsw_sp *mlxsw_sp; 67 struct mlxsw_sp_nve_mc_list *mc_list; 68 const struct mlxsw_sp_nve_mc_record_ops *ops; 69 u32 kvdl_index; 70 struct mlxsw_sp_nve_mc_entry entries[0]; 71 }; 72 73 struct mlxsw_sp_nve_mc_list { 74 struct list_head records_list; 75 struct rhash_head ht_node; 76 struct mlxsw_sp_nve_mc_list_key key; 77 }; 78 79 static const struct rhashtable_params mlxsw_sp_nve_mc_list_ht_params = { 80 .key_len = sizeof(struct mlxsw_sp_nve_mc_list_key), 81 .key_offset = offsetof(struct mlxsw_sp_nve_mc_list, key), 82 .head_offset = offsetof(struct mlxsw_sp_nve_mc_list, ht_node), 83 }; 84 85 static int 86 mlxsw_sp_nve_mc_record_ipv4_entry_add(struct mlxsw_sp_nve_mc_record *mc_record, 87 struct mlxsw_sp_nve_mc_entry *mc_entry, 88 const union mlxsw_sp_l3addr *addr) 89 { 90 mc_entry->addr4 = addr->addr4; 91 92 return 0; 93 } 94 95 static void 96 mlxsw_sp_nve_mc_record_ipv4_entry_del(const struct mlxsw_sp_nve_mc_record *mc_record, 97 const struct mlxsw_sp_nve_mc_entry *mc_entry) 98 { 99 } 100 101 static void 102 mlxsw_sp_nve_mc_record_ipv4_entry_set(const struct mlxsw_sp_nve_mc_record *mc_record, 103 const struct mlxsw_sp_nve_mc_entry *mc_entry, 104 char *tnumt_pl, unsigned int entry_index) 105 { 106 u32 udip = be32_to_cpu(mc_entry->addr4); 107 108 mlxsw_reg_tnumt_udip_set(tnumt_pl, entry_index, udip); 109 } 110 111 static bool 112 mlxsw_sp_nve_mc_record_ipv4_entry_compare(const struct mlxsw_sp_nve_mc_record *mc_record, 113 const struct mlxsw_sp_nve_mc_entry *mc_entry, 114 const union mlxsw_sp_l3addr *addr) 115 { 116 return mc_entry->addr4 == addr->addr4; 117 } 118 119 static const struct mlxsw_sp_nve_mc_record_ops 120 mlxsw_sp_nve_mc_record_ipv4_ops = { 121 .type = MLXSW_REG_TNUMT_RECORD_TYPE_IPV4, 122 .entry_add = &mlxsw_sp_nve_mc_record_ipv4_entry_add, 123 .entry_del = &mlxsw_sp_nve_mc_record_ipv4_entry_del, 124 .entry_set = &mlxsw_sp_nve_mc_record_ipv4_entry_set, 125 .entry_compare = &mlxsw_sp_nve_mc_record_ipv4_entry_compare, 126 }; 127 128 static int 129 mlxsw_sp_nve_mc_record_ipv6_entry_add(struct mlxsw_sp_nve_mc_record *mc_record, 130 struct mlxsw_sp_nve_mc_entry *mc_entry, 131 const union mlxsw_sp_l3addr *addr) 132 { 133 WARN_ON(1); 134 135 return -EINVAL; 136 } 137 138 static void 139 mlxsw_sp_nve_mc_record_ipv6_entry_del(const struct mlxsw_sp_nve_mc_record *mc_record, 140 const struct mlxsw_sp_nve_mc_entry *mc_entry) 141 { 142 } 143 144 static void 145 mlxsw_sp_nve_mc_record_ipv6_entry_set(const struct mlxsw_sp_nve_mc_record *mc_record, 146 const struct mlxsw_sp_nve_mc_entry *mc_entry, 147 char *tnumt_pl, unsigned int entry_index) 148 { 149 u32 udip_ptr = mc_entry->ipv6_entry.addr6_kvdl_index; 150 151 mlxsw_reg_tnumt_udip_ptr_set(tnumt_pl, entry_index, udip_ptr); 152 } 153 154 static bool 155 mlxsw_sp_nve_mc_record_ipv6_entry_compare(const struct mlxsw_sp_nve_mc_record *mc_record, 156 const struct mlxsw_sp_nve_mc_entry *mc_entry, 157 const union mlxsw_sp_l3addr *addr) 158 { 159 return ipv6_addr_equal(&mc_entry->ipv6_entry.addr6, &addr->addr6); 160 } 161 162 static const struct mlxsw_sp_nve_mc_record_ops 163 mlxsw_sp_nve_mc_record_ipv6_ops = { 164 .type = MLXSW_REG_TNUMT_RECORD_TYPE_IPV6, 165 .entry_add = &mlxsw_sp_nve_mc_record_ipv6_entry_add, 166 .entry_del = &mlxsw_sp_nve_mc_record_ipv6_entry_del, 167 .entry_set = &mlxsw_sp_nve_mc_record_ipv6_entry_set, 168 .entry_compare = &mlxsw_sp_nve_mc_record_ipv6_entry_compare, 169 }; 170 171 static const struct mlxsw_sp_nve_mc_record_ops * 172 mlxsw_sp_nve_mc_record_ops_arr[] = { 173 [MLXSW_SP_L3_PROTO_IPV4] = &mlxsw_sp_nve_mc_record_ipv4_ops, 174 [MLXSW_SP_L3_PROTO_IPV6] = &mlxsw_sp_nve_mc_record_ipv6_ops, 175 }; 176 177 static struct mlxsw_sp_nve_mc_list * 178 mlxsw_sp_nve_mc_list_find(struct mlxsw_sp *mlxsw_sp, 179 const struct mlxsw_sp_nve_mc_list_key *key) 180 { 181 struct mlxsw_sp_nve *nve = mlxsw_sp->nve; 182 183 return rhashtable_lookup_fast(&nve->mc_list_ht, key, 184 mlxsw_sp_nve_mc_list_ht_params); 185 } 186 187 static struct mlxsw_sp_nve_mc_list * 188 mlxsw_sp_nve_mc_list_create(struct mlxsw_sp *mlxsw_sp, 189 const struct mlxsw_sp_nve_mc_list_key *key) 190 { 191 struct mlxsw_sp_nve *nve = mlxsw_sp->nve; 192 struct mlxsw_sp_nve_mc_list *mc_list; 193 int err; 194 195 mc_list = kmalloc(sizeof(*mc_list), GFP_KERNEL); 196 if (!mc_list) 197 return ERR_PTR(-ENOMEM); 198 199 INIT_LIST_HEAD(&mc_list->records_list); 200 mc_list->key = *key; 201 202 err = rhashtable_insert_fast(&nve->mc_list_ht, &mc_list->ht_node, 203 mlxsw_sp_nve_mc_list_ht_params); 204 if (err) 205 goto err_rhashtable_insert; 206 207 return mc_list; 208 209 err_rhashtable_insert: 210 kfree(mc_list); 211 return ERR_PTR(err); 212 } 213 214 static void mlxsw_sp_nve_mc_list_destroy(struct mlxsw_sp *mlxsw_sp, 215 struct mlxsw_sp_nve_mc_list *mc_list) 216 { 217 struct mlxsw_sp_nve *nve = mlxsw_sp->nve; 218 219 rhashtable_remove_fast(&nve->mc_list_ht, &mc_list->ht_node, 220 mlxsw_sp_nve_mc_list_ht_params); 221 WARN_ON(!list_empty(&mc_list->records_list)); 222 kfree(mc_list); 223 } 224 225 static struct mlxsw_sp_nve_mc_list * 226 mlxsw_sp_nve_mc_list_get(struct mlxsw_sp *mlxsw_sp, 227 const struct mlxsw_sp_nve_mc_list_key *key) 228 { 229 struct mlxsw_sp_nve_mc_list *mc_list; 230 231 mc_list = mlxsw_sp_nve_mc_list_find(mlxsw_sp, key); 232 if (mc_list) 233 return mc_list; 234 235 return mlxsw_sp_nve_mc_list_create(mlxsw_sp, key); 236 } 237 238 static void 239 mlxsw_sp_nve_mc_list_put(struct mlxsw_sp *mlxsw_sp, 240 struct mlxsw_sp_nve_mc_list *mc_list) 241 { 242 if (!list_empty(&mc_list->records_list)) 243 return; 244 mlxsw_sp_nve_mc_list_destroy(mlxsw_sp, mc_list); 245 } 246 247 static struct mlxsw_sp_nve_mc_record * 248 mlxsw_sp_nve_mc_record_create(struct mlxsw_sp *mlxsw_sp, 249 struct mlxsw_sp_nve_mc_list *mc_list, 250 enum mlxsw_sp_l3proto proto) 251 { 252 unsigned int num_max_entries = mlxsw_sp->nve->num_max_mc_entries[proto]; 253 struct mlxsw_sp_nve_mc_record *mc_record; 254 int err; 255 256 mc_record = kzalloc(sizeof(*mc_record) + num_max_entries * 257 sizeof(struct mlxsw_sp_nve_mc_entry), GFP_KERNEL); 258 if (!mc_record) 259 return ERR_PTR(-ENOMEM); 260 261 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_TNUMT, 1, 262 &mc_record->kvdl_index); 263 if (err) 264 goto err_kvdl_alloc; 265 266 mc_record->ops = mlxsw_sp_nve_mc_record_ops_arr[proto]; 267 mc_record->mlxsw_sp = mlxsw_sp; 268 mc_record->mc_list = mc_list; 269 mc_record->proto = proto; 270 list_add_tail(&mc_record->list, &mc_list->records_list); 271 272 return mc_record; 273 274 err_kvdl_alloc: 275 kfree(mc_record); 276 return ERR_PTR(err); 277 } 278 279 static void 280 mlxsw_sp_nve_mc_record_destroy(struct mlxsw_sp_nve_mc_record *mc_record) 281 { 282 struct mlxsw_sp *mlxsw_sp = mc_record->mlxsw_sp; 283 284 list_del(&mc_record->list); 285 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_TNUMT, 1, 286 mc_record->kvdl_index); 287 WARN_ON(mc_record->num_entries); 288 kfree(mc_record); 289 } 290 291 static struct mlxsw_sp_nve_mc_record * 292 mlxsw_sp_nve_mc_record_get(struct mlxsw_sp *mlxsw_sp, 293 struct mlxsw_sp_nve_mc_list *mc_list, 294 enum mlxsw_sp_l3proto proto) 295 { 296 struct mlxsw_sp_nve_mc_record *mc_record; 297 298 list_for_each_entry_reverse(mc_record, &mc_list->records_list, list) { 299 unsigned int num_entries = mc_record->num_entries; 300 struct mlxsw_sp_nve *nve = mlxsw_sp->nve; 301 302 if (mc_record->proto == proto && 303 num_entries < nve->num_max_mc_entries[proto]) 304 return mc_record; 305 } 306 307 return mlxsw_sp_nve_mc_record_create(mlxsw_sp, mc_list, proto); 308 } 309 310 static void 311 mlxsw_sp_nve_mc_record_put(struct mlxsw_sp_nve_mc_record *mc_record) 312 { 313 if (mc_record->num_entries != 0) 314 return; 315 316 mlxsw_sp_nve_mc_record_destroy(mc_record); 317 } 318 319 static struct mlxsw_sp_nve_mc_entry * 320 mlxsw_sp_nve_mc_free_entry_find(struct mlxsw_sp_nve_mc_record *mc_record) 321 { 322 struct mlxsw_sp_nve *nve = mc_record->mlxsw_sp->nve; 323 unsigned int num_max_entries; 324 int i; 325 326 num_max_entries = nve->num_max_mc_entries[mc_record->proto]; 327 for (i = 0; i < num_max_entries; i++) { 328 if (mc_record->entries[i].valid) 329 continue; 330 return &mc_record->entries[i]; 331 } 332 333 return NULL; 334 } 335 336 static int 337 mlxsw_sp_nve_mc_record_refresh(struct mlxsw_sp_nve_mc_record *mc_record) 338 { 339 enum mlxsw_reg_tnumt_record_type type = mc_record->ops->type; 340 struct mlxsw_sp_nve_mc_list *mc_list = mc_record->mc_list; 341 struct mlxsw_sp *mlxsw_sp = mc_record->mlxsw_sp; 342 char tnumt_pl[MLXSW_REG_TNUMT_LEN]; 343 unsigned int num_max_entries; 344 unsigned int num_entries = 0; 345 u32 next_kvdl_index = 0; 346 bool next_valid = false; 347 int i; 348 349 if (!list_is_last(&mc_record->list, &mc_list->records_list)) { 350 struct mlxsw_sp_nve_mc_record *next_record; 351 352 next_record = list_next_entry(mc_record, list); 353 next_kvdl_index = next_record->kvdl_index; 354 next_valid = true; 355 } 356 357 mlxsw_reg_tnumt_pack(tnumt_pl, type, MLXSW_REG_TNUMT_TUNNEL_PORT_NVE, 358 mc_record->kvdl_index, next_valid, 359 next_kvdl_index, mc_record->num_entries); 360 361 num_max_entries = mlxsw_sp->nve->num_max_mc_entries[mc_record->proto]; 362 for (i = 0; i < num_max_entries; i++) { 363 struct mlxsw_sp_nve_mc_entry *mc_entry; 364 365 mc_entry = &mc_record->entries[i]; 366 if (!mc_entry->valid) 367 continue; 368 mc_record->ops->entry_set(mc_record, mc_entry, tnumt_pl, 369 num_entries++); 370 } 371 372 WARN_ON(num_entries != mc_record->num_entries); 373 374 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tnumt), tnumt_pl); 375 } 376 377 static bool 378 mlxsw_sp_nve_mc_record_is_first(struct mlxsw_sp_nve_mc_record *mc_record) 379 { 380 struct mlxsw_sp_nve_mc_list *mc_list = mc_record->mc_list; 381 struct mlxsw_sp_nve_mc_record *first_record; 382 383 first_record = list_first_entry(&mc_list->records_list, 384 struct mlxsw_sp_nve_mc_record, list); 385 386 return mc_record == first_record; 387 } 388 389 static struct mlxsw_sp_nve_mc_entry * 390 mlxsw_sp_nve_mc_entry_find(struct mlxsw_sp_nve_mc_record *mc_record, 391 union mlxsw_sp_l3addr *addr) 392 { 393 struct mlxsw_sp_nve *nve = mc_record->mlxsw_sp->nve; 394 unsigned int num_max_entries; 395 int i; 396 397 num_max_entries = nve->num_max_mc_entries[mc_record->proto]; 398 for (i = 0; i < num_max_entries; i++) { 399 struct mlxsw_sp_nve_mc_entry *mc_entry; 400 401 mc_entry = &mc_record->entries[i]; 402 if (!mc_entry->valid) 403 continue; 404 if (mc_record->ops->entry_compare(mc_record, mc_entry, addr)) 405 return mc_entry; 406 } 407 408 return NULL; 409 } 410 411 static int 412 mlxsw_sp_nve_mc_record_ip_add(struct mlxsw_sp_nve_mc_record *mc_record, 413 union mlxsw_sp_l3addr *addr) 414 { 415 struct mlxsw_sp_nve_mc_entry *mc_entry = NULL; 416 int err; 417 418 mc_entry = mlxsw_sp_nve_mc_free_entry_find(mc_record); 419 if (WARN_ON(!mc_entry)) 420 return -EINVAL; 421 422 err = mc_record->ops->entry_add(mc_record, mc_entry, addr); 423 if (err) 424 return err; 425 mc_record->num_entries++; 426 mc_entry->valid = true; 427 428 err = mlxsw_sp_nve_mc_record_refresh(mc_record); 429 if (err) 430 goto err_record_refresh; 431 432 /* If this is a new record and not the first one, then we need to 433 * update the next pointer of the previous entry 434 */ 435 if (mc_record->num_entries != 1 || 436 mlxsw_sp_nve_mc_record_is_first(mc_record)) 437 return 0; 438 439 err = mlxsw_sp_nve_mc_record_refresh(list_prev_entry(mc_record, list)); 440 if (err) 441 goto err_prev_record_refresh; 442 443 return 0; 444 445 err_prev_record_refresh: 446 err_record_refresh: 447 mc_entry->valid = false; 448 mc_record->num_entries--; 449 mc_record->ops->entry_del(mc_record, mc_entry); 450 return err; 451 } 452 453 static void 454 mlxsw_sp_nve_mc_record_entry_del(struct mlxsw_sp_nve_mc_record *mc_record, 455 struct mlxsw_sp_nve_mc_entry *mc_entry) 456 { 457 struct mlxsw_sp_nve_mc_list *mc_list = mc_record->mc_list; 458 459 mc_entry->valid = false; 460 mc_record->num_entries--; 461 462 /* When the record continues to exist we only need to invalidate 463 * the requested entry 464 */ 465 if (mc_record->num_entries != 0) { 466 mlxsw_sp_nve_mc_record_refresh(mc_record); 467 mc_record->ops->entry_del(mc_record, mc_entry); 468 return; 469 } 470 471 /* If the record needs to be deleted, but it is not the first, 472 * then we need to make sure that the previous record no longer 473 * points to it. Remove deleted record from the list to reflect 474 * that and then re-add it at the end, so that it could be 475 * properly removed by the record destruction code 476 */ 477 if (!mlxsw_sp_nve_mc_record_is_first(mc_record)) { 478 struct mlxsw_sp_nve_mc_record *prev_record; 479 480 prev_record = list_prev_entry(mc_record, list); 481 list_del(&mc_record->list); 482 mlxsw_sp_nve_mc_record_refresh(prev_record); 483 list_add_tail(&mc_record->list, &mc_list->records_list); 484 mc_record->ops->entry_del(mc_record, mc_entry); 485 return; 486 } 487 488 /* If the first record needs to be deleted, but the list is not 489 * singular, then the second record needs to be written in the 490 * first record's address, as this address is stored as a property 491 * of the FID 492 */ 493 if (mlxsw_sp_nve_mc_record_is_first(mc_record) && 494 !list_is_singular(&mc_list->records_list)) { 495 struct mlxsw_sp_nve_mc_record *next_record; 496 497 next_record = list_next_entry(mc_record, list); 498 swap(mc_record->kvdl_index, next_record->kvdl_index); 499 mlxsw_sp_nve_mc_record_refresh(next_record); 500 mc_record->ops->entry_del(mc_record, mc_entry); 501 return; 502 } 503 504 /* This is the last case where the last remaining record needs to 505 * be deleted. Simply delete the entry 506 */ 507 mc_record->ops->entry_del(mc_record, mc_entry); 508 } 509 510 static struct mlxsw_sp_nve_mc_record * 511 mlxsw_sp_nve_mc_record_find(struct mlxsw_sp_nve_mc_list *mc_list, 512 enum mlxsw_sp_l3proto proto, 513 union mlxsw_sp_l3addr *addr, 514 struct mlxsw_sp_nve_mc_entry **mc_entry) 515 { 516 struct mlxsw_sp_nve_mc_record *mc_record; 517 518 list_for_each_entry(mc_record, &mc_list->records_list, list) { 519 if (mc_record->proto != proto) 520 continue; 521 522 *mc_entry = mlxsw_sp_nve_mc_entry_find(mc_record, addr); 523 if (*mc_entry) 524 return mc_record; 525 } 526 527 return NULL; 528 } 529 530 static int mlxsw_sp_nve_mc_list_ip_add(struct mlxsw_sp *mlxsw_sp, 531 struct mlxsw_sp_nve_mc_list *mc_list, 532 enum mlxsw_sp_l3proto proto, 533 union mlxsw_sp_l3addr *addr) 534 { 535 struct mlxsw_sp_nve_mc_record *mc_record; 536 int err; 537 538 mc_record = mlxsw_sp_nve_mc_record_get(mlxsw_sp, mc_list, proto); 539 if (IS_ERR(mc_record)) 540 return PTR_ERR(mc_record); 541 542 err = mlxsw_sp_nve_mc_record_ip_add(mc_record, addr); 543 if (err) 544 goto err_ip_add; 545 546 return 0; 547 548 err_ip_add: 549 mlxsw_sp_nve_mc_record_put(mc_record); 550 return err; 551 } 552 553 static void mlxsw_sp_nve_mc_list_ip_del(struct mlxsw_sp *mlxsw_sp, 554 struct mlxsw_sp_nve_mc_list *mc_list, 555 enum mlxsw_sp_l3proto proto, 556 union mlxsw_sp_l3addr *addr) 557 { 558 struct mlxsw_sp_nve_mc_record *mc_record; 559 struct mlxsw_sp_nve_mc_entry *mc_entry; 560 561 mc_record = mlxsw_sp_nve_mc_record_find(mc_list, proto, addr, 562 &mc_entry); 563 if (!mc_record) 564 return; 565 566 mlxsw_sp_nve_mc_record_entry_del(mc_record, mc_entry); 567 mlxsw_sp_nve_mc_record_put(mc_record); 568 } 569 570 static int 571 mlxsw_sp_nve_fid_flood_index_set(struct mlxsw_sp_fid *fid, 572 struct mlxsw_sp_nve_mc_list *mc_list) 573 { 574 struct mlxsw_sp_nve_mc_record *mc_record; 575 576 /* The address of the first record in the list is a property of 577 * the FID and we never change it. It only needs to be set when 578 * a new list is created 579 */ 580 if (mlxsw_sp_fid_nve_flood_index_is_set(fid)) 581 return 0; 582 583 mc_record = list_first_entry(&mc_list->records_list, 584 struct mlxsw_sp_nve_mc_record, list); 585 586 return mlxsw_sp_fid_nve_flood_index_set(fid, mc_record->kvdl_index); 587 } 588 589 static void 590 mlxsw_sp_nve_fid_flood_index_clear(struct mlxsw_sp_fid *fid, 591 struct mlxsw_sp_nve_mc_list *mc_list) 592 { 593 struct mlxsw_sp_nve_mc_record *mc_record; 594 595 /* The address of the first record needs to be invalidated only when 596 * the last record is about to be removed 597 */ 598 if (!list_is_singular(&mc_list->records_list)) 599 return; 600 601 mc_record = list_first_entry(&mc_list->records_list, 602 struct mlxsw_sp_nve_mc_record, list); 603 if (mc_record->num_entries != 1) 604 return; 605 606 return mlxsw_sp_fid_nve_flood_index_clear(fid); 607 } 608 609 int mlxsw_sp_nve_flood_ip_add(struct mlxsw_sp *mlxsw_sp, 610 struct mlxsw_sp_fid *fid, 611 enum mlxsw_sp_l3proto proto, 612 union mlxsw_sp_l3addr *addr) 613 { 614 struct mlxsw_sp_nve_mc_list_key key = { 0 }; 615 struct mlxsw_sp_nve_mc_list *mc_list; 616 int err; 617 618 key.fid_index = mlxsw_sp_fid_index(fid); 619 mc_list = mlxsw_sp_nve_mc_list_get(mlxsw_sp, &key); 620 if (IS_ERR(mc_list)) 621 return PTR_ERR(mc_list); 622 623 err = mlxsw_sp_nve_mc_list_ip_add(mlxsw_sp, mc_list, proto, addr); 624 if (err) 625 goto err_add_ip; 626 627 err = mlxsw_sp_nve_fid_flood_index_set(fid, mc_list); 628 if (err) 629 goto err_fid_flood_index_set; 630 631 return 0; 632 633 err_fid_flood_index_set: 634 mlxsw_sp_nve_mc_list_ip_del(mlxsw_sp, mc_list, proto, addr); 635 err_add_ip: 636 mlxsw_sp_nve_mc_list_put(mlxsw_sp, mc_list); 637 return err; 638 } 639 640 void mlxsw_sp_nve_flood_ip_del(struct mlxsw_sp *mlxsw_sp, 641 struct mlxsw_sp_fid *fid, 642 enum mlxsw_sp_l3proto proto, 643 union mlxsw_sp_l3addr *addr) 644 { 645 struct mlxsw_sp_nve_mc_list_key key = { 0 }; 646 struct mlxsw_sp_nve_mc_list *mc_list; 647 648 key.fid_index = mlxsw_sp_fid_index(fid); 649 mc_list = mlxsw_sp_nve_mc_list_find(mlxsw_sp, &key); 650 if (!mc_list) 651 return; 652 653 mlxsw_sp_nve_fid_flood_index_clear(fid, mc_list); 654 mlxsw_sp_nve_mc_list_ip_del(mlxsw_sp, mc_list, proto, addr); 655 mlxsw_sp_nve_mc_list_put(mlxsw_sp, mc_list); 656 } 657 658 static void 659 mlxsw_sp_nve_mc_record_delete(struct mlxsw_sp_nve_mc_record *mc_record) 660 { 661 struct mlxsw_sp_nve *nve = mc_record->mlxsw_sp->nve; 662 unsigned int num_max_entries; 663 int i; 664 665 num_max_entries = nve->num_max_mc_entries[mc_record->proto]; 666 for (i = 0; i < num_max_entries; i++) { 667 struct mlxsw_sp_nve_mc_entry *mc_entry = &mc_record->entries[i]; 668 669 if (!mc_entry->valid) 670 continue; 671 mlxsw_sp_nve_mc_record_entry_del(mc_record, mc_entry); 672 } 673 674 WARN_ON(mc_record->num_entries); 675 mlxsw_sp_nve_mc_record_put(mc_record); 676 } 677 678 static void mlxsw_sp_nve_flood_ip_flush(struct mlxsw_sp *mlxsw_sp, 679 struct mlxsw_sp_fid *fid) 680 { 681 struct mlxsw_sp_nve_mc_record *mc_record, *tmp; 682 struct mlxsw_sp_nve_mc_list_key key = { 0 }; 683 struct mlxsw_sp_nve_mc_list *mc_list; 684 685 if (!mlxsw_sp_fid_nve_flood_index_is_set(fid)) 686 return; 687 688 mlxsw_sp_fid_nve_flood_index_clear(fid); 689 690 key.fid_index = mlxsw_sp_fid_index(fid); 691 mc_list = mlxsw_sp_nve_mc_list_find(mlxsw_sp, &key); 692 if (WARN_ON(!mc_list)) 693 return; 694 695 list_for_each_entry_safe(mc_record, tmp, &mc_list->records_list, list) 696 mlxsw_sp_nve_mc_record_delete(mc_record); 697 698 WARN_ON(!list_empty(&mc_list->records_list)); 699 mlxsw_sp_nve_mc_list_put(mlxsw_sp, mc_list); 700 } 701 702 u32 mlxsw_sp_nve_decap_tunnel_index_get(const struct mlxsw_sp *mlxsw_sp) 703 { 704 WARN_ON(mlxsw_sp->nve->num_nve_tunnels == 0); 705 706 return mlxsw_sp->nve->tunnel_index; 707 } 708 709 bool mlxsw_sp_nve_ipv4_route_is_decap(const struct mlxsw_sp *mlxsw_sp, 710 u32 tb_id, __be32 addr) 711 { 712 struct mlxsw_sp_nve *nve = mlxsw_sp->nve; 713 struct mlxsw_sp_nve_config *config = &nve->config; 714 715 if (nve->num_nve_tunnels && 716 config->ul_proto == MLXSW_SP_L3_PROTO_IPV4 && 717 config->ul_sip.addr4 == addr && config->ul_tb_id == tb_id) 718 return true; 719 720 return false; 721 } 722 723 static int mlxsw_sp_nve_tunnel_init(struct mlxsw_sp *mlxsw_sp, 724 struct mlxsw_sp_nve_config *config) 725 { 726 struct mlxsw_sp_nve *nve = mlxsw_sp->nve; 727 const struct mlxsw_sp_nve_ops *ops; 728 int err; 729 730 if (nve->num_nve_tunnels++ != 0) 731 return 0; 732 733 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 734 &nve->tunnel_index); 735 if (err) 736 goto err_kvdl_alloc; 737 738 ops = nve->nve_ops_arr[config->type]; 739 err = ops->init(nve, config); 740 if (err) 741 goto err_ops_init; 742 743 return 0; 744 745 err_ops_init: 746 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 747 nve->tunnel_index); 748 err_kvdl_alloc: 749 nve->num_nve_tunnels--; 750 return err; 751 } 752 753 static void mlxsw_sp_nve_tunnel_fini(struct mlxsw_sp *mlxsw_sp) 754 { 755 struct mlxsw_sp_nve *nve = mlxsw_sp->nve; 756 const struct mlxsw_sp_nve_ops *ops; 757 758 ops = nve->nve_ops_arr[nve->config.type]; 759 760 if (mlxsw_sp->nve->num_nve_tunnels == 1) { 761 ops->fini(nve); 762 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 763 nve->tunnel_index); 764 } 765 nve->num_nve_tunnels--; 766 } 767 768 static void mlxsw_sp_nve_fdb_flush_by_fid(struct mlxsw_sp *mlxsw_sp, 769 u16 fid_index) 770 { 771 char sfdf_pl[MLXSW_REG_SFDF_LEN]; 772 773 mlxsw_reg_sfdf_pack(sfdf_pl, MLXSW_REG_SFDF_FLUSH_PER_NVE_AND_FID); 774 mlxsw_reg_sfdf_fid_set(sfdf_pl, fid_index); 775 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sfdf), sfdf_pl); 776 } 777 778 int mlxsw_sp_nve_fid_enable(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fid *fid, 779 struct mlxsw_sp_nve_params *params, 780 struct netlink_ext_ack *extack) 781 { 782 struct mlxsw_sp_nve *nve = mlxsw_sp->nve; 783 const struct mlxsw_sp_nve_ops *ops; 784 struct mlxsw_sp_nve_config config; 785 int err; 786 787 ops = nve->nve_ops_arr[params->type]; 788 789 if (!ops->can_offload(nve, params->dev, extack)) 790 return -EOPNOTSUPP; 791 792 memset(&config, 0, sizeof(config)); 793 ops->nve_config(nve, params->dev, &config); 794 if (nve->num_nve_tunnels && 795 memcmp(&config, &nve->config, sizeof(config))) { 796 NL_SET_ERR_MSG_MOD(extack, "Conflicting NVE tunnels configuration"); 797 return -EOPNOTSUPP; 798 } 799 800 err = mlxsw_sp_nve_tunnel_init(mlxsw_sp, &config); 801 if (err) { 802 NL_SET_ERR_MSG_MOD(extack, "Failed to initialize NVE tunnel"); 803 return err; 804 } 805 806 err = mlxsw_sp_fid_vni_set(fid, params->vni); 807 if (err) { 808 NL_SET_ERR_MSG_MOD(extack, "Failed to set VNI on FID"); 809 goto err_fid_vni_set; 810 } 811 812 nve->config = config; 813 814 return 0; 815 816 err_fid_vni_set: 817 mlxsw_sp_nve_tunnel_fini(mlxsw_sp); 818 return err; 819 } 820 821 void mlxsw_sp_nve_fid_disable(struct mlxsw_sp *mlxsw_sp, 822 struct mlxsw_sp_fid *fid) 823 { 824 u16 fid_index = mlxsw_sp_fid_index(fid); 825 826 mlxsw_sp_nve_flood_ip_flush(mlxsw_sp, fid); 827 mlxsw_sp_nve_fdb_flush_by_fid(mlxsw_sp, fid_index); 828 mlxsw_sp_fid_vni_clear(fid); 829 mlxsw_sp_nve_tunnel_fini(mlxsw_sp); 830 } 831 832 int mlxsw_sp_port_nve_init(struct mlxsw_sp_port *mlxsw_sp_port) 833 { 834 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 835 char tnqdr_pl[MLXSW_REG_TNQDR_LEN]; 836 837 mlxsw_reg_tnqdr_pack(tnqdr_pl, mlxsw_sp_port->local_port); 838 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tnqdr), tnqdr_pl); 839 } 840 841 void mlxsw_sp_port_nve_fini(struct mlxsw_sp_port *mlxsw_sp_port) 842 { 843 } 844 845 static int mlxsw_sp_nve_qos_init(struct mlxsw_sp *mlxsw_sp) 846 { 847 char tnqcr_pl[MLXSW_REG_TNQCR_LEN]; 848 849 mlxsw_reg_tnqcr_pack(tnqcr_pl); 850 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tnqcr), tnqcr_pl); 851 } 852 853 static int mlxsw_sp_nve_ecn_encap_init(struct mlxsw_sp *mlxsw_sp) 854 { 855 int i; 856 857 /* Iterate over inner ECN values */ 858 for (i = INET_ECN_NOT_ECT; i <= INET_ECN_CE; i++) { 859 u8 outer_ecn = INET_ECN_encapsulate(0, i); 860 char tneem_pl[MLXSW_REG_TNEEM_LEN]; 861 int err; 862 863 mlxsw_reg_tneem_pack(tneem_pl, i, outer_ecn); 864 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tneem), 865 tneem_pl); 866 if (err) 867 return err; 868 } 869 870 return 0; 871 } 872 873 static int __mlxsw_sp_nve_ecn_decap_init(struct mlxsw_sp *mlxsw_sp, 874 u8 inner_ecn, u8 outer_ecn) 875 { 876 char tndem_pl[MLXSW_REG_TNDEM_LEN]; 877 bool trap_en, set_ce = false; 878 u8 new_inner_ecn; 879 880 trap_en = !!__INET_ECN_decapsulate(outer_ecn, inner_ecn, &set_ce); 881 new_inner_ecn = set_ce ? INET_ECN_CE : inner_ecn; 882 883 mlxsw_reg_tndem_pack(tndem_pl, outer_ecn, inner_ecn, new_inner_ecn, 884 trap_en, trap_en ? MLXSW_TRAP_ID_DECAP_ECN0 : 0); 885 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tndem), tndem_pl); 886 } 887 888 static int mlxsw_sp_nve_ecn_decap_init(struct mlxsw_sp *mlxsw_sp) 889 { 890 int i; 891 892 /* Iterate over inner ECN values */ 893 for (i = INET_ECN_NOT_ECT; i <= INET_ECN_CE; i++) { 894 int j; 895 896 /* Iterate over outer ECN values */ 897 for (j = INET_ECN_NOT_ECT; j <= INET_ECN_CE; j++) { 898 int err; 899 900 err = __mlxsw_sp_nve_ecn_decap_init(mlxsw_sp, i, j); 901 if (err) 902 return err; 903 } 904 } 905 906 return 0; 907 } 908 909 static int mlxsw_sp_nve_ecn_init(struct mlxsw_sp *mlxsw_sp) 910 { 911 int err; 912 913 err = mlxsw_sp_nve_ecn_encap_init(mlxsw_sp); 914 if (err) 915 return err; 916 917 return mlxsw_sp_nve_ecn_decap_init(mlxsw_sp); 918 } 919 920 static int mlxsw_sp_nve_resources_query(struct mlxsw_sp *mlxsw_sp) 921 { 922 unsigned int max; 923 924 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_NVE_MC_ENTRIES_IPV4) || 925 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_NVE_MC_ENTRIES_IPV6)) 926 return -EIO; 927 max = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_NVE_MC_ENTRIES_IPV4); 928 mlxsw_sp->nve->num_max_mc_entries[MLXSW_SP_L3_PROTO_IPV4] = max; 929 max = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_NVE_MC_ENTRIES_IPV6); 930 mlxsw_sp->nve->num_max_mc_entries[MLXSW_SP_L3_PROTO_IPV6] = max; 931 932 return 0; 933 } 934 935 int mlxsw_sp_nve_init(struct mlxsw_sp *mlxsw_sp) 936 { 937 struct mlxsw_sp_nve *nve; 938 int err; 939 940 nve = kzalloc(sizeof(*mlxsw_sp->nve), GFP_KERNEL); 941 if (!nve) 942 return -ENOMEM; 943 mlxsw_sp->nve = nve; 944 nve->mlxsw_sp = mlxsw_sp; 945 nve->nve_ops_arr = mlxsw_sp->nve_ops_arr; 946 947 err = rhashtable_init(&nve->mc_list_ht, 948 &mlxsw_sp_nve_mc_list_ht_params); 949 if (err) 950 goto err_rhashtable_init; 951 952 err = mlxsw_sp_nve_qos_init(mlxsw_sp); 953 if (err) 954 goto err_nve_qos_init; 955 956 err = mlxsw_sp_nve_ecn_init(mlxsw_sp); 957 if (err) 958 goto err_nve_ecn_init; 959 960 err = mlxsw_sp_nve_resources_query(mlxsw_sp); 961 if (err) 962 goto err_nve_resources_query; 963 964 return 0; 965 966 err_nve_resources_query: 967 err_nve_ecn_init: 968 err_nve_qos_init: 969 rhashtable_destroy(&nve->mc_list_ht); 970 err_rhashtable_init: 971 mlxsw_sp->nve = NULL; 972 kfree(nve); 973 return err; 974 } 975 976 void mlxsw_sp_nve_fini(struct mlxsw_sp *mlxsw_sp) 977 { 978 WARN_ON(mlxsw_sp->nve->num_nve_tunnels); 979 rhashtable_destroy(&mlxsw_sp->nve->mc_list_ht); 980 mlxsw_sp->nve = NULL; 981 kfree(mlxsw_sp->nve); 982 } 983