1 // SPDX-License-Identifier: BSD-3-Clause OR GPL-2.0 2 /* Copyright (c) 2018 Mellanox Technologies. All rights reserved */ 3 4 #include <linux/err.h> 5 #include <linux/gfp.h> 6 #include <linux/kernel.h> 7 #include <linux/list.h> 8 #include <linux/netlink.h> 9 #include <linux/rtnetlink.h> 10 #include <linux/slab.h> 11 #include <net/inet_ecn.h> 12 #include <net/ipv6.h> 13 14 #include "reg.h" 15 #include "spectrum.h" 16 #include "spectrum_nve.h" 17 18 const struct mlxsw_sp_nve_ops *mlxsw_sp1_nve_ops_arr[] = { 19 [MLXSW_SP_NVE_TYPE_VXLAN] = &mlxsw_sp1_nve_vxlan_ops, 20 }; 21 22 const struct mlxsw_sp_nve_ops *mlxsw_sp2_nve_ops_arr[] = { 23 [MLXSW_SP_NVE_TYPE_VXLAN] = &mlxsw_sp2_nve_vxlan_ops, 24 }; 25 26 struct mlxsw_sp_nve_mc_entry; 27 struct mlxsw_sp_nve_mc_record; 28 struct mlxsw_sp_nve_mc_list; 29 30 struct mlxsw_sp_nve_mc_record_ops { 31 enum mlxsw_reg_tnumt_record_type type; 32 int (*entry_add)(struct mlxsw_sp_nve_mc_record *mc_record, 33 struct mlxsw_sp_nve_mc_entry *mc_entry, 34 const union mlxsw_sp_l3addr *addr); 35 void (*entry_del)(const struct mlxsw_sp_nve_mc_record *mc_record, 36 const struct mlxsw_sp_nve_mc_entry *mc_entry); 37 void (*entry_set)(const struct mlxsw_sp_nve_mc_record *mc_record, 38 const struct mlxsw_sp_nve_mc_entry *mc_entry, 39 char *tnumt_pl, unsigned int entry_index); 40 bool (*entry_compare)(const struct mlxsw_sp_nve_mc_record *mc_record, 41 const struct mlxsw_sp_nve_mc_entry *mc_entry, 42 const union mlxsw_sp_l3addr *addr); 43 }; 44 45 struct mlxsw_sp_nve_mc_list_key { 46 u16 fid_index; 47 }; 48 49 struct mlxsw_sp_nve_mc_ipv6_entry { 50 struct in6_addr addr6; 51 u32 addr6_kvdl_index; 52 }; 53 54 struct mlxsw_sp_nve_mc_entry { 55 union { 56 __be32 addr4; 57 struct mlxsw_sp_nve_mc_ipv6_entry ipv6_entry; 58 }; 59 u8 valid:1; 60 }; 61 62 struct mlxsw_sp_nve_mc_record { 63 struct list_head list; 64 enum mlxsw_sp_l3proto proto; 65 unsigned int num_entries; 66 struct mlxsw_sp *mlxsw_sp; 67 struct mlxsw_sp_nve_mc_list *mc_list; 68 const struct mlxsw_sp_nve_mc_record_ops *ops; 69 u32 kvdl_index; 70 struct mlxsw_sp_nve_mc_entry entries[0]; 71 }; 72 73 struct mlxsw_sp_nve_mc_list { 74 struct list_head records_list; 75 struct rhash_head ht_node; 76 struct mlxsw_sp_nve_mc_list_key key; 77 }; 78 79 static const struct rhashtable_params mlxsw_sp_nve_mc_list_ht_params = { 80 .key_len = sizeof(struct mlxsw_sp_nve_mc_list_key), 81 .key_offset = offsetof(struct mlxsw_sp_nve_mc_list, key), 82 .head_offset = offsetof(struct mlxsw_sp_nve_mc_list, ht_node), 83 }; 84 85 static int 86 mlxsw_sp_nve_mc_record_ipv4_entry_add(struct mlxsw_sp_nve_mc_record *mc_record, 87 struct mlxsw_sp_nve_mc_entry *mc_entry, 88 const union mlxsw_sp_l3addr *addr) 89 { 90 mc_entry->addr4 = addr->addr4; 91 92 return 0; 93 } 94 95 static void 96 mlxsw_sp_nve_mc_record_ipv4_entry_del(const struct mlxsw_sp_nve_mc_record *mc_record, 97 const struct mlxsw_sp_nve_mc_entry *mc_entry) 98 { 99 } 100 101 static void 102 mlxsw_sp_nve_mc_record_ipv4_entry_set(const struct mlxsw_sp_nve_mc_record *mc_record, 103 const struct mlxsw_sp_nve_mc_entry *mc_entry, 104 char *tnumt_pl, unsigned int entry_index) 105 { 106 u32 udip = be32_to_cpu(mc_entry->addr4); 107 108 mlxsw_reg_tnumt_udip_set(tnumt_pl, entry_index, udip); 109 } 110 111 static bool 112 mlxsw_sp_nve_mc_record_ipv4_entry_compare(const struct mlxsw_sp_nve_mc_record *mc_record, 113 const struct mlxsw_sp_nve_mc_entry *mc_entry, 114 const union mlxsw_sp_l3addr *addr) 115 { 116 return mc_entry->addr4 == addr->addr4; 117 } 118 119 static const struct mlxsw_sp_nve_mc_record_ops 120 mlxsw_sp_nve_mc_record_ipv4_ops = { 121 .type = MLXSW_REG_TNUMT_RECORD_TYPE_IPV4, 122 .entry_add = &mlxsw_sp_nve_mc_record_ipv4_entry_add, 123 .entry_del = &mlxsw_sp_nve_mc_record_ipv4_entry_del, 124 .entry_set = &mlxsw_sp_nve_mc_record_ipv4_entry_set, 125 .entry_compare = &mlxsw_sp_nve_mc_record_ipv4_entry_compare, 126 }; 127 128 static int 129 mlxsw_sp_nve_mc_record_ipv6_entry_add(struct mlxsw_sp_nve_mc_record *mc_record, 130 struct mlxsw_sp_nve_mc_entry *mc_entry, 131 const union mlxsw_sp_l3addr *addr) 132 { 133 WARN_ON(1); 134 135 return -EINVAL; 136 } 137 138 static void 139 mlxsw_sp_nve_mc_record_ipv6_entry_del(const struct mlxsw_sp_nve_mc_record *mc_record, 140 const struct mlxsw_sp_nve_mc_entry *mc_entry) 141 { 142 } 143 144 static void 145 mlxsw_sp_nve_mc_record_ipv6_entry_set(const struct mlxsw_sp_nve_mc_record *mc_record, 146 const struct mlxsw_sp_nve_mc_entry *mc_entry, 147 char *tnumt_pl, unsigned int entry_index) 148 { 149 u32 udip_ptr = mc_entry->ipv6_entry.addr6_kvdl_index; 150 151 mlxsw_reg_tnumt_udip_ptr_set(tnumt_pl, entry_index, udip_ptr); 152 } 153 154 static bool 155 mlxsw_sp_nve_mc_record_ipv6_entry_compare(const struct mlxsw_sp_nve_mc_record *mc_record, 156 const struct mlxsw_sp_nve_mc_entry *mc_entry, 157 const union mlxsw_sp_l3addr *addr) 158 { 159 return ipv6_addr_equal(&mc_entry->ipv6_entry.addr6, &addr->addr6); 160 } 161 162 static const struct mlxsw_sp_nve_mc_record_ops 163 mlxsw_sp_nve_mc_record_ipv6_ops = { 164 .type = MLXSW_REG_TNUMT_RECORD_TYPE_IPV6, 165 .entry_add = &mlxsw_sp_nve_mc_record_ipv6_entry_add, 166 .entry_del = &mlxsw_sp_nve_mc_record_ipv6_entry_del, 167 .entry_set = &mlxsw_sp_nve_mc_record_ipv6_entry_set, 168 .entry_compare = &mlxsw_sp_nve_mc_record_ipv6_entry_compare, 169 }; 170 171 static const struct mlxsw_sp_nve_mc_record_ops * 172 mlxsw_sp_nve_mc_record_ops_arr[] = { 173 [MLXSW_SP_L3_PROTO_IPV4] = &mlxsw_sp_nve_mc_record_ipv4_ops, 174 [MLXSW_SP_L3_PROTO_IPV6] = &mlxsw_sp_nve_mc_record_ipv6_ops, 175 }; 176 177 int mlxsw_sp_nve_learned_ip_resolve(struct mlxsw_sp *mlxsw_sp, u32 uip, 178 enum mlxsw_sp_l3proto proto, 179 union mlxsw_sp_l3addr *addr) 180 { 181 switch (proto) { 182 case MLXSW_SP_L3_PROTO_IPV4: 183 addr->addr4 = cpu_to_be32(uip); 184 return 0; 185 default: 186 WARN_ON(1); 187 return -EINVAL; 188 } 189 } 190 191 static struct mlxsw_sp_nve_mc_list * 192 mlxsw_sp_nve_mc_list_find(struct mlxsw_sp *mlxsw_sp, 193 const struct mlxsw_sp_nve_mc_list_key *key) 194 { 195 struct mlxsw_sp_nve *nve = mlxsw_sp->nve; 196 197 return rhashtable_lookup_fast(&nve->mc_list_ht, key, 198 mlxsw_sp_nve_mc_list_ht_params); 199 } 200 201 static struct mlxsw_sp_nve_mc_list * 202 mlxsw_sp_nve_mc_list_create(struct mlxsw_sp *mlxsw_sp, 203 const struct mlxsw_sp_nve_mc_list_key *key) 204 { 205 struct mlxsw_sp_nve *nve = mlxsw_sp->nve; 206 struct mlxsw_sp_nve_mc_list *mc_list; 207 int err; 208 209 mc_list = kmalloc(sizeof(*mc_list), GFP_KERNEL); 210 if (!mc_list) 211 return ERR_PTR(-ENOMEM); 212 213 INIT_LIST_HEAD(&mc_list->records_list); 214 mc_list->key = *key; 215 216 err = rhashtable_insert_fast(&nve->mc_list_ht, &mc_list->ht_node, 217 mlxsw_sp_nve_mc_list_ht_params); 218 if (err) 219 goto err_rhashtable_insert; 220 221 return mc_list; 222 223 err_rhashtable_insert: 224 kfree(mc_list); 225 return ERR_PTR(err); 226 } 227 228 static void mlxsw_sp_nve_mc_list_destroy(struct mlxsw_sp *mlxsw_sp, 229 struct mlxsw_sp_nve_mc_list *mc_list) 230 { 231 struct mlxsw_sp_nve *nve = mlxsw_sp->nve; 232 233 rhashtable_remove_fast(&nve->mc_list_ht, &mc_list->ht_node, 234 mlxsw_sp_nve_mc_list_ht_params); 235 WARN_ON(!list_empty(&mc_list->records_list)); 236 kfree(mc_list); 237 } 238 239 static struct mlxsw_sp_nve_mc_list * 240 mlxsw_sp_nve_mc_list_get(struct mlxsw_sp *mlxsw_sp, 241 const struct mlxsw_sp_nve_mc_list_key *key) 242 { 243 struct mlxsw_sp_nve_mc_list *mc_list; 244 245 mc_list = mlxsw_sp_nve_mc_list_find(mlxsw_sp, key); 246 if (mc_list) 247 return mc_list; 248 249 return mlxsw_sp_nve_mc_list_create(mlxsw_sp, key); 250 } 251 252 static void 253 mlxsw_sp_nve_mc_list_put(struct mlxsw_sp *mlxsw_sp, 254 struct mlxsw_sp_nve_mc_list *mc_list) 255 { 256 if (!list_empty(&mc_list->records_list)) 257 return; 258 mlxsw_sp_nve_mc_list_destroy(mlxsw_sp, mc_list); 259 } 260 261 static struct mlxsw_sp_nve_mc_record * 262 mlxsw_sp_nve_mc_record_create(struct mlxsw_sp *mlxsw_sp, 263 struct mlxsw_sp_nve_mc_list *mc_list, 264 enum mlxsw_sp_l3proto proto) 265 { 266 unsigned int num_max_entries = mlxsw_sp->nve->num_max_mc_entries[proto]; 267 struct mlxsw_sp_nve_mc_record *mc_record; 268 int err; 269 270 mc_record = kzalloc(struct_size(mc_record, entries, num_max_entries), 271 GFP_KERNEL); 272 if (!mc_record) 273 return ERR_PTR(-ENOMEM); 274 275 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_TNUMT, 1, 276 &mc_record->kvdl_index); 277 if (err) 278 goto err_kvdl_alloc; 279 280 mc_record->ops = mlxsw_sp_nve_mc_record_ops_arr[proto]; 281 mc_record->mlxsw_sp = mlxsw_sp; 282 mc_record->mc_list = mc_list; 283 mc_record->proto = proto; 284 list_add_tail(&mc_record->list, &mc_list->records_list); 285 286 return mc_record; 287 288 err_kvdl_alloc: 289 kfree(mc_record); 290 return ERR_PTR(err); 291 } 292 293 static void 294 mlxsw_sp_nve_mc_record_destroy(struct mlxsw_sp_nve_mc_record *mc_record) 295 { 296 struct mlxsw_sp *mlxsw_sp = mc_record->mlxsw_sp; 297 298 list_del(&mc_record->list); 299 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_TNUMT, 1, 300 mc_record->kvdl_index); 301 WARN_ON(mc_record->num_entries); 302 kfree(mc_record); 303 } 304 305 static struct mlxsw_sp_nve_mc_record * 306 mlxsw_sp_nve_mc_record_get(struct mlxsw_sp *mlxsw_sp, 307 struct mlxsw_sp_nve_mc_list *mc_list, 308 enum mlxsw_sp_l3proto proto) 309 { 310 struct mlxsw_sp_nve_mc_record *mc_record; 311 312 list_for_each_entry_reverse(mc_record, &mc_list->records_list, list) { 313 unsigned int num_entries = mc_record->num_entries; 314 struct mlxsw_sp_nve *nve = mlxsw_sp->nve; 315 316 if (mc_record->proto == proto && 317 num_entries < nve->num_max_mc_entries[proto]) 318 return mc_record; 319 } 320 321 return mlxsw_sp_nve_mc_record_create(mlxsw_sp, mc_list, proto); 322 } 323 324 static void 325 mlxsw_sp_nve_mc_record_put(struct mlxsw_sp_nve_mc_record *mc_record) 326 { 327 if (mc_record->num_entries != 0) 328 return; 329 330 mlxsw_sp_nve_mc_record_destroy(mc_record); 331 } 332 333 static struct mlxsw_sp_nve_mc_entry * 334 mlxsw_sp_nve_mc_free_entry_find(struct mlxsw_sp_nve_mc_record *mc_record) 335 { 336 struct mlxsw_sp_nve *nve = mc_record->mlxsw_sp->nve; 337 unsigned int num_max_entries; 338 int i; 339 340 num_max_entries = nve->num_max_mc_entries[mc_record->proto]; 341 for (i = 0; i < num_max_entries; i++) { 342 if (mc_record->entries[i].valid) 343 continue; 344 return &mc_record->entries[i]; 345 } 346 347 return NULL; 348 } 349 350 static int 351 mlxsw_sp_nve_mc_record_refresh(struct mlxsw_sp_nve_mc_record *mc_record) 352 { 353 enum mlxsw_reg_tnumt_record_type type = mc_record->ops->type; 354 struct mlxsw_sp_nve_mc_list *mc_list = mc_record->mc_list; 355 struct mlxsw_sp *mlxsw_sp = mc_record->mlxsw_sp; 356 char tnumt_pl[MLXSW_REG_TNUMT_LEN]; 357 unsigned int num_max_entries; 358 unsigned int num_entries = 0; 359 u32 next_kvdl_index = 0; 360 bool next_valid = false; 361 int i; 362 363 if (!list_is_last(&mc_record->list, &mc_list->records_list)) { 364 struct mlxsw_sp_nve_mc_record *next_record; 365 366 next_record = list_next_entry(mc_record, list); 367 next_kvdl_index = next_record->kvdl_index; 368 next_valid = true; 369 } 370 371 mlxsw_reg_tnumt_pack(tnumt_pl, type, MLXSW_REG_TNUMT_TUNNEL_PORT_NVE, 372 mc_record->kvdl_index, next_valid, 373 next_kvdl_index, mc_record->num_entries); 374 375 num_max_entries = mlxsw_sp->nve->num_max_mc_entries[mc_record->proto]; 376 for (i = 0; i < num_max_entries; i++) { 377 struct mlxsw_sp_nve_mc_entry *mc_entry; 378 379 mc_entry = &mc_record->entries[i]; 380 if (!mc_entry->valid) 381 continue; 382 mc_record->ops->entry_set(mc_record, mc_entry, tnumt_pl, 383 num_entries++); 384 } 385 386 WARN_ON(num_entries != mc_record->num_entries); 387 388 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tnumt), tnumt_pl); 389 } 390 391 static bool 392 mlxsw_sp_nve_mc_record_is_first(struct mlxsw_sp_nve_mc_record *mc_record) 393 { 394 struct mlxsw_sp_nve_mc_list *mc_list = mc_record->mc_list; 395 struct mlxsw_sp_nve_mc_record *first_record; 396 397 first_record = list_first_entry(&mc_list->records_list, 398 struct mlxsw_sp_nve_mc_record, list); 399 400 return mc_record == first_record; 401 } 402 403 static struct mlxsw_sp_nve_mc_entry * 404 mlxsw_sp_nve_mc_entry_find(struct mlxsw_sp_nve_mc_record *mc_record, 405 union mlxsw_sp_l3addr *addr) 406 { 407 struct mlxsw_sp_nve *nve = mc_record->mlxsw_sp->nve; 408 unsigned int num_max_entries; 409 int i; 410 411 num_max_entries = nve->num_max_mc_entries[mc_record->proto]; 412 for (i = 0; i < num_max_entries; i++) { 413 struct mlxsw_sp_nve_mc_entry *mc_entry; 414 415 mc_entry = &mc_record->entries[i]; 416 if (!mc_entry->valid) 417 continue; 418 if (mc_record->ops->entry_compare(mc_record, mc_entry, addr)) 419 return mc_entry; 420 } 421 422 return NULL; 423 } 424 425 static int 426 mlxsw_sp_nve_mc_record_ip_add(struct mlxsw_sp_nve_mc_record *mc_record, 427 union mlxsw_sp_l3addr *addr) 428 { 429 struct mlxsw_sp_nve_mc_entry *mc_entry = NULL; 430 int err; 431 432 mc_entry = mlxsw_sp_nve_mc_free_entry_find(mc_record); 433 if (WARN_ON(!mc_entry)) 434 return -EINVAL; 435 436 err = mc_record->ops->entry_add(mc_record, mc_entry, addr); 437 if (err) 438 return err; 439 mc_record->num_entries++; 440 mc_entry->valid = true; 441 442 err = mlxsw_sp_nve_mc_record_refresh(mc_record); 443 if (err) 444 goto err_record_refresh; 445 446 /* If this is a new record and not the first one, then we need to 447 * update the next pointer of the previous entry 448 */ 449 if (mc_record->num_entries != 1 || 450 mlxsw_sp_nve_mc_record_is_first(mc_record)) 451 return 0; 452 453 err = mlxsw_sp_nve_mc_record_refresh(list_prev_entry(mc_record, list)); 454 if (err) 455 goto err_prev_record_refresh; 456 457 return 0; 458 459 err_prev_record_refresh: 460 err_record_refresh: 461 mc_entry->valid = false; 462 mc_record->num_entries--; 463 mc_record->ops->entry_del(mc_record, mc_entry); 464 return err; 465 } 466 467 static void 468 mlxsw_sp_nve_mc_record_entry_del(struct mlxsw_sp_nve_mc_record *mc_record, 469 struct mlxsw_sp_nve_mc_entry *mc_entry) 470 { 471 struct mlxsw_sp_nve_mc_list *mc_list = mc_record->mc_list; 472 473 mc_entry->valid = false; 474 mc_record->num_entries--; 475 476 /* When the record continues to exist we only need to invalidate 477 * the requested entry 478 */ 479 if (mc_record->num_entries != 0) { 480 mlxsw_sp_nve_mc_record_refresh(mc_record); 481 mc_record->ops->entry_del(mc_record, mc_entry); 482 return; 483 } 484 485 /* If the record needs to be deleted, but it is not the first, 486 * then we need to make sure that the previous record no longer 487 * points to it. Remove deleted record from the list to reflect 488 * that and then re-add it at the end, so that it could be 489 * properly removed by the record destruction code 490 */ 491 if (!mlxsw_sp_nve_mc_record_is_first(mc_record)) { 492 struct mlxsw_sp_nve_mc_record *prev_record; 493 494 prev_record = list_prev_entry(mc_record, list); 495 list_del(&mc_record->list); 496 mlxsw_sp_nve_mc_record_refresh(prev_record); 497 list_add_tail(&mc_record->list, &mc_list->records_list); 498 mc_record->ops->entry_del(mc_record, mc_entry); 499 return; 500 } 501 502 /* If the first record needs to be deleted, but the list is not 503 * singular, then the second record needs to be written in the 504 * first record's address, as this address is stored as a property 505 * of the FID 506 */ 507 if (mlxsw_sp_nve_mc_record_is_first(mc_record) && 508 !list_is_singular(&mc_list->records_list)) { 509 struct mlxsw_sp_nve_mc_record *next_record; 510 511 next_record = list_next_entry(mc_record, list); 512 swap(mc_record->kvdl_index, next_record->kvdl_index); 513 mlxsw_sp_nve_mc_record_refresh(next_record); 514 mc_record->ops->entry_del(mc_record, mc_entry); 515 return; 516 } 517 518 /* This is the last case where the last remaining record needs to 519 * be deleted. Simply delete the entry 520 */ 521 mc_record->ops->entry_del(mc_record, mc_entry); 522 } 523 524 static struct mlxsw_sp_nve_mc_record * 525 mlxsw_sp_nve_mc_record_find(struct mlxsw_sp_nve_mc_list *mc_list, 526 enum mlxsw_sp_l3proto proto, 527 union mlxsw_sp_l3addr *addr, 528 struct mlxsw_sp_nve_mc_entry **mc_entry) 529 { 530 struct mlxsw_sp_nve_mc_record *mc_record; 531 532 list_for_each_entry(mc_record, &mc_list->records_list, list) { 533 if (mc_record->proto != proto) 534 continue; 535 536 *mc_entry = mlxsw_sp_nve_mc_entry_find(mc_record, addr); 537 if (*mc_entry) 538 return mc_record; 539 } 540 541 return NULL; 542 } 543 544 static int mlxsw_sp_nve_mc_list_ip_add(struct mlxsw_sp *mlxsw_sp, 545 struct mlxsw_sp_nve_mc_list *mc_list, 546 enum mlxsw_sp_l3proto proto, 547 union mlxsw_sp_l3addr *addr) 548 { 549 struct mlxsw_sp_nve_mc_record *mc_record; 550 int err; 551 552 mc_record = mlxsw_sp_nve_mc_record_get(mlxsw_sp, mc_list, proto); 553 if (IS_ERR(mc_record)) 554 return PTR_ERR(mc_record); 555 556 err = mlxsw_sp_nve_mc_record_ip_add(mc_record, addr); 557 if (err) 558 goto err_ip_add; 559 560 return 0; 561 562 err_ip_add: 563 mlxsw_sp_nve_mc_record_put(mc_record); 564 return err; 565 } 566 567 static void mlxsw_sp_nve_mc_list_ip_del(struct mlxsw_sp *mlxsw_sp, 568 struct mlxsw_sp_nve_mc_list *mc_list, 569 enum mlxsw_sp_l3proto proto, 570 union mlxsw_sp_l3addr *addr) 571 { 572 struct mlxsw_sp_nve_mc_record *mc_record; 573 struct mlxsw_sp_nve_mc_entry *mc_entry; 574 575 mc_record = mlxsw_sp_nve_mc_record_find(mc_list, proto, addr, 576 &mc_entry); 577 if (!mc_record) 578 return; 579 580 mlxsw_sp_nve_mc_record_entry_del(mc_record, mc_entry); 581 mlxsw_sp_nve_mc_record_put(mc_record); 582 } 583 584 static int 585 mlxsw_sp_nve_fid_flood_index_set(struct mlxsw_sp_fid *fid, 586 struct mlxsw_sp_nve_mc_list *mc_list) 587 { 588 struct mlxsw_sp_nve_mc_record *mc_record; 589 590 /* The address of the first record in the list is a property of 591 * the FID and we never change it. It only needs to be set when 592 * a new list is created 593 */ 594 if (mlxsw_sp_fid_nve_flood_index_is_set(fid)) 595 return 0; 596 597 mc_record = list_first_entry(&mc_list->records_list, 598 struct mlxsw_sp_nve_mc_record, list); 599 600 return mlxsw_sp_fid_nve_flood_index_set(fid, mc_record->kvdl_index); 601 } 602 603 static void 604 mlxsw_sp_nve_fid_flood_index_clear(struct mlxsw_sp_fid *fid, 605 struct mlxsw_sp_nve_mc_list *mc_list) 606 { 607 struct mlxsw_sp_nve_mc_record *mc_record; 608 609 /* The address of the first record needs to be invalidated only when 610 * the last record is about to be removed 611 */ 612 if (!list_is_singular(&mc_list->records_list)) 613 return; 614 615 mc_record = list_first_entry(&mc_list->records_list, 616 struct mlxsw_sp_nve_mc_record, list); 617 if (mc_record->num_entries != 1) 618 return; 619 620 return mlxsw_sp_fid_nve_flood_index_clear(fid); 621 } 622 623 int mlxsw_sp_nve_flood_ip_add(struct mlxsw_sp *mlxsw_sp, 624 struct mlxsw_sp_fid *fid, 625 enum mlxsw_sp_l3proto proto, 626 union mlxsw_sp_l3addr *addr) 627 { 628 struct mlxsw_sp_nve_mc_list_key key = { 0 }; 629 struct mlxsw_sp_nve_mc_list *mc_list; 630 int err; 631 632 key.fid_index = mlxsw_sp_fid_index(fid); 633 mc_list = mlxsw_sp_nve_mc_list_get(mlxsw_sp, &key); 634 if (IS_ERR(mc_list)) 635 return PTR_ERR(mc_list); 636 637 err = mlxsw_sp_nve_mc_list_ip_add(mlxsw_sp, mc_list, proto, addr); 638 if (err) 639 goto err_add_ip; 640 641 err = mlxsw_sp_nve_fid_flood_index_set(fid, mc_list); 642 if (err) 643 goto err_fid_flood_index_set; 644 645 return 0; 646 647 err_fid_flood_index_set: 648 mlxsw_sp_nve_mc_list_ip_del(mlxsw_sp, mc_list, proto, addr); 649 err_add_ip: 650 mlxsw_sp_nve_mc_list_put(mlxsw_sp, mc_list); 651 return err; 652 } 653 654 void mlxsw_sp_nve_flood_ip_del(struct mlxsw_sp *mlxsw_sp, 655 struct mlxsw_sp_fid *fid, 656 enum mlxsw_sp_l3proto proto, 657 union mlxsw_sp_l3addr *addr) 658 { 659 struct mlxsw_sp_nve_mc_list_key key = { 0 }; 660 struct mlxsw_sp_nve_mc_list *mc_list; 661 662 key.fid_index = mlxsw_sp_fid_index(fid); 663 mc_list = mlxsw_sp_nve_mc_list_find(mlxsw_sp, &key); 664 if (!mc_list) 665 return; 666 667 mlxsw_sp_nve_fid_flood_index_clear(fid, mc_list); 668 mlxsw_sp_nve_mc_list_ip_del(mlxsw_sp, mc_list, proto, addr); 669 mlxsw_sp_nve_mc_list_put(mlxsw_sp, mc_list); 670 } 671 672 static void 673 mlxsw_sp_nve_mc_record_delete(struct mlxsw_sp_nve_mc_record *mc_record) 674 { 675 struct mlxsw_sp_nve *nve = mc_record->mlxsw_sp->nve; 676 unsigned int num_max_entries; 677 int i; 678 679 num_max_entries = nve->num_max_mc_entries[mc_record->proto]; 680 for (i = 0; i < num_max_entries; i++) { 681 struct mlxsw_sp_nve_mc_entry *mc_entry = &mc_record->entries[i]; 682 683 if (!mc_entry->valid) 684 continue; 685 mlxsw_sp_nve_mc_record_entry_del(mc_record, mc_entry); 686 } 687 688 WARN_ON(mc_record->num_entries); 689 mlxsw_sp_nve_mc_record_put(mc_record); 690 } 691 692 static void mlxsw_sp_nve_flood_ip_flush(struct mlxsw_sp *mlxsw_sp, 693 struct mlxsw_sp_fid *fid) 694 { 695 struct mlxsw_sp_nve_mc_record *mc_record, *tmp; 696 struct mlxsw_sp_nve_mc_list_key key = { 0 }; 697 struct mlxsw_sp_nve_mc_list *mc_list; 698 699 if (!mlxsw_sp_fid_nve_flood_index_is_set(fid)) 700 return; 701 702 mlxsw_sp_fid_nve_flood_index_clear(fid); 703 704 key.fid_index = mlxsw_sp_fid_index(fid); 705 mc_list = mlxsw_sp_nve_mc_list_find(mlxsw_sp, &key); 706 if (WARN_ON(!mc_list)) 707 return; 708 709 list_for_each_entry_safe(mc_record, tmp, &mc_list->records_list, list) 710 mlxsw_sp_nve_mc_record_delete(mc_record); 711 712 WARN_ON(!list_empty(&mc_list->records_list)); 713 mlxsw_sp_nve_mc_list_put(mlxsw_sp, mc_list); 714 } 715 716 u32 mlxsw_sp_nve_decap_tunnel_index_get(const struct mlxsw_sp *mlxsw_sp) 717 { 718 WARN_ON(mlxsw_sp->nve->num_nve_tunnels == 0); 719 720 return mlxsw_sp->nve->tunnel_index; 721 } 722 723 bool mlxsw_sp_nve_ipv4_route_is_decap(const struct mlxsw_sp *mlxsw_sp, 724 u32 tb_id, __be32 addr) 725 { 726 struct mlxsw_sp_nve *nve = mlxsw_sp->nve; 727 struct mlxsw_sp_nve_config *config = &nve->config; 728 729 if (nve->num_nve_tunnels && 730 config->ul_proto == MLXSW_SP_L3_PROTO_IPV4 && 731 config->ul_sip.addr4 == addr && config->ul_tb_id == tb_id) 732 return true; 733 734 return false; 735 } 736 737 static int mlxsw_sp_nve_tunnel_init(struct mlxsw_sp *mlxsw_sp, 738 struct mlxsw_sp_nve_config *config) 739 { 740 struct mlxsw_sp_nve *nve = mlxsw_sp->nve; 741 const struct mlxsw_sp_nve_ops *ops; 742 int err; 743 744 if (nve->num_nve_tunnels++ != 0) 745 return 0; 746 747 err = mlxsw_sp_kvdl_alloc(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 748 &nve->tunnel_index); 749 if (err) 750 goto err_kvdl_alloc; 751 752 ops = nve->nve_ops_arr[config->type]; 753 err = ops->init(nve, config); 754 if (err) 755 goto err_ops_init; 756 757 return 0; 758 759 err_ops_init: 760 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 761 nve->tunnel_index); 762 err_kvdl_alloc: 763 nve->num_nve_tunnels--; 764 return err; 765 } 766 767 static void mlxsw_sp_nve_tunnel_fini(struct mlxsw_sp *mlxsw_sp) 768 { 769 struct mlxsw_sp_nve *nve = mlxsw_sp->nve; 770 const struct mlxsw_sp_nve_ops *ops; 771 772 ops = nve->nve_ops_arr[nve->config.type]; 773 774 if (mlxsw_sp->nve->num_nve_tunnels == 1) { 775 ops->fini(nve); 776 mlxsw_sp_kvdl_free(mlxsw_sp, MLXSW_SP_KVDL_ENTRY_TYPE_ADJ, 1, 777 nve->tunnel_index); 778 } 779 nve->num_nve_tunnels--; 780 } 781 782 static void mlxsw_sp_nve_fdb_flush_by_fid(struct mlxsw_sp *mlxsw_sp, 783 u16 fid_index) 784 { 785 char sfdf_pl[MLXSW_REG_SFDF_LEN]; 786 787 mlxsw_reg_sfdf_pack(sfdf_pl, MLXSW_REG_SFDF_FLUSH_PER_NVE_AND_FID); 788 mlxsw_reg_sfdf_fid_set(sfdf_pl, fid_index); 789 mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(sfdf), sfdf_pl); 790 } 791 792 static void mlxsw_sp_nve_fdb_clear_offload(struct mlxsw_sp *mlxsw_sp, 793 const struct mlxsw_sp_fid *fid, 794 const struct net_device *nve_dev, 795 __be32 vni) 796 { 797 const struct mlxsw_sp_nve_ops *ops; 798 enum mlxsw_sp_nve_type type; 799 800 if (WARN_ON(mlxsw_sp_fid_nve_type(fid, &type))) 801 return; 802 803 ops = mlxsw_sp->nve->nve_ops_arr[type]; 804 ops->fdb_clear_offload(nve_dev, vni); 805 } 806 807 int mlxsw_sp_nve_fid_enable(struct mlxsw_sp *mlxsw_sp, struct mlxsw_sp_fid *fid, 808 struct mlxsw_sp_nve_params *params, 809 struct netlink_ext_ack *extack) 810 { 811 struct mlxsw_sp_nve *nve = mlxsw_sp->nve; 812 const struct mlxsw_sp_nve_ops *ops; 813 struct mlxsw_sp_nve_config config; 814 int err; 815 816 ops = nve->nve_ops_arr[params->type]; 817 818 if (!ops->can_offload(nve, params->dev, extack)) 819 return -EINVAL; 820 821 memset(&config, 0, sizeof(config)); 822 ops->nve_config(nve, params->dev, &config); 823 if (nve->num_nve_tunnels && 824 memcmp(&config, &nve->config, sizeof(config))) { 825 NL_SET_ERR_MSG_MOD(extack, "Conflicting NVE tunnels configuration"); 826 return -EINVAL; 827 } 828 829 err = mlxsw_sp_nve_tunnel_init(mlxsw_sp, &config); 830 if (err) { 831 NL_SET_ERR_MSG_MOD(extack, "Failed to initialize NVE tunnel"); 832 return err; 833 } 834 835 err = mlxsw_sp_fid_vni_set(fid, params->type, params->vni, 836 params->dev->ifindex); 837 if (err) { 838 NL_SET_ERR_MSG_MOD(extack, "Failed to set VNI on FID"); 839 goto err_fid_vni_set; 840 } 841 842 nve->config = config; 843 844 err = ops->fdb_replay(params->dev, params->vni, extack); 845 if (err) 846 goto err_fdb_replay; 847 848 return 0; 849 850 err_fdb_replay: 851 mlxsw_sp_fid_vni_clear(fid); 852 err_fid_vni_set: 853 mlxsw_sp_nve_tunnel_fini(mlxsw_sp); 854 return err; 855 } 856 857 void mlxsw_sp_nve_fid_disable(struct mlxsw_sp *mlxsw_sp, 858 struct mlxsw_sp_fid *fid) 859 { 860 u16 fid_index = mlxsw_sp_fid_index(fid); 861 struct net_device *nve_dev; 862 int nve_ifindex; 863 __be32 vni; 864 865 mlxsw_sp_nve_flood_ip_flush(mlxsw_sp, fid); 866 mlxsw_sp_nve_fdb_flush_by_fid(mlxsw_sp, fid_index); 867 868 if (WARN_ON(mlxsw_sp_fid_nve_ifindex(fid, &nve_ifindex) || 869 mlxsw_sp_fid_vni(fid, &vni))) 870 goto out; 871 872 nve_dev = dev_get_by_index(&init_net, nve_ifindex); 873 if (!nve_dev) 874 goto out; 875 876 mlxsw_sp_nve_fdb_clear_offload(mlxsw_sp, fid, nve_dev, vni); 877 mlxsw_sp_fid_fdb_clear_offload(fid, nve_dev); 878 879 dev_put(nve_dev); 880 881 out: 882 mlxsw_sp_fid_vni_clear(fid); 883 mlxsw_sp_nve_tunnel_fini(mlxsw_sp); 884 } 885 886 int mlxsw_sp_port_nve_init(struct mlxsw_sp_port *mlxsw_sp_port) 887 { 888 struct mlxsw_sp *mlxsw_sp = mlxsw_sp_port->mlxsw_sp; 889 char tnqdr_pl[MLXSW_REG_TNQDR_LEN]; 890 891 mlxsw_reg_tnqdr_pack(tnqdr_pl, mlxsw_sp_port->local_port); 892 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tnqdr), tnqdr_pl); 893 } 894 895 void mlxsw_sp_port_nve_fini(struct mlxsw_sp_port *mlxsw_sp_port) 896 { 897 } 898 899 static int mlxsw_sp_nve_qos_init(struct mlxsw_sp *mlxsw_sp) 900 { 901 char tnqcr_pl[MLXSW_REG_TNQCR_LEN]; 902 903 mlxsw_reg_tnqcr_pack(tnqcr_pl); 904 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tnqcr), tnqcr_pl); 905 } 906 907 static int mlxsw_sp_nve_ecn_encap_init(struct mlxsw_sp *mlxsw_sp) 908 { 909 int i; 910 911 /* Iterate over inner ECN values */ 912 for (i = INET_ECN_NOT_ECT; i <= INET_ECN_CE; i++) { 913 u8 outer_ecn = INET_ECN_encapsulate(0, i); 914 char tneem_pl[MLXSW_REG_TNEEM_LEN]; 915 int err; 916 917 mlxsw_reg_tneem_pack(tneem_pl, i, outer_ecn); 918 err = mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tneem), 919 tneem_pl); 920 if (err) 921 return err; 922 } 923 924 return 0; 925 } 926 927 static int __mlxsw_sp_nve_ecn_decap_init(struct mlxsw_sp *mlxsw_sp, 928 u8 inner_ecn, u8 outer_ecn) 929 { 930 char tndem_pl[MLXSW_REG_TNDEM_LEN]; 931 bool trap_en, set_ce = false; 932 u8 new_inner_ecn; 933 934 trap_en = !!__INET_ECN_decapsulate(outer_ecn, inner_ecn, &set_ce); 935 new_inner_ecn = set_ce ? INET_ECN_CE : inner_ecn; 936 937 mlxsw_reg_tndem_pack(tndem_pl, outer_ecn, inner_ecn, new_inner_ecn, 938 trap_en, trap_en ? MLXSW_TRAP_ID_DECAP_ECN0 : 0); 939 return mlxsw_reg_write(mlxsw_sp->core, MLXSW_REG(tndem), tndem_pl); 940 } 941 942 static int mlxsw_sp_nve_ecn_decap_init(struct mlxsw_sp *mlxsw_sp) 943 { 944 int i; 945 946 /* Iterate over inner ECN values */ 947 for (i = INET_ECN_NOT_ECT; i <= INET_ECN_CE; i++) { 948 int j; 949 950 /* Iterate over outer ECN values */ 951 for (j = INET_ECN_NOT_ECT; j <= INET_ECN_CE; j++) { 952 int err; 953 954 err = __mlxsw_sp_nve_ecn_decap_init(mlxsw_sp, i, j); 955 if (err) 956 return err; 957 } 958 } 959 960 return 0; 961 } 962 963 static int mlxsw_sp_nve_ecn_init(struct mlxsw_sp *mlxsw_sp) 964 { 965 int err; 966 967 err = mlxsw_sp_nve_ecn_encap_init(mlxsw_sp); 968 if (err) 969 return err; 970 971 return mlxsw_sp_nve_ecn_decap_init(mlxsw_sp); 972 } 973 974 static int mlxsw_sp_nve_resources_query(struct mlxsw_sp *mlxsw_sp) 975 { 976 unsigned int max; 977 978 if (!MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_NVE_MC_ENTRIES_IPV4) || 979 !MLXSW_CORE_RES_VALID(mlxsw_sp->core, MAX_NVE_MC_ENTRIES_IPV6)) 980 return -EIO; 981 max = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_NVE_MC_ENTRIES_IPV4); 982 mlxsw_sp->nve->num_max_mc_entries[MLXSW_SP_L3_PROTO_IPV4] = max; 983 max = MLXSW_CORE_RES_GET(mlxsw_sp->core, MAX_NVE_MC_ENTRIES_IPV6); 984 mlxsw_sp->nve->num_max_mc_entries[MLXSW_SP_L3_PROTO_IPV6] = max; 985 986 return 0; 987 } 988 989 int mlxsw_sp_nve_init(struct mlxsw_sp *mlxsw_sp) 990 { 991 struct mlxsw_sp_nve *nve; 992 int err; 993 994 nve = kzalloc(sizeof(*mlxsw_sp->nve), GFP_KERNEL); 995 if (!nve) 996 return -ENOMEM; 997 mlxsw_sp->nve = nve; 998 nve->mlxsw_sp = mlxsw_sp; 999 nve->nve_ops_arr = mlxsw_sp->nve_ops_arr; 1000 1001 err = rhashtable_init(&nve->mc_list_ht, 1002 &mlxsw_sp_nve_mc_list_ht_params); 1003 if (err) 1004 goto err_rhashtable_init; 1005 1006 err = mlxsw_sp_nve_qos_init(mlxsw_sp); 1007 if (err) 1008 goto err_nve_qos_init; 1009 1010 err = mlxsw_sp_nve_ecn_init(mlxsw_sp); 1011 if (err) 1012 goto err_nve_ecn_init; 1013 1014 err = mlxsw_sp_nve_resources_query(mlxsw_sp); 1015 if (err) 1016 goto err_nve_resources_query; 1017 1018 return 0; 1019 1020 err_nve_resources_query: 1021 err_nve_ecn_init: 1022 err_nve_qos_init: 1023 rhashtable_destroy(&nve->mc_list_ht); 1024 err_rhashtable_init: 1025 mlxsw_sp->nve = NULL; 1026 kfree(nve); 1027 return err; 1028 } 1029 1030 void mlxsw_sp_nve_fini(struct mlxsw_sp *mlxsw_sp) 1031 { 1032 WARN_ON(mlxsw_sp->nve->num_nve_tunnels); 1033 rhashtable_destroy(&mlxsw_sp->nve->mc_list_ht); 1034 kfree(mlxsw_sp->nve); 1035 mlxsw_sp->nve = NULL; 1036 } 1037