1 /* 2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34 #include <linux/bpf.h> 35 #include <linux/etherdevice.h> 36 #include <linux/tcp.h> 37 #include <linux/if_vlan.h> 38 #include <linux/delay.h> 39 #include <linux/slab.h> 40 #include <linux/hash.h> 41 #include <net/ip.h> 42 #include <net/busy_poll.h> 43 #include <net/vxlan.h> 44 #include <net/devlink.h> 45 46 #include <linux/mlx4/driver.h> 47 #include <linux/mlx4/device.h> 48 #include <linux/mlx4/cmd.h> 49 #include <linux/mlx4/cq.h> 50 51 #include "mlx4_en.h" 52 #include "en_port.h" 53 54 int mlx4_en_setup_tc(struct net_device *dev, u8 up) 55 { 56 struct mlx4_en_priv *priv = netdev_priv(dev); 57 int i; 58 unsigned int offset = 0; 59 60 if (up && up != MLX4_EN_NUM_UP) 61 return -EINVAL; 62 63 netdev_set_num_tc(dev, up); 64 65 /* Partition Tx queues evenly amongst UP's */ 66 for (i = 0; i < up; i++) { 67 netdev_set_tc_queue(dev, i, priv->num_tx_rings_p_up, offset); 68 offset += priv->num_tx_rings_p_up; 69 } 70 71 #ifdef CONFIG_MLX4_EN_DCB 72 if (!mlx4_is_slave(priv->mdev->dev)) { 73 if (up) { 74 priv->flags |= MLX4_EN_FLAG_DCB_ENABLED; 75 } else { 76 priv->flags &= ~MLX4_EN_FLAG_DCB_ENABLED; 77 priv->cee_params.dcb_cfg.pfc_state = false; 78 } 79 } 80 #endif /* CONFIG_MLX4_EN_DCB */ 81 82 return 0; 83 } 84 85 static int __mlx4_en_setup_tc(struct net_device *dev, u32 handle, __be16 proto, 86 struct tc_to_netdev *tc) 87 { 88 if (tc->type != TC_SETUP_MQPRIO) 89 return -EINVAL; 90 91 return mlx4_en_setup_tc(dev, tc->tc); 92 } 93 94 #ifdef CONFIG_RFS_ACCEL 95 96 struct mlx4_en_filter { 97 struct list_head next; 98 struct work_struct work; 99 100 u8 ip_proto; 101 __be32 src_ip; 102 __be32 dst_ip; 103 __be16 src_port; 104 __be16 dst_port; 105 106 int rxq_index; 107 struct mlx4_en_priv *priv; 108 u32 flow_id; /* RFS infrastructure id */ 109 int id; /* mlx4_en driver id */ 110 u64 reg_id; /* Flow steering API id */ 111 u8 activated; /* Used to prevent expiry before filter 112 * is attached 113 */ 114 struct hlist_node filter_chain; 115 }; 116 117 static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv); 118 119 static enum mlx4_net_trans_rule_id mlx4_ip_proto_to_trans_rule_id(u8 ip_proto) 120 { 121 switch (ip_proto) { 122 case IPPROTO_UDP: 123 return MLX4_NET_TRANS_RULE_ID_UDP; 124 case IPPROTO_TCP: 125 return MLX4_NET_TRANS_RULE_ID_TCP; 126 default: 127 return MLX4_NET_TRANS_RULE_NUM; 128 } 129 }; 130 131 static void mlx4_en_filter_work(struct work_struct *work) 132 { 133 struct mlx4_en_filter *filter = container_of(work, 134 struct mlx4_en_filter, 135 work); 136 struct mlx4_en_priv *priv = filter->priv; 137 struct mlx4_spec_list spec_tcp_udp = { 138 .id = mlx4_ip_proto_to_trans_rule_id(filter->ip_proto), 139 { 140 .tcp_udp = { 141 .dst_port = filter->dst_port, 142 .dst_port_msk = (__force __be16)-1, 143 .src_port = filter->src_port, 144 .src_port_msk = (__force __be16)-1, 145 }, 146 }, 147 }; 148 struct mlx4_spec_list spec_ip = { 149 .id = MLX4_NET_TRANS_RULE_ID_IPV4, 150 { 151 .ipv4 = { 152 .dst_ip = filter->dst_ip, 153 .dst_ip_msk = (__force __be32)-1, 154 .src_ip = filter->src_ip, 155 .src_ip_msk = (__force __be32)-1, 156 }, 157 }, 158 }; 159 struct mlx4_spec_list spec_eth = { 160 .id = MLX4_NET_TRANS_RULE_ID_ETH, 161 }; 162 struct mlx4_net_trans_rule rule = { 163 .list = LIST_HEAD_INIT(rule.list), 164 .queue_mode = MLX4_NET_TRANS_Q_LIFO, 165 .exclusive = 1, 166 .allow_loopback = 1, 167 .promisc_mode = MLX4_FS_REGULAR, 168 .port = priv->port, 169 .priority = MLX4_DOMAIN_RFS, 170 }; 171 int rc; 172 __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); 173 174 if (spec_tcp_udp.id >= MLX4_NET_TRANS_RULE_NUM) { 175 en_warn(priv, "RFS: ignoring unsupported ip protocol (%d)\n", 176 filter->ip_proto); 177 goto ignore; 178 } 179 list_add_tail(&spec_eth.list, &rule.list); 180 list_add_tail(&spec_ip.list, &rule.list); 181 list_add_tail(&spec_tcp_udp.list, &rule.list); 182 183 rule.qpn = priv->rss_map.qps[filter->rxq_index].qpn; 184 memcpy(spec_eth.eth.dst_mac, priv->dev->dev_addr, ETH_ALEN); 185 memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN); 186 187 filter->activated = 0; 188 189 if (filter->reg_id) { 190 rc = mlx4_flow_detach(priv->mdev->dev, filter->reg_id); 191 if (rc && rc != -ENOENT) 192 en_err(priv, "Error detaching flow. rc = %d\n", rc); 193 } 194 195 rc = mlx4_flow_attach(priv->mdev->dev, &rule, &filter->reg_id); 196 if (rc) 197 en_err(priv, "Error attaching flow. err = %d\n", rc); 198 199 ignore: 200 mlx4_en_filter_rfs_expire(priv); 201 202 filter->activated = 1; 203 } 204 205 static inline struct hlist_head * 206 filter_hash_bucket(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip, 207 __be16 src_port, __be16 dst_port) 208 { 209 unsigned long l; 210 int bucket_idx; 211 212 l = (__force unsigned long)src_port | 213 ((__force unsigned long)dst_port << 2); 214 l ^= (__force unsigned long)(src_ip ^ dst_ip); 215 216 bucket_idx = hash_long(l, MLX4_EN_FILTER_HASH_SHIFT); 217 218 return &priv->filter_hash[bucket_idx]; 219 } 220 221 static struct mlx4_en_filter * 222 mlx4_en_filter_alloc(struct mlx4_en_priv *priv, int rxq_index, __be32 src_ip, 223 __be32 dst_ip, u8 ip_proto, __be16 src_port, 224 __be16 dst_port, u32 flow_id) 225 { 226 struct mlx4_en_filter *filter = NULL; 227 228 filter = kzalloc(sizeof(struct mlx4_en_filter), GFP_ATOMIC); 229 if (!filter) 230 return NULL; 231 232 filter->priv = priv; 233 filter->rxq_index = rxq_index; 234 INIT_WORK(&filter->work, mlx4_en_filter_work); 235 236 filter->src_ip = src_ip; 237 filter->dst_ip = dst_ip; 238 filter->ip_proto = ip_proto; 239 filter->src_port = src_port; 240 filter->dst_port = dst_port; 241 242 filter->flow_id = flow_id; 243 244 filter->id = priv->last_filter_id++ % RPS_NO_FILTER; 245 246 list_add_tail(&filter->next, &priv->filters); 247 hlist_add_head(&filter->filter_chain, 248 filter_hash_bucket(priv, src_ip, dst_ip, src_port, 249 dst_port)); 250 251 return filter; 252 } 253 254 static void mlx4_en_filter_free(struct mlx4_en_filter *filter) 255 { 256 struct mlx4_en_priv *priv = filter->priv; 257 int rc; 258 259 list_del(&filter->next); 260 261 rc = mlx4_flow_detach(priv->mdev->dev, filter->reg_id); 262 if (rc && rc != -ENOENT) 263 en_err(priv, "Error detaching flow. rc = %d\n", rc); 264 265 kfree(filter); 266 } 267 268 static inline struct mlx4_en_filter * 269 mlx4_en_filter_find(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip, 270 u8 ip_proto, __be16 src_port, __be16 dst_port) 271 { 272 struct mlx4_en_filter *filter; 273 struct mlx4_en_filter *ret = NULL; 274 275 hlist_for_each_entry(filter, 276 filter_hash_bucket(priv, src_ip, dst_ip, 277 src_port, dst_port), 278 filter_chain) { 279 if (filter->src_ip == src_ip && 280 filter->dst_ip == dst_ip && 281 filter->ip_proto == ip_proto && 282 filter->src_port == src_port && 283 filter->dst_port == dst_port) { 284 ret = filter; 285 break; 286 } 287 } 288 289 return ret; 290 } 291 292 static int 293 mlx4_en_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, 294 u16 rxq_index, u32 flow_id) 295 { 296 struct mlx4_en_priv *priv = netdev_priv(net_dev); 297 struct mlx4_en_filter *filter; 298 const struct iphdr *ip; 299 const __be16 *ports; 300 u8 ip_proto; 301 __be32 src_ip; 302 __be32 dst_ip; 303 __be16 src_port; 304 __be16 dst_port; 305 int nhoff = skb_network_offset(skb); 306 int ret = 0; 307 308 if (skb->protocol != htons(ETH_P_IP)) 309 return -EPROTONOSUPPORT; 310 311 ip = (const struct iphdr *)(skb->data + nhoff); 312 if (ip_is_fragment(ip)) 313 return -EPROTONOSUPPORT; 314 315 if ((ip->protocol != IPPROTO_TCP) && (ip->protocol != IPPROTO_UDP)) 316 return -EPROTONOSUPPORT; 317 ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl); 318 319 ip_proto = ip->protocol; 320 src_ip = ip->saddr; 321 dst_ip = ip->daddr; 322 src_port = ports[0]; 323 dst_port = ports[1]; 324 325 spin_lock_bh(&priv->filters_lock); 326 filter = mlx4_en_filter_find(priv, src_ip, dst_ip, ip_proto, 327 src_port, dst_port); 328 if (filter) { 329 if (filter->rxq_index == rxq_index) 330 goto out; 331 332 filter->rxq_index = rxq_index; 333 } else { 334 filter = mlx4_en_filter_alloc(priv, rxq_index, 335 src_ip, dst_ip, ip_proto, 336 src_port, dst_port, flow_id); 337 if (!filter) { 338 ret = -ENOMEM; 339 goto err; 340 } 341 } 342 343 queue_work(priv->mdev->workqueue, &filter->work); 344 345 out: 346 ret = filter->id; 347 err: 348 spin_unlock_bh(&priv->filters_lock); 349 350 return ret; 351 } 352 353 void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv) 354 { 355 struct mlx4_en_filter *filter, *tmp; 356 LIST_HEAD(del_list); 357 358 spin_lock_bh(&priv->filters_lock); 359 list_for_each_entry_safe(filter, tmp, &priv->filters, next) { 360 list_move(&filter->next, &del_list); 361 hlist_del(&filter->filter_chain); 362 } 363 spin_unlock_bh(&priv->filters_lock); 364 365 list_for_each_entry_safe(filter, tmp, &del_list, next) { 366 cancel_work_sync(&filter->work); 367 mlx4_en_filter_free(filter); 368 } 369 } 370 371 static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv) 372 { 373 struct mlx4_en_filter *filter = NULL, *tmp, *last_filter = NULL; 374 LIST_HEAD(del_list); 375 int i = 0; 376 377 spin_lock_bh(&priv->filters_lock); 378 list_for_each_entry_safe(filter, tmp, &priv->filters, next) { 379 if (i > MLX4_EN_FILTER_EXPIRY_QUOTA) 380 break; 381 382 if (filter->activated && 383 !work_pending(&filter->work) && 384 rps_may_expire_flow(priv->dev, 385 filter->rxq_index, filter->flow_id, 386 filter->id)) { 387 list_move(&filter->next, &del_list); 388 hlist_del(&filter->filter_chain); 389 } else 390 last_filter = filter; 391 392 i++; 393 } 394 395 if (last_filter && (&last_filter->next != priv->filters.next)) 396 list_move(&priv->filters, &last_filter->next); 397 398 spin_unlock_bh(&priv->filters_lock); 399 400 list_for_each_entry_safe(filter, tmp, &del_list, next) 401 mlx4_en_filter_free(filter); 402 } 403 #endif 404 405 static int mlx4_en_vlan_rx_add_vid(struct net_device *dev, 406 __be16 proto, u16 vid) 407 { 408 struct mlx4_en_priv *priv = netdev_priv(dev); 409 struct mlx4_en_dev *mdev = priv->mdev; 410 int err; 411 int idx; 412 413 en_dbg(HW, priv, "adding VLAN:%d\n", vid); 414 415 set_bit(vid, priv->active_vlans); 416 417 /* Add VID to port VLAN filter */ 418 mutex_lock(&mdev->state_lock); 419 if (mdev->device_up && priv->port_up) { 420 err = mlx4_SET_VLAN_FLTR(mdev->dev, priv); 421 if (err) { 422 en_err(priv, "Failed configuring VLAN filter\n"); 423 goto out; 424 } 425 } 426 err = mlx4_register_vlan(mdev->dev, priv->port, vid, &idx); 427 if (err) 428 en_dbg(HW, priv, "Failed adding vlan %d\n", vid); 429 430 out: 431 mutex_unlock(&mdev->state_lock); 432 return err; 433 } 434 435 static int mlx4_en_vlan_rx_kill_vid(struct net_device *dev, 436 __be16 proto, u16 vid) 437 { 438 struct mlx4_en_priv *priv = netdev_priv(dev); 439 struct mlx4_en_dev *mdev = priv->mdev; 440 int err = 0; 441 442 en_dbg(HW, priv, "Killing VID:%d\n", vid); 443 444 clear_bit(vid, priv->active_vlans); 445 446 /* Remove VID from port VLAN filter */ 447 mutex_lock(&mdev->state_lock); 448 mlx4_unregister_vlan(mdev->dev, priv->port, vid); 449 450 if (mdev->device_up && priv->port_up) { 451 err = mlx4_SET_VLAN_FLTR(mdev->dev, priv); 452 if (err) 453 en_err(priv, "Failed configuring VLAN filter\n"); 454 } 455 mutex_unlock(&mdev->state_lock); 456 457 return err; 458 } 459 460 static void mlx4_en_u64_to_mac(unsigned char dst_mac[ETH_ALEN + 2], u64 src_mac) 461 { 462 int i; 463 for (i = ETH_ALEN - 1; i >= 0; --i) { 464 dst_mac[i] = src_mac & 0xff; 465 src_mac >>= 8; 466 } 467 memset(&dst_mac[ETH_ALEN], 0, 2); 468 } 469 470 471 static int mlx4_en_tunnel_steer_add(struct mlx4_en_priv *priv, unsigned char *addr, 472 int qpn, u64 *reg_id) 473 { 474 int err; 475 476 if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN || 477 priv->mdev->dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_STATIC) 478 return 0; /* do nothing */ 479 480 err = mlx4_tunnel_steer_add(priv->mdev->dev, addr, priv->port, qpn, 481 MLX4_DOMAIN_NIC, reg_id); 482 if (err) { 483 en_err(priv, "failed to add vxlan steering rule, err %d\n", err); 484 return err; 485 } 486 en_dbg(DRV, priv, "added vxlan steering rule, mac %pM reg_id %llx\n", addr, *reg_id); 487 return 0; 488 } 489 490 491 static int mlx4_en_uc_steer_add(struct mlx4_en_priv *priv, 492 unsigned char *mac, int *qpn, u64 *reg_id) 493 { 494 struct mlx4_en_dev *mdev = priv->mdev; 495 struct mlx4_dev *dev = mdev->dev; 496 int err; 497 498 switch (dev->caps.steering_mode) { 499 case MLX4_STEERING_MODE_B0: { 500 struct mlx4_qp qp; 501 u8 gid[16] = {0}; 502 503 qp.qpn = *qpn; 504 memcpy(&gid[10], mac, ETH_ALEN); 505 gid[5] = priv->port; 506 507 err = mlx4_unicast_attach(dev, &qp, gid, 0, MLX4_PROT_ETH); 508 break; 509 } 510 case MLX4_STEERING_MODE_DEVICE_MANAGED: { 511 struct mlx4_spec_list spec_eth = { {NULL} }; 512 __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); 513 514 struct mlx4_net_trans_rule rule = { 515 .queue_mode = MLX4_NET_TRANS_Q_FIFO, 516 .exclusive = 0, 517 .allow_loopback = 1, 518 .promisc_mode = MLX4_FS_REGULAR, 519 .priority = MLX4_DOMAIN_NIC, 520 }; 521 522 rule.port = priv->port; 523 rule.qpn = *qpn; 524 INIT_LIST_HEAD(&rule.list); 525 526 spec_eth.id = MLX4_NET_TRANS_RULE_ID_ETH; 527 memcpy(spec_eth.eth.dst_mac, mac, ETH_ALEN); 528 memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN); 529 list_add_tail(&spec_eth.list, &rule.list); 530 531 err = mlx4_flow_attach(dev, &rule, reg_id); 532 break; 533 } 534 default: 535 return -EINVAL; 536 } 537 if (err) 538 en_warn(priv, "Failed Attaching Unicast\n"); 539 540 return err; 541 } 542 543 static void mlx4_en_uc_steer_release(struct mlx4_en_priv *priv, 544 unsigned char *mac, int qpn, u64 reg_id) 545 { 546 struct mlx4_en_dev *mdev = priv->mdev; 547 struct mlx4_dev *dev = mdev->dev; 548 549 switch (dev->caps.steering_mode) { 550 case MLX4_STEERING_MODE_B0: { 551 struct mlx4_qp qp; 552 u8 gid[16] = {0}; 553 554 qp.qpn = qpn; 555 memcpy(&gid[10], mac, ETH_ALEN); 556 gid[5] = priv->port; 557 558 mlx4_unicast_detach(dev, &qp, gid, MLX4_PROT_ETH); 559 break; 560 } 561 case MLX4_STEERING_MODE_DEVICE_MANAGED: { 562 mlx4_flow_detach(dev, reg_id); 563 break; 564 } 565 default: 566 en_err(priv, "Invalid steering mode.\n"); 567 } 568 } 569 570 static int mlx4_en_get_qp(struct mlx4_en_priv *priv) 571 { 572 struct mlx4_en_dev *mdev = priv->mdev; 573 struct mlx4_dev *dev = mdev->dev; 574 int index = 0; 575 int err = 0; 576 int *qpn = &priv->base_qpn; 577 u64 mac = mlx4_mac_to_u64(priv->dev->dev_addr); 578 579 en_dbg(DRV, priv, "Registering MAC: %pM for adding\n", 580 priv->dev->dev_addr); 581 index = mlx4_register_mac(dev, priv->port, mac); 582 if (index < 0) { 583 err = index; 584 en_err(priv, "Failed adding MAC: %pM\n", 585 priv->dev->dev_addr); 586 return err; 587 } 588 589 if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) { 590 int base_qpn = mlx4_get_base_qpn(dev, priv->port); 591 *qpn = base_qpn + index; 592 return 0; 593 } 594 595 err = mlx4_qp_reserve_range(dev, 1, 1, qpn, MLX4_RESERVE_A0_QP); 596 en_dbg(DRV, priv, "Reserved qp %d\n", *qpn); 597 if (err) { 598 en_err(priv, "Failed to reserve qp for mac registration\n"); 599 mlx4_unregister_mac(dev, priv->port, mac); 600 return err; 601 } 602 603 return 0; 604 } 605 606 static void mlx4_en_put_qp(struct mlx4_en_priv *priv) 607 { 608 struct mlx4_en_dev *mdev = priv->mdev; 609 struct mlx4_dev *dev = mdev->dev; 610 int qpn = priv->base_qpn; 611 612 if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) { 613 u64 mac = mlx4_mac_to_u64(priv->dev->dev_addr); 614 en_dbg(DRV, priv, "Registering MAC: %pM for deleting\n", 615 priv->dev->dev_addr); 616 mlx4_unregister_mac(dev, priv->port, mac); 617 } else { 618 en_dbg(DRV, priv, "Releasing qp: port %d, qpn %d\n", 619 priv->port, qpn); 620 mlx4_qp_release_range(dev, qpn, 1); 621 priv->flags &= ~MLX4_EN_FLAG_FORCE_PROMISC; 622 } 623 } 624 625 static int mlx4_en_replace_mac(struct mlx4_en_priv *priv, int qpn, 626 unsigned char *new_mac, unsigned char *prev_mac) 627 { 628 struct mlx4_en_dev *mdev = priv->mdev; 629 struct mlx4_dev *dev = mdev->dev; 630 int err = 0; 631 u64 new_mac_u64 = mlx4_mac_to_u64(new_mac); 632 633 if (dev->caps.steering_mode != MLX4_STEERING_MODE_A0) { 634 struct hlist_head *bucket; 635 unsigned int mac_hash; 636 struct mlx4_mac_entry *entry; 637 struct hlist_node *tmp; 638 u64 prev_mac_u64 = mlx4_mac_to_u64(prev_mac); 639 640 bucket = &priv->mac_hash[prev_mac[MLX4_EN_MAC_HASH_IDX]]; 641 hlist_for_each_entry_safe(entry, tmp, bucket, hlist) { 642 if (ether_addr_equal_64bits(entry->mac, prev_mac)) { 643 mlx4_en_uc_steer_release(priv, entry->mac, 644 qpn, entry->reg_id); 645 mlx4_unregister_mac(dev, priv->port, 646 prev_mac_u64); 647 hlist_del_rcu(&entry->hlist); 648 synchronize_rcu(); 649 memcpy(entry->mac, new_mac, ETH_ALEN); 650 entry->reg_id = 0; 651 mac_hash = new_mac[MLX4_EN_MAC_HASH_IDX]; 652 hlist_add_head_rcu(&entry->hlist, 653 &priv->mac_hash[mac_hash]); 654 mlx4_register_mac(dev, priv->port, new_mac_u64); 655 err = mlx4_en_uc_steer_add(priv, new_mac, 656 &qpn, 657 &entry->reg_id); 658 if (err) 659 return err; 660 if (priv->tunnel_reg_id) { 661 mlx4_flow_detach(priv->mdev->dev, priv->tunnel_reg_id); 662 priv->tunnel_reg_id = 0; 663 } 664 err = mlx4_en_tunnel_steer_add(priv, new_mac, qpn, 665 &priv->tunnel_reg_id); 666 return err; 667 } 668 } 669 return -EINVAL; 670 } 671 672 return __mlx4_replace_mac(dev, priv->port, qpn, new_mac_u64); 673 } 674 675 static int mlx4_en_do_set_mac(struct mlx4_en_priv *priv, 676 unsigned char new_mac[ETH_ALEN + 2]) 677 { 678 int err = 0; 679 680 if (priv->port_up) { 681 /* Remove old MAC and insert the new one */ 682 err = mlx4_en_replace_mac(priv, priv->base_qpn, 683 new_mac, priv->current_mac); 684 if (err) 685 en_err(priv, "Failed changing HW MAC address\n"); 686 } else 687 en_dbg(HW, priv, "Port is down while registering mac, exiting...\n"); 688 689 if (!err) 690 memcpy(priv->current_mac, new_mac, sizeof(priv->current_mac)); 691 692 return err; 693 } 694 695 static int mlx4_en_set_mac(struct net_device *dev, void *addr) 696 { 697 struct mlx4_en_priv *priv = netdev_priv(dev); 698 struct mlx4_en_dev *mdev = priv->mdev; 699 struct sockaddr *saddr = addr; 700 unsigned char new_mac[ETH_ALEN + 2]; 701 int err; 702 703 if (!is_valid_ether_addr(saddr->sa_data)) 704 return -EADDRNOTAVAIL; 705 706 mutex_lock(&mdev->state_lock); 707 memcpy(new_mac, saddr->sa_data, ETH_ALEN); 708 err = mlx4_en_do_set_mac(priv, new_mac); 709 if (!err) 710 memcpy(dev->dev_addr, saddr->sa_data, ETH_ALEN); 711 mutex_unlock(&mdev->state_lock); 712 713 return err; 714 } 715 716 static void mlx4_en_clear_list(struct net_device *dev) 717 { 718 struct mlx4_en_priv *priv = netdev_priv(dev); 719 struct mlx4_en_mc_list *tmp, *mc_to_del; 720 721 list_for_each_entry_safe(mc_to_del, tmp, &priv->mc_list, list) { 722 list_del(&mc_to_del->list); 723 kfree(mc_to_del); 724 } 725 } 726 727 static void mlx4_en_cache_mclist(struct net_device *dev) 728 { 729 struct mlx4_en_priv *priv = netdev_priv(dev); 730 struct netdev_hw_addr *ha; 731 struct mlx4_en_mc_list *tmp; 732 733 mlx4_en_clear_list(dev); 734 netdev_for_each_mc_addr(ha, dev) { 735 tmp = kzalloc(sizeof(struct mlx4_en_mc_list), GFP_ATOMIC); 736 if (!tmp) { 737 mlx4_en_clear_list(dev); 738 return; 739 } 740 memcpy(tmp->addr, ha->addr, ETH_ALEN); 741 list_add_tail(&tmp->list, &priv->mc_list); 742 } 743 } 744 745 static void update_mclist_flags(struct mlx4_en_priv *priv, 746 struct list_head *dst, 747 struct list_head *src) 748 { 749 struct mlx4_en_mc_list *dst_tmp, *src_tmp, *new_mc; 750 bool found; 751 752 /* Find all the entries that should be removed from dst, 753 * These are the entries that are not found in src 754 */ 755 list_for_each_entry(dst_tmp, dst, list) { 756 found = false; 757 list_for_each_entry(src_tmp, src, list) { 758 if (ether_addr_equal(dst_tmp->addr, src_tmp->addr)) { 759 found = true; 760 break; 761 } 762 } 763 if (!found) 764 dst_tmp->action = MCLIST_REM; 765 } 766 767 /* Add entries that exist in src but not in dst 768 * mark them as need to add 769 */ 770 list_for_each_entry(src_tmp, src, list) { 771 found = false; 772 list_for_each_entry(dst_tmp, dst, list) { 773 if (ether_addr_equal(dst_tmp->addr, src_tmp->addr)) { 774 dst_tmp->action = MCLIST_NONE; 775 found = true; 776 break; 777 } 778 } 779 if (!found) { 780 new_mc = kmemdup(src_tmp, 781 sizeof(struct mlx4_en_mc_list), 782 GFP_KERNEL); 783 if (!new_mc) 784 return; 785 786 new_mc->action = MCLIST_ADD; 787 list_add_tail(&new_mc->list, dst); 788 } 789 } 790 } 791 792 static void mlx4_en_set_rx_mode(struct net_device *dev) 793 { 794 struct mlx4_en_priv *priv = netdev_priv(dev); 795 796 if (!priv->port_up) 797 return; 798 799 queue_work(priv->mdev->workqueue, &priv->rx_mode_task); 800 } 801 802 static void mlx4_en_set_promisc_mode(struct mlx4_en_priv *priv, 803 struct mlx4_en_dev *mdev) 804 { 805 int err = 0; 806 807 if (!(priv->flags & MLX4_EN_FLAG_PROMISC)) { 808 if (netif_msg_rx_status(priv)) 809 en_warn(priv, "Entering promiscuous mode\n"); 810 priv->flags |= MLX4_EN_FLAG_PROMISC; 811 812 /* Enable promiscouos mode */ 813 switch (mdev->dev->caps.steering_mode) { 814 case MLX4_STEERING_MODE_DEVICE_MANAGED: 815 err = mlx4_flow_steer_promisc_add(mdev->dev, 816 priv->port, 817 priv->base_qpn, 818 MLX4_FS_ALL_DEFAULT); 819 if (err) 820 en_err(priv, "Failed enabling promiscuous mode\n"); 821 priv->flags |= MLX4_EN_FLAG_MC_PROMISC; 822 break; 823 824 case MLX4_STEERING_MODE_B0: 825 err = mlx4_unicast_promisc_add(mdev->dev, 826 priv->base_qpn, 827 priv->port); 828 if (err) 829 en_err(priv, "Failed enabling unicast promiscuous mode\n"); 830 831 /* Add the default qp number as multicast 832 * promisc 833 */ 834 if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) { 835 err = mlx4_multicast_promisc_add(mdev->dev, 836 priv->base_qpn, 837 priv->port); 838 if (err) 839 en_err(priv, "Failed enabling multicast promiscuous mode\n"); 840 priv->flags |= MLX4_EN_FLAG_MC_PROMISC; 841 } 842 break; 843 844 case MLX4_STEERING_MODE_A0: 845 err = mlx4_SET_PORT_qpn_calc(mdev->dev, 846 priv->port, 847 priv->base_qpn, 848 1); 849 if (err) 850 en_err(priv, "Failed enabling promiscuous mode\n"); 851 break; 852 } 853 854 /* Disable port multicast filter (unconditionally) */ 855 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 856 0, MLX4_MCAST_DISABLE); 857 if (err) 858 en_err(priv, "Failed disabling multicast filter\n"); 859 } 860 } 861 862 static void mlx4_en_clear_promisc_mode(struct mlx4_en_priv *priv, 863 struct mlx4_en_dev *mdev) 864 { 865 int err = 0; 866 867 if (netif_msg_rx_status(priv)) 868 en_warn(priv, "Leaving promiscuous mode\n"); 869 priv->flags &= ~MLX4_EN_FLAG_PROMISC; 870 871 /* Disable promiscouos mode */ 872 switch (mdev->dev->caps.steering_mode) { 873 case MLX4_STEERING_MODE_DEVICE_MANAGED: 874 err = mlx4_flow_steer_promisc_remove(mdev->dev, 875 priv->port, 876 MLX4_FS_ALL_DEFAULT); 877 if (err) 878 en_err(priv, "Failed disabling promiscuous mode\n"); 879 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; 880 break; 881 882 case MLX4_STEERING_MODE_B0: 883 err = mlx4_unicast_promisc_remove(mdev->dev, 884 priv->base_qpn, 885 priv->port); 886 if (err) 887 en_err(priv, "Failed disabling unicast promiscuous mode\n"); 888 /* Disable Multicast promisc */ 889 if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { 890 err = mlx4_multicast_promisc_remove(mdev->dev, 891 priv->base_qpn, 892 priv->port); 893 if (err) 894 en_err(priv, "Failed disabling multicast promiscuous mode\n"); 895 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; 896 } 897 break; 898 899 case MLX4_STEERING_MODE_A0: 900 err = mlx4_SET_PORT_qpn_calc(mdev->dev, 901 priv->port, 902 priv->base_qpn, 0); 903 if (err) 904 en_err(priv, "Failed disabling promiscuous mode\n"); 905 break; 906 } 907 } 908 909 static void mlx4_en_do_multicast(struct mlx4_en_priv *priv, 910 struct net_device *dev, 911 struct mlx4_en_dev *mdev) 912 { 913 struct mlx4_en_mc_list *mclist, *tmp; 914 u64 mcast_addr = 0; 915 u8 mc_list[16] = {0}; 916 int err = 0; 917 918 /* Enable/disable the multicast filter according to IFF_ALLMULTI */ 919 if (dev->flags & IFF_ALLMULTI) { 920 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 921 0, MLX4_MCAST_DISABLE); 922 if (err) 923 en_err(priv, "Failed disabling multicast filter\n"); 924 925 /* Add the default qp number as multicast promisc */ 926 if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) { 927 switch (mdev->dev->caps.steering_mode) { 928 case MLX4_STEERING_MODE_DEVICE_MANAGED: 929 err = mlx4_flow_steer_promisc_add(mdev->dev, 930 priv->port, 931 priv->base_qpn, 932 MLX4_FS_MC_DEFAULT); 933 break; 934 935 case MLX4_STEERING_MODE_B0: 936 err = mlx4_multicast_promisc_add(mdev->dev, 937 priv->base_qpn, 938 priv->port); 939 break; 940 941 case MLX4_STEERING_MODE_A0: 942 break; 943 } 944 if (err) 945 en_err(priv, "Failed entering multicast promisc mode\n"); 946 priv->flags |= MLX4_EN_FLAG_MC_PROMISC; 947 } 948 } else { 949 /* Disable Multicast promisc */ 950 if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { 951 switch (mdev->dev->caps.steering_mode) { 952 case MLX4_STEERING_MODE_DEVICE_MANAGED: 953 err = mlx4_flow_steer_promisc_remove(mdev->dev, 954 priv->port, 955 MLX4_FS_MC_DEFAULT); 956 break; 957 958 case MLX4_STEERING_MODE_B0: 959 err = mlx4_multicast_promisc_remove(mdev->dev, 960 priv->base_qpn, 961 priv->port); 962 break; 963 964 case MLX4_STEERING_MODE_A0: 965 break; 966 } 967 if (err) 968 en_err(priv, "Failed disabling multicast promiscuous mode\n"); 969 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; 970 } 971 972 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 973 0, MLX4_MCAST_DISABLE); 974 if (err) 975 en_err(priv, "Failed disabling multicast filter\n"); 976 977 /* Flush mcast filter and init it with broadcast address */ 978 mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, ETH_BCAST, 979 1, MLX4_MCAST_CONFIG); 980 981 /* Update multicast list - we cache all addresses so they won't 982 * change while HW is updated holding the command semaphor */ 983 netif_addr_lock_bh(dev); 984 mlx4_en_cache_mclist(dev); 985 netif_addr_unlock_bh(dev); 986 list_for_each_entry(mclist, &priv->mc_list, list) { 987 mcast_addr = mlx4_mac_to_u64(mclist->addr); 988 mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 989 mcast_addr, 0, MLX4_MCAST_CONFIG); 990 } 991 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 992 0, MLX4_MCAST_ENABLE); 993 if (err) 994 en_err(priv, "Failed enabling multicast filter\n"); 995 996 update_mclist_flags(priv, &priv->curr_list, &priv->mc_list); 997 list_for_each_entry_safe(mclist, tmp, &priv->curr_list, list) { 998 if (mclist->action == MCLIST_REM) { 999 /* detach this address and delete from list */ 1000 memcpy(&mc_list[10], mclist->addr, ETH_ALEN); 1001 mc_list[5] = priv->port; 1002 err = mlx4_multicast_detach(mdev->dev, 1003 &priv->rss_map.indir_qp, 1004 mc_list, 1005 MLX4_PROT_ETH, 1006 mclist->reg_id); 1007 if (err) 1008 en_err(priv, "Fail to detach multicast address\n"); 1009 1010 if (mclist->tunnel_reg_id) { 1011 err = mlx4_flow_detach(priv->mdev->dev, mclist->tunnel_reg_id); 1012 if (err) 1013 en_err(priv, "Failed to detach multicast address\n"); 1014 } 1015 1016 /* remove from list */ 1017 list_del(&mclist->list); 1018 kfree(mclist); 1019 } else if (mclist->action == MCLIST_ADD) { 1020 /* attach the address */ 1021 memcpy(&mc_list[10], mclist->addr, ETH_ALEN); 1022 /* needed for B0 steering support */ 1023 mc_list[5] = priv->port; 1024 err = mlx4_multicast_attach(mdev->dev, 1025 &priv->rss_map.indir_qp, 1026 mc_list, 1027 priv->port, 0, 1028 MLX4_PROT_ETH, 1029 &mclist->reg_id); 1030 if (err) 1031 en_err(priv, "Fail to attach multicast address\n"); 1032 1033 err = mlx4_en_tunnel_steer_add(priv, &mc_list[10], priv->base_qpn, 1034 &mclist->tunnel_reg_id); 1035 if (err) 1036 en_err(priv, "Failed to attach multicast address\n"); 1037 } 1038 } 1039 } 1040 } 1041 1042 static void mlx4_en_do_uc_filter(struct mlx4_en_priv *priv, 1043 struct net_device *dev, 1044 struct mlx4_en_dev *mdev) 1045 { 1046 struct netdev_hw_addr *ha; 1047 struct mlx4_mac_entry *entry; 1048 struct hlist_node *tmp; 1049 bool found; 1050 u64 mac; 1051 int err = 0; 1052 struct hlist_head *bucket; 1053 unsigned int i; 1054 int removed = 0; 1055 u32 prev_flags; 1056 1057 /* Note that we do not need to protect our mac_hash traversal with rcu, 1058 * since all modification code is protected by mdev->state_lock 1059 */ 1060 1061 /* find what to remove */ 1062 for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) { 1063 bucket = &priv->mac_hash[i]; 1064 hlist_for_each_entry_safe(entry, tmp, bucket, hlist) { 1065 found = false; 1066 netdev_for_each_uc_addr(ha, dev) { 1067 if (ether_addr_equal_64bits(entry->mac, 1068 ha->addr)) { 1069 found = true; 1070 break; 1071 } 1072 } 1073 1074 /* MAC address of the port is not in uc list */ 1075 if (ether_addr_equal_64bits(entry->mac, 1076 priv->current_mac)) 1077 found = true; 1078 1079 if (!found) { 1080 mac = mlx4_mac_to_u64(entry->mac); 1081 mlx4_en_uc_steer_release(priv, entry->mac, 1082 priv->base_qpn, 1083 entry->reg_id); 1084 mlx4_unregister_mac(mdev->dev, priv->port, mac); 1085 1086 hlist_del_rcu(&entry->hlist); 1087 kfree_rcu(entry, rcu); 1088 en_dbg(DRV, priv, "Removed MAC %pM on port:%d\n", 1089 entry->mac, priv->port); 1090 ++removed; 1091 } 1092 } 1093 } 1094 1095 /* if we didn't remove anything, there is no use in trying to add 1096 * again once we are in a forced promisc mode state 1097 */ 1098 if ((priv->flags & MLX4_EN_FLAG_FORCE_PROMISC) && 0 == removed) 1099 return; 1100 1101 prev_flags = priv->flags; 1102 priv->flags &= ~MLX4_EN_FLAG_FORCE_PROMISC; 1103 1104 /* find what to add */ 1105 netdev_for_each_uc_addr(ha, dev) { 1106 found = false; 1107 bucket = &priv->mac_hash[ha->addr[MLX4_EN_MAC_HASH_IDX]]; 1108 hlist_for_each_entry(entry, bucket, hlist) { 1109 if (ether_addr_equal_64bits(entry->mac, ha->addr)) { 1110 found = true; 1111 break; 1112 } 1113 } 1114 1115 if (!found) { 1116 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 1117 if (!entry) { 1118 en_err(priv, "Failed adding MAC %pM on port:%d (out of memory)\n", 1119 ha->addr, priv->port); 1120 priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC; 1121 break; 1122 } 1123 mac = mlx4_mac_to_u64(ha->addr); 1124 memcpy(entry->mac, ha->addr, ETH_ALEN); 1125 err = mlx4_register_mac(mdev->dev, priv->port, mac); 1126 if (err < 0) { 1127 en_err(priv, "Failed registering MAC %pM on port %d: %d\n", 1128 ha->addr, priv->port, err); 1129 kfree(entry); 1130 priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC; 1131 break; 1132 } 1133 err = mlx4_en_uc_steer_add(priv, ha->addr, 1134 &priv->base_qpn, 1135 &entry->reg_id); 1136 if (err) { 1137 en_err(priv, "Failed adding MAC %pM on port %d: %d\n", 1138 ha->addr, priv->port, err); 1139 mlx4_unregister_mac(mdev->dev, priv->port, mac); 1140 kfree(entry); 1141 priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC; 1142 break; 1143 } else { 1144 unsigned int mac_hash; 1145 en_dbg(DRV, priv, "Added MAC %pM on port:%d\n", 1146 ha->addr, priv->port); 1147 mac_hash = ha->addr[MLX4_EN_MAC_HASH_IDX]; 1148 bucket = &priv->mac_hash[mac_hash]; 1149 hlist_add_head_rcu(&entry->hlist, bucket); 1150 } 1151 } 1152 } 1153 1154 if (priv->flags & MLX4_EN_FLAG_FORCE_PROMISC) { 1155 en_warn(priv, "Forcing promiscuous mode on port:%d\n", 1156 priv->port); 1157 } else if (prev_flags & MLX4_EN_FLAG_FORCE_PROMISC) { 1158 en_warn(priv, "Stop forcing promiscuous mode on port:%d\n", 1159 priv->port); 1160 } 1161 } 1162 1163 static void mlx4_en_do_set_rx_mode(struct work_struct *work) 1164 { 1165 struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, 1166 rx_mode_task); 1167 struct mlx4_en_dev *mdev = priv->mdev; 1168 struct net_device *dev = priv->dev; 1169 1170 mutex_lock(&mdev->state_lock); 1171 if (!mdev->device_up) { 1172 en_dbg(HW, priv, "Card is not up, ignoring rx mode change.\n"); 1173 goto out; 1174 } 1175 if (!priv->port_up) { 1176 en_dbg(HW, priv, "Port is down, ignoring rx mode change.\n"); 1177 goto out; 1178 } 1179 1180 if (!netif_carrier_ok(dev)) { 1181 if (!mlx4_en_QUERY_PORT(mdev, priv->port)) { 1182 if (priv->port_state.link_state) { 1183 priv->last_link_state = MLX4_DEV_EVENT_PORT_UP; 1184 netif_carrier_on(dev); 1185 en_dbg(LINK, priv, "Link Up\n"); 1186 } 1187 } 1188 } 1189 1190 if (dev->priv_flags & IFF_UNICAST_FLT) 1191 mlx4_en_do_uc_filter(priv, dev, mdev); 1192 1193 /* Promsicuous mode: disable all filters */ 1194 if ((dev->flags & IFF_PROMISC) || 1195 (priv->flags & MLX4_EN_FLAG_FORCE_PROMISC)) { 1196 mlx4_en_set_promisc_mode(priv, mdev); 1197 goto out; 1198 } 1199 1200 /* Not in promiscuous mode */ 1201 if (priv->flags & MLX4_EN_FLAG_PROMISC) 1202 mlx4_en_clear_promisc_mode(priv, mdev); 1203 1204 mlx4_en_do_multicast(priv, dev, mdev); 1205 out: 1206 mutex_unlock(&mdev->state_lock); 1207 } 1208 1209 #ifdef CONFIG_NET_POLL_CONTROLLER 1210 static void mlx4_en_netpoll(struct net_device *dev) 1211 { 1212 struct mlx4_en_priv *priv = netdev_priv(dev); 1213 struct mlx4_en_cq *cq; 1214 int i; 1215 1216 for (i = 0; i < priv->tx_ring_num; i++) { 1217 cq = priv->tx_cq[i]; 1218 napi_schedule(&cq->napi); 1219 } 1220 } 1221 #endif 1222 1223 static int mlx4_en_set_rss_steer_rules(struct mlx4_en_priv *priv) 1224 { 1225 u64 reg_id; 1226 int err = 0; 1227 int *qpn = &priv->base_qpn; 1228 struct mlx4_mac_entry *entry; 1229 1230 err = mlx4_en_uc_steer_add(priv, priv->dev->dev_addr, qpn, ®_id); 1231 if (err) 1232 return err; 1233 1234 err = mlx4_en_tunnel_steer_add(priv, priv->dev->dev_addr, *qpn, 1235 &priv->tunnel_reg_id); 1236 if (err) 1237 goto tunnel_err; 1238 1239 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 1240 if (!entry) { 1241 err = -ENOMEM; 1242 goto alloc_err; 1243 } 1244 1245 memcpy(entry->mac, priv->dev->dev_addr, sizeof(entry->mac)); 1246 memcpy(priv->current_mac, entry->mac, sizeof(priv->current_mac)); 1247 entry->reg_id = reg_id; 1248 hlist_add_head_rcu(&entry->hlist, 1249 &priv->mac_hash[entry->mac[MLX4_EN_MAC_HASH_IDX]]); 1250 1251 return 0; 1252 1253 alloc_err: 1254 if (priv->tunnel_reg_id) 1255 mlx4_flow_detach(priv->mdev->dev, priv->tunnel_reg_id); 1256 1257 tunnel_err: 1258 mlx4_en_uc_steer_release(priv, priv->dev->dev_addr, *qpn, reg_id); 1259 return err; 1260 } 1261 1262 static void mlx4_en_delete_rss_steer_rules(struct mlx4_en_priv *priv) 1263 { 1264 u64 mac; 1265 unsigned int i; 1266 int qpn = priv->base_qpn; 1267 struct hlist_head *bucket; 1268 struct hlist_node *tmp; 1269 struct mlx4_mac_entry *entry; 1270 1271 for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) { 1272 bucket = &priv->mac_hash[i]; 1273 hlist_for_each_entry_safe(entry, tmp, bucket, hlist) { 1274 mac = mlx4_mac_to_u64(entry->mac); 1275 en_dbg(DRV, priv, "Registering MAC:%pM for deleting\n", 1276 entry->mac); 1277 mlx4_en_uc_steer_release(priv, entry->mac, 1278 qpn, entry->reg_id); 1279 1280 mlx4_unregister_mac(priv->mdev->dev, priv->port, mac); 1281 hlist_del_rcu(&entry->hlist); 1282 kfree_rcu(entry, rcu); 1283 } 1284 } 1285 1286 if (priv->tunnel_reg_id) { 1287 mlx4_flow_detach(priv->mdev->dev, priv->tunnel_reg_id); 1288 priv->tunnel_reg_id = 0; 1289 } 1290 } 1291 1292 static void mlx4_en_tx_timeout(struct net_device *dev) 1293 { 1294 struct mlx4_en_priv *priv = netdev_priv(dev); 1295 struct mlx4_en_dev *mdev = priv->mdev; 1296 int i; 1297 1298 if (netif_msg_timer(priv)) 1299 en_warn(priv, "Tx timeout called on port:%d\n", priv->port); 1300 1301 for (i = 0; i < priv->tx_ring_num; i++) { 1302 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, i))) 1303 continue; 1304 en_warn(priv, "TX timeout on queue: %d, QP: 0x%x, CQ: 0x%x, Cons: 0x%x, Prod: 0x%x\n", 1305 i, priv->tx_ring[i]->qpn, priv->tx_ring[i]->cqn, 1306 priv->tx_ring[i]->cons, priv->tx_ring[i]->prod); 1307 } 1308 1309 priv->port_stats.tx_timeout++; 1310 en_dbg(DRV, priv, "Scheduling watchdog\n"); 1311 queue_work(mdev->workqueue, &priv->watchdog_task); 1312 } 1313 1314 1315 static struct rtnl_link_stats64 * 1316 mlx4_en_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 1317 { 1318 struct mlx4_en_priv *priv = netdev_priv(dev); 1319 1320 spin_lock_bh(&priv->stats_lock); 1321 netdev_stats_to_stats64(stats, &dev->stats); 1322 spin_unlock_bh(&priv->stats_lock); 1323 1324 return stats; 1325 } 1326 1327 static void mlx4_en_set_default_moderation(struct mlx4_en_priv *priv) 1328 { 1329 struct mlx4_en_cq *cq; 1330 int i; 1331 1332 /* If we haven't received a specific coalescing setting 1333 * (module param), we set the moderation parameters as follows: 1334 * - moder_cnt is set to the number of mtu sized packets to 1335 * satisfy our coalescing target. 1336 * - moder_time is set to a fixed value. 1337 */ 1338 priv->rx_frames = MLX4_EN_RX_COAL_TARGET; 1339 priv->rx_usecs = MLX4_EN_RX_COAL_TIME; 1340 priv->tx_frames = MLX4_EN_TX_COAL_PKTS; 1341 priv->tx_usecs = MLX4_EN_TX_COAL_TIME; 1342 en_dbg(INTR, priv, "Default coalesing params for mtu:%d - rx_frames:%d rx_usecs:%d\n", 1343 priv->dev->mtu, priv->rx_frames, priv->rx_usecs); 1344 1345 /* Setup cq moderation params */ 1346 for (i = 0; i < priv->rx_ring_num; i++) { 1347 cq = priv->rx_cq[i]; 1348 cq->moder_cnt = priv->rx_frames; 1349 cq->moder_time = priv->rx_usecs; 1350 priv->last_moder_time[i] = MLX4_EN_AUTO_CONF; 1351 priv->last_moder_packets[i] = 0; 1352 priv->last_moder_bytes[i] = 0; 1353 } 1354 1355 for (i = 0; i < priv->tx_ring_num; i++) { 1356 cq = priv->tx_cq[i]; 1357 cq->moder_cnt = priv->tx_frames; 1358 cq->moder_time = priv->tx_usecs; 1359 } 1360 1361 /* Reset auto-moderation params */ 1362 priv->pkt_rate_low = MLX4_EN_RX_RATE_LOW; 1363 priv->rx_usecs_low = MLX4_EN_RX_COAL_TIME_LOW; 1364 priv->pkt_rate_high = MLX4_EN_RX_RATE_HIGH; 1365 priv->rx_usecs_high = MLX4_EN_RX_COAL_TIME_HIGH; 1366 priv->sample_interval = MLX4_EN_SAMPLE_INTERVAL; 1367 priv->adaptive_rx_coal = 1; 1368 priv->last_moder_jiffies = 0; 1369 priv->last_moder_tx_packets = 0; 1370 } 1371 1372 static void mlx4_en_auto_moderation(struct mlx4_en_priv *priv) 1373 { 1374 unsigned long period = (unsigned long) (jiffies - priv->last_moder_jiffies); 1375 struct mlx4_en_cq *cq; 1376 unsigned long packets; 1377 unsigned long rate; 1378 unsigned long avg_pkt_size; 1379 unsigned long rx_packets; 1380 unsigned long rx_bytes; 1381 unsigned long rx_pkt_diff; 1382 int moder_time; 1383 int ring, err; 1384 1385 if (!priv->adaptive_rx_coal || period < priv->sample_interval * HZ) 1386 return; 1387 1388 for (ring = 0; ring < priv->rx_ring_num; ring++) { 1389 spin_lock_bh(&priv->stats_lock); 1390 rx_packets = priv->rx_ring[ring]->packets; 1391 rx_bytes = priv->rx_ring[ring]->bytes; 1392 spin_unlock_bh(&priv->stats_lock); 1393 1394 rx_pkt_diff = ((unsigned long) (rx_packets - 1395 priv->last_moder_packets[ring])); 1396 packets = rx_pkt_diff; 1397 rate = packets * HZ / period; 1398 avg_pkt_size = packets ? ((unsigned long) (rx_bytes - 1399 priv->last_moder_bytes[ring])) / packets : 0; 1400 1401 /* Apply auto-moderation only when packet rate 1402 * exceeds a rate that it matters */ 1403 if (rate > (MLX4_EN_RX_RATE_THRESH / priv->rx_ring_num) && 1404 avg_pkt_size > MLX4_EN_AVG_PKT_SMALL) { 1405 if (rate < priv->pkt_rate_low) 1406 moder_time = priv->rx_usecs_low; 1407 else if (rate > priv->pkt_rate_high) 1408 moder_time = priv->rx_usecs_high; 1409 else 1410 moder_time = (rate - priv->pkt_rate_low) * 1411 (priv->rx_usecs_high - priv->rx_usecs_low) / 1412 (priv->pkt_rate_high - priv->pkt_rate_low) + 1413 priv->rx_usecs_low; 1414 } else { 1415 moder_time = priv->rx_usecs_low; 1416 } 1417 1418 if (moder_time != priv->last_moder_time[ring]) { 1419 priv->last_moder_time[ring] = moder_time; 1420 cq = priv->rx_cq[ring]; 1421 cq->moder_time = moder_time; 1422 cq->moder_cnt = priv->rx_frames; 1423 err = mlx4_en_set_cq_moder(priv, cq); 1424 if (err) 1425 en_err(priv, "Failed modifying moderation for cq:%d\n", 1426 ring); 1427 } 1428 priv->last_moder_packets[ring] = rx_packets; 1429 priv->last_moder_bytes[ring] = rx_bytes; 1430 } 1431 1432 priv->last_moder_jiffies = jiffies; 1433 } 1434 1435 static void mlx4_en_do_get_stats(struct work_struct *work) 1436 { 1437 struct delayed_work *delay = to_delayed_work(work); 1438 struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv, 1439 stats_task); 1440 struct mlx4_en_dev *mdev = priv->mdev; 1441 int err; 1442 1443 mutex_lock(&mdev->state_lock); 1444 if (mdev->device_up) { 1445 if (priv->port_up) { 1446 err = mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 0); 1447 if (err) 1448 en_dbg(HW, priv, "Could not update stats\n"); 1449 1450 mlx4_en_auto_moderation(priv); 1451 } 1452 1453 queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY); 1454 } 1455 if (mdev->mac_removed[MLX4_MAX_PORTS + 1 - priv->port]) { 1456 mlx4_en_do_set_mac(priv, priv->current_mac); 1457 mdev->mac_removed[MLX4_MAX_PORTS + 1 - priv->port] = 0; 1458 } 1459 mutex_unlock(&mdev->state_lock); 1460 } 1461 1462 /* mlx4_en_service_task - Run service task for tasks that needed to be done 1463 * periodically 1464 */ 1465 static void mlx4_en_service_task(struct work_struct *work) 1466 { 1467 struct delayed_work *delay = to_delayed_work(work); 1468 struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv, 1469 service_task); 1470 struct mlx4_en_dev *mdev = priv->mdev; 1471 1472 mutex_lock(&mdev->state_lock); 1473 if (mdev->device_up) { 1474 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) 1475 mlx4_en_ptp_overflow_check(mdev); 1476 1477 mlx4_en_recover_from_oom(priv); 1478 queue_delayed_work(mdev->workqueue, &priv->service_task, 1479 SERVICE_TASK_DELAY); 1480 } 1481 mutex_unlock(&mdev->state_lock); 1482 } 1483 1484 static void mlx4_en_linkstate(struct work_struct *work) 1485 { 1486 struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, 1487 linkstate_task); 1488 struct mlx4_en_dev *mdev = priv->mdev; 1489 int linkstate = priv->link_state; 1490 1491 mutex_lock(&mdev->state_lock); 1492 /* If observable port state changed set carrier state and 1493 * report to system log */ 1494 if (priv->last_link_state != linkstate) { 1495 if (linkstate == MLX4_DEV_EVENT_PORT_DOWN) { 1496 en_info(priv, "Link Down\n"); 1497 netif_carrier_off(priv->dev); 1498 } else { 1499 en_info(priv, "Link Up\n"); 1500 netif_carrier_on(priv->dev); 1501 } 1502 } 1503 priv->last_link_state = linkstate; 1504 mutex_unlock(&mdev->state_lock); 1505 } 1506 1507 static int mlx4_en_init_affinity_hint(struct mlx4_en_priv *priv, int ring_idx) 1508 { 1509 struct mlx4_en_rx_ring *ring = priv->rx_ring[ring_idx]; 1510 int numa_node = priv->mdev->dev->numa_node; 1511 1512 if (!zalloc_cpumask_var(&ring->affinity_mask, GFP_KERNEL)) 1513 return -ENOMEM; 1514 1515 cpumask_set_cpu(cpumask_local_spread(ring_idx, numa_node), 1516 ring->affinity_mask); 1517 return 0; 1518 } 1519 1520 static void mlx4_en_free_affinity_hint(struct mlx4_en_priv *priv, int ring_idx) 1521 { 1522 free_cpumask_var(priv->rx_ring[ring_idx]->affinity_mask); 1523 } 1524 1525 static void mlx4_en_init_recycle_ring(struct mlx4_en_priv *priv, 1526 int tx_ring_idx) 1527 { 1528 struct mlx4_en_tx_ring *tx_ring = priv->tx_ring[tx_ring_idx]; 1529 int rr_index; 1530 1531 rr_index = (priv->xdp_ring_num - priv->tx_ring_num) + tx_ring_idx; 1532 if (rr_index >= 0) { 1533 tx_ring->free_tx_desc = mlx4_en_recycle_tx_desc; 1534 tx_ring->recycle_ring = priv->rx_ring[rr_index]; 1535 en_dbg(DRV, priv, 1536 "Set tx_ring[%d]->recycle_ring = rx_ring[%d]\n", 1537 tx_ring_idx, rr_index); 1538 } else { 1539 tx_ring->recycle_ring = NULL; 1540 } 1541 } 1542 1543 int mlx4_en_start_port(struct net_device *dev) 1544 { 1545 struct mlx4_en_priv *priv = netdev_priv(dev); 1546 struct mlx4_en_dev *mdev = priv->mdev; 1547 struct mlx4_en_cq *cq; 1548 struct mlx4_en_tx_ring *tx_ring; 1549 int rx_index = 0; 1550 int tx_index = 0; 1551 int err = 0; 1552 int i; 1553 int j; 1554 u8 mc_list[16] = {0}; 1555 1556 if (priv->port_up) { 1557 en_dbg(DRV, priv, "start port called while port already up\n"); 1558 return 0; 1559 } 1560 1561 INIT_LIST_HEAD(&priv->mc_list); 1562 INIT_LIST_HEAD(&priv->curr_list); 1563 INIT_LIST_HEAD(&priv->ethtool_list); 1564 memset(&priv->ethtool_rules[0], 0, 1565 sizeof(struct ethtool_flow_id) * MAX_NUM_OF_FS_RULES); 1566 1567 /* Calculate Rx buf size */ 1568 dev->mtu = min(dev->mtu, priv->max_mtu); 1569 mlx4_en_calc_rx_buf(dev); 1570 en_dbg(DRV, priv, "Rx buf size:%d\n", priv->rx_skb_size); 1571 1572 /* Configure rx cq's and rings */ 1573 err = mlx4_en_activate_rx_rings(priv); 1574 if (err) { 1575 en_err(priv, "Failed to activate RX rings\n"); 1576 return err; 1577 } 1578 for (i = 0; i < priv->rx_ring_num; i++) { 1579 cq = priv->rx_cq[i]; 1580 1581 err = mlx4_en_init_affinity_hint(priv, i); 1582 if (err) { 1583 en_err(priv, "Failed preparing IRQ affinity hint\n"); 1584 goto cq_err; 1585 } 1586 1587 err = mlx4_en_activate_cq(priv, cq, i); 1588 if (err) { 1589 en_err(priv, "Failed activating Rx CQ\n"); 1590 mlx4_en_free_affinity_hint(priv, i); 1591 goto cq_err; 1592 } 1593 1594 for (j = 0; j < cq->size; j++) { 1595 struct mlx4_cqe *cqe = NULL; 1596 1597 cqe = mlx4_en_get_cqe(cq->buf, j, priv->cqe_size) + 1598 priv->cqe_factor; 1599 cqe->owner_sr_opcode = MLX4_CQE_OWNER_MASK; 1600 } 1601 1602 err = mlx4_en_set_cq_moder(priv, cq); 1603 if (err) { 1604 en_err(priv, "Failed setting cq moderation parameters\n"); 1605 mlx4_en_deactivate_cq(priv, cq); 1606 mlx4_en_free_affinity_hint(priv, i); 1607 goto cq_err; 1608 } 1609 mlx4_en_arm_cq(priv, cq); 1610 priv->rx_ring[i]->cqn = cq->mcq.cqn; 1611 ++rx_index; 1612 } 1613 1614 /* Set qp number */ 1615 en_dbg(DRV, priv, "Getting qp number for port %d\n", priv->port); 1616 err = mlx4_en_get_qp(priv); 1617 if (err) { 1618 en_err(priv, "Failed getting eth qp\n"); 1619 goto cq_err; 1620 } 1621 mdev->mac_removed[priv->port] = 0; 1622 1623 priv->counter_index = 1624 mlx4_get_default_counter_index(mdev->dev, priv->port); 1625 1626 err = mlx4_en_config_rss_steer(priv); 1627 if (err) { 1628 en_err(priv, "Failed configuring rss steering\n"); 1629 goto mac_err; 1630 } 1631 1632 err = mlx4_en_create_drop_qp(priv); 1633 if (err) 1634 goto rss_err; 1635 1636 /* Configure tx cq's and rings */ 1637 for (i = 0; i < priv->tx_ring_num; i++) { 1638 /* Configure cq */ 1639 cq = priv->tx_cq[i]; 1640 err = mlx4_en_activate_cq(priv, cq, i); 1641 if (err) { 1642 en_err(priv, "Failed allocating Tx CQ\n"); 1643 goto tx_err; 1644 } 1645 err = mlx4_en_set_cq_moder(priv, cq); 1646 if (err) { 1647 en_err(priv, "Failed setting cq moderation parameters\n"); 1648 mlx4_en_deactivate_cq(priv, cq); 1649 goto tx_err; 1650 } 1651 en_dbg(DRV, priv, "Resetting index of collapsed CQ:%d to -1\n", i); 1652 cq->buf->wqe_index = cpu_to_be16(0xffff); 1653 1654 /* Configure ring */ 1655 tx_ring = priv->tx_ring[i]; 1656 err = mlx4_en_activate_tx_ring(priv, tx_ring, cq->mcq.cqn, 1657 i / priv->num_tx_rings_p_up); 1658 if (err) { 1659 en_err(priv, "Failed allocating Tx ring\n"); 1660 mlx4_en_deactivate_cq(priv, cq); 1661 goto tx_err; 1662 } 1663 tx_ring->tx_queue = netdev_get_tx_queue(dev, i); 1664 1665 mlx4_en_init_recycle_ring(priv, i); 1666 1667 /* Arm CQ for TX completions */ 1668 mlx4_en_arm_cq(priv, cq); 1669 1670 /* Set initial ownership of all Tx TXBBs to SW (1) */ 1671 for (j = 0; j < tx_ring->buf_size; j += STAMP_STRIDE) 1672 *((u32 *) (tx_ring->buf + j)) = 0xffffffff; 1673 ++tx_index; 1674 } 1675 1676 /* Configure port */ 1677 err = mlx4_SET_PORT_general(mdev->dev, priv->port, 1678 priv->rx_skb_size + ETH_FCS_LEN, 1679 priv->prof->tx_pause, 1680 priv->prof->tx_ppp, 1681 priv->prof->rx_pause, 1682 priv->prof->rx_ppp); 1683 if (err) { 1684 en_err(priv, "Failed setting port general configurations for port %d, with error %d\n", 1685 priv->port, err); 1686 goto tx_err; 1687 } 1688 /* Set default qp number */ 1689 err = mlx4_SET_PORT_qpn_calc(mdev->dev, priv->port, priv->base_qpn, 0); 1690 if (err) { 1691 en_err(priv, "Failed setting default qp numbers\n"); 1692 goto tx_err; 1693 } 1694 1695 if (mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) { 1696 err = mlx4_SET_PORT_VXLAN(mdev->dev, priv->port, VXLAN_STEER_BY_OUTER_MAC, 1); 1697 if (err) { 1698 en_err(priv, "Failed setting port L2 tunnel configuration, err %d\n", 1699 err); 1700 goto tx_err; 1701 } 1702 } 1703 1704 /* Init port */ 1705 en_dbg(HW, priv, "Initializing port\n"); 1706 err = mlx4_INIT_PORT(mdev->dev, priv->port); 1707 if (err) { 1708 en_err(priv, "Failed Initializing port\n"); 1709 goto tx_err; 1710 } 1711 1712 /* Set Unicast and VXLAN steering rules */ 1713 if (mdev->dev->caps.steering_mode != MLX4_STEERING_MODE_A0 && 1714 mlx4_en_set_rss_steer_rules(priv)) 1715 mlx4_warn(mdev, "Failed setting steering rules\n"); 1716 1717 /* Attach rx QP to bradcast address */ 1718 eth_broadcast_addr(&mc_list[10]); 1719 mc_list[5] = priv->port; /* needed for B0 steering support */ 1720 if (mlx4_multicast_attach(mdev->dev, &priv->rss_map.indir_qp, mc_list, 1721 priv->port, 0, MLX4_PROT_ETH, 1722 &priv->broadcast_id)) 1723 mlx4_warn(mdev, "Failed Attaching Broadcast\n"); 1724 1725 /* Must redo promiscuous mode setup. */ 1726 priv->flags &= ~(MLX4_EN_FLAG_PROMISC | MLX4_EN_FLAG_MC_PROMISC); 1727 1728 /* Schedule multicast task to populate multicast list */ 1729 queue_work(mdev->workqueue, &priv->rx_mode_task); 1730 1731 if (priv->mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) 1732 udp_tunnel_get_rx_info(dev); 1733 1734 priv->port_up = true; 1735 netif_tx_start_all_queues(dev); 1736 netif_device_attach(dev); 1737 1738 return 0; 1739 1740 tx_err: 1741 while (tx_index--) { 1742 mlx4_en_deactivate_tx_ring(priv, priv->tx_ring[tx_index]); 1743 mlx4_en_deactivate_cq(priv, priv->tx_cq[tx_index]); 1744 } 1745 mlx4_en_destroy_drop_qp(priv); 1746 rss_err: 1747 mlx4_en_release_rss_steer(priv); 1748 mac_err: 1749 mlx4_en_put_qp(priv); 1750 cq_err: 1751 while (rx_index--) { 1752 mlx4_en_deactivate_cq(priv, priv->rx_cq[rx_index]); 1753 mlx4_en_free_affinity_hint(priv, rx_index); 1754 } 1755 for (i = 0; i < priv->rx_ring_num; i++) 1756 mlx4_en_deactivate_rx_ring(priv, priv->rx_ring[i]); 1757 1758 return err; /* need to close devices */ 1759 } 1760 1761 1762 void mlx4_en_stop_port(struct net_device *dev, int detach) 1763 { 1764 struct mlx4_en_priv *priv = netdev_priv(dev); 1765 struct mlx4_en_dev *mdev = priv->mdev; 1766 struct mlx4_en_mc_list *mclist, *tmp; 1767 struct ethtool_flow_id *flow, *tmp_flow; 1768 int i; 1769 u8 mc_list[16] = {0}; 1770 1771 if (!priv->port_up) { 1772 en_dbg(DRV, priv, "stop port called while port already down\n"); 1773 return; 1774 } 1775 1776 /* close port*/ 1777 mlx4_CLOSE_PORT(mdev->dev, priv->port); 1778 1779 /* Synchronize with tx routine */ 1780 netif_tx_lock_bh(dev); 1781 if (detach) 1782 netif_device_detach(dev); 1783 netif_tx_stop_all_queues(dev); 1784 netif_tx_unlock_bh(dev); 1785 1786 netif_tx_disable(dev); 1787 1788 /* Set port as not active */ 1789 priv->port_up = false; 1790 priv->counter_index = MLX4_SINK_COUNTER_INDEX(mdev->dev); 1791 1792 /* Promsicuous mode */ 1793 if (mdev->dev->caps.steering_mode == 1794 MLX4_STEERING_MODE_DEVICE_MANAGED) { 1795 priv->flags &= ~(MLX4_EN_FLAG_PROMISC | 1796 MLX4_EN_FLAG_MC_PROMISC); 1797 mlx4_flow_steer_promisc_remove(mdev->dev, 1798 priv->port, 1799 MLX4_FS_ALL_DEFAULT); 1800 mlx4_flow_steer_promisc_remove(mdev->dev, 1801 priv->port, 1802 MLX4_FS_MC_DEFAULT); 1803 } else if (priv->flags & MLX4_EN_FLAG_PROMISC) { 1804 priv->flags &= ~MLX4_EN_FLAG_PROMISC; 1805 1806 /* Disable promiscouos mode */ 1807 mlx4_unicast_promisc_remove(mdev->dev, priv->base_qpn, 1808 priv->port); 1809 1810 /* Disable Multicast promisc */ 1811 if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { 1812 mlx4_multicast_promisc_remove(mdev->dev, priv->base_qpn, 1813 priv->port); 1814 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; 1815 } 1816 } 1817 1818 /* Detach All multicasts */ 1819 eth_broadcast_addr(&mc_list[10]); 1820 mc_list[5] = priv->port; /* needed for B0 steering support */ 1821 mlx4_multicast_detach(mdev->dev, &priv->rss_map.indir_qp, mc_list, 1822 MLX4_PROT_ETH, priv->broadcast_id); 1823 list_for_each_entry(mclist, &priv->curr_list, list) { 1824 memcpy(&mc_list[10], mclist->addr, ETH_ALEN); 1825 mc_list[5] = priv->port; 1826 mlx4_multicast_detach(mdev->dev, &priv->rss_map.indir_qp, 1827 mc_list, MLX4_PROT_ETH, mclist->reg_id); 1828 if (mclist->tunnel_reg_id) 1829 mlx4_flow_detach(mdev->dev, mclist->tunnel_reg_id); 1830 } 1831 mlx4_en_clear_list(dev); 1832 list_for_each_entry_safe(mclist, tmp, &priv->curr_list, list) { 1833 list_del(&mclist->list); 1834 kfree(mclist); 1835 } 1836 1837 /* Flush multicast filter */ 1838 mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 1, MLX4_MCAST_CONFIG); 1839 1840 /* Remove flow steering rules for the port*/ 1841 if (mdev->dev->caps.steering_mode == 1842 MLX4_STEERING_MODE_DEVICE_MANAGED) { 1843 ASSERT_RTNL(); 1844 list_for_each_entry_safe(flow, tmp_flow, 1845 &priv->ethtool_list, list) { 1846 mlx4_flow_detach(mdev->dev, flow->id); 1847 list_del(&flow->list); 1848 } 1849 } 1850 1851 mlx4_en_destroy_drop_qp(priv); 1852 1853 /* Free TX Rings */ 1854 for (i = 0; i < priv->tx_ring_num; i++) { 1855 mlx4_en_deactivate_tx_ring(priv, priv->tx_ring[i]); 1856 mlx4_en_deactivate_cq(priv, priv->tx_cq[i]); 1857 } 1858 msleep(10); 1859 1860 for (i = 0; i < priv->tx_ring_num; i++) 1861 mlx4_en_free_tx_buf(dev, priv->tx_ring[i]); 1862 1863 if (mdev->dev->caps.steering_mode != MLX4_STEERING_MODE_A0) 1864 mlx4_en_delete_rss_steer_rules(priv); 1865 1866 /* Free RSS qps */ 1867 mlx4_en_release_rss_steer(priv); 1868 1869 /* Unregister Mac address for the port */ 1870 mlx4_en_put_qp(priv); 1871 if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_REASSIGN_MAC_EN)) 1872 mdev->mac_removed[priv->port] = 1; 1873 1874 /* Free RX Rings */ 1875 for (i = 0; i < priv->rx_ring_num; i++) { 1876 struct mlx4_en_cq *cq = priv->rx_cq[i]; 1877 1878 napi_synchronize(&cq->napi); 1879 mlx4_en_deactivate_rx_ring(priv, priv->rx_ring[i]); 1880 mlx4_en_deactivate_cq(priv, cq); 1881 1882 mlx4_en_free_affinity_hint(priv, i); 1883 } 1884 } 1885 1886 static void mlx4_en_restart(struct work_struct *work) 1887 { 1888 struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, 1889 watchdog_task); 1890 struct mlx4_en_dev *mdev = priv->mdev; 1891 struct net_device *dev = priv->dev; 1892 1893 en_dbg(DRV, priv, "Watchdog task called for port %d\n", priv->port); 1894 1895 rtnl_lock(); 1896 mutex_lock(&mdev->state_lock); 1897 if (priv->port_up) { 1898 mlx4_en_stop_port(dev, 1); 1899 if (mlx4_en_start_port(dev)) 1900 en_err(priv, "Failed restarting port %d\n", priv->port); 1901 } 1902 mutex_unlock(&mdev->state_lock); 1903 rtnl_unlock(); 1904 } 1905 1906 static void mlx4_en_clear_stats(struct net_device *dev) 1907 { 1908 struct mlx4_en_priv *priv = netdev_priv(dev); 1909 struct mlx4_en_dev *mdev = priv->mdev; 1910 int i; 1911 1912 if (mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 1)) 1913 en_dbg(HW, priv, "Failed dumping statistics\n"); 1914 1915 memset(&priv->pstats, 0, sizeof(priv->pstats)); 1916 memset(&priv->pkstats, 0, sizeof(priv->pkstats)); 1917 memset(&priv->port_stats, 0, sizeof(priv->port_stats)); 1918 memset(&priv->rx_flowstats, 0, sizeof(priv->rx_flowstats)); 1919 memset(&priv->tx_flowstats, 0, sizeof(priv->tx_flowstats)); 1920 memset(&priv->rx_priority_flowstats, 0, 1921 sizeof(priv->rx_priority_flowstats)); 1922 memset(&priv->tx_priority_flowstats, 0, 1923 sizeof(priv->tx_priority_flowstats)); 1924 memset(&priv->pf_stats, 0, sizeof(priv->pf_stats)); 1925 1926 for (i = 0; i < priv->tx_ring_num; i++) { 1927 priv->tx_ring[i]->bytes = 0; 1928 priv->tx_ring[i]->packets = 0; 1929 priv->tx_ring[i]->tx_csum = 0; 1930 priv->tx_ring[i]->tx_dropped = 0; 1931 priv->tx_ring[i]->queue_stopped = 0; 1932 priv->tx_ring[i]->wake_queue = 0; 1933 priv->tx_ring[i]->tso_packets = 0; 1934 priv->tx_ring[i]->xmit_more = 0; 1935 } 1936 for (i = 0; i < priv->rx_ring_num; i++) { 1937 priv->rx_ring[i]->bytes = 0; 1938 priv->rx_ring[i]->packets = 0; 1939 priv->rx_ring[i]->csum_ok = 0; 1940 priv->rx_ring[i]->csum_none = 0; 1941 priv->rx_ring[i]->csum_complete = 0; 1942 } 1943 } 1944 1945 static int mlx4_en_open(struct net_device *dev) 1946 { 1947 struct mlx4_en_priv *priv = netdev_priv(dev); 1948 struct mlx4_en_dev *mdev = priv->mdev; 1949 int err = 0; 1950 1951 mutex_lock(&mdev->state_lock); 1952 1953 if (!mdev->device_up) { 1954 en_err(priv, "Cannot open - device down/disabled\n"); 1955 err = -EBUSY; 1956 goto out; 1957 } 1958 1959 /* Reset HW statistics and SW counters */ 1960 mlx4_en_clear_stats(dev); 1961 1962 err = mlx4_en_start_port(dev); 1963 if (err) 1964 en_err(priv, "Failed starting port:%d\n", priv->port); 1965 1966 out: 1967 mutex_unlock(&mdev->state_lock); 1968 return err; 1969 } 1970 1971 1972 static int mlx4_en_close(struct net_device *dev) 1973 { 1974 struct mlx4_en_priv *priv = netdev_priv(dev); 1975 struct mlx4_en_dev *mdev = priv->mdev; 1976 1977 en_dbg(IFDOWN, priv, "Close port called\n"); 1978 1979 mutex_lock(&mdev->state_lock); 1980 1981 mlx4_en_stop_port(dev, 0); 1982 netif_carrier_off(dev); 1983 1984 mutex_unlock(&mdev->state_lock); 1985 return 0; 1986 } 1987 1988 static void mlx4_en_free_resources(struct mlx4_en_priv *priv) 1989 { 1990 int i; 1991 1992 #ifdef CONFIG_RFS_ACCEL 1993 priv->dev->rx_cpu_rmap = NULL; 1994 #endif 1995 1996 for (i = 0; i < priv->tx_ring_num; i++) { 1997 if (priv->tx_ring && priv->tx_ring[i]) 1998 mlx4_en_destroy_tx_ring(priv, &priv->tx_ring[i]); 1999 if (priv->tx_cq && priv->tx_cq[i]) 2000 mlx4_en_destroy_cq(priv, &priv->tx_cq[i]); 2001 } 2002 2003 for (i = 0; i < priv->rx_ring_num; i++) { 2004 if (priv->rx_ring[i]) 2005 mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i], 2006 priv->prof->rx_ring_size, priv->stride); 2007 if (priv->rx_cq[i]) 2008 mlx4_en_destroy_cq(priv, &priv->rx_cq[i]); 2009 } 2010 2011 } 2012 2013 static int mlx4_en_alloc_resources(struct mlx4_en_priv *priv) 2014 { 2015 struct mlx4_en_port_profile *prof = priv->prof; 2016 int i; 2017 int node; 2018 2019 /* Create tx Rings */ 2020 for (i = 0; i < priv->tx_ring_num; i++) { 2021 node = cpu_to_node(i % num_online_cpus()); 2022 if (mlx4_en_create_cq(priv, &priv->tx_cq[i], 2023 prof->tx_ring_size, i, TX, node)) 2024 goto err; 2025 2026 if (mlx4_en_create_tx_ring(priv, &priv->tx_ring[i], 2027 prof->tx_ring_size, TXBB_SIZE, 2028 node, i)) 2029 goto err; 2030 } 2031 2032 /* Create rx Rings */ 2033 for (i = 0; i < priv->rx_ring_num; i++) { 2034 node = cpu_to_node(i % num_online_cpus()); 2035 if (mlx4_en_create_cq(priv, &priv->rx_cq[i], 2036 prof->rx_ring_size, i, RX, node)) 2037 goto err; 2038 2039 if (mlx4_en_create_rx_ring(priv, &priv->rx_ring[i], 2040 prof->rx_ring_size, priv->stride, 2041 node)) 2042 goto err; 2043 } 2044 2045 #ifdef CONFIG_RFS_ACCEL 2046 priv->dev->rx_cpu_rmap = mlx4_get_cpu_rmap(priv->mdev->dev, priv->port); 2047 #endif 2048 2049 return 0; 2050 2051 err: 2052 en_err(priv, "Failed to allocate NIC resources\n"); 2053 for (i = 0; i < priv->rx_ring_num; i++) { 2054 if (priv->rx_ring[i]) 2055 mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i], 2056 prof->rx_ring_size, 2057 priv->stride); 2058 if (priv->rx_cq[i]) 2059 mlx4_en_destroy_cq(priv, &priv->rx_cq[i]); 2060 } 2061 for (i = 0; i < priv->tx_ring_num; i++) { 2062 if (priv->tx_ring[i]) 2063 mlx4_en_destroy_tx_ring(priv, &priv->tx_ring[i]); 2064 if (priv->tx_cq[i]) 2065 mlx4_en_destroy_cq(priv, &priv->tx_cq[i]); 2066 } 2067 return -ENOMEM; 2068 } 2069 2070 static void mlx4_en_shutdown(struct net_device *dev) 2071 { 2072 rtnl_lock(); 2073 netif_device_detach(dev); 2074 mlx4_en_close(dev); 2075 rtnl_unlock(); 2076 } 2077 2078 static int mlx4_en_copy_priv(struct mlx4_en_priv *dst, 2079 struct mlx4_en_priv *src, 2080 struct mlx4_en_port_profile *prof) 2081 { 2082 memcpy(&dst->hwtstamp_config, &prof->hwtstamp_config, 2083 sizeof(dst->hwtstamp_config)); 2084 dst->num_tx_rings_p_up = src->mdev->profile.num_tx_rings_p_up; 2085 dst->tx_ring_num = prof->tx_ring_num; 2086 dst->rx_ring_num = prof->rx_ring_num; 2087 dst->flags = prof->flags; 2088 dst->mdev = src->mdev; 2089 dst->port = src->port; 2090 dst->dev = src->dev; 2091 dst->prof = prof; 2092 dst->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + 2093 DS_SIZE * MLX4_EN_MAX_RX_FRAGS); 2094 2095 dst->tx_ring = kzalloc(sizeof(struct mlx4_en_tx_ring *) * MAX_TX_RINGS, 2096 GFP_KERNEL); 2097 if (!dst->tx_ring) 2098 return -ENOMEM; 2099 2100 dst->tx_cq = kzalloc(sizeof(struct mlx4_en_cq *) * MAX_TX_RINGS, 2101 GFP_KERNEL); 2102 if (!dst->tx_cq) { 2103 kfree(dst->tx_ring); 2104 return -ENOMEM; 2105 } 2106 return 0; 2107 } 2108 2109 static void mlx4_en_update_priv(struct mlx4_en_priv *dst, 2110 struct mlx4_en_priv *src) 2111 { 2112 memcpy(dst->rx_ring, src->rx_ring, 2113 sizeof(struct mlx4_en_rx_ring *) * src->rx_ring_num); 2114 memcpy(dst->rx_cq, src->rx_cq, 2115 sizeof(struct mlx4_en_cq *) * src->rx_ring_num); 2116 memcpy(&dst->hwtstamp_config, &src->hwtstamp_config, 2117 sizeof(dst->hwtstamp_config)); 2118 dst->tx_ring_num = src->tx_ring_num; 2119 dst->rx_ring_num = src->rx_ring_num; 2120 dst->tx_ring = src->tx_ring; 2121 dst->tx_cq = src->tx_cq; 2122 memcpy(dst->prof, src->prof, sizeof(struct mlx4_en_port_profile)); 2123 } 2124 2125 int mlx4_en_try_alloc_resources(struct mlx4_en_priv *priv, 2126 struct mlx4_en_priv *tmp, 2127 struct mlx4_en_port_profile *prof) 2128 { 2129 mlx4_en_copy_priv(tmp, priv, prof); 2130 2131 if (mlx4_en_alloc_resources(tmp)) { 2132 en_warn(priv, 2133 "%s: Resource allocation failed, using previous configuration\n", 2134 __func__); 2135 kfree(tmp->tx_ring); 2136 kfree(tmp->tx_cq); 2137 return -ENOMEM; 2138 } 2139 return 0; 2140 } 2141 2142 void mlx4_en_safe_replace_resources(struct mlx4_en_priv *priv, 2143 struct mlx4_en_priv *tmp) 2144 { 2145 mlx4_en_free_resources(priv); 2146 mlx4_en_update_priv(priv, tmp); 2147 } 2148 2149 void mlx4_en_destroy_netdev(struct net_device *dev) 2150 { 2151 struct mlx4_en_priv *priv = netdev_priv(dev); 2152 struct mlx4_en_dev *mdev = priv->mdev; 2153 bool shutdown = mdev->dev->persist->interface_state & 2154 MLX4_INTERFACE_STATE_SHUTDOWN; 2155 2156 en_dbg(DRV, priv, "Destroying netdev on port:%d\n", priv->port); 2157 2158 /* Unregister device - this will close the port if it was up */ 2159 if (priv->registered) { 2160 devlink_port_type_clear(mlx4_get_devlink_port(mdev->dev, 2161 priv->port)); 2162 if (shutdown) 2163 mlx4_en_shutdown(dev); 2164 else 2165 unregister_netdev(dev); 2166 } 2167 2168 if (priv->allocated) 2169 mlx4_free_hwq_res(mdev->dev, &priv->res, MLX4_EN_PAGE_SIZE); 2170 2171 cancel_delayed_work(&priv->stats_task); 2172 cancel_delayed_work(&priv->service_task); 2173 /* flush any pending task for this netdev */ 2174 flush_workqueue(mdev->workqueue); 2175 2176 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) 2177 mlx4_en_remove_timestamp(mdev); 2178 2179 /* Detach the netdev so tasks would not attempt to access it */ 2180 mutex_lock(&mdev->state_lock); 2181 mdev->pndev[priv->port] = NULL; 2182 mdev->upper[priv->port] = NULL; 2183 mutex_unlock(&mdev->state_lock); 2184 2185 #ifdef CONFIG_RFS_ACCEL 2186 mlx4_en_cleanup_filters(priv); 2187 #endif 2188 2189 mlx4_en_free_resources(priv); 2190 2191 kfree(priv->tx_ring); 2192 kfree(priv->tx_cq); 2193 2194 if (!shutdown) 2195 free_netdev(dev); 2196 } 2197 2198 static int mlx4_en_change_mtu(struct net_device *dev, int new_mtu) 2199 { 2200 struct mlx4_en_priv *priv = netdev_priv(dev); 2201 struct mlx4_en_dev *mdev = priv->mdev; 2202 int err = 0; 2203 2204 en_dbg(DRV, priv, "Change MTU called - current:%d new:%d\n", 2205 dev->mtu, new_mtu); 2206 2207 if ((new_mtu < MLX4_EN_MIN_MTU) || (new_mtu > priv->max_mtu)) { 2208 en_err(priv, "Bad MTU size:%d.\n", new_mtu); 2209 return -EPERM; 2210 } 2211 if (priv->xdp_ring_num && MLX4_EN_EFF_MTU(new_mtu) > FRAG_SZ0) { 2212 en_err(priv, "MTU size:%d requires frags but XDP running\n", 2213 new_mtu); 2214 return -EOPNOTSUPP; 2215 } 2216 dev->mtu = new_mtu; 2217 2218 if (netif_running(dev)) { 2219 mutex_lock(&mdev->state_lock); 2220 if (!mdev->device_up) { 2221 /* NIC is probably restarting - let watchdog task reset 2222 * the port */ 2223 en_dbg(DRV, priv, "Change MTU called with card down!?\n"); 2224 } else { 2225 mlx4_en_stop_port(dev, 1); 2226 err = mlx4_en_start_port(dev); 2227 if (err) { 2228 en_err(priv, "Failed restarting port:%d\n", 2229 priv->port); 2230 queue_work(mdev->workqueue, &priv->watchdog_task); 2231 } 2232 } 2233 mutex_unlock(&mdev->state_lock); 2234 } 2235 return 0; 2236 } 2237 2238 static int mlx4_en_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) 2239 { 2240 struct mlx4_en_priv *priv = netdev_priv(dev); 2241 struct mlx4_en_dev *mdev = priv->mdev; 2242 struct hwtstamp_config config; 2243 2244 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 2245 return -EFAULT; 2246 2247 /* reserved for future extensions */ 2248 if (config.flags) 2249 return -EINVAL; 2250 2251 /* device doesn't support time stamping */ 2252 if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS)) 2253 return -EINVAL; 2254 2255 /* TX HW timestamp */ 2256 switch (config.tx_type) { 2257 case HWTSTAMP_TX_OFF: 2258 case HWTSTAMP_TX_ON: 2259 break; 2260 default: 2261 return -ERANGE; 2262 } 2263 2264 /* RX HW timestamp */ 2265 switch (config.rx_filter) { 2266 case HWTSTAMP_FILTER_NONE: 2267 break; 2268 case HWTSTAMP_FILTER_ALL: 2269 case HWTSTAMP_FILTER_SOME: 2270 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 2271 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 2272 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 2273 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 2274 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 2275 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 2276 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 2277 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 2278 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 2279 case HWTSTAMP_FILTER_PTP_V2_EVENT: 2280 case HWTSTAMP_FILTER_PTP_V2_SYNC: 2281 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 2282 config.rx_filter = HWTSTAMP_FILTER_ALL; 2283 break; 2284 default: 2285 return -ERANGE; 2286 } 2287 2288 if (mlx4_en_reset_config(dev, config, dev->features)) { 2289 config.tx_type = HWTSTAMP_TX_OFF; 2290 config.rx_filter = HWTSTAMP_FILTER_NONE; 2291 } 2292 2293 return copy_to_user(ifr->ifr_data, &config, 2294 sizeof(config)) ? -EFAULT : 0; 2295 } 2296 2297 static int mlx4_en_hwtstamp_get(struct net_device *dev, struct ifreq *ifr) 2298 { 2299 struct mlx4_en_priv *priv = netdev_priv(dev); 2300 2301 return copy_to_user(ifr->ifr_data, &priv->hwtstamp_config, 2302 sizeof(priv->hwtstamp_config)) ? -EFAULT : 0; 2303 } 2304 2305 static int mlx4_en_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 2306 { 2307 switch (cmd) { 2308 case SIOCSHWTSTAMP: 2309 return mlx4_en_hwtstamp_set(dev, ifr); 2310 case SIOCGHWTSTAMP: 2311 return mlx4_en_hwtstamp_get(dev, ifr); 2312 default: 2313 return -EOPNOTSUPP; 2314 } 2315 } 2316 2317 static netdev_features_t mlx4_en_fix_features(struct net_device *netdev, 2318 netdev_features_t features) 2319 { 2320 struct mlx4_en_priv *en_priv = netdev_priv(netdev); 2321 struct mlx4_en_dev *mdev = en_priv->mdev; 2322 2323 /* Since there is no support for separate RX C-TAG/S-TAG vlan accel 2324 * enable/disable make sure S-TAG flag is always in same state as 2325 * C-TAG. 2326 */ 2327 if (features & NETIF_F_HW_VLAN_CTAG_RX && 2328 !(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN)) 2329 features |= NETIF_F_HW_VLAN_STAG_RX; 2330 else 2331 features &= ~NETIF_F_HW_VLAN_STAG_RX; 2332 2333 return features; 2334 } 2335 2336 static int mlx4_en_set_features(struct net_device *netdev, 2337 netdev_features_t features) 2338 { 2339 struct mlx4_en_priv *priv = netdev_priv(netdev); 2340 bool reset = false; 2341 int ret = 0; 2342 2343 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_RXFCS)) { 2344 en_info(priv, "Turn %s RX-FCS\n", 2345 (features & NETIF_F_RXFCS) ? "ON" : "OFF"); 2346 reset = true; 2347 } 2348 2349 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_RXALL)) { 2350 u8 ignore_fcs_value = (features & NETIF_F_RXALL) ? 1 : 0; 2351 2352 en_info(priv, "Turn %s RX-ALL\n", 2353 ignore_fcs_value ? "ON" : "OFF"); 2354 ret = mlx4_SET_PORT_fcs_check(priv->mdev->dev, 2355 priv->port, ignore_fcs_value); 2356 if (ret) 2357 return ret; 2358 } 2359 2360 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_HW_VLAN_CTAG_RX)) { 2361 en_info(priv, "Turn %s RX vlan strip offload\n", 2362 (features & NETIF_F_HW_VLAN_CTAG_RX) ? "ON" : "OFF"); 2363 reset = true; 2364 } 2365 2366 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_HW_VLAN_CTAG_TX)) 2367 en_info(priv, "Turn %s TX vlan strip offload\n", 2368 (features & NETIF_F_HW_VLAN_CTAG_TX) ? "ON" : "OFF"); 2369 2370 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_HW_VLAN_STAG_TX)) 2371 en_info(priv, "Turn %s TX S-VLAN strip offload\n", 2372 (features & NETIF_F_HW_VLAN_STAG_TX) ? "ON" : "OFF"); 2373 2374 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_LOOPBACK)) { 2375 en_info(priv, "Turn %s loopback\n", 2376 (features & NETIF_F_LOOPBACK) ? "ON" : "OFF"); 2377 mlx4_en_update_loopback_state(netdev, features); 2378 } 2379 2380 if (reset) { 2381 ret = mlx4_en_reset_config(netdev, priv->hwtstamp_config, 2382 features); 2383 if (ret) 2384 return ret; 2385 } 2386 2387 return 0; 2388 } 2389 2390 static int mlx4_en_set_vf_mac(struct net_device *dev, int queue, u8 *mac) 2391 { 2392 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2393 struct mlx4_en_dev *mdev = en_priv->mdev; 2394 u64 mac_u64 = mlx4_mac_to_u64(mac); 2395 2396 if (is_multicast_ether_addr(mac)) 2397 return -EINVAL; 2398 2399 return mlx4_set_vf_mac(mdev->dev, en_priv->port, queue, mac_u64); 2400 } 2401 2402 static int mlx4_en_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos) 2403 { 2404 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2405 struct mlx4_en_dev *mdev = en_priv->mdev; 2406 2407 return mlx4_set_vf_vlan(mdev->dev, en_priv->port, vf, vlan, qos); 2408 } 2409 2410 static int mlx4_en_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, 2411 int max_tx_rate) 2412 { 2413 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2414 struct mlx4_en_dev *mdev = en_priv->mdev; 2415 2416 return mlx4_set_vf_rate(mdev->dev, en_priv->port, vf, min_tx_rate, 2417 max_tx_rate); 2418 } 2419 2420 static int mlx4_en_set_vf_spoofchk(struct net_device *dev, int vf, bool setting) 2421 { 2422 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2423 struct mlx4_en_dev *mdev = en_priv->mdev; 2424 2425 return mlx4_set_vf_spoofchk(mdev->dev, en_priv->port, vf, setting); 2426 } 2427 2428 static int mlx4_en_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivf) 2429 { 2430 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2431 struct mlx4_en_dev *mdev = en_priv->mdev; 2432 2433 return mlx4_get_vf_config(mdev->dev, en_priv->port, vf, ivf); 2434 } 2435 2436 static int mlx4_en_set_vf_link_state(struct net_device *dev, int vf, int link_state) 2437 { 2438 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2439 struct mlx4_en_dev *mdev = en_priv->mdev; 2440 2441 return mlx4_set_vf_link_state(mdev->dev, en_priv->port, vf, link_state); 2442 } 2443 2444 static int mlx4_en_get_vf_stats(struct net_device *dev, int vf, 2445 struct ifla_vf_stats *vf_stats) 2446 { 2447 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2448 struct mlx4_en_dev *mdev = en_priv->mdev; 2449 2450 return mlx4_get_vf_stats(mdev->dev, en_priv->port, vf, vf_stats); 2451 } 2452 2453 #define PORT_ID_BYTE_LEN 8 2454 static int mlx4_en_get_phys_port_id(struct net_device *dev, 2455 struct netdev_phys_item_id *ppid) 2456 { 2457 struct mlx4_en_priv *priv = netdev_priv(dev); 2458 struct mlx4_dev *mdev = priv->mdev->dev; 2459 int i; 2460 u64 phys_port_id = mdev->caps.phys_port_id[priv->port]; 2461 2462 if (!phys_port_id) 2463 return -EOPNOTSUPP; 2464 2465 ppid->id_len = sizeof(phys_port_id); 2466 for (i = PORT_ID_BYTE_LEN - 1; i >= 0; --i) { 2467 ppid->id[i] = phys_port_id & 0xff; 2468 phys_port_id >>= 8; 2469 } 2470 return 0; 2471 } 2472 2473 static void mlx4_en_add_vxlan_offloads(struct work_struct *work) 2474 { 2475 int ret; 2476 struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, 2477 vxlan_add_task); 2478 2479 ret = mlx4_config_vxlan_port(priv->mdev->dev, priv->vxlan_port); 2480 if (ret) 2481 goto out; 2482 2483 ret = mlx4_SET_PORT_VXLAN(priv->mdev->dev, priv->port, 2484 VXLAN_STEER_BY_OUTER_MAC, 1); 2485 out: 2486 if (ret) { 2487 en_err(priv, "failed setting L2 tunnel configuration ret %d\n", ret); 2488 return; 2489 } 2490 2491 /* set offloads */ 2492 priv->dev->hw_enc_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 2493 NETIF_F_RXCSUM | 2494 NETIF_F_TSO | NETIF_F_TSO6 | 2495 NETIF_F_GSO_UDP_TUNNEL | 2496 NETIF_F_GSO_UDP_TUNNEL_CSUM | 2497 NETIF_F_GSO_PARTIAL; 2498 } 2499 2500 static void mlx4_en_del_vxlan_offloads(struct work_struct *work) 2501 { 2502 int ret; 2503 struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, 2504 vxlan_del_task); 2505 /* unset offloads */ 2506 priv->dev->hw_enc_features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 2507 NETIF_F_RXCSUM | 2508 NETIF_F_TSO | NETIF_F_TSO6 | 2509 NETIF_F_GSO_UDP_TUNNEL | 2510 NETIF_F_GSO_UDP_TUNNEL_CSUM | 2511 NETIF_F_GSO_PARTIAL); 2512 2513 ret = mlx4_SET_PORT_VXLAN(priv->mdev->dev, priv->port, 2514 VXLAN_STEER_BY_OUTER_MAC, 0); 2515 if (ret) 2516 en_err(priv, "failed setting L2 tunnel configuration ret %d\n", ret); 2517 2518 priv->vxlan_port = 0; 2519 } 2520 2521 static void mlx4_en_add_vxlan_port(struct net_device *dev, 2522 struct udp_tunnel_info *ti) 2523 { 2524 struct mlx4_en_priv *priv = netdev_priv(dev); 2525 __be16 port = ti->port; 2526 __be16 current_port; 2527 2528 if (ti->type != UDP_TUNNEL_TYPE_VXLAN) 2529 return; 2530 2531 if (ti->sa_family != AF_INET) 2532 return; 2533 2534 if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) 2535 return; 2536 2537 current_port = priv->vxlan_port; 2538 if (current_port && current_port != port) { 2539 en_warn(priv, "vxlan port %d configured, can't add port %d\n", 2540 ntohs(current_port), ntohs(port)); 2541 return; 2542 } 2543 2544 priv->vxlan_port = port; 2545 queue_work(priv->mdev->workqueue, &priv->vxlan_add_task); 2546 } 2547 2548 static void mlx4_en_del_vxlan_port(struct net_device *dev, 2549 struct udp_tunnel_info *ti) 2550 { 2551 struct mlx4_en_priv *priv = netdev_priv(dev); 2552 __be16 port = ti->port; 2553 __be16 current_port; 2554 2555 if (ti->type != UDP_TUNNEL_TYPE_VXLAN) 2556 return; 2557 2558 if (ti->sa_family != AF_INET) 2559 return; 2560 2561 if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) 2562 return; 2563 2564 current_port = priv->vxlan_port; 2565 if (current_port != port) { 2566 en_dbg(DRV, priv, "vxlan port %d isn't configured, ignoring\n", ntohs(port)); 2567 return; 2568 } 2569 2570 queue_work(priv->mdev->workqueue, &priv->vxlan_del_task); 2571 } 2572 2573 static netdev_features_t mlx4_en_features_check(struct sk_buff *skb, 2574 struct net_device *dev, 2575 netdev_features_t features) 2576 { 2577 features = vlan_features_check(skb, features); 2578 features = vxlan_features_check(skb, features); 2579 2580 /* The ConnectX-3 doesn't support outer IPv6 checksums but it does 2581 * support inner IPv6 checksums and segmentation so we need to 2582 * strip that feature if this is an IPv6 encapsulated frame. 2583 */ 2584 if (skb->encapsulation && 2585 (skb->ip_summed == CHECKSUM_PARTIAL)) { 2586 struct mlx4_en_priv *priv = netdev_priv(dev); 2587 2588 if (!priv->vxlan_port || 2589 (ip_hdr(skb)->version != 4) || 2590 (udp_hdr(skb)->dest != priv->vxlan_port)) 2591 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 2592 } 2593 2594 return features; 2595 } 2596 2597 static int mlx4_en_set_tx_maxrate(struct net_device *dev, int queue_index, u32 maxrate) 2598 { 2599 struct mlx4_en_priv *priv = netdev_priv(dev); 2600 struct mlx4_en_tx_ring *tx_ring = priv->tx_ring[queue_index]; 2601 struct mlx4_update_qp_params params; 2602 int err; 2603 2604 if (!(priv->mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_QP_RATE_LIMIT)) 2605 return -EOPNOTSUPP; 2606 2607 /* rate provided to us in Mbs, check if it fits into 12 bits, if not use Gbs */ 2608 if (maxrate >> 12) { 2609 params.rate_unit = MLX4_QP_RATE_LIMIT_GBS; 2610 params.rate_val = maxrate / 1000; 2611 } else if (maxrate) { 2612 params.rate_unit = MLX4_QP_RATE_LIMIT_MBS; 2613 params.rate_val = maxrate; 2614 } else { /* zero serves to revoke the QP rate-limitation */ 2615 params.rate_unit = 0; 2616 params.rate_val = 0; 2617 } 2618 2619 err = mlx4_update_qp(priv->mdev->dev, tx_ring->qpn, MLX4_UPDATE_QP_RATE_LIMIT, 2620 ¶ms); 2621 return err; 2622 } 2623 2624 static int mlx4_xdp_set(struct net_device *dev, struct bpf_prog *prog) 2625 { 2626 struct mlx4_en_priv *priv = netdev_priv(dev); 2627 struct mlx4_en_dev *mdev = priv->mdev; 2628 struct bpf_prog *old_prog; 2629 int xdp_ring_num; 2630 int port_up = 0; 2631 int err; 2632 int i; 2633 2634 xdp_ring_num = prog ? ALIGN(priv->rx_ring_num, MLX4_EN_NUM_UP) : 0; 2635 2636 /* No need to reconfigure buffers when simply swapping the 2637 * program for a new one. 2638 */ 2639 if (priv->xdp_ring_num == xdp_ring_num) { 2640 if (prog) { 2641 prog = bpf_prog_add(prog, priv->rx_ring_num - 1); 2642 if (IS_ERR(prog)) 2643 return PTR_ERR(prog); 2644 } 2645 for (i = 0; i < priv->rx_ring_num; i++) { 2646 /* This xchg is paired with READ_ONCE in the fastpath */ 2647 old_prog = xchg(&priv->rx_ring[i]->xdp_prog, prog); 2648 if (old_prog) 2649 bpf_prog_put(old_prog); 2650 } 2651 return 0; 2652 } 2653 2654 if (priv->num_frags > 1) { 2655 en_err(priv, "Cannot set XDP if MTU requires multiple frags\n"); 2656 return -EOPNOTSUPP; 2657 } 2658 2659 if (priv->tx_ring_num < xdp_ring_num + MLX4_EN_NUM_UP) { 2660 en_err(priv, 2661 "Minimum %d tx channels required to run XDP\n", 2662 (xdp_ring_num + MLX4_EN_NUM_UP) / MLX4_EN_NUM_UP); 2663 return -EINVAL; 2664 } 2665 2666 if (prog) { 2667 prog = bpf_prog_add(prog, priv->rx_ring_num - 1); 2668 if (IS_ERR(prog)) 2669 return PTR_ERR(prog); 2670 } 2671 2672 mutex_lock(&mdev->state_lock); 2673 if (priv->port_up) { 2674 port_up = 1; 2675 mlx4_en_stop_port(dev, 1); 2676 } 2677 2678 priv->xdp_ring_num = xdp_ring_num; 2679 netif_set_real_num_tx_queues(dev, priv->tx_ring_num - 2680 priv->xdp_ring_num); 2681 2682 for (i = 0; i < priv->rx_ring_num; i++) { 2683 old_prog = xchg(&priv->rx_ring[i]->xdp_prog, prog); 2684 if (old_prog) 2685 bpf_prog_put(old_prog); 2686 } 2687 2688 if (port_up) { 2689 err = mlx4_en_start_port(dev); 2690 if (err) { 2691 en_err(priv, "Failed starting port %d for XDP change\n", 2692 priv->port); 2693 queue_work(mdev->workqueue, &priv->watchdog_task); 2694 } 2695 } 2696 2697 mutex_unlock(&mdev->state_lock); 2698 return 0; 2699 } 2700 2701 static bool mlx4_xdp_attached(struct net_device *dev) 2702 { 2703 struct mlx4_en_priv *priv = netdev_priv(dev); 2704 2705 return !!priv->xdp_ring_num; 2706 } 2707 2708 static int mlx4_xdp(struct net_device *dev, struct netdev_xdp *xdp) 2709 { 2710 switch (xdp->command) { 2711 case XDP_SETUP_PROG: 2712 return mlx4_xdp_set(dev, xdp->prog); 2713 case XDP_QUERY_PROG: 2714 xdp->prog_attached = mlx4_xdp_attached(dev); 2715 return 0; 2716 default: 2717 return -EINVAL; 2718 } 2719 } 2720 2721 static const struct net_device_ops mlx4_netdev_ops = { 2722 .ndo_open = mlx4_en_open, 2723 .ndo_stop = mlx4_en_close, 2724 .ndo_start_xmit = mlx4_en_xmit, 2725 .ndo_select_queue = mlx4_en_select_queue, 2726 .ndo_get_stats64 = mlx4_en_get_stats64, 2727 .ndo_set_rx_mode = mlx4_en_set_rx_mode, 2728 .ndo_set_mac_address = mlx4_en_set_mac, 2729 .ndo_validate_addr = eth_validate_addr, 2730 .ndo_change_mtu = mlx4_en_change_mtu, 2731 .ndo_do_ioctl = mlx4_en_ioctl, 2732 .ndo_tx_timeout = mlx4_en_tx_timeout, 2733 .ndo_vlan_rx_add_vid = mlx4_en_vlan_rx_add_vid, 2734 .ndo_vlan_rx_kill_vid = mlx4_en_vlan_rx_kill_vid, 2735 #ifdef CONFIG_NET_POLL_CONTROLLER 2736 .ndo_poll_controller = mlx4_en_netpoll, 2737 #endif 2738 .ndo_set_features = mlx4_en_set_features, 2739 .ndo_fix_features = mlx4_en_fix_features, 2740 .ndo_setup_tc = __mlx4_en_setup_tc, 2741 #ifdef CONFIG_RFS_ACCEL 2742 .ndo_rx_flow_steer = mlx4_en_filter_rfs, 2743 #endif 2744 .ndo_get_phys_port_id = mlx4_en_get_phys_port_id, 2745 .ndo_udp_tunnel_add = mlx4_en_add_vxlan_port, 2746 .ndo_udp_tunnel_del = mlx4_en_del_vxlan_port, 2747 .ndo_features_check = mlx4_en_features_check, 2748 .ndo_set_tx_maxrate = mlx4_en_set_tx_maxrate, 2749 .ndo_xdp = mlx4_xdp, 2750 }; 2751 2752 static const struct net_device_ops mlx4_netdev_ops_master = { 2753 .ndo_open = mlx4_en_open, 2754 .ndo_stop = mlx4_en_close, 2755 .ndo_start_xmit = mlx4_en_xmit, 2756 .ndo_select_queue = mlx4_en_select_queue, 2757 .ndo_get_stats64 = mlx4_en_get_stats64, 2758 .ndo_set_rx_mode = mlx4_en_set_rx_mode, 2759 .ndo_set_mac_address = mlx4_en_set_mac, 2760 .ndo_validate_addr = eth_validate_addr, 2761 .ndo_change_mtu = mlx4_en_change_mtu, 2762 .ndo_tx_timeout = mlx4_en_tx_timeout, 2763 .ndo_vlan_rx_add_vid = mlx4_en_vlan_rx_add_vid, 2764 .ndo_vlan_rx_kill_vid = mlx4_en_vlan_rx_kill_vid, 2765 .ndo_set_vf_mac = mlx4_en_set_vf_mac, 2766 .ndo_set_vf_vlan = mlx4_en_set_vf_vlan, 2767 .ndo_set_vf_rate = mlx4_en_set_vf_rate, 2768 .ndo_set_vf_spoofchk = mlx4_en_set_vf_spoofchk, 2769 .ndo_set_vf_link_state = mlx4_en_set_vf_link_state, 2770 .ndo_get_vf_stats = mlx4_en_get_vf_stats, 2771 .ndo_get_vf_config = mlx4_en_get_vf_config, 2772 #ifdef CONFIG_NET_POLL_CONTROLLER 2773 .ndo_poll_controller = mlx4_en_netpoll, 2774 #endif 2775 .ndo_set_features = mlx4_en_set_features, 2776 .ndo_fix_features = mlx4_en_fix_features, 2777 .ndo_setup_tc = __mlx4_en_setup_tc, 2778 #ifdef CONFIG_RFS_ACCEL 2779 .ndo_rx_flow_steer = mlx4_en_filter_rfs, 2780 #endif 2781 .ndo_get_phys_port_id = mlx4_en_get_phys_port_id, 2782 .ndo_udp_tunnel_add = mlx4_en_add_vxlan_port, 2783 .ndo_udp_tunnel_del = mlx4_en_del_vxlan_port, 2784 .ndo_features_check = mlx4_en_features_check, 2785 .ndo_set_tx_maxrate = mlx4_en_set_tx_maxrate, 2786 .ndo_xdp = mlx4_xdp, 2787 }; 2788 2789 struct mlx4_en_bond { 2790 struct work_struct work; 2791 struct mlx4_en_priv *priv; 2792 int is_bonded; 2793 struct mlx4_port_map port_map; 2794 }; 2795 2796 static void mlx4_en_bond_work(struct work_struct *work) 2797 { 2798 struct mlx4_en_bond *bond = container_of(work, 2799 struct mlx4_en_bond, 2800 work); 2801 int err = 0; 2802 struct mlx4_dev *dev = bond->priv->mdev->dev; 2803 2804 if (bond->is_bonded) { 2805 if (!mlx4_is_bonded(dev)) { 2806 err = mlx4_bond(dev); 2807 if (err) 2808 en_err(bond->priv, "Fail to bond device\n"); 2809 } 2810 if (!err) { 2811 err = mlx4_port_map_set(dev, &bond->port_map); 2812 if (err) 2813 en_err(bond->priv, "Fail to set port map [%d][%d]: %d\n", 2814 bond->port_map.port1, 2815 bond->port_map.port2, 2816 err); 2817 } 2818 } else if (mlx4_is_bonded(dev)) { 2819 err = mlx4_unbond(dev); 2820 if (err) 2821 en_err(bond->priv, "Fail to unbond device\n"); 2822 } 2823 dev_put(bond->priv->dev); 2824 kfree(bond); 2825 } 2826 2827 static int mlx4_en_queue_bond_work(struct mlx4_en_priv *priv, int is_bonded, 2828 u8 v2p_p1, u8 v2p_p2) 2829 { 2830 struct mlx4_en_bond *bond = NULL; 2831 2832 bond = kzalloc(sizeof(*bond), GFP_ATOMIC); 2833 if (!bond) 2834 return -ENOMEM; 2835 2836 INIT_WORK(&bond->work, mlx4_en_bond_work); 2837 bond->priv = priv; 2838 bond->is_bonded = is_bonded; 2839 bond->port_map.port1 = v2p_p1; 2840 bond->port_map.port2 = v2p_p2; 2841 dev_hold(priv->dev); 2842 queue_work(priv->mdev->workqueue, &bond->work); 2843 return 0; 2844 } 2845 2846 int mlx4_en_netdev_event(struct notifier_block *this, 2847 unsigned long event, void *ptr) 2848 { 2849 struct net_device *ndev = netdev_notifier_info_to_dev(ptr); 2850 u8 port = 0; 2851 struct mlx4_en_dev *mdev; 2852 struct mlx4_dev *dev; 2853 int i, num_eth_ports = 0; 2854 bool do_bond = true; 2855 struct mlx4_en_priv *priv; 2856 u8 v2p_port1 = 0; 2857 u8 v2p_port2 = 0; 2858 2859 if (!net_eq(dev_net(ndev), &init_net)) 2860 return NOTIFY_DONE; 2861 2862 mdev = container_of(this, struct mlx4_en_dev, nb); 2863 dev = mdev->dev; 2864 2865 /* Go into this mode only when two network devices set on two ports 2866 * of the same mlx4 device are slaves of the same bonding master 2867 */ 2868 mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { 2869 ++num_eth_ports; 2870 if (!port && (mdev->pndev[i] == ndev)) 2871 port = i; 2872 mdev->upper[i] = mdev->pndev[i] ? 2873 netdev_master_upper_dev_get(mdev->pndev[i]) : NULL; 2874 /* condition not met: network device is a slave */ 2875 if (!mdev->upper[i]) 2876 do_bond = false; 2877 if (num_eth_ports < 2) 2878 continue; 2879 /* condition not met: same master */ 2880 if (mdev->upper[i] != mdev->upper[i-1]) 2881 do_bond = false; 2882 } 2883 /* condition not met: 2 salves */ 2884 do_bond = (num_eth_ports == 2) ? do_bond : false; 2885 2886 /* handle only events that come with enough info */ 2887 if ((do_bond && (event != NETDEV_BONDING_INFO)) || !port) 2888 return NOTIFY_DONE; 2889 2890 priv = netdev_priv(ndev); 2891 if (do_bond) { 2892 struct netdev_notifier_bonding_info *notifier_info = ptr; 2893 struct netdev_bonding_info *bonding_info = 2894 ¬ifier_info->bonding_info; 2895 2896 /* required mode 1, 2 or 4 */ 2897 if ((bonding_info->master.bond_mode != BOND_MODE_ACTIVEBACKUP) && 2898 (bonding_info->master.bond_mode != BOND_MODE_XOR) && 2899 (bonding_info->master.bond_mode != BOND_MODE_8023AD)) 2900 do_bond = false; 2901 2902 /* require exactly 2 slaves */ 2903 if (bonding_info->master.num_slaves != 2) 2904 do_bond = false; 2905 2906 /* calc v2p */ 2907 if (do_bond) { 2908 if (bonding_info->master.bond_mode == 2909 BOND_MODE_ACTIVEBACKUP) { 2910 /* in active-backup mode virtual ports are 2911 * mapped to the physical port of the active 2912 * slave */ 2913 if (bonding_info->slave.state == 2914 BOND_STATE_BACKUP) { 2915 if (port == 1) { 2916 v2p_port1 = 2; 2917 v2p_port2 = 2; 2918 } else { 2919 v2p_port1 = 1; 2920 v2p_port2 = 1; 2921 } 2922 } else { /* BOND_STATE_ACTIVE */ 2923 if (port == 1) { 2924 v2p_port1 = 1; 2925 v2p_port2 = 1; 2926 } else { 2927 v2p_port1 = 2; 2928 v2p_port2 = 2; 2929 } 2930 } 2931 } else { /* Active-Active */ 2932 /* in active-active mode a virtual port is 2933 * mapped to the native physical port if and only 2934 * if the physical port is up */ 2935 __s8 link = bonding_info->slave.link; 2936 2937 if (port == 1) 2938 v2p_port2 = 2; 2939 else 2940 v2p_port1 = 1; 2941 if ((link == BOND_LINK_UP) || 2942 (link == BOND_LINK_FAIL)) { 2943 if (port == 1) 2944 v2p_port1 = 1; 2945 else 2946 v2p_port2 = 2; 2947 } else { /* BOND_LINK_DOWN || BOND_LINK_BACK */ 2948 if (port == 1) 2949 v2p_port1 = 2; 2950 else 2951 v2p_port2 = 1; 2952 } 2953 } 2954 } 2955 } 2956 2957 mlx4_en_queue_bond_work(priv, do_bond, 2958 v2p_port1, v2p_port2); 2959 2960 return NOTIFY_DONE; 2961 } 2962 2963 void mlx4_en_update_pfc_stats_bitmap(struct mlx4_dev *dev, 2964 struct mlx4_en_stats_bitmap *stats_bitmap, 2965 u8 rx_ppp, u8 rx_pause, 2966 u8 tx_ppp, u8 tx_pause) 2967 { 2968 int last_i = NUM_MAIN_STATS + NUM_PORT_STATS + NUM_PF_STATS; 2969 2970 if (!mlx4_is_slave(dev) && 2971 (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN)) { 2972 mutex_lock(&stats_bitmap->mutex); 2973 bitmap_clear(stats_bitmap->bitmap, last_i, NUM_FLOW_STATS); 2974 2975 if (rx_ppp) 2976 bitmap_set(stats_bitmap->bitmap, last_i, 2977 NUM_FLOW_PRIORITY_STATS_RX); 2978 last_i += NUM_FLOW_PRIORITY_STATS_RX; 2979 2980 if (rx_pause && !(rx_ppp)) 2981 bitmap_set(stats_bitmap->bitmap, last_i, 2982 NUM_FLOW_STATS_RX); 2983 last_i += NUM_FLOW_STATS_RX; 2984 2985 if (tx_ppp) 2986 bitmap_set(stats_bitmap->bitmap, last_i, 2987 NUM_FLOW_PRIORITY_STATS_TX); 2988 last_i += NUM_FLOW_PRIORITY_STATS_TX; 2989 2990 if (tx_pause && !(tx_ppp)) 2991 bitmap_set(stats_bitmap->bitmap, last_i, 2992 NUM_FLOW_STATS_TX); 2993 last_i += NUM_FLOW_STATS_TX; 2994 2995 mutex_unlock(&stats_bitmap->mutex); 2996 } 2997 } 2998 2999 void mlx4_en_set_stats_bitmap(struct mlx4_dev *dev, 3000 struct mlx4_en_stats_bitmap *stats_bitmap, 3001 u8 rx_ppp, u8 rx_pause, 3002 u8 tx_ppp, u8 tx_pause) 3003 { 3004 int last_i = 0; 3005 3006 mutex_init(&stats_bitmap->mutex); 3007 bitmap_zero(stats_bitmap->bitmap, NUM_ALL_STATS); 3008 3009 if (mlx4_is_slave(dev)) { 3010 bitmap_set(stats_bitmap->bitmap, last_i + 3011 MLX4_FIND_NETDEV_STAT(rx_packets), 1); 3012 bitmap_set(stats_bitmap->bitmap, last_i + 3013 MLX4_FIND_NETDEV_STAT(tx_packets), 1); 3014 bitmap_set(stats_bitmap->bitmap, last_i + 3015 MLX4_FIND_NETDEV_STAT(rx_bytes), 1); 3016 bitmap_set(stats_bitmap->bitmap, last_i + 3017 MLX4_FIND_NETDEV_STAT(tx_bytes), 1); 3018 bitmap_set(stats_bitmap->bitmap, last_i + 3019 MLX4_FIND_NETDEV_STAT(rx_dropped), 1); 3020 bitmap_set(stats_bitmap->bitmap, last_i + 3021 MLX4_FIND_NETDEV_STAT(tx_dropped), 1); 3022 } else { 3023 bitmap_set(stats_bitmap->bitmap, last_i, NUM_MAIN_STATS); 3024 } 3025 last_i += NUM_MAIN_STATS; 3026 3027 bitmap_set(stats_bitmap->bitmap, last_i, NUM_PORT_STATS); 3028 last_i += NUM_PORT_STATS; 3029 3030 if (mlx4_is_master(dev)) 3031 bitmap_set(stats_bitmap->bitmap, last_i, 3032 NUM_PF_STATS); 3033 last_i += NUM_PF_STATS; 3034 3035 mlx4_en_update_pfc_stats_bitmap(dev, stats_bitmap, 3036 rx_ppp, rx_pause, 3037 tx_ppp, tx_pause); 3038 last_i += NUM_FLOW_STATS; 3039 3040 if (!mlx4_is_slave(dev)) 3041 bitmap_set(stats_bitmap->bitmap, last_i, NUM_PKT_STATS); 3042 } 3043 3044 int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, 3045 struct mlx4_en_port_profile *prof) 3046 { 3047 struct net_device *dev; 3048 struct mlx4_en_priv *priv; 3049 int i; 3050 int err; 3051 #ifdef CONFIG_MLX4_EN_DCB 3052 struct tc_configuration *tc; 3053 #endif 3054 3055 dev = alloc_etherdev_mqs(sizeof(struct mlx4_en_priv), 3056 MAX_TX_RINGS, MAX_RX_RINGS); 3057 if (dev == NULL) 3058 return -ENOMEM; 3059 3060 netif_set_real_num_tx_queues(dev, prof->tx_ring_num); 3061 netif_set_real_num_rx_queues(dev, prof->rx_ring_num); 3062 3063 SET_NETDEV_DEV(dev, &mdev->dev->persist->pdev->dev); 3064 dev->dev_port = port - 1; 3065 3066 /* 3067 * Initialize driver private data 3068 */ 3069 3070 priv = netdev_priv(dev); 3071 memset(priv, 0, sizeof(struct mlx4_en_priv)); 3072 priv->counter_index = MLX4_SINK_COUNTER_INDEX(mdev->dev); 3073 spin_lock_init(&priv->stats_lock); 3074 INIT_WORK(&priv->rx_mode_task, mlx4_en_do_set_rx_mode); 3075 INIT_WORK(&priv->watchdog_task, mlx4_en_restart); 3076 INIT_WORK(&priv->linkstate_task, mlx4_en_linkstate); 3077 INIT_DELAYED_WORK(&priv->stats_task, mlx4_en_do_get_stats); 3078 INIT_DELAYED_WORK(&priv->service_task, mlx4_en_service_task); 3079 INIT_WORK(&priv->vxlan_add_task, mlx4_en_add_vxlan_offloads); 3080 INIT_WORK(&priv->vxlan_del_task, mlx4_en_del_vxlan_offloads); 3081 #ifdef CONFIG_RFS_ACCEL 3082 INIT_LIST_HEAD(&priv->filters); 3083 spin_lock_init(&priv->filters_lock); 3084 #endif 3085 3086 priv->dev = dev; 3087 priv->mdev = mdev; 3088 priv->ddev = &mdev->pdev->dev; 3089 priv->prof = prof; 3090 priv->port = port; 3091 priv->port_up = false; 3092 priv->flags = prof->flags; 3093 priv->pflags = MLX4_EN_PRIV_FLAGS_BLUEFLAME; 3094 priv->ctrl_flags = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE | 3095 MLX4_WQE_CTRL_SOLICITED); 3096 priv->num_tx_rings_p_up = mdev->profile.num_tx_rings_p_up; 3097 priv->tx_ring_num = prof->tx_ring_num; 3098 priv->tx_work_limit = MLX4_EN_DEFAULT_TX_WORK; 3099 netdev_rss_key_fill(priv->rss_key, sizeof(priv->rss_key)); 3100 3101 priv->tx_ring = kzalloc(sizeof(struct mlx4_en_tx_ring *) * MAX_TX_RINGS, 3102 GFP_KERNEL); 3103 if (!priv->tx_ring) { 3104 err = -ENOMEM; 3105 goto out; 3106 } 3107 priv->tx_cq = kzalloc(sizeof(struct mlx4_en_cq *) * MAX_TX_RINGS, 3108 GFP_KERNEL); 3109 if (!priv->tx_cq) { 3110 err = -ENOMEM; 3111 goto out; 3112 } 3113 priv->rx_ring_num = prof->rx_ring_num; 3114 priv->cqe_factor = (mdev->dev->caps.cqe_size == 64) ? 1 : 0; 3115 priv->cqe_size = mdev->dev->caps.cqe_size; 3116 priv->mac_index = -1; 3117 priv->msg_enable = MLX4_EN_MSG_LEVEL; 3118 #ifdef CONFIG_MLX4_EN_DCB 3119 if (!mlx4_is_slave(priv->mdev->dev)) { 3120 priv->cee_params.dcbx_cap = DCB_CAP_DCBX_VER_CEE | 3121 DCB_CAP_DCBX_HOST | 3122 DCB_CAP_DCBX_VER_IEEE; 3123 priv->flags |= MLX4_EN_DCB_ENABLED; 3124 priv->cee_params.dcb_cfg.pfc_state = false; 3125 3126 for (i = 0; i < MLX4_EN_NUM_UP; i++) { 3127 tc = &priv->cee_params.dcb_cfg.tc_config[i]; 3128 tc->dcb_pfc = pfc_disabled; 3129 } 3130 3131 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETS_CFG) { 3132 dev->dcbnl_ops = &mlx4_en_dcbnl_ops; 3133 } else { 3134 en_info(priv, "enabling only PFC DCB ops\n"); 3135 dev->dcbnl_ops = &mlx4_en_dcbnl_pfc_ops; 3136 } 3137 } 3138 #endif 3139 3140 for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) 3141 INIT_HLIST_HEAD(&priv->mac_hash[i]); 3142 3143 /* Query for default mac and max mtu */ 3144 priv->max_mtu = mdev->dev->caps.eth_mtu_cap[priv->port]; 3145 3146 if (mdev->dev->caps.rx_checksum_flags_port[priv->port] & 3147 MLX4_RX_CSUM_MODE_VAL_NON_TCP_UDP) 3148 priv->flags |= MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP; 3149 3150 /* Set default MAC */ 3151 dev->addr_len = ETH_ALEN; 3152 mlx4_en_u64_to_mac(dev->dev_addr, mdev->dev->caps.def_mac[priv->port]); 3153 if (!is_valid_ether_addr(dev->dev_addr)) { 3154 en_err(priv, "Port: %d, invalid mac burned: %pM, quiting\n", 3155 priv->port, dev->dev_addr); 3156 err = -EINVAL; 3157 goto out; 3158 } else if (mlx4_is_slave(priv->mdev->dev) && 3159 (priv->mdev->dev->port_random_macs & 1 << priv->port)) { 3160 /* Random MAC was assigned in mlx4_slave_cap 3161 * in mlx4_core module 3162 */ 3163 dev->addr_assign_type |= NET_ADDR_RANDOM; 3164 en_warn(priv, "Assigned random MAC address %pM\n", dev->dev_addr); 3165 } 3166 3167 memcpy(priv->current_mac, dev->dev_addr, sizeof(priv->current_mac)); 3168 3169 priv->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + 3170 DS_SIZE * MLX4_EN_MAX_RX_FRAGS); 3171 err = mlx4_en_alloc_resources(priv); 3172 if (err) 3173 goto out; 3174 3175 /* Initialize time stamping config */ 3176 priv->hwtstamp_config.flags = 0; 3177 priv->hwtstamp_config.tx_type = HWTSTAMP_TX_OFF; 3178 priv->hwtstamp_config.rx_filter = HWTSTAMP_FILTER_NONE; 3179 3180 /* Allocate page for receive rings */ 3181 err = mlx4_alloc_hwq_res(mdev->dev, &priv->res, 3182 MLX4_EN_PAGE_SIZE); 3183 if (err) { 3184 en_err(priv, "Failed to allocate page for rx qps\n"); 3185 goto out; 3186 } 3187 priv->allocated = 1; 3188 3189 /* 3190 * Initialize netdev entry points 3191 */ 3192 if (mlx4_is_master(priv->mdev->dev)) 3193 dev->netdev_ops = &mlx4_netdev_ops_master; 3194 else 3195 dev->netdev_ops = &mlx4_netdev_ops; 3196 dev->watchdog_timeo = MLX4_EN_WATCHDOG_TIMEOUT; 3197 netif_set_real_num_tx_queues(dev, priv->tx_ring_num); 3198 netif_set_real_num_rx_queues(dev, priv->rx_ring_num); 3199 3200 dev->ethtool_ops = &mlx4_en_ethtool_ops; 3201 3202 /* 3203 * Set driver features 3204 */ 3205 dev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 3206 if (mdev->LSO_support) 3207 dev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 3208 3209 dev->vlan_features = dev->hw_features; 3210 3211 dev->hw_features |= NETIF_F_RXCSUM | NETIF_F_RXHASH; 3212 dev->features = dev->hw_features | NETIF_F_HIGHDMA | 3213 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | 3214 NETIF_F_HW_VLAN_CTAG_FILTER; 3215 dev->hw_features |= NETIF_F_LOOPBACK | 3216 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; 3217 3218 if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN)) { 3219 dev->features |= NETIF_F_HW_VLAN_STAG_RX | 3220 NETIF_F_HW_VLAN_STAG_FILTER; 3221 dev->hw_features |= NETIF_F_HW_VLAN_STAG_RX; 3222 } 3223 3224 if (mlx4_is_slave(mdev->dev)) { 3225 int phv; 3226 3227 err = get_phv_bit(mdev->dev, port, &phv); 3228 if (!err && phv) { 3229 dev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; 3230 priv->pflags |= MLX4_EN_PRIV_FLAGS_PHV; 3231 } 3232 } else { 3233 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PHV_EN && 3234 !(mdev->dev->caps.flags2 & 3235 MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN)) 3236 dev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; 3237 } 3238 3239 if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) 3240 dev->hw_features |= NETIF_F_RXFCS; 3241 3242 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_IGNORE_FCS) 3243 dev->hw_features |= NETIF_F_RXALL; 3244 3245 if (mdev->dev->caps.steering_mode == 3246 MLX4_STEERING_MODE_DEVICE_MANAGED && 3247 mdev->dev->caps.dmfs_high_steer_mode != MLX4_STEERING_DMFS_A0_STATIC) 3248 dev->hw_features |= NETIF_F_NTUPLE; 3249 3250 if (mdev->dev->caps.steering_mode != MLX4_STEERING_MODE_A0) 3251 dev->priv_flags |= IFF_UNICAST_FLT; 3252 3253 /* Setting a default hash function value */ 3254 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_TOP) { 3255 priv->rss_hash_fn = ETH_RSS_HASH_TOP; 3256 } else if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_XOR) { 3257 priv->rss_hash_fn = ETH_RSS_HASH_XOR; 3258 } else { 3259 en_warn(priv, 3260 "No RSS hash capabilities exposed, using Toeplitz\n"); 3261 priv->rss_hash_fn = ETH_RSS_HASH_TOP; 3262 } 3263 3264 if (mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) { 3265 dev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 3266 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3267 NETIF_F_GSO_PARTIAL; 3268 dev->features |= NETIF_F_GSO_UDP_TUNNEL | 3269 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3270 NETIF_F_GSO_PARTIAL; 3271 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM; 3272 } 3273 3274 mdev->pndev[port] = dev; 3275 mdev->upper[port] = NULL; 3276 3277 netif_carrier_off(dev); 3278 mlx4_en_set_default_moderation(priv); 3279 3280 en_warn(priv, "Using %d TX rings\n", prof->tx_ring_num); 3281 en_warn(priv, "Using %d RX rings\n", prof->rx_ring_num); 3282 3283 mlx4_en_update_loopback_state(priv->dev, priv->dev->features); 3284 3285 /* Configure port */ 3286 mlx4_en_calc_rx_buf(dev); 3287 err = mlx4_SET_PORT_general(mdev->dev, priv->port, 3288 priv->rx_skb_size + ETH_FCS_LEN, 3289 prof->tx_pause, prof->tx_ppp, 3290 prof->rx_pause, prof->rx_ppp); 3291 if (err) { 3292 en_err(priv, "Failed setting port general configurations for port %d, with error %d\n", 3293 priv->port, err); 3294 goto out; 3295 } 3296 3297 if (mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) { 3298 err = mlx4_SET_PORT_VXLAN(mdev->dev, priv->port, VXLAN_STEER_BY_OUTER_MAC, 1); 3299 if (err) { 3300 en_err(priv, "Failed setting port L2 tunnel configuration, err %d\n", 3301 err); 3302 goto out; 3303 } 3304 } 3305 3306 /* Init port */ 3307 en_warn(priv, "Initializing port\n"); 3308 err = mlx4_INIT_PORT(mdev->dev, priv->port); 3309 if (err) { 3310 en_err(priv, "Failed Initializing port\n"); 3311 goto out; 3312 } 3313 queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY); 3314 3315 /* Initialize time stamp mechanism */ 3316 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) 3317 mlx4_en_init_timestamp(mdev); 3318 3319 queue_delayed_work(mdev->workqueue, &priv->service_task, 3320 SERVICE_TASK_DELAY); 3321 3322 mlx4_en_set_stats_bitmap(mdev->dev, &priv->stats_bitmap, 3323 mdev->profile.prof[priv->port].rx_ppp, 3324 mdev->profile.prof[priv->port].rx_pause, 3325 mdev->profile.prof[priv->port].tx_ppp, 3326 mdev->profile.prof[priv->port].tx_pause); 3327 3328 err = register_netdev(dev); 3329 if (err) { 3330 en_err(priv, "Netdev registration failed for port %d\n", port); 3331 goto out; 3332 } 3333 3334 priv->registered = 1; 3335 devlink_port_type_eth_set(mlx4_get_devlink_port(mdev->dev, priv->port), 3336 dev); 3337 3338 return 0; 3339 3340 out: 3341 mlx4_en_destroy_netdev(dev); 3342 return err; 3343 } 3344 3345 int mlx4_en_reset_config(struct net_device *dev, 3346 struct hwtstamp_config ts_config, 3347 netdev_features_t features) 3348 { 3349 struct mlx4_en_priv *priv = netdev_priv(dev); 3350 struct mlx4_en_dev *mdev = priv->mdev; 3351 struct mlx4_en_port_profile new_prof; 3352 struct mlx4_en_priv *tmp; 3353 int port_up = 0; 3354 int err = 0; 3355 3356 if (priv->hwtstamp_config.tx_type == ts_config.tx_type && 3357 priv->hwtstamp_config.rx_filter == ts_config.rx_filter && 3358 !DEV_FEATURE_CHANGED(dev, features, NETIF_F_HW_VLAN_CTAG_RX) && 3359 !DEV_FEATURE_CHANGED(dev, features, NETIF_F_RXFCS)) 3360 return 0; /* Nothing to change */ 3361 3362 if (DEV_FEATURE_CHANGED(dev, features, NETIF_F_HW_VLAN_CTAG_RX) && 3363 (features & NETIF_F_HW_VLAN_CTAG_RX) && 3364 (priv->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE)) { 3365 en_warn(priv, "Can't turn ON rx vlan offload while time-stamping rx filter is ON\n"); 3366 return -EINVAL; 3367 } 3368 3369 tmp = kzalloc(sizeof(*tmp), GFP_KERNEL); 3370 if (!tmp) 3371 return -ENOMEM; 3372 3373 mutex_lock(&mdev->state_lock); 3374 3375 memcpy(&new_prof, priv->prof, sizeof(struct mlx4_en_port_profile)); 3376 memcpy(&new_prof.hwtstamp_config, &ts_config, sizeof(ts_config)); 3377 3378 err = mlx4_en_try_alloc_resources(priv, tmp, &new_prof); 3379 if (err) 3380 goto out; 3381 3382 if (priv->port_up) { 3383 port_up = 1; 3384 mlx4_en_stop_port(dev, 1); 3385 } 3386 3387 en_warn(priv, "Changing device configuration rx filter(%x) rx vlan(%x)\n", 3388 ts_config.rx_filter, 3389 !!(features & NETIF_F_HW_VLAN_CTAG_RX)); 3390 3391 mlx4_en_safe_replace_resources(priv, tmp); 3392 3393 if (DEV_FEATURE_CHANGED(dev, features, NETIF_F_HW_VLAN_CTAG_RX)) { 3394 if (features & NETIF_F_HW_VLAN_CTAG_RX) 3395 dev->features |= NETIF_F_HW_VLAN_CTAG_RX; 3396 else 3397 dev->features &= ~NETIF_F_HW_VLAN_CTAG_RX; 3398 } else if (ts_config.rx_filter == HWTSTAMP_FILTER_NONE) { 3399 /* RX time-stamping is OFF, update the RX vlan offload 3400 * to the latest wanted state 3401 */ 3402 if (dev->wanted_features & NETIF_F_HW_VLAN_CTAG_RX) 3403 dev->features |= NETIF_F_HW_VLAN_CTAG_RX; 3404 else 3405 dev->features &= ~NETIF_F_HW_VLAN_CTAG_RX; 3406 } 3407 3408 if (DEV_FEATURE_CHANGED(dev, features, NETIF_F_RXFCS)) { 3409 if (features & NETIF_F_RXFCS) 3410 dev->features |= NETIF_F_RXFCS; 3411 else 3412 dev->features &= ~NETIF_F_RXFCS; 3413 } 3414 3415 /* RX vlan offload and RX time-stamping can't co-exist ! 3416 * Regardless of the caller's choice, 3417 * Turn Off RX vlan offload in case of time-stamping is ON 3418 */ 3419 if (ts_config.rx_filter != HWTSTAMP_FILTER_NONE) { 3420 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX) 3421 en_warn(priv, "Turning off RX vlan offload since RX time-stamping is ON\n"); 3422 dev->features &= ~NETIF_F_HW_VLAN_CTAG_RX; 3423 } 3424 3425 if (port_up) { 3426 err = mlx4_en_start_port(dev); 3427 if (err) 3428 en_err(priv, "Failed starting port\n"); 3429 } 3430 3431 out: 3432 mutex_unlock(&mdev->state_lock); 3433 kfree(tmp); 3434 if (!err) 3435 netdev_features_change(dev); 3436 return err; 3437 } 3438