1 /* 2 * Copyright (c) 2007 Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 * 32 */ 33 34 #include <linux/bpf.h> 35 #include <linux/etherdevice.h> 36 #include <linux/tcp.h> 37 #include <linux/if_vlan.h> 38 #include <linux/delay.h> 39 #include <linux/slab.h> 40 #include <linux/hash.h> 41 #include <net/ip.h> 42 #include <net/vxlan.h> 43 #include <net/devlink.h> 44 45 #include <linux/mlx4/driver.h> 46 #include <linux/mlx4/device.h> 47 #include <linux/mlx4/cmd.h> 48 #include <linux/mlx4/cq.h> 49 50 #include "mlx4_en.h" 51 #include "en_port.h" 52 53 #define MLX4_EN_MAX_XDP_MTU ((int)(PAGE_SIZE - ETH_HLEN - (2 * VLAN_HLEN) - \ 54 XDP_PACKET_HEADROOM)) 55 56 int mlx4_en_setup_tc(struct net_device *dev, u8 up) 57 { 58 struct mlx4_en_priv *priv = netdev_priv(dev); 59 int i; 60 unsigned int offset = 0; 61 62 if (up && up != MLX4_EN_NUM_UP_HIGH) 63 return -EINVAL; 64 65 netdev_set_num_tc(dev, up); 66 netif_set_real_num_tx_queues(dev, priv->tx_ring_num[TX]); 67 /* Partition Tx queues evenly amongst UP's */ 68 for (i = 0; i < up; i++) { 69 netdev_set_tc_queue(dev, i, priv->num_tx_rings_p_up, offset); 70 offset += priv->num_tx_rings_p_up; 71 } 72 73 #ifdef CONFIG_MLX4_EN_DCB 74 if (!mlx4_is_slave(priv->mdev->dev)) { 75 if (up) { 76 if (priv->dcbx_cap) 77 priv->flags |= MLX4_EN_FLAG_DCB_ENABLED; 78 } else { 79 priv->flags &= ~MLX4_EN_FLAG_DCB_ENABLED; 80 priv->cee_config.pfc_state = false; 81 } 82 } 83 #endif /* CONFIG_MLX4_EN_DCB */ 84 85 return 0; 86 } 87 88 int mlx4_en_alloc_tx_queue_per_tc(struct net_device *dev, u8 tc) 89 { 90 struct mlx4_en_priv *priv = netdev_priv(dev); 91 struct mlx4_en_dev *mdev = priv->mdev; 92 struct mlx4_en_port_profile new_prof; 93 struct mlx4_en_priv *tmp; 94 int total_count; 95 int port_up = 0; 96 int err = 0; 97 98 tmp = kzalloc(sizeof(*tmp), GFP_KERNEL); 99 if (!tmp) 100 return -ENOMEM; 101 102 mutex_lock(&mdev->state_lock); 103 memcpy(&new_prof, priv->prof, sizeof(struct mlx4_en_port_profile)); 104 new_prof.num_up = (tc == 0) ? MLX4_EN_NUM_UP_LOW : 105 MLX4_EN_NUM_UP_HIGH; 106 new_prof.tx_ring_num[TX] = new_prof.num_tx_rings_p_up * 107 new_prof.num_up; 108 total_count = new_prof.tx_ring_num[TX] + new_prof.tx_ring_num[TX_XDP]; 109 if (total_count > MAX_TX_RINGS) { 110 err = -EINVAL; 111 en_err(priv, 112 "Total number of TX and XDP rings (%d) exceeds the maximum supported (%d)\n", 113 total_count, MAX_TX_RINGS); 114 goto out; 115 } 116 err = mlx4_en_try_alloc_resources(priv, tmp, &new_prof, true); 117 if (err) 118 goto out; 119 120 if (priv->port_up) { 121 port_up = 1; 122 mlx4_en_stop_port(dev, 1); 123 } 124 125 mlx4_en_safe_replace_resources(priv, tmp); 126 if (port_up) { 127 err = mlx4_en_start_port(dev); 128 if (err) { 129 en_err(priv, "Failed starting port for setup TC\n"); 130 goto out; 131 } 132 } 133 134 err = mlx4_en_setup_tc(dev, tc); 135 out: 136 mutex_unlock(&mdev->state_lock); 137 kfree(tmp); 138 return err; 139 } 140 141 static int __mlx4_en_setup_tc(struct net_device *dev, enum tc_setup_type type, 142 void *type_data) 143 { 144 struct tc_mqprio_qopt *mqprio = type_data; 145 146 if (type != TC_SETUP_QDISC_MQPRIO) 147 return -EOPNOTSUPP; 148 149 if (mqprio->num_tc && mqprio->num_tc != MLX4_EN_NUM_UP_HIGH) 150 return -EINVAL; 151 152 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 153 154 return mlx4_en_alloc_tx_queue_per_tc(dev, mqprio->num_tc); 155 } 156 157 #ifdef CONFIG_RFS_ACCEL 158 159 struct mlx4_en_filter { 160 struct list_head next; 161 struct work_struct work; 162 163 u8 ip_proto; 164 __be32 src_ip; 165 __be32 dst_ip; 166 __be16 src_port; 167 __be16 dst_port; 168 169 int rxq_index; 170 struct mlx4_en_priv *priv; 171 u32 flow_id; /* RFS infrastructure id */ 172 int id; /* mlx4_en driver id */ 173 u64 reg_id; /* Flow steering API id */ 174 u8 activated; /* Used to prevent expiry before filter 175 * is attached 176 */ 177 struct hlist_node filter_chain; 178 }; 179 180 static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv); 181 182 static enum mlx4_net_trans_rule_id mlx4_ip_proto_to_trans_rule_id(u8 ip_proto) 183 { 184 switch (ip_proto) { 185 case IPPROTO_UDP: 186 return MLX4_NET_TRANS_RULE_ID_UDP; 187 case IPPROTO_TCP: 188 return MLX4_NET_TRANS_RULE_ID_TCP; 189 default: 190 return MLX4_NET_TRANS_RULE_NUM; 191 } 192 }; 193 194 /* Must not acquire state_lock, as its corresponding work_sync 195 * is done under it. 196 */ 197 static void mlx4_en_filter_work(struct work_struct *work) 198 { 199 struct mlx4_en_filter *filter = container_of(work, 200 struct mlx4_en_filter, 201 work); 202 struct mlx4_en_priv *priv = filter->priv; 203 struct mlx4_spec_list spec_tcp_udp = { 204 .id = mlx4_ip_proto_to_trans_rule_id(filter->ip_proto), 205 { 206 .tcp_udp = { 207 .dst_port = filter->dst_port, 208 .dst_port_msk = (__force __be16)-1, 209 .src_port = filter->src_port, 210 .src_port_msk = (__force __be16)-1, 211 }, 212 }, 213 }; 214 struct mlx4_spec_list spec_ip = { 215 .id = MLX4_NET_TRANS_RULE_ID_IPV4, 216 { 217 .ipv4 = { 218 .dst_ip = filter->dst_ip, 219 .dst_ip_msk = (__force __be32)-1, 220 .src_ip = filter->src_ip, 221 .src_ip_msk = (__force __be32)-1, 222 }, 223 }, 224 }; 225 struct mlx4_spec_list spec_eth = { 226 .id = MLX4_NET_TRANS_RULE_ID_ETH, 227 }; 228 struct mlx4_net_trans_rule rule = { 229 .list = LIST_HEAD_INIT(rule.list), 230 .queue_mode = MLX4_NET_TRANS_Q_LIFO, 231 .exclusive = 1, 232 .allow_loopback = 1, 233 .promisc_mode = MLX4_FS_REGULAR, 234 .port = priv->port, 235 .priority = MLX4_DOMAIN_RFS, 236 }; 237 int rc; 238 __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); 239 240 if (spec_tcp_udp.id >= MLX4_NET_TRANS_RULE_NUM) { 241 en_warn(priv, "RFS: ignoring unsupported ip protocol (%d)\n", 242 filter->ip_proto); 243 goto ignore; 244 } 245 list_add_tail(&spec_eth.list, &rule.list); 246 list_add_tail(&spec_ip.list, &rule.list); 247 list_add_tail(&spec_tcp_udp.list, &rule.list); 248 249 rule.qpn = priv->rss_map.qps[filter->rxq_index].qpn; 250 memcpy(spec_eth.eth.dst_mac, priv->dev->dev_addr, ETH_ALEN); 251 memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN); 252 253 filter->activated = 0; 254 255 if (filter->reg_id) { 256 rc = mlx4_flow_detach(priv->mdev->dev, filter->reg_id); 257 if (rc && rc != -ENOENT) 258 en_err(priv, "Error detaching flow. rc = %d\n", rc); 259 } 260 261 rc = mlx4_flow_attach(priv->mdev->dev, &rule, &filter->reg_id); 262 if (rc) 263 en_err(priv, "Error attaching flow. err = %d\n", rc); 264 265 ignore: 266 mlx4_en_filter_rfs_expire(priv); 267 268 filter->activated = 1; 269 } 270 271 static inline struct hlist_head * 272 filter_hash_bucket(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip, 273 __be16 src_port, __be16 dst_port) 274 { 275 unsigned long l; 276 int bucket_idx; 277 278 l = (__force unsigned long)src_port | 279 ((__force unsigned long)dst_port << 2); 280 l ^= (__force unsigned long)(src_ip ^ dst_ip); 281 282 bucket_idx = hash_long(l, MLX4_EN_FILTER_HASH_SHIFT); 283 284 return &priv->filter_hash[bucket_idx]; 285 } 286 287 static struct mlx4_en_filter * 288 mlx4_en_filter_alloc(struct mlx4_en_priv *priv, int rxq_index, __be32 src_ip, 289 __be32 dst_ip, u8 ip_proto, __be16 src_port, 290 __be16 dst_port, u32 flow_id) 291 { 292 struct mlx4_en_filter *filter = NULL; 293 294 filter = kzalloc(sizeof(struct mlx4_en_filter), GFP_ATOMIC); 295 if (!filter) 296 return NULL; 297 298 filter->priv = priv; 299 filter->rxq_index = rxq_index; 300 INIT_WORK(&filter->work, mlx4_en_filter_work); 301 302 filter->src_ip = src_ip; 303 filter->dst_ip = dst_ip; 304 filter->ip_proto = ip_proto; 305 filter->src_port = src_port; 306 filter->dst_port = dst_port; 307 308 filter->flow_id = flow_id; 309 310 filter->id = priv->last_filter_id++ % RPS_NO_FILTER; 311 312 list_add_tail(&filter->next, &priv->filters); 313 hlist_add_head(&filter->filter_chain, 314 filter_hash_bucket(priv, src_ip, dst_ip, src_port, 315 dst_port)); 316 317 return filter; 318 } 319 320 static void mlx4_en_filter_free(struct mlx4_en_filter *filter) 321 { 322 struct mlx4_en_priv *priv = filter->priv; 323 int rc; 324 325 list_del(&filter->next); 326 327 rc = mlx4_flow_detach(priv->mdev->dev, filter->reg_id); 328 if (rc && rc != -ENOENT) 329 en_err(priv, "Error detaching flow. rc = %d\n", rc); 330 331 kfree(filter); 332 } 333 334 static inline struct mlx4_en_filter * 335 mlx4_en_filter_find(struct mlx4_en_priv *priv, __be32 src_ip, __be32 dst_ip, 336 u8 ip_proto, __be16 src_port, __be16 dst_port) 337 { 338 struct mlx4_en_filter *filter; 339 struct mlx4_en_filter *ret = NULL; 340 341 hlist_for_each_entry(filter, 342 filter_hash_bucket(priv, src_ip, dst_ip, 343 src_port, dst_port), 344 filter_chain) { 345 if (filter->src_ip == src_ip && 346 filter->dst_ip == dst_ip && 347 filter->ip_proto == ip_proto && 348 filter->src_port == src_port && 349 filter->dst_port == dst_port) { 350 ret = filter; 351 break; 352 } 353 } 354 355 return ret; 356 } 357 358 static int 359 mlx4_en_filter_rfs(struct net_device *net_dev, const struct sk_buff *skb, 360 u16 rxq_index, u32 flow_id) 361 { 362 struct mlx4_en_priv *priv = netdev_priv(net_dev); 363 struct mlx4_en_filter *filter; 364 const struct iphdr *ip; 365 const __be16 *ports; 366 u8 ip_proto; 367 __be32 src_ip; 368 __be32 dst_ip; 369 __be16 src_port; 370 __be16 dst_port; 371 int nhoff = skb_network_offset(skb); 372 int ret = 0; 373 374 if (skb->protocol != htons(ETH_P_IP)) 375 return -EPROTONOSUPPORT; 376 377 ip = (const struct iphdr *)(skb->data + nhoff); 378 if (ip_is_fragment(ip)) 379 return -EPROTONOSUPPORT; 380 381 if ((ip->protocol != IPPROTO_TCP) && (ip->protocol != IPPROTO_UDP)) 382 return -EPROTONOSUPPORT; 383 ports = (const __be16 *)(skb->data + nhoff + 4 * ip->ihl); 384 385 ip_proto = ip->protocol; 386 src_ip = ip->saddr; 387 dst_ip = ip->daddr; 388 src_port = ports[0]; 389 dst_port = ports[1]; 390 391 spin_lock_bh(&priv->filters_lock); 392 filter = mlx4_en_filter_find(priv, src_ip, dst_ip, ip_proto, 393 src_port, dst_port); 394 if (filter) { 395 if (filter->rxq_index == rxq_index) 396 goto out; 397 398 filter->rxq_index = rxq_index; 399 } else { 400 filter = mlx4_en_filter_alloc(priv, rxq_index, 401 src_ip, dst_ip, ip_proto, 402 src_port, dst_port, flow_id); 403 if (!filter) { 404 ret = -ENOMEM; 405 goto err; 406 } 407 } 408 409 queue_work(priv->mdev->workqueue, &filter->work); 410 411 out: 412 ret = filter->id; 413 err: 414 spin_unlock_bh(&priv->filters_lock); 415 416 return ret; 417 } 418 419 void mlx4_en_cleanup_filters(struct mlx4_en_priv *priv) 420 { 421 struct mlx4_en_filter *filter, *tmp; 422 LIST_HEAD(del_list); 423 424 spin_lock_bh(&priv->filters_lock); 425 list_for_each_entry_safe(filter, tmp, &priv->filters, next) { 426 list_move(&filter->next, &del_list); 427 hlist_del(&filter->filter_chain); 428 } 429 spin_unlock_bh(&priv->filters_lock); 430 431 list_for_each_entry_safe(filter, tmp, &del_list, next) { 432 cancel_work_sync(&filter->work); 433 mlx4_en_filter_free(filter); 434 } 435 } 436 437 static void mlx4_en_filter_rfs_expire(struct mlx4_en_priv *priv) 438 { 439 struct mlx4_en_filter *filter = NULL, *tmp, *last_filter = NULL; 440 LIST_HEAD(del_list); 441 int i = 0; 442 443 spin_lock_bh(&priv->filters_lock); 444 list_for_each_entry_safe(filter, tmp, &priv->filters, next) { 445 if (i > MLX4_EN_FILTER_EXPIRY_QUOTA) 446 break; 447 448 if (filter->activated && 449 !work_pending(&filter->work) && 450 rps_may_expire_flow(priv->dev, 451 filter->rxq_index, filter->flow_id, 452 filter->id)) { 453 list_move(&filter->next, &del_list); 454 hlist_del(&filter->filter_chain); 455 } else 456 last_filter = filter; 457 458 i++; 459 } 460 461 if (last_filter && (&last_filter->next != priv->filters.next)) 462 list_move(&priv->filters, &last_filter->next); 463 464 spin_unlock_bh(&priv->filters_lock); 465 466 list_for_each_entry_safe(filter, tmp, &del_list, next) 467 mlx4_en_filter_free(filter); 468 } 469 #endif 470 471 static int mlx4_en_vlan_rx_add_vid(struct net_device *dev, 472 __be16 proto, u16 vid) 473 { 474 struct mlx4_en_priv *priv = netdev_priv(dev); 475 struct mlx4_en_dev *mdev = priv->mdev; 476 int err; 477 int idx; 478 479 en_dbg(HW, priv, "adding VLAN:%d\n", vid); 480 481 set_bit(vid, priv->active_vlans); 482 483 /* Add VID to port VLAN filter */ 484 mutex_lock(&mdev->state_lock); 485 if (mdev->device_up && priv->port_up) { 486 err = mlx4_SET_VLAN_FLTR(mdev->dev, priv); 487 if (err) { 488 en_err(priv, "Failed configuring VLAN filter\n"); 489 goto out; 490 } 491 } 492 err = mlx4_register_vlan(mdev->dev, priv->port, vid, &idx); 493 if (err) 494 en_dbg(HW, priv, "Failed adding vlan %d\n", vid); 495 496 out: 497 mutex_unlock(&mdev->state_lock); 498 return err; 499 } 500 501 static int mlx4_en_vlan_rx_kill_vid(struct net_device *dev, 502 __be16 proto, u16 vid) 503 { 504 struct mlx4_en_priv *priv = netdev_priv(dev); 505 struct mlx4_en_dev *mdev = priv->mdev; 506 int err = 0; 507 508 en_dbg(HW, priv, "Killing VID:%d\n", vid); 509 510 clear_bit(vid, priv->active_vlans); 511 512 /* Remove VID from port VLAN filter */ 513 mutex_lock(&mdev->state_lock); 514 mlx4_unregister_vlan(mdev->dev, priv->port, vid); 515 516 if (mdev->device_up && priv->port_up) { 517 err = mlx4_SET_VLAN_FLTR(mdev->dev, priv); 518 if (err) 519 en_err(priv, "Failed configuring VLAN filter\n"); 520 } 521 mutex_unlock(&mdev->state_lock); 522 523 return err; 524 } 525 526 static void mlx4_en_u64_to_mac(unsigned char dst_mac[ETH_ALEN + 2], u64 src_mac) 527 { 528 int i; 529 for (i = ETH_ALEN - 1; i >= 0; --i) { 530 dst_mac[i] = src_mac & 0xff; 531 src_mac >>= 8; 532 } 533 memset(&dst_mac[ETH_ALEN], 0, 2); 534 } 535 536 537 static int mlx4_en_tunnel_steer_add(struct mlx4_en_priv *priv, unsigned char *addr, 538 int qpn, u64 *reg_id) 539 { 540 int err; 541 542 if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN || 543 priv->mdev->dev->caps.dmfs_high_steer_mode == MLX4_STEERING_DMFS_A0_STATIC) 544 return 0; /* do nothing */ 545 546 err = mlx4_tunnel_steer_add(priv->mdev->dev, addr, priv->port, qpn, 547 MLX4_DOMAIN_NIC, reg_id); 548 if (err) { 549 en_err(priv, "failed to add vxlan steering rule, err %d\n", err); 550 return err; 551 } 552 en_dbg(DRV, priv, "added vxlan steering rule, mac %pM reg_id %llx\n", addr, *reg_id); 553 return 0; 554 } 555 556 557 static int mlx4_en_uc_steer_add(struct mlx4_en_priv *priv, 558 unsigned char *mac, int *qpn, u64 *reg_id) 559 { 560 struct mlx4_en_dev *mdev = priv->mdev; 561 struct mlx4_dev *dev = mdev->dev; 562 int err; 563 564 switch (dev->caps.steering_mode) { 565 case MLX4_STEERING_MODE_B0: { 566 struct mlx4_qp qp; 567 u8 gid[16] = {0}; 568 569 qp.qpn = *qpn; 570 memcpy(&gid[10], mac, ETH_ALEN); 571 gid[5] = priv->port; 572 573 err = mlx4_unicast_attach(dev, &qp, gid, 0, MLX4_PROT_ETH); 574 break; 575 } 576 case MLX4_STEERING_MODE_DEVICE_MANAGED: { 577 struct mlx4_spec_list spec_eth = { {NULL} }; 578 __be64 mac_mask = cpu_to_be64(MLX4_MAC_MASK << 16); 579 580 struct mlx4_net_trans_rule rule = { 581 .queue_mode = MLX4_NET_TRANS_Q_FIFO, 582 .exclusive = 0, 583 .allow_loopback = 1, 584 .promisc_mode = MLX4_FS_REGULAR, 585 .priority = MLX4_DOMAIN_NIC, 586 }; 587 588 rule.port = priv->port; 589 rule.qpn = *qpn; 590 INIT_LIST_HEAD(&rule.list); 591 592 spec_eth.id = MLX4_NET_TRANS_RULE_ID_ETH; 593 memcpy(spec_eth.eth.dst_mac, mac, ETH_ALEN); 594 memcpy(spec_eth.eth.dst_mac_msk, &mac_mask, ETH_ALEN); 595 list_add_tail(&spec_eth.list, &rule.list); 596 597 err = mlx4_flow_attach(dev, &rule, reg_id); 598 break; 599 } 600 default: 601 return -EINVAL; 602 } 603 if (err) 604 en_warn(priv, "Failed Attaching Unicast\n"); 605 606 return err; 607 } 608 609 static void mlx4_en_uc_steer_release(struct mlx4_en_priv *priv, 610 unsigned char *mac, int qpn, u64 reg_id) 611 { 612 struct mlx4_en_dev *mdev = priv->mdev; 613 struct mlx4_dev *dev = mdev->dev; 614 615 switch (dev->caps.steering_mode) { 616 case MLX4_STEERING_MODE_B0: { 617 struct mlx4_qp qp; 618 u8 gid[16] = {0}; 619 620 qp.qpn = qpn; 621 memcpy(&gid[10], mac, ETH_ALEN); 622 gid[5] = priv->port; 623 624 mlx4_unicast_detach(dev, &qp, gid, MLX4_PROT_ETH); 625 break; 626 } 627 case MLX4_STEERING_MODE_DEVICE_MANAGED: { 628 mlx4_flow_detach(dev, reg_id); 629 break; 630 } 631 default: 632 en_err(priv, "Invalid steering mode.\n"); 633 } 634 } 635 636 static int mlx4_en_get_qp(struct mlx4_en_priv *priv) 637 { 638 struct mlx4_en_dev *mdev = priv->mdev; 639 struct mlx4_dev *dev = mdev->dev; 640 int index = 0; 641 int err = 0; 642 int *qpn = &priv->base_qpn; 643 u64 mac = mlx4_mac_to_u64(priv->dev->dev_addr); 644 645 en_dbg(DRV, priv, "Registering MAC: %pM for adding\n", 646 priv->dev->dev_addr); 647 index = mlx4_register_mac(dev, priv->port, mac); 648 if (index < 0) { 649 err = index; 650 en_err(priv, "Failed adding MAC: %pM\n", 651 priv->dev->dev_addr); 652 return err; 653 } 654 655 en_info(priv, "Steering Mode %d\n", dev->caps.steering_mode); 656 657 if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) { 658 int base_qpn = mlx4_get_base_qpn(dev, priv->port); 659 *qpn = base_qpn + index; 660 return 0; 661 } 662 663 err = mlx4_qp_reserve_range(dev, 1, 1, qpn, MLX4_RESERVE_A0_QP, 664 MLX4_RES_USAGE_DRIVER); 665 en_dbg(DRV, priv, "Reserved qp %d\n", *qpn); 666 if (err) { 667 en_err(priv, "Failed to reserve qp for mac registration\n"); 668 mlx4_unregister_mac(dev, priv->port, mac); 669 return err; 670 } 671 672 return 0; 673 } 674 675 static void mlx4_en_put_qp(struct mlx4_en_priv *priv) 676 { 677 struct mlx4_en_dev *mdev = priv->mdev; 678 struct mlx4_dev *dev = mdev->dev; 679 int qpn = priv->base_qpn; 680 681 if (dev->caps.steering_mode == MLX4_STEERING_MODE_A0) { 682 u64 mac = mlx4_mac_to_u64(priv->dev->dev_addr); 683 en_dbg(DRV, priv, "Registering MAC: %pM for deleting\n", 684 priv->dev->dev_addr); 685 mlx4_unregister_mac(dev, priv->port, mac); 686 } else { 687 en_dbg(DRV, priv, "Releasing qp: port %d, qpn %d\n", 688 priv->port, qpn); 689 mlx4_qp_release_range(dev, qpn, 1); 690 priv->flags &= ~MLX4_EN_FLAG_FORCE_PROMISC; 691 } 692 } 693 694 static int mlx4_en_replace_mac(struct mlx4_en_priv *priv, int qpn, 695 unsigned char *new_mac, unsigned char *prev_mac) 696 { 697 struct mlx4_en_dev *mdev = priv->mdev; 698 struct mlx4_dev *dev = mdev->dev; 699 int err = 0; 700 u64 new_mac_u64 = mlx4_mac_to_u64(new_mac); 701 702 if (dev->caps.steering_mode != MLX4_STEERING_MODE_A0) { 703 struct hlist_head *bucket; 704 unsigned int mac_hash; 705 struct mlx4_mac_entry *entry; 706 struct hlist_node *tmp; 707 u64 prev_mac_u64 = mlx4_mac_to_u64(prev_mac); 708 709 bucket = &priv->mac_hash[prev_mac[MLX4_EN_MAC_HASH_IDX]]; 710 hlist_for_each_entry_safe(entry, tmp, bucket, hlist) { 711 if (ether_addr_equal_64bits(entry->mac, prev_mac)) { 712 mlx4_en_uc_steer_release(priv, entry->mac, 713 qpn, entry->reg_id); 714 mlx4_unregister_mac(dev, priv->port, 715 prev_mac_u64); 716 hlist_del_rcu(&entry->hlist); 717 synchronize_rcu(); 718 memcpy(entry->mac, new_mac, ETH_ALEN); 719 entry->reg_id = 0; 720 mac_hash = new_mac[MLX4_EN_MAC_HASH_IDX]; 721 hlist_add_head_rcu(&entry->hlist, 722 &priv->mac_hash[mac_hash]); 723 mlx4_register_mac(dev, priv->port, new_mac_u64); 724 err = mlx4_en_uc_steer_add(priv, new_mac, 725 &qpn, 726 &entry->reg_id); 727 if (err) 728 return err; 729 if (priv->tunnel_reg_id) { 730 mlx4_flow_detach(priv->mdev->dev, priv->tunnel_reg_id); 731 priv->tunnel_reg_id = 0; 732 } 733 err = mlx4_en_tunnel_steer_add(priv, new_mac, qpn, 734 &priv->tunnel_reg_id); 735 return err; 736 } 737 } 738 return -EINVAL; 739 } 740 741 return __mlx4_replace_mac(dev, priv->port, qpn, new_mac_u64); 742 } 743 744 static void mlx4_en_update_user_mac(struct mlx4_en_priv *priv, 745 unsigned char new_mac[ETH_ALEN + 2]) 746 { 747 struct mlx4_en_dev *mdev = priv->mdev; 748 int err; 749 750 if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_USER_MAC_EN)) 751 return; 752 753 err = mlx4_SET_PORT_user_mac(mdev->dev, priv->port, new_mac); 754 if (err) 755 en_err(priv, "Failed to pass user MAC(%pM) to Firmware for port %d, with error %d\n", 756 new_mac, priv->port, err); 757 } 758 759 static int mlx4_en_do_set_mac(struct mlx4_en_priv *priv, 760 unsigned char new_mac[ETH_ALEN + 2]) 761 { 762 int err = 0; 763 764 if (priv->port_up) { 765 /* Remove old MAC and insert the new one */ 766 err = mlx4_en_replace_mac(priv, priv->base_qpn, 767 new_mac, priv->current_mac); 768 if (err) 769 en_err(priv, "Failed changing HW MAC address\n"); 770 } else 771 en_dbg(HW, priv, "Port is down while registering mac, exiting...\n"); 772 773 if (!err) 774 memcpy(priv->current_mac, new_mac, sizeof(priv->current_mac)); 775 776 return err; 777 } 778 779 static int mlx4_en_set_mac(struct net_device *dev, void *addr) 780 { 781 struct mlx4_en_priv *priv = netdev_priv(dev); 782 struct mlx4_en_dev *mdev = priv->mdev; 783 struct sockaddr *saddr = addr; 784 unsigned char new_mac[ETH_ALEN + 2]; 785 int err; 786 787 if (!is_valid_ether_addr(saddr->sa_data)) 788 return -EADDRNOTAVAIL; 789 790 mutex_lock(&mdev->state_lock); 791 memcpy(new_mac, saddr->sa_data, ETH_ALEN); 792 err = mlx4_en_do_set_mac(priv, new_mac); 793 if (err) 794 goto out; 795 796 memcpy(dev->dev_addr, saddr->sa_data, ETH_ALEN); 797 mlx4_en_update_user_mac(priv, new_mac); 798 out: 799 mutex_unlock(&mdev->state_lock); 800 801 return err; 802 } 803 804 static void mlx4_en_clear_list(struct net_device *dev) 805 { 806 struct mlx4_en_priv *priv = netdev_priv(dev); 807 struct mlx4_en_mc_list *tmp, *mc_to_del; 808 809 list_for_each_entry_safe(mc_to_del, tmp, &priv->mc_list, list) { 810 list_del(&mc_to_del->list); 811 kfree(mc_to_del); 812 } 813 } 814 815 static void mlx4_en_cache_mclist(struct net_device *dev) 816 { 817 struct mlx4_en_priv *priv = netdev_priv(dev); 818 struct netdev_hw_addr *ha; 819 struct mlx4_en_mc_list *tmp; 820 821 mlx4_en_clear_list(dev); 822 netdev_for_each_mc_addr(ha, dev) { 823 tmp = kzalloc(sizeof(struct mlx4_en_mc_list), GFP_ATOMIC); 824 if (!tmp) { 825 mlx4_en_clear_list(dev); 826 return; 827 } 828 memcpy(tmp->addr, ha->addr, ETH_ALEN); 829 list_add_tail(&tmp->list, &priv->mc_list); 830 } 831 } 832 833 static void update_mclist_flags(struct mlx4_en_priv *priv, 834 struct list_head *dst, 835 struct list_head *src) 836 { 837 struct mlx4_en_mc_list *dst_tmp, *src_tmp, *new_mc; 838 bool found; 839 840 /* Find all the entries that should be removed from dst, 841 * These are the entries that are not found in src 842 */ 843 list_for_each_entry(dst_tmp, dst, list) { 844 found = false; 845 list_for_each_entry(src_tmp, src, list) { 846 if (ether_addr_equal(dst_tmp->addr, src_tmp->addr)) { 847 found = true; 848 break; 849 } 850 } 851 if (!found) 852 dst_tmp->action = MCLIST_REM; 853 } 854 855 /* Add entries that exist in src but not in dst 856 * mark them as need to add 857 */ 858 list_for_each_entry(src_tmp, src, list) { 859 found = false; 860 list_for_each_entry(dst_tmp, dst, list) { 861 if (ether_addr_equal(dst_tmp->addr, src_tmp->addr)) { 862 dst_tmp->action = MCLIST_NONE; 863 found = true; 864 break; 865 } 866 } 867 if (!found) { 868 new_mc = kmemdup(src_tmp, 869 sizeof(struct mlx4_en_mc_list), 870 GFP_KERNEL); 871 if (!new_mc) 872 return; 873 874 new_mc->action = MCLIST_ADD; 875 list_add_tail(&new_mc->list, dst); 876 } 877 } 878 } 879 880 static void mlx4_en_set_rx_mode(struct net_device *dev) 881 { 882 struct mlx4_en_priv *priv = netdev_priv(dev); 883 884 if (!priv->port_up) 885 return; 886 887 queue_work(priv->mdev->workqueue, &priv->rx_mode_task); 888 } 889 890 static void mlx4_en_set_promisc_mode(struct mlx4_en_priv *priv, 891 struct mlx4_en_dev *mdev) 892 { 893 int err = 0; 894 895 if (!(priv->flags & MLX4_EN_FLAG_PROMISC)) { 896 if (netif_msg_rx_status(priv)) 897 en_warn(priv, "Entering promiscuous mode\n"); 898 priv->flags |= MLX4_EN_FLAG_PROMISC; 899 900 /* Enable promiscouos mode */ 901 switch (mdev->dev->caps.steering_mode) { 902 case MLX4_STEERING_MODE_DEVICE_MANAGED: 903 err = mlx4_flow_steer_promisc_add(mdev->dev, 904 priv->port, 905 priv->base_qpn, 906 MLX4_FS_ALL_DEFAULT); 907 if (err) 908 en_err(priv, "Failed enabling promiscuous mode\n"); 909 priv->flags |= MLX4_EN_FLAG_MC_PROMISC; 910 break; 911 912 case MLX4_STEERING_MODE_B0: 913 err = mlx4_unicast_promisc_add(mdev->dev, 914 priv->base_qpn, 915 priv->port); 916 if (err) 917 en_err(priv, "Failed enabling unicast promiscuous mode\n"); 918 919 /* Add the default qp number as multicast 920 * promisc 921 */ 922 if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) { 923 err = mlx4_multicast_promisc_add(mdev->dev, 924 priv->base_qpn, 925 priv->port); 926 if (err) 927 en_err(priv, "Failed enabling multicast promiscuous mode\n"); 928 priv->flags |= MLX4_EN_FLAG_MC_PROMISC; 929 } 930 break; 931 932 case MLX4_STEERING_MODE_A0: 933 err = mlx4_SET_PORT_qpn_calc(mdev->dev, 934 priv->port, 935 priv->base_qpn, 936 1); 937 if (err) 938 en_err(priv, "Failed enabling promiscuous mode\n"); 939 break; 940 } 941 942 /* Disable port multicast filter (unconditionally) */ 943 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 944 0, MLX4_MCAST_DISABLE); 945 if (err) 946 en_err(priv, "Failed disabling multicast filter\n"); 947 } 948 } 949 950 static void mlx4_en_clear_promisc_mode(struct mlx4_en_priv *priv, 951 struct mlx4_en_dev *mdev) 952 { 953 int err = 0; 954 955 if (netif_msg_rx_status(priv)) 956 en_warn(priv, "Leaving promiscuous mode\n"); 957 priv->flags &= ~MLX4_EN_FLAG_PROMISC; 958 959 /* Disable promiscouos mode */ 960 switch (mdev->dev->caps.steering_mode) { 961 case MLX4_STEERING_MODE_DEVICE_MANAGED: 962 err = mlx4_flow_steer_promisc_remove(mdev->dev, 963 priv->port, 964 MLX4_FS_ALL_DEFAULT); 965 if (err) 966 en_err(priv, "Failed disabling promiscuous mode\n"); 967 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; 968 break; 969 970 case MLX4_STEERING_MODE_B0: 971 err = mlx4_unicast_promisc_remove(mdev->dev, 972 priv->base_qpn, 973 priv->port); 974 if (err) 975 en_err(priv, "Failed disabling unicast promiscuous mode\n"); 976 /* Disable Multicast promisc */ 977 if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { 978 err = mlx4_multicast_promisc_remove(mdev->dev, 979 priv->base_qpn, 980 priv->port); 981 if (err) 982 en_err(priv, "Failed disabling multicast promiscuous mode\n"); 983 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; 984 } 985 break; 986 987 case MLX4_STEERING_MODE_A0: 988 err = mlx4_SET_PORT_qpn_calc(mdev->dev, 989 priv->port, 990 priv->base_qpn, 0); 991 if (err) 992 en_err(priv, "Failed disabling promiscuous mode\n"); 993 break; 994 } 995 } 996 997 static void mlx4_en_do_multicast(struct mlx4_en_priv *priv, 998 struct net_device *dev, 999 struct mlx4_en_dev *mdev) 1000 { 1001 struct mlx4_en_mc_list *mclist, *tmp; 1002 u64 mcast_addr = 0; 1003 u8 mc_list[16] = {0}; 1004 int err = 0; 1005 1006 /* Enable/disable the multicast filter according to IFF_ALLMULTI */ 1007 if (dev->flags & IFF_ALLMULTI) { 1008 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 1009 0, MLX4_MCAST_DISABLE); 1010 if (err) 1011 en_err(priv, "Failed disabling multicast filter\n"); 1012 1013 /* Add the default qp number as multicast promisc */ 1014 if (!(priv->flags & MLX4_EN_FLAG_MC_PROMISC)) { 1015 switch (mdev->dev->caps.steering_mode) { 1016 case MLX4_STEERING_MODE_DEVICE_MANAGED: 1017 err = mlx4_flow_steer_promisc_add(mdev->dev, 1018 priv->port, 1019 priv->base_qpn, 1020 MLX4_FS_MC_DEFAULT); 1021 break; 1022 1023 case MLX4_STEERING_MODE_B0: 1024 err = mlx4_multicast_promisc_add(mdev->dev, 1025 priv->base_qpn, 1026 priv->port); 1027 break; 1028 1029 case MLX4_STEERING_MODE_A0: 1030 break; 1031 } 1032 if (err) 1033 en_err(priv, "Failed entering multicast promisc mode\n"); 1034 priv->flags |= MLX4_EN_FLAG_MC_PROMISC; 1035 } 1036 } else { 1037 /* Disable Multicast promisc */ 1038 if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { 1039 switch (mdev->dev->caps.steering_mode) { 1040 case MLX4_STEERING_MODE_DEVICE_MANAGED: 1041 err = mlx4_flow_steer_promisc_remove(mdev->dev, 1042 priv->port, 1043 MLX4_FS_MC_DEFAULT); 1044 break; 1045 1046 case MLX4_STEERING_MODE_B0: 1047 err = mlx4_multicast_promisc_remove(mdev->dev, 1048 priv->base_qpn, 1049 priv->port); 1050 break; 1051 1052 case MLX4_STEERING_MODE_A0: 1053 break; 1054 } 1055 if (err) 1056 en_err(priv, "Failed disabling multicast promiscuous mode\n"); 1057 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; 1058 } 1059 1060 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 1061 0, MLX4_MCAST_DISABLE); 1062 if (err) 1063 en_err(priv, "Failed disabling multicast filter\n"); 1064 1065 /* Flush mcast filter and init it with broadcast address */ 1066 mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, ETH_BCAST, 1067 1, MLX4_MCAST_CONFIG); 1068 1069 /* Update multicast list - we cache all addresses so they won't 1070 * change while HW is updated holding the command semaphor */ 1071 netif_addr_lock_bh(dev); 1072 mlx4_en_cache_mclist(dev); 1073 netif_addr_unlock_bh(dev); 1074 list_for_each_entry(mclist, &priv->mc_list, list) { 1075 mcast_addr = mlx4_mac_to_u64(mclist->addr); 1076 mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 1077 mcast_addr, 0, MLX4_MCAST_CONFIG); 1078 } 1079 err = mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 1080 0, MLX4_MCAST_ENABLE); 1081 if (err) 1082 en_err(priv, "Failed enabling multicast filter\n"); 1083 1084 update_mclist_flags(priv, &priv->curr_list, &priv->mc_list); 1085 list_for_each_entry_safe(mclist, tmp, &priv->curr_list, list) { 1086 if (mclist->action == MCLIST_REM) { 1087 /* detach this address and delete from list */ 1088 memcpy(&mc_list[10], mclist->addr, ETH_ALEN); 1089 mc_list[5] = priv->port; 1090 err = mlx4_multicast_detach(mdev->dev, 1091 priv->rss_map.indir_qp, 1092 mc_list, 1093 MLX4_PROT_ETH, 1094 mclist->reg_id); 1095 if (err) 1096 en_err(priv, "Fail to detach multicast address\n"); 1097 1098 if (mclist->tunnel_reg_id) { 1099 err = mlx4_flow_detach(priv->mdev->dev, mclist->tunnel_reg_id); 1100 if (err) 1101 en_err(priv, "Failed to detach multicast address\n"); 1102 } 1103 1104 /* remove from list */ 1105 list_del(&mclist->list); 1106 kfree(mclist); 1107 } else if (mclist->action == MCLIST_ADD) { 1108 /* attach the address */ 1109 memcpy(&mc_list[10], mclist->addr, ETH_ALEN); 1110 /* needed for B0 steering support */ 1111 mc_list[5] = priv->port; 1112 err = mlx4_multicast_attach(mdev->dev, 1113 priv->rss_map.indir_qp, 1114 mc_list, 1115 priv->port, 0, 1116 MLX4_PROT_ETH, 1117 &mclist->reg_id); 1118 if (err) 1119 en_err(priv, "Fail to attach multicast address\n"); 1120 1121 err = mlx4_en_tunnel_steer_add(priv, &mc_list[10], priv->base_qpn, 1122 &mclist->tunnel_reg_id); 1123 if (err) 1124 en_err(priv, "Failed to attach multicast address\n"); 1125 } 1126 } 1127 } 1128 } 1129 1130 static void mlx4_en_do_uc_filter(struct mlx4_en_priv *priv, 1131 struct net_device *dev, 1132 struct mlx4_en_dev *mdev) 1133 { 1134 struct netdev_hw_addr *ha; 1135 struct mlx4_mac_entry *entry; 1136 struct hlist_node *tmp; 1137 bool found; 1138 u64 mac; 1139 int err = 0; 1140 struct hlist_head *bucket; 1141 unsigned int i; 1142 int removed = 0; 1143 u32 prev_flags; 1144 1145 /* Note that we do not need to protect our mac_hash traversal with rcu, 1146 * since all modification code is protected by mdev->state_lock 1147 */ 1148 1149 /* find what to remove */ 1150 for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) { 1151 bucket = &priv->mac_hash[i]; 1152 hlist_for_each_entry_safe(entry, tmp, bucket, hlist) { 1153 found = false; 1154 netdev_for_each_uc_addr(ha, dev) { 1155 if (ether_addr_equal_64bits(entry->mac, 1156 ha->addr)) { 1157 found = true; 1158 break; 1159 } 1160 } 1161 1162 /* MAC address of the port is not in uc list */ 1163 if (ether_addr_equal_64bits(entry->mac, 1164 priv->current_mac)) 1165 found = true; 1166 1167 if (!found) { 1168 mac = mlx4_mac_to_u64(entry->mac); 1169 mlx4_en_uc_steer_release(priv, entry->mac, 1170 priv->base_qpn, 1171 entry->reg_id); 1172 mlx4_unregister_mac(mdev->dev, priv->port, mac); 1173 1174 hlist_del_rcu(&entry->hlist); 1175 kfree_rcu(entry, rcu); 1176 en_dbg(DRV, priv, "Removed MAC %pM on port:%d\n", 1177 entry->mac, priv->port); 1178 ++removed; 1179 } 1180 } 1181 } 1182 1183 /* if we didn't remove anything, there is no use in trying to add 1184 * again once we are in a forced promisc mode state 1185 */ 1186 if ((priv->flags & MLX4_EN_FLAG_FORCE_PROMISC) && 0 == removed) 1187 return; 1188 1189 prev_flags = priv->flags; 1190 priv->flags &= ~MLX4_EN_FLAG_FORCE_PROMISC; 1191 1192 /* find what to add */ 1193 netdev_for_each_uc_addr(ha, dev) { 1194 found = false; 1195 bucket = &priv->mac_hash[ha->addr[MLX4_EN_MAC_HASH_IDX]]; 1196 hlist_for_each_entry(entry, bucket, hlist) { 1197 if (ether_addr_equal_64bits(entry->mac, ha->addr)) { 1198 found = true; 1199 break; 1200 } 1201 } 1202 1203 if (!found) { 1204 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 1205 if (!entry) { 1206 en_err(priv, "Failed adding MAC %pM on port:%d (out of memory)\n", 1207 ha->addr, priv->port); 1208 priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC; 1209 break; 1210 } 1211 mac = mlx4_mac_to_u64(ha->addr); 1212 memcpy(entry->mac, ha->addr, ETH_ALEN); 1213 err = mlx4_register_mac(mdev->dev, priv->port, mac); 1214 if (err < 0) { 1215 en_err(priv, "Failed registering MAC %pM on port %d: %d\n", 1216 ha->addr, priv->port, err); 1217 kfree(entry); 1218 priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC; 1219 break; 1220 } 1221 err = mlx4_en_uc_steer_add(priv, ha->addr, 1222 &priv->base_qpn, 1223 &entry->reg_id); 1224 if (err) { 1225 en_err(priv, "Failed adding MAC %pM on port %d: %d\n", 1226 ha->addr, priv->port, err); 1227 mlx4_unregister_mac(mdev->dev, priv->port, mac); 1228 kfree(entry); 1229 priv->flags |= MLX4_EN_FLAG_FORCE_PROMISC; 1230 break; 1231 } else { 1232 unsigned int mac_hash; 1233 en_dbg(DRV, priv, "Added MAC %pM on port:%d\n", 1234 ha->addr, priv->port); 1235 mac_hash = ha->addr[MLX4_EN_MAC_HASH_IDX]; 1236 bucket = &priv->mac_hash[mac_hash]; 1237 hlist_add_head_rcu(&entry->hlist, bucket); 1238 } 1239 } 1240 } 1241 1242 if (priv->flags & MLX4_EN_FLAG_FORCE_PROMISC) { 1243 en_warn(priv, "Forcing promiscuous mode on port:%d\n", 1244 priv->port); 1245 } else if (prev_flags & MLX4_EN_FLAG_FORCE_PROMISC) { 1246 en_warn(priv, "Stop forcing promiscuous mode on port:%d\n", 1247 priv->port); 1248 } 1249 } 1250 1251 static void mlx4_en_do_set_rx_mode(struct work_struct *work) 1252 { 1253 struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, 1254 rx_mode_task); 1255 struct mlx4_en_dev *mdev = priv->mdev; 1256 struct net_device *dev = priv->dev; 1257 1258 mutex_lock(&mdev->state_lock); 1259 if (!mdev->device_up) { 1260 en_dbg(HW, priv, "Card is not up, ignoring rx mode change.\n"); 1261 goto out; 1262 } 1263 if (!priv->port_up) { 1264 en_dbg(HW, priv, "Port is down, ignoring rx mode change.\n"); 1265 goto out; 1266 } 1267 1268 if (!netif_carrier_ok(dev)) { 1269 if (!mlx4_en_QUERY_PORT(mdev, priv->port)) { 1270 if (priv->port_state.link_state) { 1271 priv->last_link_state = MLX4_DEV_EVENT_PORT_UP; 1272 netif_carrier_on(dev); 1273 en_dbg(LINK, priv, "Link Up\n"); 1274 } 1275 } 1276 } 1277 1278 if (dev->priv_flags & IFF_UNICAST_FLT) 1279 mlx4_en_do_uc_filter(priv, dev, mdev); 1280 1281 /* Promsicuous mode: disable all filters */ 1282 if ((dev->flags & IFF_PROMISC) || 1283 (priv->flags & MLX4_EN_FLAG_FORCE_PROMISC)) { 1284 mlx4_en_set_promisc_mode(priv, mdev); 1285 goto out; 1286 } 1287 1288 /* Not in promiscuous mode */ 1289 if (priv->flags & MLX4_EN_FLAG_PROMISC) 1290 mlx4_en_clear_promisc_mode(priv, mdev); 1291 1292 mlx4_en_do_multicast(priv, dev, mdev); 1293 out: 1294 mutex_unlock(&mdev->state_lock); 1295 } 1296 1297 static int mlx4_en_set_rss_steer_rules(struct mlx4_en_priv *priv) 1298 { 1299 u64 reg_id; 1300 int err = 0; 1301 int *qpn = &priv->base_qpn; 1302 struct mlx4_mac_entry *entry; 1303 1304 err = mlx4_en_uc_steer_add(priv, priv->dev->dev_addr, qpn, ®_id); 1305 if (err) 1306 return err; 1307 1308 err = mlx4_en_tunnel_steer_add(priv, priv->dev->dev_addr, *qpn, 1309 &priv->tunnel_reg_id); 1310 if (err) 1311 goto tunnel_err; 1312 1313 entry = kmalloc(sizeof(*entry), GFP_KERNEL); 1314 if (!entry) { 1315 err = -ENOMEM; 1316 goto alloc_err; 1317 } 1318 1319 memcpy(entry->mac, priv->dev->dev_addr, sizeof(entry->mac)); 1320 memcpy(priv->current_mac, entry->mac, sizeof(priv->current_mac)); 1321 entry->reg_id = reg_id; 1322 hlist_add_head_rcu(&entry->hlist, 1323 &priv->mac_hash[entry->mac[MLX4_EN_MAC_HASH_IDX]]); 1324 1325 return 0; 1326 1327 alloc_err: 1328 if (priv->tunnel_reg_id) 1329 mlx4_flow_detach(priv->mdev->dev, priv->tunnel_reg_id); 1330 1331 tunnel_err: 1332 mlx4_en_uc_steer_release(priv, priv->dev->dev_addr, *qpn, reg_id); 1333 return err; 1334 } 1335 1336 static void mlx4_en_delete_rss_steer_rules(struct mlx4_en_priv *priv) 1337 { 1338 u64 mac; 1339 unsigned int i; 1340 int qpn = priv->base_qpn; 1341 struct hlist_head *bucket; 1342 struct hlist_node *tmp; 1343 struct mlx4_mac_entry *entry; 1344 1345 for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) { 1346 bucket = &priv->mac_hash[i]; 1347 hlist_for_each_entry_safe(entry, tmp, bucket, hlist) { 1348 mac = mlx4_mac_to_u64(entry->mac); 1349 en_dbg(DRV, priv, "Registering MAC:%pM for deleting\n", 1350 entry->mac); 1351 mlx4_en_uc_steer_release(priv, entry->mac, 1352 qpn, entry->reg_id); 1353 1354 mlx4_unregister_mac(priv->mdev->dev, priv->port, mac); 1355 hlist_del_rcu(&entry->hlist); 1356 kfree_rcu(entry, rcu); 1357 } 1358 } 1359 1360 if (priv->tunnel_reg_id) { 1361 mlx4_flow_detach(priv->mdev->dev, priv->tunnel_reg_id); 1362 priv->tunnel_reg_id = 0; 1363 } 1364 } 1365 1366 static void mlx4_en_tx_timeout(struct net_device *dev, unsigned int txqueue) 1367 { 1368 struct mlx4_en_priv *priv = netdev_priv(dev); 1369 struct mlx4_en_dev *mdev = priv->mdev; 1370 struct mlx4_en_tx_ring *tx_ring = priv->tx_ring[TX][txqueue]; 1371 1372 if (netif_msg_timer(priv)) 1373 en_warn(priv, "Tx timeout called on port:%d\n", priv->port); 1374 1375 en_warn(priv, "TX timeout on queue: %d, QP: 0x%x, CQ: 0x%x, Cons: 0x%x, Prod: 0x%x\n", 1376 txqueue, tx_ring->qpn, tx_ring->sp_cqn, 1377 tx_ring->cons, tx_ring->prod); 1378 1379 priv->port_stats.tx_timeout++; 1380 en_dbg(DRV, priv, "Scheduling watchdog\n"); 1381 queue_work(mdev->workqueue, &priv->watchdog_task); 1382 } 1383 1384 1385 static void 1386 mlx4_en_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats) 1387 { 1388 struct mlx4_en_priv *priv = netdev_priv(dev); 1389 1390 spin_lock_bh(&priv->stats_lock); 1391 mlx4_en_fold_software_stats(dev); 1392 netdev_stats_to_stats64(stats, &dev->stats); 1393 spin_unlock_bh(&priv->stats_lock); 1394 } 1395 1396 static void mlx4_en_set_default_moderation(struct mlx4_en_priv *priv) 1397 { 1398 struct mlx4_en_cq *cq; 1399 int i, t; 1400 1401 /* If we haven't received a specific coalescing setting 1402 * (module param), we set the moderation parameters as follows: 1403 * - moder_cnt is set to the number of mtu sized packets to 1404 * satisfy our coalescing target. 1405 * - moder_time is set to a fixed value. 1406 */ 1407 priv->rx_frames = MLX4_EN_RX_COAL_TARGET; 1408 priv->rx_usecs = MLX4_EN_RX_COAL_TIME; 1409 priv->tx_frames = MLX4_EN_TX_COAL_PKTS; 1410 priv->tx_usecs = MLX4_EN_TX_COAL_TIME; 1411 en_dbg(INTR, priv, "Default coalescing params for mtu:%d - rx_frames:%d rx_usecs:%d\n", 1412 priv->dev->mtu, priv->rx_frames, priv->rx_usecs); 1413 1414 /* Setup cq moderation params */ 1415 for (i = 0; i < priv->rx_ring_num; i++) { 1416 cq = priv->rx_cq[i]; 1417 cq->moder_cnt = priv->rx_frames; 1418 cq->moder_time = priv->rx_usecs; 1419 priv->last_moder_time[i] = MLX4_EN_AUTO_CONF; 1420 priv->last_moder_packets[i] = 0; 1421 priv->last_moder_bytes[i] = 0; 1422 } 1423 1424 for (t = 0 ; t < MLX4_EN_NUM_TX_TYPES; t++) { 1425 for (i = 0; i < priv->tx_ring_num[t]; i++) { 1426 cq = priv->tx_cq[t][i]; 1427 cq->moder_cnt = priv->tx_frames; 1428 cq->moder_time = priv->tx_usecs; 1429 } 1430 } 1431 1432 /* Reset auto-moderation params */ 1433 priv->pkt_rate_low = MLX4_EN_RX_RATE_LOW; 1434 priv->rx_usecs_low = MLX4_EN_RX_COAL_TIME_LOW; 1435 priv->pkt_rate_high = MLX4_EN_RX_RATE_HIGH; 1436 priv->rx_usecs_high = MLX4_EN_RX_COAL_TIME_HIGH; 1437 priv->sample_interval = MLX4_EN_SAMPLE_INTERVAL; 1438 priv->adaptive_rx_coal = 1; 1439 priv->last_moder_jiffies = 0; 1440 priv->last_moder_tx_packets = 0; 1441 } 1442 1443 static void mlx4_en_auto_moderation(struct mlx4_en_priv *priv) 1444 { 1445 unsigned long period = (unsigned long) (jiffies - priv->last_moder_jiffies); 1446 u32 pkt_rate_high, pkt_rate_low; 1447 struct mlx4_en_cq *cq; 1448 unsigned long packets; 1449 unsigned long rate; 1450 unsigned long avg_pkt_size; 1451 unsigned long rx_packets; 1452 unsigned long rx_bytes; 1453 unsigned long rx_pkt_diff; 1454 int moder_time; 1455 int ring, err; 1456 1457 if (!priv->adaptive_rx_coal || period < priv->sample_interval * HZ) 1458 return; 1459 1460 pkt_rate_low = READ_ONCE(priv->pkt_rate_low); 1461 pkt_rate_high = READ_ONCE(priv->pkt_rate_high); 1462 1463 for (ring = 0; ring < priv->rx_ring_num; ring++) { 1464 rx_packets = READ_ONCE(priv->rx_ring[ring]->packets); 1465 rx_bytes = READ_ONCE(priv->rx_ring[ring]->bytes); 1466 1467 rx_pkt_diff = rx_packets - priv->last_moder_packets[ring]; 1468 packets = rx_pkt_diff; 1469 rate = packets * HZ / period; 1470 avg_pkt_size = packets ? (rx_bytes - 1471 priv->last_moder_bytes[ring]) / packets : 0; 1472 1473 /* Apply auto-moderation only when packet rate 1474 * exceeds a rate that it matters */ 1475 if (rate > (MLX4_EN_RX_RATE_THRESH / priv->rx_ring_num) && 1476 avg_pkt_size > MLX4_EN_AVG_PKT_SMALL) { 1477 if (rate <= pkt_rate_low) 1478 moder_time = priv->rx_usecs_low; 1479 else if (rate >= pkt_rate_high) 1480 moder_time = priv->rx_usecs_high; 1481 else 1482 moder_time = (rate - pkt_rate_low) * 1483 (priv->rx_usecs_high - priv->rx_usecs_low) / 1484 (pkt_rate_high - pkt_rate_low) + 1485 priv->rx_usecs_low; 1486 } else { 1487 moder_time = priv->rx_usecs_low; 1488 } 1489 1490 cq = priv->rx_cq[ring]; 1491 if (moder_time != priv->last_moder_time[ring] || 1492 cq->moder_cnt != priv->rx_frames) { 1493 priv->last_moder_time[ring] = moder_time; 1494 cq->moder_time = moder_time; 1495 cq->moder_cnt = priv->rx_frames; 1496 err = mlx4_en_set_cq_moder(priv, cq); 1497 if (err) 1498 en_err(priv, "Failed modifying moderation for cq:%d\n", 1499 ring); 1500 } 1501 priv->last_moder_packets[ring] = rx_packets; 1502 priv->last_moder_bytes[ring] = rx_bytes; 1503 } 1504 1505 priv->last_moder_jiffies = jiffies; 1506 } 1507 1508 static void mlx4_en_do_get_stats(struct work_struct *work) 1509 { 1510 struct delayed_work *delay = to_delayed_work(work); 1511 struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv, 1512 stats_task); 1513 struct mlx4_en_dev *mdev = priv->mdev; 1514 int err; 1515 1516 mutex_lock(&mdev->state_lock); 1517 if (mdev->device_up) { 1518 if (priv->port_up) { 1519 err = mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 0); 1520 if (err) 1521 en_dbg(HW, priv, "Could not update stats\n"); 1522 1523 mlx4_en_auto_moderation(priv); 1524 } 1525 1526 queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY); 1527 } 1528 if (mdev->mac_removed[MLX4_MAX_PORTS + 1 - priv->port]) { 1529 mlx4_en_do_set_mac(priv, priv->current_mac); 1530 mdev->mac_removed[MLX4_MAX_PORTS + 1 - priv->port] = 0; 1531 } 1532 mutex_unlock(&mdev->state_lock); 1533 } 1534 1535 /* mlx4_en_service_task - Run service task for tasks that needed to be done 1536 * periodically 1537 */ 1538 static void mlx4_en_service_task(struct work_struct *work) 1539 { 1540 struct delayed_work *delay = to_delayed_work(work); 1541 struct mlx4_en_priv *priv = container_of(delay, struct mlx4_en_priv, 1542 service_task); 1543 struct mlx4_en_dev *mdev = priv->mdev; 1544 1545 mutex_lock(&mdev->state_lock); 1546 if (mdev->device_up) { 1547 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) 1548 mlx4_en_ptp_overflow_check(mdev); 1549 1550 mlx4_en_recover_from_oom(priv); 1551 queue_delayed_work(mdev->workqueue, &priv->service_task, 1552 SERVICE_TASK_DELAY); 1553 } 1554 mutex_unlock(&mdev->state_lock); 1555 } 1556 1557 static void mlx4_en_linkstate(struct work_struct *work) 1558 { 1559 struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, 1560 linkstate_task); 1561 struct mlx4_en_dev *mdev = priv->mdev; 1562 int linkstate = priv->link_state; 1563 1564 mutex_lock(&mdev->state_lock); 1565 /* If observable port state changed set carrier state and 1566 * report to system log */ 1567 if (priv->last_link_state != linkstate) { 1568 if (linkstate == MLX4_DEV_EVENT_PORT_DOWN) { 1569 en_info(priv, "Link Down\n"); 1570 netif_carrier_off(priv->dev); 1571 } else { 1572 en_info(priv, "Link Up\n"); 1573 netif_carrier_on(priv->dev); 1574 } 1575 } 1576 priv->last_link_state = linkstate; 1577 mutex_unlock(&mdev->state_lock); 1578 } 1579 1580 static int mlx4_en_init_affinity_hint(struct mlx4_en_priv *priv, int ring_idx) 1581 { 1582 struct mlx4_en_rx_ring *ring = priv->rx_ring[ring_idx]; 1583 int numa_node = priv->mdev->dev->numa_node; 1584 1585 if (!zalloc_cpumask_var(&ring->affinity_mask, GFP_KERNEL)) 1586 return -ENOMEM; 1587 1588 cpumask_set_cpu(cpumask_local_spread(ring_idx, numa_node), 1589 ring->affinity_mask); 1590 return 0; 1591 } 1592 1593 static void mlx4_en_free_affinity_hint(struct mlx4_en_priv *priv, int ring_idx) 1594 { 1595 free_cpumask_var(priv->rx_ring[ring_idx]->affinity_mask); 1596 } 1597 1598 static void mlx4_en_init_recycle_ring(struct mlx4_en_priv *priv, 1599 int tx_ring_idx) 1600 { 1601 struct mlx4_en_tx_ring *tx_ring = priv->tx_ring[TX_XDP][tx_ring_idx]; 1602 int rr_index = tx_ring_idx; 1603 1604 tx_ring->free_tx_desc = mlx4_en_recycle_tx_desc; 1605 tx_ring->recycle_ring = priv->rx_ring[rr_index]; 1606 en_dbg(DRV, priv, "Set tx_ring[%d][%d]->recycle_ring = rx_ring[%d]\n", 1607 TX_XDP, tx_ring_idx, rr_index); 1608 } 1609 1610 int mlx4_en_start_port(struct net_device *dev) 1611 { 1612 struct mlx4_en_priv *priv = netdev_priv(dev); 1613 struct mlx4_en_dev *mdev = priv->mdev; 1614 struct mlx4_en_cq *cq; 1615 struct mlx4_en_tx_ring *tx_ring; 1616 int rx_index = 0; 1617 int err = 0; 1618 int i, t; 1619 int j; 1620 u8 mc_list[16] = {0}; 1621 1622 if (priv->port_up) { 1623 en_dbg(DRV, priv, "start port called while port already up\n"); 1624 return 0; 1625 } 1626 1627 INIT_LIST_HEAD(&priv->mc_list); 1628 INIT_LIST_HEAD(&priv->curr_list); 1629 INIT_LIST_HEAD(&priv->ethtool_list); 1630 memset(&priv->ethtool_rules[0], 0, 1631 sizeof(struct ethtool_flow_id) * MAX_NUM_OF_FS_RULES); 1632 1633 /* Calculate Rx buf size */ 1634 dev->mtu = min(dev->mtu, priv->max_mtu); 1635 mlx4_en_calc_rx_buf(dev); 1636 en_dbg(DRV, priv, "Rx buf size:%d\n", priv->rx_skb_size); 1637 1638 /* Configure rx cq's and rings */ 1639 err = mlx4_en_activate_rx_rings(priv); 1640 if (err) { 1641 en_err(priv, "Failed to activate RX rings\n"); 1642 return err; 1643 } 1644 for (i = 0; i < priv->rx_ring_num; i++) { 1645 cq = priv->rx_cq[i]; 1646 1647 err = mlx4_en_init_affinity_hint(priv, i); 1648 if (err) { 1649 en_err(priv, "Failed preparing IRQ affinity hint\n"); 1650 goto cq_err; 1651 } 1652 1653 err = mlx4_en_activate_cq(priv, cq, i); 1654 if (err) { 1655 en_err(priv, "Failed activating Rx CQ\n"); 1656 mlx4_en_free_affinity_hint(priv, i); 1657 goto cq_err; 1658 } 1659 1660 for (j = 0; j < cq->size; j++) { 1661 struct mlx4_cqe *cqe = NULL; 1662 1663 cqe = mlx4_en_get_cqe(cq->buf, j, priv->cqe_size) + 1664 priv->cqe_factor; 1665 cqe->owner_sr_opcode = MLX4_CQE_OWNER_MASK; 1666 } 1667 1668 err = mlx4_en_set_cq_moder(priv, cq); 1669 if (err) { 1670 en_err(priv, "Failed setting cq moderation parameters\n"); 1671 mlx4_en_deactivate_cq(priv, cq); 1672 mlx4_en_free_affinity_hint(priv, i); 1673 goto cq_err; 1674 } 1675 mlx4_en_arm_cq(priv, cq); 1676 priv->rx_ring[i]->cqn = cq->mcq.cqn; 1677 ++rx_index; 1678 } 1679 1680 /* Set qp number */ 1681 en_dbg(DRV, priv, "Getting qp number for port %d\n", priv->port); 1682 err = mlx4_en_get_qp(priv); 1683 if (err) { 1684 en_err(priv, "Failed getting eth qp\n"); 1685 goto cq_err; 1686 } 1687 mdev->mac_removed[priv->port] = 0; 1688 1689 priv->counter_index = 1690 mlx4_get_default_counter_index(mdev->dev, priv->port); 1691 1692 err = mlx4_en_config_rss_steer(priv); 1693 if (err) { 1694 en_err(priv, "Failed configuring rss steering\n"); 1695 goto mac_err; 1696 } 1697 1698 err = mlx4_en_create_drop_qp(priv); 1699 if (err) 1700 goto rss_err; 1701 1702 /* Configure tx cq's and rings */ 1703 for (t = 0 ; t < MLX4_EN_NUM_TX_TYPES; t++) { 1704 u8 num_tx_rings_p_up = t == TX ? 1705 priv->num_tx_rings_p_up : priv->tx_ring_num[t]; 1706 1707 for (i = 0; i < priv->tx_ring_num[t]; i++) { 1708 /* Configure cq */ 1709 cq = priv->tx_cq[t][i]; 1710 err = mlx4_en_activate_cq(priv, cq, i); 1711 if (err) { 1712 en_err(priv, "Failed allocating Tx CQ\n"); 1713 goto tx_err; 1714 } 1715 err = mlx4_en_set_cq_moder(priv, cq); 1716 if (err) { 1717 en_err(priv, "Failed setting cq moderation parameters\n"); 1718 mlx4_en_deactivate_cq(priv, cq); 1719 goto tx_err; 1720 } 1721 en_dbg(DRV, priv, 1722 "Resetting index of collapsed CQ:%d to -1\n", i); 1723 cq->buf->wqe_index = cpu_to_be16(0xffff); 1724 1725 /* Configure ring */ 1726 tx_ring = priv->tx_ring[t][i]; 1727 err = mlx4_en_activate_tx_ring(priv, tx_ring, 1728 cq->mcq.cqn, 1729 i / num_tx_rings_p_up); 1730 if (err) { 1731 en_err(priv, "Failed allocating Tx ring\n"); 1732 mlx4_en_deactivate_cq(priv, cq); 1733 goto tx_err; 1734 } 1735 if (t != TX_XDP) { 1736 tx_ring->tx_queue = netdev_get_tx_queue(dev, i); 1737 tx_ring->recycle_ring = NULL; 1738 1739 /* Arm CQ for TX completions */ 1740 mlx4_en_arm_cq(priv, cq); 1741 1742 } else { 1743 mlx4_en_init_tx_xdp_ring_descs(priv, tx_ring); 1744 mlx4_en_init_recycle_ring(priv, i); 1745 /* XDP TX CQ should never be armed */ 1746 } 1747 1748 /* Set initial ownership of all Tx TXBBs to SW (1) */ 1749 for (j = 0; j < tx_ring->buf_size; j += STAMP_STRIDE) 1750 *((u32 *)(tx_ring->buf + j)) = 0xffffffff; 1751 } 1752 } 1753 1754 /* Configure port */ 1755 err = mlx4_SET_PORT_general(mdev->dev, priv->port, 1756 priv->rx_skb_size + ETH_FCS_LEN, 1757 priv->prof->tx_pause, 1758 priv->prof->tx_ppp, 1759 priv->prof->rx_pause, 1760 priv->prof->rx_ppp); 1761 if (err) { 1762 en_err(priv, "Failed setting port general configurations for port %d, with error %d\n", 1763 priv->port, err); 1764 goto tx_err; 1765 } 1766 1767 err = mlx4_SET_PORT_user_mtu(mdev->dev, priv->port, dev->mtu); 1768 if (err) { 1769 en_err(priv, "Failed to pass user MTU(%d) to Firmware for port %d, with error %d\n", 1770 dev->mtu, priv->port, err); 1771 goto tx_err; 1772 } 1773 1774 /* Set default qp number */ 1775 err = mlx4_SET_PORT_qpn_calc(mdev->dev, priv->port, priv->base_qpn, 0); 1776 if (err) { 1777 en_err(priv, "Failed setting default qp numbers\n"); 1778 goto tx_err; 1779 } 1780 1781 if (mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) { 1782 err = mlx4_SET_PORT_VXLAN(mdev->dev, priv->port, VXLAN_STEER_BY_OUTER_MAC, 1); 1783 if (err) { 1784 en_err(priv, "Failed setting port L2 tunnel configuration, err %d\n", 1785 err); 1786 goto tx_err; 1787 } 1788 } 1789 1790 /* Init port */ 1791 en_dbg(HW, priv, "Initializing port\n"); 1792 err = mlx4_INIT_PORT(mdev->dev, priv->port); 1793 if (err) { 1794 en_err(priv, "Failed Initializing port\n"); 1795 goto tx_err; 1796 } 1797 1798 /* Set Unicast and VXLAN steering rules */ 1799 if (mdev->dev->caps.steering_mode != MLX4_STEERING_MODE_A0 && 1800 mlx4_en_set_rss_steer_rules(priv)) 1801 mlx4_warn(mdev, "Failed setting steering rules\n"); 1802 1803 /* Attach rx QP to bradcast address */ 1804 eth_broadcast_addr(&mc_list[10]); 1805 mc_list[5] = priv->port; /* needed for B0 steering support */ 1806 if (mlx4_multicast_attach(mdev->dev, priv->rss_map.indir_qp, mc_list, 1807 priv->port, 0, MLX4_PROT_ETH, 1808 &priv->broadcast_id)) 1809 mlx4_warn(mdev, "Failed Attaching Broadcast\n"); 1810 1811 /* Must redo promiscuous mode setup. */ 1812 priv->flags &= ~(MLX4_EN_FLAG_PROMISC | MLX4_EN_FLAG_MC_PROMISC); 1813 1814 /* Schedule multicast task to populate multicast list */ 1815 queue_work(mdev->workqueue, &priv->rx_mode_task); 1816 1817 if (priv->mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) 1818 udp_tunnel_get_rx_info(dev); 1819 1820 priv->port_up = true; 1821 1822 /* Process all completions if exist to prevent 1823 * the queues freezing if they are full 1824 */ 1825 for (i = 0; i < priv->rx_ring_num; i++) { 1826 local_bh_disable(); 1827 napi_schedule(&priv->rx_cq[i]->napi); 1828 local_bh_enable(); 1829 } 1830 1831 netif_tx_start_all_queues(dev); 1832 netif_device_attach(dev); 1833 1834 return 0; 1835 1836 tx_err: 1837 if (t == MLX4_EN_NUM_TX_TYPES) { 1838 t--; 1839 i = priv->tx_ring_num[t]; 1840 } 1841 while (t >= 0) { 1842 while (i--) { 1843 mlx4_en_deactivate_tx_ring(priv, priv->tx_ring[t][i]); 1844 mlx4_en_deactivate_cq(priv, priv->tx_cq[t][i]); 1845 } 1846 if (!t--) 1847 break; 1848 i = priv->tx_ring_num[t]; 1849 } 1850 mlx4_en_destroy_drop_qp(priv); 1851 rss_err: 1852 mlx4_en_release_rss_steer(priv); 1853 mac_err: 1854 mlx4_en_put_qp(priv); 1855 cq_err: 1856 while (rx_index--) { 1857 mlx4_en_deactivate_cq(priv, priv->rx_cq[rx_index]); 1858 mlx4_en_free_affinity_hint(priv, rx_index); 1859 } 1860 for (i = 0; i < priv->rx_ring_num; i++) 1861 mlx4_en_deactivate_rx_ring(priv, priv->rx_ring[i]); 1862 1863 return err; /* need to close devices */ 1864 } 1865 1866 1867 void mlx4_en_stop_port(struct net_device *dev, int detach) 1868 { 1869 struct mlx4_en_priv *priv = netdev_priv(dev); 1870 struct mlx4_en_dev *mdev = priv->mdev; 1871 struct mlx4_en_mc_list *mclist, *tmp; 1872 struct ethtool_flow_id *flow, *tmp_flow; 1873 int i, t; 1874 u8 mc_list[16] = {0}; 1875 1876 if (!priv->port_up) { 1877 en_dbg(DRV, priv, "stop port called while port already down\n"); 1878 return; 1879 } 1880 1881 /* close port*/ 1882 mlx4_CLOSE_PORT(mdev->dev, priv->port); 1883 1884 /* Synchronize with tx routine */ 1885 netif_tx_lock_bh(dev); 1886 if (detach) 1887 netif_device_detach(dev); 1888 netif_tx_stop_all_queues(dev); 1889 netif_tx_unlock_bh(dev); 1890 1891 netif_tx_disable(dev); 1892 1893 spin_lock_bh(&priv->stats_lock); 1894 mlx4_en_fold_software_stats(dev); 1895 /* Set port as not active */ 1896 priv->port_up = false; 1897 spin_unlock_bh(&priv->stats_lock); 1898 1899 priv->counter_index = MLX4_SINK_COUNTER_INDEX(mdev->dev); 1900 1901 /* Promsicuous mode */ 1902 if (mdev->dev->caps.steering_mode == 1903 MLX4_STEERING_MODE_DEVICE_MANAGED) { 1904 priv->flags &= ~(MLX4_EN_FLAG_PROMISC | 1905 MLX4_EN_FLAG_MC_PROMISC); 1906 mlx4_flow_steer_promisc_remove(mdev->dev, 1907 priv->port, 1908 MLX4_FS_ALL_DEFAULT); 1909 mlx4_flow_steer_promisc_remove(mdev->dev, 1910 priv->port, 1911 MLX4_FS_MC_DEFAULT); 1912 } else if (priv->flags & MLX4_EN_FLAG_PROMISC) { 1913 priv->flags &= ~MLX4_EN_FLAG_PROMISC; 1914 1915 /* Disable promiscouos mode */ 1916 mlx4_unicast_promisc_remove(mdev->dev, priv->base_qpn, 1917 priv->port); 1918 1919 /* Disable Multicast promisc */ 1920 if (priv->flags & MLX4_EN_FLAG_MC_PROMISC) { 1921 mlx4_multicast_promisc_remove(mdev->dev, priv->base_qpn, 1922 priv->port); 1923 priv->flags &= ~MLX4_EN_FLAG_MC_PROMISC; 1924 } 1925 } 1926 1927 /* Detach All multicasts */ 1928 eth_broadcast_addr(&mc_list[10]); 1929 mc_list[5] = priv->port; /* needed for B0 steering support */ 1930 mlx4_multicast_detach(mdev->dev, priv->rss_map.indir_qp, mc_list, 1931 MLX4_PROT_ETH, priv->broadcast_id); 1932 list_for_each_entry(mclist, &priv->curr_list, list) { 1933 memcpy(&mc_list[10], mclist->addr, ETH_ALEN); 1934 mc_list[5] = priv->port; 1935 mlx4_multicast_detach(mdev->dev, priv->rss_map.indir_qp, 1936 mc_list, MLX4_PROT_ETH, mclist->reg_id); 1937 if (mclist->tunnel_reg_id) 1938 mlx4_flow_detach(mdev->dev, mclist->tunnel_reg_id); 1939 } 1940 mlx4_en_clear_list(dev); 1941 list_for_each_entry_safe(mclist, tmp, &priv->curr_list, list) { 1942 list_del(&mclist->list); 1943 kfree(mclist); 1944 } 1945 1946 /* Flush multicast filter */ 1947 mlx4_SET_MCAST_FLTR(mdev->dev, priv->port, 0, 1, MLX4_MCAST_CONFIG); 1948 1949 /* Remove flow steering rules for the port*/ 1950 if (mdev->dev->caps.steering_mode == 1951 MLX4_STEERING_MODE_DEVICE_MANAGED) { 1952 ASSERT_RTNL(); 1953 list_for_each_entry_safe(flow, tmp_flow, 1954 &priv->ethtool_list, list) { 1955 mlx4_flow_detach(mdev->dev, flow->id); 1956 list_del(&flow->list); 1957 } 1958 } 1959 1960 mlx4_en_destroy_drop_qp(priv); 1961 1962 /* Free TX Rings */ 1963 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { 1964 for (i = 0; i < priv->tx_ring_num[t]; i++) { 1965 mlx4_en_deactivate_tx_ring(priv, priv->tx_ring[t][i]); 1966 mlx4_en_deactivate_cq(priv, priv->tx_cq[t][i]); 1967 } 1968 } 1969 msleep(10); 1970 1971 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) 1972 for (i = 0; i < priv->tx_ring_num[t]; i++) 1973 mlx4_en_free_tx_buf(dev, priv->tx_ring[t][i]); 1974 1975 if (mdev->dev->caps.steering_mode != MLX4_STEERING_MODE_A0) 1976 mlx4_en_delete_rss_steer_rules(priv); 1977 1978 /* Free RSS qps */ 1979 mlx4_en_release_rss_steer(priv); 1980 1981 /* Unregister Mac address for the port */ 1982 mlx4_en_put_qp(priv); 1983 if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_REASSIGN_MAC_EN)) 1984 mdev->mac_removed[priv->port] = 1; 1985 1986 /* Free RX Rings */ 1987 for (i = 0; i < priv->rx_ring_num; i++) { 1988 struct mlx4_en_cq *cq = priv->rx_cq[i]; 1989 1990 napi_synchronize(&cq->napi); 1991 mlx4_en_deactivate_rx_ring(priv, priv->rx_ring[i]); 1992 mlx4_en_deactivate_cq(priv, cq); 1993 1994 mlx4_en_free_affinity_hint(priv, i); 1995 } 1996 } 1997 1998 static void mlx4_en_restart(struct work_struct *work) 1999 { 2000 struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, 2001 watchdog_task); 2002 struct mlx4_en_dev *mdev = priv->mdev; 2003 struct net_device *dev = priv->dev; 2004 2005 en_dbg(DRV, priv, "Watchdog task called for port %d\n", priv->port); 2006 2007 rtnl_lock(); 2008 mutex_lock(&mdev->state_lock); 2009 if (priv->port_up) { 2010 mlx4_en_stop_port(dev, 1); 2011 if (mlx4_en_start_port(dev)) 2012 en_err(priv, "Failed restarting port %d\n", priv->port); 2013 } 2014 mutex_unlock(&mdev->state_lock); 2015 rtnl_unlock(); 2016 } 2017 2018 static void mlx4_en_clear_stats(struct net_device *dev) 2019 { 2020 struct mlx4_en_priv *priv = netdev_priv(dev); 2021 struct mlx4_en_dev *mdev = priv->mdev; 2022 struct mlx4_en_tx_ring **tx_ring; 2023 int i; 2024 2025 if (!mlx4_is_slave(mdev->dev)) 2026 if (mlx4_en_DUMP_ETH_STATS(mdev, priv->port, 1)) 2027 en_dbg(HW, priv, "Failed dumping statistics\n"); 2028 2029 memset(&priv->pstats, 0, sizeof(priv->pstats)); 2030 memset(&priv->pkstats, 0, sizeof(priv->pkstats)); 2031 memset(&priv->port_stats, 0, sizeof(priv->port_stats)); 2032 memset(&priv->rx_flowstats, 0, sizeof(priv->rx_flowstats)); 2033 memset(&priv->tx_flowstats, 0, sizeof(priv->tx_flowstats)); 2034 memset(&priv->rx_priority_flowstats, 0, 2035 sizeof(priv->rx_priority_flowstats)); 2036 memset(&priv->tx_priority_flowstats, 0, 2037 sizeof(priv->tx_priority_flowstats)); 2038 memset(&priv->pf_stats, 0, sizeof(priv->pf_stats)); 2039 2040 tx_ring = priv->tx_ring[TX]; 2041 for (i = 0; i < priv->tx_ring_num[TX]; i++) { 2042 tx_ring[i]->bytes = 0; 2043 tx_ring[i]->packets = 0; 2044 tx_ring[i]->tx_csum = 0; 2045 tx_ring[i]->tx_dropped = 0; 2046 tx_ring[i]->queue_stopped = 0; 2047 tx_ring[i]->wake_queue = 0; 2048 tx_ring[i]->tso_packets = 0; 2049 tx_ring[i]->xmit_more = 0; 2050 } 2051 for (i = 0; i < priv->rx_ring_num; i++) { 2052 priv->rx_ring[i]->bytes = 0; 2053 priv->rx_ring[i]->packets = 0; 2054 priv->rx_ring[i]->csum_ok = 0; 2055 priv->rx_ring[i]->csum_none = 0; 2056 priv->rx_ring[i]->csum_complete = 0; 2057 } 2058 } 2059 2060 static int mlx4_en_open(struct net_device *dev) 2061 { 2062 struct mlx4_en_priv *priv = netdev_priv(dev); 2063 struct mlx4_en_dev *mdev = priv->mdev; 2064 int err = 0; 2065 2066 mutex_lock(&mdev->state_lock); 2067 2068 if (!mdev->device_up) { 2069 en_err(priv, "Cannot open - device down/disabled\n"); 2070 err = -EBUSY; 2071 goto out; 2072 } 2073 2074 /* Reset HW statistics and SW counters */ 2075 mlx4_en_clear_stats(dev); 2076 2077 err = mlx4_en_start_port(dev); 2078 if (err) 2079 en_err(priv, "Failed starting port:%d\n", priv->port); 2080 2081 out: 2082 mutex_unlock(&mdev->state_lock); 2083 return err; 2084 } 2085 2086 2087 static int mlx4_en_close(struct net_device *dev) 2088 { 2089 struct mlx4_en_priv *priv = netdev_priv(dev); 2090 struct mlx4_en_dev *mdev = priv->mdev; 2091 2092 en_dbg(IFDOWN, priv, "Close port called\n"); 2093 2094 mutex_lock(&mdev->state_lock); 2095 2096 mlx4_en_stop_port(dev, 0); 2097 netif_carrier_off(dev); 2098 2099 mutex_unlock(&mdev->state_lock); 2100 return 0; 2101 } 2102 2103 static void mlx4_en_free_resources(struct mlx4_en_priv *priv) 2104 { 2105 int i, t; 2106 2107 #ifdef CONFIG_RFS_ACCEL 2108 priv->dev->rx_cpu_rmap = NULL; 2109 #endif 2110 2111 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { 2112 for (i = 0; i < priv->tx_ring_num[t]; i++) { 2113 if (priv->tx_ring[t] && priv->tx_ring[t][i]) 2114 mlx4_en_destroy_tx_ring(priv, 2115 &priv->tx_ring[t][i]); 2116 if (priv->tx_cq[t] && priv->tx_cq[t][i]) 2117 mlx4_en_destroy_cq(priv, &priv->tx_cq[t][i]); 2118 } 2119 kfree(priv->tx_ring[t]); 2120 kfree(priv->tx_cq[t]); 2121 } 2122 2123 for (i = 0; i < priv->rx_ring_num; i++) { 2124 if (priv->rx_ring[i]) 2125 mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i], 2126 priv->prof->rx_ring_size, priv->stride); 2127 if (priv->rx_cq[i]) 2128 mlx4_en_destroy_cq(priv, &priv->rx_cq[i]); 2129 } 2130 2131 } 2132 2133 static int mlx4_en_alloc_resources(struct mlx4_en_priv *priv) 2134 { 2135 struct mlx4_en_port_profile *prof = priv->prof; 2136 int i, t; 2137 int node; 2138 2139 /* Create tx Rings */ 2140 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { 2141 for (i = 0; i < priv->tx_ring_num[t]; i++) { 2142 node = cpu_to_node(i % num_online_cpus()); 2143 if (mlx4_en_create_cq(priv, &priv->tx_cq[t][i], 2144 prof->tx_ring_size, i, t, node)) 2145 goto err; 2146 2147 if (mlx4_en_create_tx_ring(priv, &priv->tx_ring[t][i], 2148 prof->tx_ring_size, 2149 TXBB_SIZE, node, i)) 2150 goto err; 2151 } 2152 } 2153 2154 /* Create rx Rings */ 2155 for (i = 0; i < priv->rx_ring_num; i++) { 2156 node = cpu_to_node(i % num_online_cpus()); 2157 if (mlx4_en_create_cq(priv, &priv->rx_cq[i], 2158 prof->rx_ring_size, i, RX, node)) 2159 goto err; 2160 2161 if (mlx4_en_create_rx_ring(priv, &priv->rx_ring[i], 2162 prof->rx_ring_size, priv->stride, 2163 node, i)) 2164 goto err; 2165 2166 } 2167 2168 #ifdef CONFIG_RFS_ACCEL 2169 priv->dev->rx_cpu_rmap = mlx4_get_cpu_rmap(priv->mdev->dev, priv->port); 2170 #endif 2171 2172 return 0; 2173 2174 err: 2175 en_err(priv, "Failed to allocate NIC resources\n"); 2176 for (i = 0; i < priv->rx_ring_num; i++) { 2177 if (priv->rx_ring[i]) 2178 mlx4_en_destroy_rx_ring(priv, &priv->rx_ring[i], 2179 prof->rx_ring_size, 2180 priv->stride); 2181 if (priv->rx_cq[i]) 2182 mlx4_en_destroy_cq(priv, &priv->rx_cq[i]); 2183 } 2184 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { 2185 for (i = 0; i < priv->tx_ring_num[t]; i++) { 2186 if (priv->tx_ring[t][i]) 2187 mlx4_en_destroy_tx_ring(priv, 2188 &priv->tx_ring[t][i]); 2189 if (priv->tx_cq[t][i]) 2190 mlx4_en_destroy_cq(priv, &priv->tx_cq[t][i]); 2191 } 2192 } 2193 return -ENOMEM; 2194 } 2195 2196 2197 static int mlx4_en_copy_priv(struct mlx4_en_priv *dst, 2198 struct mlx4_en_priv *src, 2199 struct mlx4_en_port_profile *prof) 2200 { 2201 int t; 2202 2203 memcpy(&dst->hwtstamp_config, &prof->hwtstamp_config, 2204 sizeof(dst->hwtstamp_config)); 2205 dst->num_tx_rings_p_up = prof->num_tx_rings_p_up; 2206 dst->rx_ring_num = prof->rx_ring_num; 2207 dst->flags = prof->flags; 2208 dst->mdev = src->mdev; 2209 dst->port = src->port; 2210 dst->dev = src->dev; 2211 dst->prof = prof; 2212 dst->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + 2213 DS_SIZE * MLX4_EN_MAX_RX_FRAGS); 2214 2215 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { 2216 dst->tx_ring_num[t] = prof->tx_ring_num[t]; 2217 if (!dst->tx_ring_num[t]) 2218 continue; 2219 2220 dst->tx_ring[t] = kcalloc(MAX_TX_RINGS, 2221 sizeof(struct mlx4_en_tx_ring *), 2222 GFP_KERNEL); 2223 if (!dst->tx_ring[t]) 2224 goto err_free_tx; 2225 2226 dst->tx_cq[t] = kcalloc(MAX_TX_RINGS, 2227 sizeof(struct mlx4_en_cq *), 2228 GFP_KERNEL); 2229 if (!dst->tx_cq[t]) { 2230 kfree(dst->tx_ring[t]); 2231 goto err_free_tx; 2232 } 2233 } 2234 2235 return 0; 2236 2237 err_free_tx: 2238 while (t--) { 2239 kfree(dst->tx_ring[t]); 2240 kfree(dst->tx_cq[t]); 2241 } 2242 return -ENOMEM; 2243 } 2244 2245 static void mlx4_en_update_priv(struct mlx4_en_priv *dst, 2246 struct mlx4_en_priv *src) 2247 { 2248 int t; 2249 memcpy(dst->rx_ring, src->rx_ring, 2250 sizeof(struct mlx4_en_rx_ring *) * src->rx_ring_num); 2251 memcpy(dst->rx_cq, src->rx_cq, 2252 sizeof(struct mlx4_en_cq *) * src->rx_ring_num); 2253 memcpy(&dst->hwtstamp_config, &src->hwtstamp_config, 2254 sizeof(dst->hwtstamp_config)); 2255 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { 2256 dst->tx_ring_num[t] = src->tx_ring_num[t]; 2257 dst->tx_ring[t] = src->tx_ring[t]; 2258 dst->tx_cq[t] = src->tx_cq[t]; 2259 } 2260 dst->num_tx_rings_p_up = src->num_tx_rings_p_up; 2261 dst->rx_ring_num = src->rx_ring_num; 2262 memcpy(dst->prof, src->prof, sizeof(struct mlx4_en_port_profile)); 2263 } 2264 2265 int mlx4_en_try_alloc_resources(struct mlx4_en_priv *priv, 2266 struct mlx4_en_priv *tmp, 2267 struct mlx4_en_port_profile *prof, 2268 bool carry_xdp_prog) 2269 { 2270 struct bpf_prog *xdp_prog; 2271 int i, t; 2272 2273 mlx4_en_copy_priv(tmp, priv, prof); 2274 2275 if (mlx4_en_alloc_resources(tmp)) { 2276 en_warn(priv, 2277 "%s: Resource allocation failed, using previous configuration\n", 2278 __func__); 2279 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { 2280 kfree(tmp->tx_ring[t]); 2281 kfree(tmp->tx_cq[t]); 2282 } 2283 return -ENOMEM; 2284 } 2285 2286 /* All rx_rings has the same xdp_prog. Pick the first one. */ 2287 xdp_prog = rcu_dereference_protected( 2288 priv->rx_ring[0]->xdp_prog, 2289 lockdep_is_held(&priv->mdev->state_lock)); 2290 2291 if (xdp_prog && carry_xdp_prog) { 2292 bpf_prog_add(xdp_prog, tmp->rx_ring_num); 2293 for (i = 0; i < tmp->rx_ring_num; i++) 2294 rcu_assign_pointer(tmp->rx_ring[i]->xdp_prog, 2295 xdp_prog); 2296 } 2297 2298 return 0; 2299 } 2300 2301 void mlx4_en_safe_replace_resources(struct mlx4_en_priv *priv, 2302 struct mlx4_en_priv *tmp) 2303 { 2304 mlx4_en_free_resources(priv); 2305 mlx4_en_update_priv(priv, tmp); 2306 } 2307 2308 void mlx4_en_destroy_netdev(struct net_device *dev) 2309 { 2310 struct mlx4_en_priv *priv = netdev_priv(dev); 2311 struct mlx4_en_dev *mdev = priv->mdev; 2312 2313 en_dbg(DRV, priv, "Destroying netdev on port:%d\n", priv->port); 2314 2315 /* Unregister device - this will close the port if it was up */ 2316 if (priv->registered) { 2317 devlink_port_type_clear(mlx4_get_devlink_port(mdev->dev, 2318 priv->port)); 2319 unregister_netdev(dev); 2320 } 2321 2322 if (priv->allocated) 2323 mlx4_free_hwq_res(mdev->dev, &priv->res, MLX4_EN_PAGE_SIZE); 2324 2325 cancel_delayed_work(&priv->stats_task); 2326 cancel_delayed_work(&priv->service_task); 2327 /* flush any pending task for this netdev */ 2328 flush_workqueue(mdev->workqueue); 2329 2330 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) 2331 mlx4_en_remove_timestamp(mdev); 2332 2333 /* Detach the netdev so tasks would not attempt to access it */ 2334 mutex_lock(&mdev->state_lock); 2335 mdev->pndev[priv->port] = NULL; 2336 mdev->upper[priv->port] = NULL; 2337 2338 #ifdef CONFIG_RFS_ACCEL 2339 mlx4_en_cleanup_filters(priv); 2340 #endif 2341 2342 mlx4_en_free_resources(priv); 2343 mutex_unlock(&mdev->state_lock); 2344 2345 free_netdev(dev); 2346 } 2347 2348 static bool mlx4_en_check_xdp_mtu(struct net_device *dev, int mtu) 2349 { 2350 struct mlx4_en_priv *priv = netdev_priv(dev); 2351 2352 if (mtu > MLX4_EN_MAX_XDP_MTU) { 2353 en_err(priv, "mtu:%d > max:%d when XDP prog is attached\n", 2354 mtu, MLX4_EN_MAX_XDP_MTU); 2355 return false; 2356 } 2357 2358 return true; 2359 } 2360 2361 static int mlx4_en_change_mtu(struct net_device *dev, int new_mtu) 2362 { 2363 struct mlx4_en_priv *priv = netdev_priv(dev); 2364 struct mlx4_en_dev *mdev = priv->mdev; 2365 int err = 0; 2366 2367 en_dbg(DRV, priv, "Change MTU called - current:%d new:%d\n", 2368 dev->mtu, new_mtu); 2369 2370 if (priv->tx_ring_num[TX_XDP] && 2371 !mlx4_en_check_xdp_mtu(dev, new_mtu)) 2372 return -EOPNOTSUPP; 2373 2374 dev->mtu = new_mtu; 2375 2376 if (netif_running(dev)) { 2377 mutex_lock(&mdev->state_lock); 2378 if (!mdev->device_up) { 2379 /* NIC is probably restarting - let watchdog task reset 2380 * the port */ 2381 en_dbg(DRV, priv, "Change MTU called with card down!?\n"); 2382 } else { 2383 mlx4_en_stop_port(dev, 1); 2384 err = mlx4_en_start_port(dev); 2385 if (err) { 2386 en_err(priv, "Failed restarting port:%d\n", 2387 priv->port); 2388 queue_work(mdev->workqueue, &priv->watchdog_task); 2389 } 2390 } 2391 mutex_unlock(&mdev->state_lock); 2392 } 2393 return 0; 2394 } 2395 2396 static int mlx4_en_hwtstamp_set(struct net_device *dev, struct ifreq *ifr) 2397 { 2398 struct mlx4_en_priv *priv = netdev_priv(dev); 2399 struct mlx4_en_dev *mdev = priv->mdev; 2400 struct hwtstamp_config config; 2401 2402 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 2403 return -EFAULT; 2404 2405 /* reserved for future extensions */ 2406 if (config.flags) 2407 return -EINVAL; 2408 2409 /* device doesn't support time stamping */ 2410 if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS)) 2411 return -EINVAL; 2412 2413 /* TX HW timestamp */ 2414 switch (config.tx_type) { 2415 case HWTSTAMP_TX_OFF: 2416 case HWTSTAMP_TX_ON: 2417 break; 2418 default: 2419 return -ERANGE; 2420 } 2421 2422 /* RX HW timestamp */ 2423 switch (config.rx_filter) { 2424 case HWTSTAMP_FILTER_NONE: 2425 break; 2426 case HWTSTAMP_FILTER_ALL: 2427 case HWTSTAMP_FILTER_SOME: 2428 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 2429 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 2430 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 2431 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 2432 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 2433 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 2434 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 2435 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 2436 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 2437 case HWTSTAMP_FILTER_PTP_V2_EVENT: 2438 case HWTSTAMP_FILTER_PTP_V2_SYNC: 2439 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 2440 case HWTSTAMP_FILTER_NTP_ALL: 2441 config.rx_filter = HWTSTAMP_FILTER_ALL; 2442 break; 2443 default: 2444 return -ERANGE; 2445 } 2446 2447 if (mlx4_en_reset_config(dev, config, dev->features)) { 2448 config.tx_type = HWTSTAMP_TX_OFF; 2449 config.rx_filter = HWTSTAMP_FILTER_NONE; 2450 } 2451 2452 return copy_to_user(ifr->ifr_data, &config, 2453 sizeof(config)) ? -EFAULT : 0; 2454 } 2455 2456 static int mlx4_en_hwtstamp_get(struct net_device *dev, struct ifreq *ifr) 2457 { 2458 struct mlx4_en_priv *priv = netdev_priv(dev); 2459 2460 return copy_to_user(ifr->ifr_data, &priv->hwtstamp_config, 2461 sizeof(priv->hwtstamp_config)) ? -EFAULT : 0; 2462 } 2463 2464 static int mlx4_en_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 2465 { 2466 switch (cmd) { 2467 case SIOCSHWTSTAMP: 2468 return mlx4_en_hwtstamp_set(dev, ifr); 2469 case SIOCGHWTSTAMP: 2470 return mlx4_en_hwtstamp_get(dev, ifr); 2471 default: 2472 return -EOPNOTSUPP; 2473 } 2474 } 2475 2476 static netdev_features_t mlx4_en_fix_features(struct net_device *netdev, 2477 netdev_features_t features) 2478 { 2479 struct mlx4_en_priv *en_priv = netdev_priv(netdev); 2480 struct mlx4_en_dev *mdev = en_priv->mdev; 2481 2482 /* Since there is no support for separate RX C-TAG/S-TAG vlan accel 2483 * enable/disable make sure S-TAG flag is always in same state as 2484 * C-TAG. 2485 */ 2486 if (features & NETIF_F_HW_VLAN_CTAG_RX && 2487 !(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN)) 2488 features |= NETIF_F_HW_VLAN_STAG_RX; 2489 else 2490 features &= ~NETIF_F_HW_VLAN_STAG_RX; 2491 2492 return features; 2493 } 2494 2495 static int mlx4_en_set_features(struct net_device *netdev, 2496 netdev_features_t features) 2497 { 2498 struct mlx4_en_priv *priv = netdev_priv(netdev); 2499 bool reset = false; 2500 int ret = 0; 2501 2502 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_RXFCS)) { 2503 en_info(priv, "Turn %s RX-FCS\n", 2504 (features & NETIF_F_RXFCS) ? "ON" : "OFF"); 2505 reset = true; 2506 } 2507 2508 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_RXALL)) { 2509 u8 ignore_fcs_value = (features & NETIF_F_RXALL) ? 1 : 0; 2510 2511 en_info(priv, "Turn %s RX-ALL\n", 2512 ignore_fcs_value ? "ON" : "OFF"); 2513 ret = mlx4_SET_PORT_fcs_check(priv->mdev->dev, 2514 priv->port, ignore_fcs_value); 2515 if (ret) 2516 return ret; 2517 } 2518 2519 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_HW_VLAN_CTAG_RX)) { 2520 en_info(priv, "Turn %s RX vlan strip offload\n", 2521 (features & NETIF_F_HW_VLAN_CTAG_RX) ? "ON" : "OFF"); 2522 reset = true; 2523 } 2524 2525 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_HW_VLAN_CTAG_TX)) 2526 en_info(priv, "Turn %s TX vlan strip offload\n", 2527 (features & NETIF_F_HW_VLAN_CTAG_TX) ? "ON" : "OFF"); 2528 2529 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_HW_VLAN_STAG_TX)) 2530 en_info(priv, "Turn %s TX S-VLAN strip offload\n", 2531 (features & NETIF_F_HW_VLAN_STAG_TX) ? "ON" : "OFF"); 2532 2533 if (DEV_FEATURE_CHANGED(netdev, features, NETIF_F_LOOPBACK)) { 2534 en_info(priv, "Turn %s loopback\n", 2535 (features & NETIF_F_LOOPBACK) ? "ON" : "OFF"); 2536 mlx4_en_update_loopback_state(netdev, features); 2537 } 2538 2539 if (reset) { 2540 ret = mlx4_en_reset_config(netdev, priv->hwtstamp_config, 2541 features); 2542 if (ret) 2543 return ret; 2544 } 2545 2546 return 0; 2547 } 2548 2549 static int mlx4_en_set_vf_mac(struct net_device *dev, int queue, u8 *mac) 2550 { 2551 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2552 struct mlx4_en_dev *mdev = en_priv->mdev; 2553 2554 return mlx4_set_vf_mac(mdev->dev, en_priv->port, queue, mac); 2555 } 2556 2557 static int mlx4_en_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos, 2558 __be16 vlan_proto) 2559 { 2560 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2561 struct mlx4_en_dev *mdev = en_priv->mdev; 2562 2563 return mlx4_set_vf_vlan(mdev->dev, en_priv->port, vf, vlan, qos, 2564 vlan_proto); 2565 } 2566 2567 static int mlx4_en_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, 2568 int max_tx_rate) 2569 { 2570 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2571 struct mlx4_en_dev *mdev = en_priv->mdev; 2572 2573 return mlx4_set_vf_rate(mdev->dev, en_priv->port, vf, min_tx_rate, 2574 max_tx_rate); 2575 } 2576 2577 static int mlx4_en_set_vf_spoofchk(struct net_device *dev, int vf, bool setting) 2578 { 2579 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2580 struct mlx4_en_dev *mdev = en_priv->mdev; 2581 2582 return mlx4_set_vf_spoofchk(mdev->dev, en_priv->port, vf, setting); 2583 } 2584 2585 static int mlx4_en_get_vf_config(struct net_device *dev, int vf, struct ifla_vf_info *ivf) 2586 { 2587 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2588 struct mlx4_en_dev *mdev = en_priv->mdev; 2589 2590 return mlx4_get_vf_config(mdev->dev, en_priv->port, vf, ivf); 2591 } 2592 2593 static int mlx4_en_set_vf_link_state(struct net_device *dev, int vf, int link_state) 2594 { 2595 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2596 struct mlx4_en_dev *mdev = en_priv->mdev; 2597 2598 return mlx4_set_vf_link_state(mdev->dev, en_priv->port, vf, link_state); 2599 } 2600 2601 static int mlx4_en_get_vf_stats(struct net_device *dev, int vf, 2602 struct ifla_vf_stats *vf_stats) 2603 { 2604 struct mlx4_en_priv *en_priv = netdev_priv(dev); 2605 struct mlx4_en_dev *mdev = en_priv->mdev; 2606 2607 return mlx4_get_vf_stats(mdev->dev, en_priv->port, vf, vf_stats); 2608 } 2609 2610 #define PORT_ID_BYTE_LEN 8 2611 static int mlx4_en_get_phys_port_id(struct net_device *dev, 2612 struct netdev_phys_item_id *ppid) 2613 { 2614 struct mlx4_en_priv *priv = netdev_priv(dev); 2615 struct mlx4_dev *mdev = priv->mdev->dev; 2616 int i; 2617 u64 phys_port_id = mdev->caps.phys_port_id[priv->port]; 2618 2619 if (!phys_port_id) 2620 return -EOPNOTSUPP; 2621 2622 ppid->id_len = sizeof(phys_port_id); 2623 for (i = PORT_ID_BYTE_LEN - 1; i >= 0; --i) { 2624 ppid->id[i] = phys_port_id & 0xff; 2625 phys_port_id >>= 8; 2626 } 2627 return 0; 2628 } 2629 2630 static void mlx4_en_add_vxlan_offloads(struct work_struct *work) 2631 { 2632 int ret; 2633 struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, 2634 vxlan_add_task); 2635 2636 ret = mlx4_config_vxlan_port(priv->mdev->dev, priv->vxlan_port); 2637 if (ret) 2638 goto out; 2639 2640 ret = mlx4_SET_PORT_VXLAN(priv->mdev->dev, priv->port, 2641 VXLAN_STEER_BY_OUTER_MAC, 1); 2642 out: 2643 if (ret) { 2644 en_err(priv, "failed setting L2 tunnel configuration ret %d\n", ret); 2645 return; 2646 } 2647 } 2648 2649 static void mlx4_en_del_vxlan_offloads(struct work_struct *work) 2650 { 2651 int ret; 2652 struct mlx4_en_priv *priv = container_of(work, struct mlx4_en_priv, 2653 vxlan_del_task); 2654 ret = mlx4_SET_PORT_VXLAN(priv->mdev->dev, priv->port, 2655 VXLAN_STEER_BY_OUTER_MAC, 0); 2656 if (ret) 2657 en_err(priv, "failed setting L2 tunnel configuration ret %d\n", ret); 2658 2659 priv->vxlan_port = 0; 2660 } 2661 2662 static void mlx4_en_add_vxlan_port(struct net_device *dev, 2663 struct udp_tunnel_info *ti) 2664 { 2665 struct mlx4_en_priv *priv = netdev_priv(dev); 2666 __be16 port = ti->port; 2667 __be16 current_port; 2668 2669 if (ti->type != UDP_TUNNEL_TYPE_VXLAN) 2670 return; 2671 2672 if (ti->sa_family != AF_INET) 2673 return; 2674 2675 if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) 2676 return; 2677 2678 current_port = priv->vxlan_port; 2679 if (current_port && current_port != port) { 2680 en_warn(priv, "vxlan port %d configured, can't add port %d\n", 2681 ntohs(current_port), ntohs(port)); 2682 return; 2683 } 2684 2685 priv->vxlan_port = port; 2686 queue_work(priv->mdev->workqueue, &priv->vxlan_add_task); 2687 } 2688 2689 static void mlx4_en_del_vxlan_port(struct net_device *dev, 2690 struct udp_tunnel_info *ti) 2691 { 2692 struct mlx4_en_priv *priv = netdev_priv(dev); 2693 __be16 port = ti->port; 2694 __be16 current_port; 2695 2696 if (ti->type != UDP_TUNNEL_TYPE_VXLAN) 2697 return; 2698 2699 if (ti->sa_family != AF_INET) 2700 return; 2701 2702 if (priv->mdev->dev->caps.tunnel_offload_mode != MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) 2703 return; 2704 2705 current_port = priv->vxlan_port; 2706 if (current_port != port) { 2707 en_dbg(DRV, priv, "vxlan port %d isn't configured, ignoring\n", ntohs(port)); 2708 return; 2709 } 2710 2711 queue_work(priv->mdev->workqueue, &priv->vxlan_del_task); 2712 } 2713 2714 static netdev_features_t mlx4_en_features_check(struct sk_buff *skb, 2715 struct net_device *dev, 2716 netdev_features_t features) 2717 { 2718 features = vlan_features_check(skb, features); 2719 features = vxlan_features_check(skb, features); 2720 2721 /* The ConnectX-3 doesn't support outer IPv6 checksums but it does 2722 * support inner IPv6 checksums and segmentation so we need to 2723 * strip that feature if this is an IPv6 encapsulated frame. 2724 */ 2725 if (skb->encapsulation && 2726 (skb->ip_summed == CHECKSUM_PARTIAL)) { 2727 struct mlx4_en_priv *priv = netdev_priv(dev); 2728 2729 if (!priv->vxlan_port || 2730 (ip_hdr(skb)->version != 4) || 2731 (udp_hdr(skb)->dest != priv->vxlan_port)) 2732 features &= ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 2733 } 2734 2735 return features; 2736 } 2737 2738 static int mlx4_en_set_tx_maxrate(struct net_device *dev, int queue_index, u32 maxrate) 2739 { 2740 struct mlx4_en_priv *priv = netdev_priv(dev); 2741 struct mlx4_en_tx_ring *tx_ring = priv->tx_ring[TX][queue_index]; 2742 struct mlx4_update_qp_params params; 2743 int err; 2744 2745 if (!(priv->mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_QP_RATE_LIMIT)) 2746 return -EOPNOTSUPP; 2747 2748 /* rate provided to us in Mbs, check if it fits into 12 bits, if not use Gbs */ 2749 if (maxrate >> 12) { 2750 params.rate_unit = MLX4_QP_RATE_LIMIT_GBS; 2751 params.rate_val = maxrate / 1000; 2752 } else if (maxrate) { 2753 params.rate_unit = MLX4_QP_RATE_LIMIT_MBS; 2754 params.rate_val = maxrate; 2755 } else { /* zero serves to revoke the QP rate-limitation */ 2756 params.rate_unit = 0; 2757 params.rate_val = 0; 2758 } 2759 2760 err = mlx4_update_qp(priv->mdev->dev, tx_ring->qpn, MLX4_UPDATE_QP_RATE_LIMIT, 2761 ¶ms); 2762 return err; 2763 } 2764 2765 static int mlx4_xdp_set(struct net_device *dev, struct bpf_prog *prog) 2766 { 2767 struct mlx4_en_priv *priv = netdev_priv(dev); 2768 struct mlx4_en_dev *mdev = priv->mdev; 2769 struct mlx4_en_port_profile new_prof; 2770 struct bpf_prog *old_prog; 2771 struct mlx4_en_priv *tmp; 2772 int tx_changed = 0; 2773 int xdp_ring_num; 2774 int port_up = 0; 2775 int err; 2776 int i; 2777 2778 xdp_ring_num = prog ? priv->rx_ring_num : 0; 2779 2780 /* No need to reconfigure buffers when simply swapping the 2781 * program for a new one. 2782 */ 2783 if (priv->tx_ring_num[TX_XDP] == xdp_ring_num) { 2784 if (prog) 2785 bpf_prog_add(prog, priv->rx_ring_num - 1); 2786 2787 mutex_lock(&mdev->state_lock); 2788 for (i = 0; i < priv->rx_ring_num; i++) { 2789 old_prog = rcu_dereference_protected( 2790 priv->rx_ring[i]->xdp_prog, 2791 lockdep_is_held(&mdev->state_lock)); 2792 rcu_assign_pointer(priv->rx_ring[i]->xdp_prog, prog); 2793 if (old_prog) 2794 bpf_prog_put(old_prog); 2795 } 2796 mutex_unlock(&mdev->state_lock); 2797 return 0; 2798 } 2799 2800 if (!mlx4_en_check_xdp_mtu(dev, dev->mtu)) 2801 return -EOPNOTSUPP; 2802 2803 tmp = kzalloc(sizeof(*tmp), GFP_KERNEL); 2804 if (!tmp) 2805 return -ENOMEM; 2806 2807 if (prog) 2808 bpf_prog_add(prog, priv->rx_ring_num - 1); 2809 2810 mutex_lock(&mdev->state_lock); 2811 memcpy(&new_prof, priv->prof, sizeof(struct mlx4_en_port_profile)); 2812 new_prof.tx_ring_num[TX_XDP] = xdp_ring_num; 2813 2814 if (priv->tx_ring_num[TX] + xdp_ring_num > MAX_TX_RINGS) { 2815 tx_changed = 1; 2816 new_prof.tx_ring_num[TX] = 2817 MAX_TX_RINGS - ALIGN(xdp_ring_num, priv->prof->num_up); 2818 en_warn(priv, "Reducing the number of TX rings, to not exceed the max total rings number.\n"); 2819 } 2820 2821 err = mlx4_en_try_alloc_resources(priv, tmp, &new_prof, false); 2822 if (err) { 2823 if (prog) 2824 bpf_prog_sub(prog, priv->rx_ring_num - 1); 2825 goto unlock_out; 2826 } 2827 2828 if (priv->port_up) { 2829 port_up = 1; 2830 mlx4_en_stop_port(dev, 1); 2831 } 2832 2833 mlx4_en_safe_replace_resources(priv, tmp); 2834 if (tx_changed) 2835 netif_set_real_num_tx_queues(dev, priv->tx_ring_num[TX]); 2836 2837 for (i = 0; i < priv->rx_ring_num; i++) { 2838 old_prog = rcu_dereference_protected( 2839 priv->rx_ring[i]->xdp_prog, 2840 lockdep_is_held(&mdev->state_lock)); 2841 rcu_assign_pointer(priv->rx_ring[i]->xdp_prog, prog); 2842 if (old_prog) 2843 bpf_prog_put(old_prog); 2844 } 2845 2846 if (port_up) { 2847 err = mlx4_en_start_port(dev); 2848 if (err) { 2849 en_err(priv, "Failed starting port %d for XDP change\n", 2850 priv->port); 2851 queue_work(mdev->workqueue, &priv->watchdog_task); 2852 } 2853 } 2854 2855 unlock_out: 2856 mutex_unlock(&mdev->state_lock); 2857 kfree(tmp); 2858 return err; 2859 } 2860 2861 static u32 mlx4_xdp_query(struct net_device *dev) 2862 { 2863 struct mlx4_en_priv *priv = netdev_priv(dev); 2864 struct mlx4_en_dev *mdev = priv->mdev; 2865 const struct bpf_prog *xdp_prog; 2866 u32 prog_id = 0; 2867 2868 if (!priv->tx_ring_num[TX_XDP]) 2869 return prog_id; 2870 2871 mutex_lock(&mdev->state_lock); 2872 xdp_prog = rcu_dereference_protected( 2873 priv->rx_ring[0]->xdp_prog, 2874 lockdep_is_held(&mdev->state_lock)); 2875 if (xdp_prog) 2876 prog_id = xdp_prog->aux->id; 2877 mutex_unlock(&mdev->state_lock); 2878 2879 return prog_id; 2880 } 2881 2882 static int mlx4_xdp(struct net_device *dev, struct netdev_bpf *xdp) 2883 { 2884 switch (xdp->command) { 2885 case XDP_SETUP_PROG: 2886 return mlx4_xdp_set(dev, xdp->prog); 2887 case XDP_QUERY_PROG: 2888 xdp->prog_id = mlx4_xdp_query(dev); 2889 return 0; 2890 default: 2891 return -EINVAL; 2892 } 2893 } 2894 2895 static const struct net_device_ops mlx4_netdev_ops = { 2896 .ndo_open = mlx4_en_open, 2897 .ndo_stop = mlx4_en_close, 2898 .ndo_start_xmit = mlx4_en_xmit, 2899 .ndo_select_queue = mlx4_en_select_queue, 2900 .ndo_get_stats64 = mlx4_en_get_stats64, 2901 .ndo_set_rx_mode = mlx4_en_set_rx_mode, 2902 .ndo_set_mac_address = mlx4_en_set_mac, 2903 .ndo_validate_addr = eth_validate_addr, 2904 .ndo_change_mtu = mlx4_en_change_mtu, 2905 .ndo_do_ioctl = mlx4_en_ioctl, 2906 .ndo_tx_timeout = mlx4_en_tx_timeout, 2907 .ndo_vlan_rx_add_vid = mlx4_en_vlan_rx_add_vid, 2908 .ndo_vlan_rx_kill_vid = mlx4_en_vlan_rx_kill_vid, 2909 .ndo_set_features = mlx4_en_set_features, 2910 .ndo_fix_features = mlx4_en_fix_features, 2911 .ndo_setup_tc = __mlx4_en_setup_tc, 2912 #ifdef CONFIG_RFS_ACCEL 2913 .ndo_rx_flow_steer = mlx4_en_filter_rfs, 2914 #endif 2915 .ndo_get_phys_port_id = mlx4_en_get_phys_port_id, 2916 .ndo_udp_tunnel_add = mlx4_en_add_vxlan_port, 2917 .ndo_udp_tunnel_del = mlx4_en_del_vxlan_port, 2918 .ndo_features_check = mlx4_en_features_check, 2919 .ndo_set_tx_maxrate = mlx4_en_set_tx_maxrate, 2920 .ndo_bpf = mlx4_xdp, 2921 }; 2922 2923 static const struct net_device_ops mlx4_netdev_ops_master = { 2924 .ndo_open = mlx4_en_open, 2925 .ndo_stop = mlx4_en_close, 2926 .ndo_start_xmit = mlx4_en_xmit, 2927 .ndo_select_queue = mlx4_en_select_queue, 2928 .ndo_get_stats64 = mlx4_en_get_stats64, 2929 .ndo_set_rx_mode = mlx4_en_set_rx_mode, 2930 .ndo_set_mac_address = mlx4_en_set_mac, 2931 .ndo_validate_addr = eth_validate_addr, 2932 .ndo_change_mtu = mlx4_en_change_mtu, 2933 .ndo_tx_timeout = mlx4_en_tx_timeout, 2934 .ndo_vlan_rx_add_vid = mlx4_en_vlan_rx_add_vid, 2935 .ndo_vlan_rx_kill_vid = mlx4_en_vlan_rx_kill_vid, 2936 .ndo_set_vf_mac = mlx4_en_set_vf_mac, 2937 .ndo_set_vf_vlan = mlx4_en_set_vf_vlan, 2938 .ndo_set_vf_rate = mlx4_en_set_vf_rate, 2939 .ndo_set_vf_spoofchk = mlx4_en_set_vf_spoofchk, 2940 .ndo_set_vf_link_state = mlx4_en_set_vf_link_state, 2941 .ndo_get_vf_stats = mlx4_en_get_vf_stats, 2942 .ndo_get_vf_config = mlx4_en_get_vf_config, 2943 .ndo_set_features = mlx4_en_set_features, 2944 .ndo_fix_features = mlx4_en_fix_features, 2945 .ndo_setup_tc = __mlx4_en_setup_tc, 2946 #ifdef CONFIG_RFS_ACCEL 2947 .ndo_rx_flow_steer = mlx4_en_filter_rfs, 2948 #endif 2949 .ndo_get_phys_port_id = mlx4_en_get_phys_port_id, 2950 .ndo_udp_tunnel_add = mlx4_en_add_vxlan_port, 2951 .ndo_udp_tunnel_del = mlx4_en_del_vxlan_port, 2952 .ndo_features_check = mlx4_en_features_check, 2953 .ndo_set_tx_maxrate = mlx4_en_set_tx_maxrate, 2954 .ndo_bpf = mlx4_xdp, 2955 }; 2956 2957 struct mlx4_en_bond { 2958 struct work_struct work; 2959 struct mlx4_en_priv *priv; 2960 int is_bonded; 2961 struct mlx4_port_map port_map; 2962 }; 2963 2964 static void mlx4_en_bond_work(struct work_struct *work) 2965 { 2966 struct mlx4_en_bond *bond = container_of(work, 2967 struct mlx4_en_bond, 2968 work); 2969 int err = 0; 2970 struct mlx4_dev *dev = bond->priv->mdev->dev; 2971 2972 if (bond->is_bonded) { 2973 if (!mlx4_is_bonded(dev)) { 2974 err = mlx4_bond(dev); 2975 if (err) 2976 en_err(bond->priv, "Fail to bond device\n"); 2977 } 2978 if (!err) { 2979 err = mlx4_port_map_set(dev, &bond->port_map); 2980 if (err) 2981 en_err(bond->priv, "Fail to set port map [%d][%d]: %d\n", 2982 bond->port_map.port1, 2983 bond->port_map.port2, 2984 err); 2985 } 2986 } else if (mlx4_is_bonded(dev)) { 2987 err = mlx4_unbond(dev); 2988 if (err) 2989 en_err(bond->priv, "Fail to unbond device\n"); 2990 } 2991 dev_put(bond->priv->dev); 2992 kfree(bond); 2993 } 2994 2995 static int mlx4_en_queue_bond_work(struct mlx4_en_priv *priv, int is_bonded, 2996 u8 v2p_p1, u8 v2p_p2) 2997 { 2998 struct mlx4_en_bond *bond = NULL; 2999 3000 bond = kzalloc(sizeof(*bond), GFP_ATOMIC); 3001 if (!bond) 3002 return -ENOMEM; 3003 3004 INIT_WORK(&bond->work, mlx4_en_bond_work); 3005 bond->priv = priv; 3006 bond->is_bonded = is_bonded; 3007 bond->port_map.port1 = v2p_p1; 3008 bond->port_map.port2 = v2p_p2; 3009 dev_hold(priv->dev); 3010 queue_work(priv->mdev->workqueue, &bond->work); 3011 return 0; 3012 } 3013 3014 int mlx4_en_netdev_event(struct notifier_block *this, 3015 unsigned long event, void *ptr) 3016 { 3017 struct net_device *ndev = netdev_notifier_info_to_dev(ptr); 3018 u8 port = 0; 3019 struct mlx4_en_dev *mdev; 3020 struct mlx4_dev *dev; 3021 int i, num_eth_ports = 0; 3022 bool do_bond = true; 3023 struct mlx4_en_priv *priv; 3024 u8 v2p_port1 = 0; 3025 u8 v2p_port2 = 0; 3026 3027 if (!net_eq(dev_net(ndev), &init_net)) 3028 return NOTIFY_DONE; 3029 3030 mdev = container_of(this, struct mlx4_en_dev, nb); 3031 dev = mdev->dev; 3032 3033 /* Go into this mode only when two network devices set on two ports 3034 * of the same mlx4 device are slaves of the same bonding master 3035 */ 3036 mlx4_foreach_port(i, dev, MLX4_PORT_TYPE_ETH) { 3037 ++num_eth_ports; 3038 if (!port && (mdev->pndev[i] == ndev)) 3039 port = i; 3040 mdev->upper[i] = mdev->pndev[i] ? 3041 netdev_master_upper_dev_get(mdev->pndev[i]) : NULL; 3042 /* condition not met: network device is a slave */ 3043 if (!mdev->upper[i]) 3044 do_bond = false; 3045 if (num_eth_ports < 2) 3046 continue; 3047 /* condition not met: same master */ 3048 if (mdev->upper[i] != mdev->upper[i-1]) 3049 do_bond = false; 3050 } 3051 /* condition not met: 2 salves */ 3052 do_bond = (num_eth_ports == 2) ? do_bond : false; 3053 3054 /* handle only events that come with enough info */ 3055 if ((do_bond && (event != NETDEV_BONDING_INFO)) || !port) 3056 return NOTIFY_DONE; 3057 3058 priv = netdev_priv(ndev); 3059 if (do_bond) { 3060 struct netdev_notifier_bonding_info *notifier_info = ptr; 3061 struct netdev_bonding_info *bonding_info = 3062 ¬ifier_info->bonding_info; 3063 3064 /* required mode 1, 2 or 4 */ 3065 if ((bonding_info->master.bond_mode != BOND_MODE_ACTIVEBACKUP) && 3066 (bonding_info->master.bond_mode != BOND_MODE_XOR) && 3067 (bonding_info->master.bond_mode != BOND_MODE_8023AD)) 3068 do_bond = false; 3069 3070 /* require exactly 2 slaves */ 3071 if (bonding_info->master.num_slaves != 2) 3072 do_bond = false; 3073 3074 /* calc v2p */ 3075 if (do_bond) { 3076 if (bonding_info->master.bond_mode == 3077 BOND_MODE_ACTIVEBACKUP) { 3078 /* in active-backup mode virtual ports are 3079 * mapped to the physical port of the active 3080 * slave */ 3081 if (bonding_info->slave.state == 3082 BOND_STATE_BACKUP) { 3083 if (port == 1) { 3084 v2p_port1 = 2; 3085 v2p_port2 = 2; 3086 } else { 3087 v2p_port1 = 1; 3088 v2p_port2 = 1; 3089 } 3090 } else { /* BOND_STATE_ACTIVE */ 3091 if (port == 1) { 3092 v2p_port1 = 1; 3093 v2p_port2 = 1; 3094 } else { 3095 v2p_port1 = 2; 3096 v2p_port2 = 2; 3097 } 3098 } 3099 } else { /* Active-Active */ 3100 /* in active-active mode a virtual port is 3101 * mapped to the native physical port if and only 3102 * if the physical port is up */ 3103 __s8 link = bonding_info->slave.link; 3104 3105 if (port == 1) 3106 v2p_port2 = 2; 3107 else 3108 v2p_port1 = 1; 3109 if ((link == BOND_LINK_UP) || 3110 (link == BOND_LINK_FAIL)) { 3111 if (port == 1) 3112 v2p_port1 = 1; 3113 else 3114 v2p_port2 = 2; 3115 } else { /* BOND_LINK_DOWN || BOND_LINK_BACK */ 3116 if (port == 1) 3117 v2p_port1 = 2; 3118 else 3119 v2p_port2 = 1; 3120 } 3121 } 3122 } 3123 } 3124 3125 mlx4_en_queue_bond_work(priv, do_bond, 3126 v2p_port1, v2p_port2); 3127 3128 return NOTIFY_DONE; 3129 } 3130 3131 void mlx4_en_update_pfc_stats_bitmap(struct mlx4_dev *dev, 3132 struct mlx4_en_stats_bitmap *stats_bitmap, 3133 u8 rx_ppp, u8 rx_pause, 3134 u8 tx_ppp, u8 tx_pause) 3135 { 3136 int last_i = NUM_MAIN_STATS + NUM_PORT_STATS + NUM_PF_STATS; 3137 3138 if (!mlx4_is_slave(dev) && 3139 (dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_FLOWSTATS_EN)) { 3140 mutex_lock(&stats_bitmap->mutex); 3141 bitmap_clear(stats_bitmap->bitmap, last_i, NUM_FLOW_STATS); 3142 3143 if (rx_ppp) 3144 bitmap_set(stats_bitmap->bitmap, last_i, 3145 NUM_FLOW_PRIORITY_STATS_RX); 3146 last_i += NUM_FLOW_PRIORITY_STATS_RX; 3147 3148 if (rx_pause && !(rx_ppp)) 3149 bitmap_set(stats_bitmap->bitmap, last_i, 3150 NUM_FLOW_STATS_RX); 3151 last_i += NUM_FLOW_STATS_RX; 3152 3153 if (tx_ppp) 3154 bitmap_set(stats_bitmap->bitmap, last_i, 3155 NUM_FLOW_PRIORITY_STATS_TX); 3156 last_i += NUM_FLOW_PRIORITY_STATS_TX; 3157 3158 if (tx_pause && !(tx_ppp)) 3159 bitmap_set(stats_bitmap->bitmap, last_i, 3160 NUM_FLOW_STATS_TX); 3161 last_i += NUM_FLOW_STATS_TX; 3162 3163 mutex_unlock(&stats_bitmap->mutex); 3164 } 3165 } 3166 3167 void mlx4_en_set_stats_bitmap(struct mlx4_dev *dev, 3168 struct mlx4_en_stats_bitmap *stats_bitmap, 3169 u8 rx_ppp, u8 rx_pause, 3170 u8 tx_ppp, u8 tx_pause) 3171 { 3172 int last_i = 0; 3173 3174 mutex_init(&stats_bitmap->mutex); 3175 bitmap_zero(stats_bitmap->bitmap, NUM_ALL_STATS); 3176 3177 if (mlx4_is_slave(dev)) { 3178 bitmap_set(stats_bitmap->bitmap, last_i + 3179 MLX4_FIND_NETDEV_STAT(rx_packets), 1); 3180 bitmap_set(stats_bitmap->bitmap, last_i + 3181 MLX4_FIND_NETDEV_STAT(tx_packets), 1); 3182 bitmap_set(stats_bitmap->bitmap, last_i + 3183 MLX4_FIND_NETDEV_STAT(rx_bytes), 1); 3184 bitmap_set(stats_bitmap->bitmap, last_i + 3185 MLX4_FIND_NETDEV_STAT(tx_bytes), 1); 3186 bitmap_set(stats_bitmap->bitmap, last_i + 3187 MLX4_FIND_NETDEV_STAT(rx_dropped), 1); 3188 bitmap_set(stats_bitmap->bitmap, last_i + 3189 MLX4_FIND_NETDEV_STAT(tx_dropped), 1); 3190 } else { 3191 bitmap_set(stats_bitmap->bitmap, last_i, NUM_MAIN_STATS); 3192 } 3193 last_i += NUM_MAIN_STATS; 3194 3195 bitmap_set(stats_bitmap->bitmap, last_i, NUM_PORT_STATS); 3196 last_i += NUM_PORT_STATS; 3197 3198 if (mlx4_is_master(dev)) 3199 bitmap_set(stats_bitmap->bitmap, last_i, 3200 NUM_PF_STATS); 3201 last_i += NUM_PF_STATS; 3202 3203 mlx4_en_update_pfc_stats_bitmap(dev, stats_bitmap, 3204 rx_ppp, rx_pause, 3205 tx_ppp, tx_pause); 3206 last_i += NUM_FLOW_STATS; 3207 3208 if (!mlx4_is_slave(dev)) 3209 bitmap_set(stats_bitmap->bitmap, last_i, NUM_PKT_STATS); 3210 last_i += NUM_PKT_STATS; 3211 3212 bitmap_set(stats_bitmap->bitmap, last_i, NUM_XDP_STATS); 3213 last_i += NUM_XDP_STATS; 3214 3215 if (!mlx4_is_slave(dev)) 3216 bitmap_set(stats_bitmap->bitmap, last_i, NUM_PHY_STATS); 3217 last_i += NUM_PHY_STATS; 3218 } 3219 3220 int mlx4_en_init_netdev(struct mlx4_en_dev *mdev, int port, 3221 struct mlx4_en_port_profile *prof) 3222 { 3223 struct net_device *dev; 3224 struct mlx4_en_priv *priv; 3225 int i, t; 3226 int err; 3227 3228 dev = alloc_etherdev_mqs(sizeof(struct mlx4_en_priv), 3229 MAX_TX_RINGS, MAX_RX_RINGS); 3230 if (dev == NULL) 3231 return -ENOMEM; 3232 3233 netif_set_real_num_tx_queues(dev, prof->tx_ring_num[TX]); 3234 netif_set_real_num_rx_queues(dev, prof->rx_ring_num); 3235 3236 SET_NETDEV_DEV(dev, &mdev->dev->persist->pdev->dev); 3237 dev->dev_port = port - 1; 3238 3239 /* 3240 * Initialize driver private data 3241 */ 3242 3243 priv = netdev_priv(dev); 3244 memset(priv, 0, sizeof(struct mlx4_en_priv)); 3245 priv->counter_index = MLX4_SINK_COUNTER_INDEX(mdev->dev); 3246 spin_lock_init(&priv->stats_lock); 3247 INIT_WORK(&priv->rx_mode_task, mlx4_en_do_set_rx_mode); 3248 INIT_WORK(&priv->watchdog_task, mlx4_en_restart); 3249 INIT_WORK(&priv->linkstate_task, mlx4_en_linkstate); 3250 INIT_DELAYED_WORK(&priv->stats_task, mlx4_en_do_get_stats); 3251 INIT_DELAYED_WORK(&priv->service_task, mlx4_en_service_task); 3252 INIT_WORK(&priv->vxlan_add_task, mlx4_en_add_vxlan_offloads); 3253 INIT_WORK(&priv->vxlan_del_task, mlx4_en_del_vxlan_offloads); 3254 #ifdef CONFIG_RFS_ACCEL 3255 INIT_LIST_HEAD(&priv->filters); 3256 spin_lock_init(&priv->filters_lock); 3257 #endif 3258 3259 priv->dev = dev; 3260 priv->mdev = mdev; 3261 priv->ddev = &mdev->pdev->dev; 3262 priv->prof = prof; 3263 priv->port = port; 3264 priv->port_up = false; 3265 priv->flags = prof->flags; 3266 priv->pflags = MLX4_EN_PRIV_FLAGS_BLUEFLAME; 3267 priv->ctrl_flags = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE | 3268 MLX4_WQE_CTRL_SOLICITED); 3269 priv->num_tx_rings_p_up = mdev->profile.max_num_tx_rings_p_up; 3270 priv->tx_work_limit = MLX4_EN_DEFAULT_TX_WORK; 3271 netdev_rss_key_fill(priv->rss_key, sizeof(priv->rss_key)); 3272 3273 for (t = 0; t < MLX4_EN_NUM_TX_TYPES; t++) { 3274 priv->tx_ring_num[t] = prof->tx_ring_num[t]; 3275 if (!priv->tx_ring_num[t]) 3276 continue; 3277 3278 priv->tx_ring[t] = kcalloc(MAX_TX_RINGS, 3279 sizeof(struct mlx4_en_tx_ring *), 3280 GFP_KERNEL); 3281 if (!priv->tx_ring[t]) { 3282 err = -ENOMEM; 3283 goto out; 3284 } 3285 priv->tx_cq[t] = kcalloc(MAX_TX_RINGS, 3286 sizeof(struct mlx4_en_cq *), 3287 GFP_KERNEL); 3288 if (!priv->tx_cq[t]) { 3289 err = -ENOMEM; 3290 goto out; 3291 } 3292 } 3293 priv->rx_ring_num = prof->rx_ring_num; 3294 priv->cqe_factor = (mdev->dev->caps.cqe_size == 64) ? 1 : 0; 3295 priv->cqe_size = mdev->dev->caps.cqe_size; 3296 priv->mac_index = -1; 3297 priv->msg_enable = MLX4_EN_MSG_LEVEL; 3298 #ifdef CONFIG_MLX4_EN_DCB 3299 if (!mlx4_is_slave(priv->mdev->dev)) { 3300 u8 prio; 3301 3302 for (prio = 0; prio < IEEE_8021QAZ_MAX_TCS; ++prio) { 3303 priv->ets.prio_tc[prio] = prio; 3304 priv->ets.tc_tsa[prio] = IEEE_8021QAZ_TSA_VENDOR; 3305 } 3306 3307 priv->dcbx_cap = DCB_CAP_DCBX_VER_CEE | DCB_CAP_DCBX_HOST | 3308 DCB_CAP_DCBX_VER_IEEE; 3309 priv->flags |= MLX4_EN_DCB_ENABLED; 3310 priv->cee_config.pfc_state = false; 3311 3312 for (i = 0; i < MLX4_EN_NUM_UP_HIGH; i++) 3313 priv->cee_config.dcb_pfc[i] = pfc_disabled; 3314 3315 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_ETS_CFG) { 3316 dev->dcbnl_ops = &mlx4_en_dcbnl_ops; 3317 } else { 3318 en_info(priv, "enabling only PFC DCB ops\n"); 3319 dev->dcbnl_ops = &mlx4_en_dcbnl_pfc_ops; 3320 } 3321 } 3322 #endif 3323 3324 for (i = 0; i < MLX4_EN_MAC_HASH_SIZE; ++i) 3325 INIT_HLIST_HEAD(&priv->mac_hash[i]); 3326 3327 /* Query for default mac and max mtu */ 3328 priv->max_mtu = mdev->dev->caps.eth_mtu_cap[priv->port]; 3329 3330 if (mdev->dev->caps.rx_checksum_flags_port[priv->port] & 3331 MLX4_RX_CSUM_MODE_VAL_NON_TCP_UDP) 3332 priv->flags |= MLX4_EN_FLAG_RX_CSUM_NON_TCP_UDP; 3333 3334 /* Set default MAC */ 3335 dev->addr_len = ETH_ALEN; 3336 mlx4_en_u64_to_mac(dev->dev_addr, mdev->dev->caps.def_mac[priv->port]); 3337 if (!is_valid_ether_addr(dev->dev_addr)) { 3338 en_err(priv, "Port: %d, invalid mac burned: %pM, quitting\n", 3339 priv->port, dev->dev_addr); 3340 err = -EINVAL; 3341 goto out; 3342 } else if (mlx4_is_slave(priv->mdev->dev) && 3343 (priv->mdev->dev->port_random_macs & 1 << priv->port)) { 3344 /* Random MAC was assigned in mlx4_slave_cap 3345 * in mlx4_core module 3346 */ 3347 dev->addr_assign_type |= NET_ADDR_RANDOM; 3348 en_warn(priv, "Assigned random MAC address %pM\n", dev->dev_addr); 3349 } 3350 3351 memcpy(priv->current_mac, dev->dev_addr, sizeof(priv->current_mac)); 3352 3353 priv->stride = roundup_pow_of_two(sizeof(struct mlx4_en_rx_desc) + 3354 DS_SIZE * MLX4_EN_MAX_RX_FRAGS); 3355 err = mlx4_en_alloc_resources(priv); 3356 if (err) 3357 goto out; 3358 3359 /* Initialize time stamping config */ 3360 priv->hwtstamp_config.flags = 0; 3361 priv->hwtstamp_config.tx_type = HWTSTAMP_TX_OFF; 3362 priv->hwtstamp_config.rx_filter = HWTSTAMP_FILTER_NONE; 3363 3364 /* Allocate page for receive rings */ 3365 err = mlx4_alloc_hwq_res(mdev->dev, &priv->res, 3366 MLX4_EN_PAGE_SIZE); 3367 if (err) { 3368 en_err(priv, "Failed to allocate page for rx qps\n"); 3369 goto out; 3370 } 3371 priv->allocated = 1; 3372 3373 /* 3374 * Initialize netdev entry points 3375 */ 3376 if (mlx4_is_master(priv->mdev->dev)) 3377 dev->netdev_ops = &mlx4_netdev_ops_master; 3378 else 3379 dev->netdev_ops = &mlx4_netdev_ops; 3380 dev->watchdog_timeo = MLX4_EN_WATCHDOG_TIMEOUT; 3381 netif_set_real_num_tx_queues(dev, priv->tx_ring_num[TX]); 3382 netif_set_real_num_rx_queues(dev, priv->rx_ring_num); 3383 3384 dev->ethtool_ops = &mlx4_en_ethtool_ops; 3385 3386 /* 3387 * Set driver features 3388 */ 3389 dev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 3390 if (mdev->LSO_support) 3391 dev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; 3392 3393 if (mdev->dev->caps.tunnel_offload_mode == 3394 MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) { 3395 dev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 3396 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3397 NETIF_F_GSO_PARTIAL; 3398 dev->features |= NETIF_F_GSO_UDP_TUNNEL | 3399 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3400 NETIF_F_GSO_PARTIAL; 3401 dev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM; 3402 dev->hw_enc_features = NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 3403 NETIF_F_RXCSUM | 3404 NETIF_F_TSO | NETIF_F_TSO6 | 3405 NETIF_F_GSO_UDP_TUNNEL | 3406 NETIF_F_GSO_UDP_TUNNEL_CSUM | 3407 NETIF_F_GSO_PARTIAL; 3408 } 3409 3410 dev->vlan_features = dev->hw_features; 3411 3412 dev->hw_features |= NETIF_F_RXCSUM | NETIF_F_RXHASH; 3413 dev->features = dev->hw_features | NETIF_F_HIGHDMA | 3414 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX | 3415 NETIF_F_HW_VLAN_CTAG_FILTER; 3416 dev->hw_features |= NETIF_F_LOOPBACK | 3417 NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_RX; 3418 3419 if (!(mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN)) { 3420 dev->features |= NETIF_F_HW_VLAN_STAG_RX | 3421 NETIF_F_HW_VLAN_STAG_FILTER; 3422 dev->hw_features |= NETIF_F_HW_VLAN_STAG_RX; 3423 } 3424 3425 if (mlx4_is_slave(mdev->dev)) { 3426 bool vlan_offload_disabled; 3427 int phv; 3428 3429 err = get_phv_bit(mdev->dev, port, &phv); 3430 if (!err && phv) { 3431 dev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; 3432 priv->pflags |= MLX4_EN_PRIV_FLAGS_PHV; 3433 } 3434 err = mlx4_get_is_vlan_offload_disabled(mdev->dev, port, 3435 &vlan_offload_disabled); 3436 if (!err && vlan_offload_disabled) { 3437 dev->hw_features &= ~(NETIF_F_HW_VLAN_CTAG_TX | 3438 NETIF_F_HW_VLAN_CTAG_RX | 3439 NETIF_F_HW_VLAN_STAG_TX | 3440 NETIF_F_HW_VLAN_STAG_RX); 3441 dev->features &= ~(NETIF_F_HW_VLAN_CTAG_TX | 3442 NETIF_F_HW_VLAN_CTAG_RX | 3443 NETIF_F_HW_VLAN_STAG_TX | 3444 NETIF_F_HW_VLAN_STAG_RX); 3445 } 3446 } else { 3447 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_PHV_EN && 3448 !(mdev->dev->caps.flags2 & 3449 MLX4_DEV_CAP_FLAG2_SKIP_OUTER_VLAN)) 3450 dev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; 3451 } 3452 3453 if (mdev->dev->caps.flags & MLX4_DEV_CAP_FLAG_FCS_KEEP) 3454 dev->hw_features |= NETIF_F_RXFCS; 3455 3456 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_IGNORE_FCS) 3457 dev->hw_features |= NETIF_F_RXALL; 3458 3459 if (mdev->dev->caps.steering_mode == 3460 MLX4_STEERING_MODE_DEVICE_MANAGED && 3461 mdev->dev->caps.dmfs_high_steer_mode != MLX4_STEERING_DMFS_A0_STATIC) 3462 dev->hw_features |= NETIF_F_NTUPLE; 3463 3464 if (mdev->dev->caps.steering_mode != MLX4_STEERING_MODE_A0) 3465 dev->priv_flags |= IFF_UNICAST_FLT; 3466 3467 /* Setting a default hash function value */ 3468 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_TOP) { 3469 priv->rss_hash_fn = ETH_RSS_HASH_TOP; 3470 } else if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_RSS_XOR) { 3471 priv->rss_hash_fn = ETH_RSS_HASH_XOR; 3472 } else { 3473 en_warn(priv, 3474 "No RSS hash capabilities exposed, using Toeplitz\n"); 3475 priv->rss_hash_fn = ETH_RSS_HASH_TOP; 3476 } 3477 3478 /* MTU range: 68 - hw-specific max */ 3479 dev->min_mtu = ETH_MIN_MTU; 3480 dev->max_mtu = priv->max_mtu; 3481 3482 mdev->pndev[port] = dev; 3483 mdev->upper[port] = NULL; 3484 3485 netif_carrier_off(dev); 3486 mlx4_en_set_default_moderation(priv); 3487 3488 en_warn(priv, "Using %d TX rings\n", prof->tx_ring_num[TX]); 3489 en_warn(priv, "Using %d RX rings\n", prof->rx_ring_num); 3490 3491 mlx4_en_update_loopback_state(priv->dev, priv->dev->features); 3492 3493 /* Configure port */ 3494 mlx4_en_calc_rx_buf(dev); 3495 err = mlx4_SET_PORT_general(mdev->dev, priv->port, 3496 priv->rx_skb_size + ETH_FCS_LEN, 3497 prof->tx_pause, prof->tx_ppp, 3498 prof->rx_pause, prof->rx_ppp); 3499 if (err) { 3500 en_err(priv, "Failed setting port general configurations for port %d, with error %d\n", 3501 priv->port, err); 3502 goto out; 3503 } 3504 3505 if (mdev->dev->caps.tunnel_offload_mode == MLX4_TUNNEL_OFFLOAD_MODE_VXLAN) { 3506 err = mlx4_SET_PORT_VXLAN(mdev->dev, priv->port, VXLAN_STEER_BY_OUTER_MAC, 1); 3507 if (err) { 3508 en_err(priv, "Failed setting port L2 tunnel configuration, err %d\n", 3509 err); 3510 goto out; 3511 } 3512 } 3513 3514 /* Init port */ 3515 en_warn(priv, "Initializing port\n"); 3516 err = mlx4_INIT_PORT(mdev->dev, priv->port); 3517 if (err) { 3518 en_err(priv, "Failed Initializing port\n"); 3519 goto out; 3520 } 3521 queue_delayed_work(mdev->workqueue, &priv->stats_task, STATS_DELAY); 3522 3523 /* Initialize time stamp mechanism */ 3524 if (mdev->dev->caps.flags2 & MLX4_DEV_CAP_FLAG2_TS) 3525 mlx4_en_init_timestamp(mdev); 3526 3527 queue_delayed_work(mdev->workqueue, &priv->service_task, 3528 SERVICE_TASK_DELAY); 3529 3530 mlx4_en_set_stats_bitmap(mdev->dev, &priv->stats_bitmap, 3531 mdev->profile.prof[priv->port].rx_ppp, 3532 mdev->profile.prof[priv->port].rx_pause, 3533 mdev->profile.prof[priv->port].tx_ppp, 3534 mdev->profile.prof[priv->port].tx_pause); 3535 3536 err = register_netdev(dev); 3537 if (err) { 3538 en_err(priv, "Netdev registration failed for port %d\n", port); 3539 goto out; 3540 } 3541 3542 priv->registered = 1; 3543 devlink_port_type_eth_set(mlx4_get_devlink_port(mdev->dev, priv->port), 3544 dev); 3545 3546 return 0; 3547 3548 out: 3549 mlx4_en_destroy_netdev(dev); 3550 return err; 3551 } 3552 3553 int mlx4_en_reset_config(struct net_device *dev, 3554 struct hwtstamp_config ts_config, 3555 netdev_features_t features) 3556 { 3557 struct mlx4_en_priv *priv = netdev_priv(dev); 3558 struct mlx4_en_dev *mdev = priv->mdev; 3559 struct mlx4_en_port_profile new_prof; 3560 struct mlx4_en_priv *tmp; 3561 int port_up = 0; 3562 int err = 0; 3563 3564 if (priv->hwtstamp_config.tx_type == ts_config.tx_type && 3565 priv->hwtstamp_config.rx_filter == ts_config.rx_filter && 3566 !DEV_FEATURE_CHANGED(dev, features, NETIF_F_HW_VLAN_CTAG_RX) && 3567 !DEV_FEATURE_CHANGED(dev, features, NETIF_F_RXFCS)) 3568 return 0; /* Nothing to change */ 3569 3570 if (DEV_FEATURE_CHANGED(dev, features, NETIF_F_HW_VLAN_CTAG_RX) && 3571 (features & NETIF_F_HW_VLAN_CTAG_RX) && 3572 (priv->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE)) { 3573 en_warn(priv, "Can't turn ON rx vlan offload while time-stamping rx filter is ON\n"); 3574 return -EINVAL; 3575 } 3576 3577 tmp = kzalloc(sizeof(*tmp), GFP_KERNEL); 3578 if (!tmp) 3579 return -ENOMEM; 3580 3581 mutex_lock(&mdev->state_lock); 3582 3583 memcpy(&new_prof, priv->prof, sizeof(struct mlx4_en_port_profile)); 3584 memcpy(&new_prof.hwtstamp_config, &ts_config, sizeof(ts_config)); 3585 3586 err = mlx4_en_try_alloc_resources(priv, tmp, &new_prof, true); 3587 if (err) 3588 goto out; 3589 3590 if (priv->port_up) { 3591 port_up = 1; 3592 mlx4_en_stop_port(dev, 1); 3593 } 3594 3595 mlx4_en_safe_replace_resources(priv, tmp); 3596 3597 if (DEV_FEATURE_CHANGED(dev, features, NETIF_F_HW_VLAN_CTAG_RX)) { 3598 if (features & NETIF_F_HW_VLAN_CTAG_RX) 3599 dev->features |= NETIF_F_HW_VLAN_CTAG_RX; 3600 else 3601 dev->features &= ~NETIF_F_HW_VLAN_CTAG_RX; 3602 } else if (ts_config.rx_filter == HWTSTAMP_FILTER_NONE) { 3603 /* RX time-stamping is OFF, update the RX vlan offload 3604 * to the latest wanted state 3605 */ 3606 if (dev->wanted_features & NETIF_F_HW_VLAN_CTAG_RX) 3607 dev->features |= NETIF_F_HW_VLAN_CTAG_RX; 3608 else 3609 dev->features &= ~NETIF_F_HW_VLAN_CTAG_RX; 3610 } 3611 3612 if (DEV_FEATURE_CHANGED(dev, features, NETIF_F_RXFCS)) { 3613 if (features & NETIF_F_RXFCS) 3614 dev->features |= NETIF_F_RXFCS; 3615 else 3616 dev->features &= ~NETIF_F_RXFCS; 3617 } 3618 3619 /* RX vlan offload and RX time-stamping can't co-exist ! 3620 * Regardless of the caller's choice, 3621 * Turn Off RX vlan offload in case of time-stamping is ON 3622 */ 3623 if (ts_config.rx_filter != HWTSTAMP_FILTER_NONE) { 3624 if (dev->features & NETIF_F_HW_VLAN_CTAG_RX) 3625 en_warn(priv, "Turning off RX vlan offload since RX time-stamping is ON\n"); 3626 dev->features &= ~NETIF_F_HW_VLAN_CTAG_RX; 3627 } 3628 3629 if (port_up) { 3630 err = mlx4_en_start_port(dev); 3631 if (err) 3632 en_err(priv, "Failed starting port\n"); 3633 } 3634 3635 out: 3636 mutex_unlock(&mdev->state_lock); 3637 kfree(tmp); 3638 if (!err) 3639 netdev_features_change(dev); 3640 return err; 3641 } 3642